differential-coverage 1.1.2__tar.gz → 1.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/PKG-INFO +29 -6
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/README.md +28 -5
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/pyproject.toml +1 -1
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/cli.py +24 -16
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/output.py +36 -11
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/readers/__init__.py +2 -2
- differential_coverage-1.2.1/src/differential_coverage/readers/libfuzzer_merge.py +41 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/readers/registry.py +4 -2
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/PKG-INFO +29 -6
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/SOURCES.txt +2 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_cli.py +12 -0
- differential_coverage-1.2.1/tests/test_libfuzzer_merge.py +121 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_llvm_cov.py +46 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_output.py +29 -2
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_readers.py +27 -4
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/setup.cfg +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/__init__.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/api.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/approach_data.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/fs.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/readers/afl_showmap.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/readers/llvm_cov.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/types.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/dependency_links.txt +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/entry_points.txt +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/requires.txt +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/top_level.txt +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_api.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_approach_data.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_docs.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_reducers.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: differential_coverage
|
|
3
|
-
Version: 1.1
|
|
3
|
+
Version: 1.2.1
|
|
4
4
|
Summary: Compute differential coverage, a better way of comparing testing tools.
|
|
5
5
|
Author-email: Valentin Huber <contact@valentinhuber.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -43,7 +43,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
|
|
|
43
43
|
|
|
44
44
|
## How?
|
|
45
45
|
|
|
46
|
-
Campaign data lives in one directory per approach, with one file per trial.
|
|
46
|
+
Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
|
|
47
|
+
|
|
48
|
+
**All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
|
|
49
|
+
|
|
50
|
+
For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
|
|
47
51
|
|
|
48
52
|
### afl-showmap
|
|
49
53
|
|
|
@@ -88,6 +92,8 @@ llvm-cov export \
|
|
|
88
92
|
|
|
89
93
|
Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
|
|
90
94
|
|
|
95
|
+
On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
|
|
96
|
+
|
|
91
97
|
**All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
|
|
92
98
|
|
|
93
99
|
#### cov-analysis
|
|
@@ -101,11 +107,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
|
|
|
101
107
|
|
|
102
108
|
Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
|
|
103
109
|
|
|
104
|
-
###
|
|
110
|
+
### libFuzzer merge control files
|
|
111
|
+
|
|
112
|
+
Same layout, with merge control files from libFuzzer's `-merge=1` mode:
|
|
105
113
|
|
|
106
|
-
|
|
114
|
+
```
|
|
115
|
+
coverage_data
|
|
116
|
+
|-- approach_1
|
|
117
|
+
| |-- trial_1.merge
|
|
118
|
+
| `-- trial_2.merge
|
|
119
|
+
`-- seeds
|
|
120
|
+
`-- seeds.merge
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
empty=$(mktemp -d)
|
|
127
|
+
./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
|
|
128
|
+
rmdir "$empty"
|
|
129
|
+
```
|
|
107
130
|
|
|
108
|
-
|
|
131
|
+
Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
|
|
109
132
|
|
|
110
133
|
### Installation
|
|
111
134
|
|
|
@@ -123,7 +146,7 @@ pip install differential-coverage[latex]
|
|
|
123
146
|
Generally, the command line interface follows the following structure:
|
|
124
147
|
|
|
125
148
|
```bash
|
|
126
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
149
|
+
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
127
150
|
```
|
|
128
151
|
|
|
129
152
|
Examples:
|
|
@@ -19,7 +19,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
|
|
|
19
19
|
|
|
20
20
|
## How?
|
|
21
21
|
|
|
22
|
-
Campaign data lives in one directory per approach, with one file per trial.
|
|
22
|
+
Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
|
|
23
|
+
|
|
24
|
+
**All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
|
|
25
|
+
|
|
26
|
+
For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
|
|
23
27
|
|
|
24
28
|
### afl-showmap
|
|
25
29
|
|
|
@@ -64,6 +68,8 @@ llvm-cov export \
|
|
|
64
68
|
|
|
65
69
|
Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
|
|
66
70
|
|
|
71
|
+
On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
|
|
72
|
+
|
|
67
73
|
**All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
|
|
68
74
|
|
|
69
75
|
#### cov-analysis
|
|
@@ -77,11 +83,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
|
|
|
77
83
|
|
|
78
84
|
Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
|
|
79
85
|
|
|
80
|
-
###
|
|
86
|
+
### libFuzzer merge control files
|
|
87
|
+
|
|
88
|
+
Same layout, with merge control files from libFuzzer's `-merge=1` mode:
|
|
81
89
|
|
|
82
|
-
|
|
90
|
+
```
|
|
91
|
+
coverage_data
|
|
92
|
+
|-- approach_1
|
|
93
|
+
| |-- trial_1.merge
|
|
94
|
+
| `-- trial_2.merge
|
|
95
|
+
`-- seeds
|
|
96
|
+
`-- seeds.merge
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
empty=$(mktemp -d)
|
|
103
|
+
./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
|
|
104
|
+
rmdir "$empty"
|
|
105
|
+
```
|
|
83
106
|
|
|
84
|
-
|
|
107
|
+
Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
|
|
85
108
|
|
|
86
109
|
### Installation
|
|
87
110
|
|
|
@@ -99,7 +122,7 @@ pip install differential-coverage[latex]
|
|
|
99
122
|
Generally, the command line interface follows the following structure:
|
|
100
123
|
|
|
101
124
|
```bash
|
|
102
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
125
|
+
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
103
126
|
```
|
|
104
127
|
|
|
105
128
|
Examples:
|
{differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/cli.py
RENAMED
|
@@ -12,7 +12,8 @@ from differential_coverage.output import print_relcov_corpus_table, print_scores
|
|
|
12
12
|
# All subcommands expect this directory layout (one campaign dir, no moving files).
|
|
13
13
|
CAMPAIGN_DIR_HELP = (
|
|
14
14
|
"Campaign directory: one subdirectory per approach, each containing "
|
|
15
|
-
"trial coverage files (afl-showmap id:count files
|
|
15
|
+
"trial coverage files (afl-showmap id:count files, llvm-cov export JSON, "
|
|
16
|
+
"or libFuzzer merge control files)."
|
|
16
17
|
)
|
|
17
18
|
|
|
18
19
|
try:
|
|
@@ -86,8 +87,9 @@ def cmd_relscore(args: argparse.Namespace) -> int:
|
|
|
86
87
|
print_scores(
|
|
87
88
|
scores,
|
|
88
89
|
output=args.output,
|
|
89
|
-
colormap=
|
|
90
|
-
latex_enable_color=
|
|
90
|
+
colormap=args.colormap,
|
|
91
|
+
latex_enable_color=args.latex_enable_color,
|
|
92
|
+
digits=args.digits,
|
|
91
93
|
)
|
|
92
94
|
return 0
|
|
93
95
|
|
|
@@ -96,11 +98,6 @@ def cmd_relcov_performance_over_approach(args: argparse.Namespace) -> int:
|
|
|
96
98
|
campaign = _load_campaign(args)
|
|
97
99
|
|
|
98
100
|
dc = DifferentialCoverage(campaign)
|
|
99
|
-
output = args.output
|
|
100
|
-
colormap = getattr(args, "colormap", "viridis")
|
|
101
|
-
latex_rotate_headers = getattr(args, "latex_rotate_headers", None)
|
|
102
|
-
latex_enable_color = getattr(args, "latex_enable_color", False)
|
|
103
|
-
|
|
104
101
|
ref_approaches = list(dc.approaches.keys())
|
|
105
102
|
table = {
|
|
106
103
|
f: {ref: dc.approaches[f].relcov(dc.approaches[ref]) for ref in ref_approaches}
|
|
@@ -109,10 +106,11 @@ def cmd_relcov_performance_over_approach(args: argparse.Namespace) -> int:
|
|
|
109
106
|
print_relcov_corpus_table(
|
|
110
107
|
ref_approaches,
|
|
111
108
|
table,
|
|
112
|
-
output=output,
|
|
113
|
-
colormap=colormap,
|
|
114
|
-
latex_rotate_headers=latex_rotate_headers,
|
|
115
|
-
latex_enable_color=latex_enable_color,
|
|
109
|
+
output=args.output,
|
|
110
|
+
colormap=args.colormap,
|
|
111
|
+
latex_rotate_headers=args.latex_rotate_headers,
|
|
112
|
+
latex_enable_color=args.latex_enable_color,
|
|
113
|
+
digits=args.digits,
|
|
116
114
|
)
|
|
117
115
|
return 0
|
|
118
116
|
|
|
@@ -133,11 +131,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
133
131
|
|
|
134
132
|
parser.add_argument(
|
|
135
133
|
"--input-format",
|
|
136
|
-
choices=["auto", "afl-showmap", "llvm-cov"],
|
|
134
|
+
choices=["auto", "afl-showmap", "llvm-cov", "libfuzzer-merge"],
|
|
137
135
|
default="auto",
|
|
138
136
|
help=(
|
|
139
137
|
"Input format for trial files: auto (default) detects from file "
|
|
140
|
-
"
|
|
138
|
+
"content; afl-showmap for id:count files; llvm-cov for export JSON; "
|
|
139
|
+
"libfuzzer-merge for -merge=1 control files."
|
|
141
140
|
),
|
|
142
141
|
)
|
|
143
142
|
parser.add_argument(
|
|
@@ -155,8 +154,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
155
154
|
default="auto",
|
|
156
155
|
help=(
|
|
157
156
|
"Coverage granularity: auto (default) uses edge for afl-showmap and "
|
|
158
|
-
"branch for llvm-cov; branch/block for llvm-cov
|
|
159
|
-
"afl-showmap bitmap tuples."
|
|
157
|
+
"libfuzzer-merge, branch for llvm-cov; branch/block for llvm-cov "
|
|
158
|
+
"exports; edge for afl-showmap bitmap tuples and libFuzzer COV lines."
|
|
160
159
|
),
|
|
161
160
|
)
|
|
162
161
|
parser.add_argument(
|
|
@@ -190,6 +189,15 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
190
189
|
'latex for LaTeX tabular (requires "latex" optional dependencies)'
|
|
191
190
|
),
|
|
192
191
|
)
|
|
192
|
+
parser.add_argument(
|
|
193
|
+
"--digits",
|
|
194
|
+
type=int,
|
|
195
|
+
default=3,
|
|
196
|
+
metavar="N",
|
|
197
|
+
help=(
|
|
198
|
+
"Number of digits after the decimal point for numeric results (default: 3)."
|
|
199
|
+
),
|
|
200
|
+
)
|
|
193
201
|
parser.add_argument(
|
|
194
202
|
"--latex-rotate-headers",
|
|
195
203
|
type=float,
|
{differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/output.py
RENAMED
|
@@ -30,12 +30,22 @@ class _PrinterIO:
|
|
|
30
30
|
return len(s)
|
|
31
31
|
|
|
32
32
|
|
|
33
|
+
# Default decimal places for numeric output.
|
|
34
|
+
_DEFAULT_DIGITS = 3
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _format_float(value: float, digits: int) -> str:
|
|
38
|
+
"""Format a float with a fixed number of digits after the decimal point."""
|
|
39
|
+
return f"{value:.{digits}f}"
|
|
40
|
+
|
|
41
|
+
|
|
33
42
|
def print_scores(
|
|
34
43
|
scores: Mapping[ApproachId, float],
|
|
35
44
|
*,
|
|
36
45
|
output: OutputFormat = "stdout",
|
|
37
46
|
colormap: str = "viridis",
|
|
38
47
|
latex_enable_color: bool = False,
|
|
48
|
+
digits: int = _DEFAULT_DIGITS,
|
|
39
49
|
printer: Callable[[str], None] = print,
|
|
40
50
|
) -> None:
|
|
41
51
|
"""Print one score per approach, in the requested output format."""
|
|
@@ -43,14 +53,15 @@ def print_scores(
|
|
|
43
53
|
performance_sorted = sorted(name_sorted, key=lambda x: x[1], reverse=True)
|
|
44
54
|
|
|
45
55
|
if output == "stdout":
|
|
46
|
-
_print_scores_plain(performance_sorted, printer=printer)
|
|
56
|
+
_print_scores_plain(performance_sorted, digits=digits, printer=printer)
|
|
47
57
|
elif output == "csv":
|
|
48
|
-
_print_scores_csv(performance_sorted, printer=printer)
|
|
58
|
+
_print_scores_csv(performance_sorted, digits=digits, printer=printer)
|
|
49
59
|
elif output == "latex":
|
|
50
60
|
_print_scores_latex(
|
|
51
61
|
performance_sorted,
|
|
52
62
|
enable_color=latex_enable_color,
|
|
53
63
|
colormap=colormap,
|
|
64
|
+
digits=digits,
|
|
54
65
|
printer=printer,
|
|
55
66
|
)
|
|
56
67
|
else:
|
|
@@ -60,21 +71,23 @@ def print_scores(
|
|
|
60
71
|
def _print_scores_plain(
|
|
61
72
|
performance_sorted: Sequence[tuple[ApproachId, float]],
|
|
62
73
|
*,
|
|
74
|
+
digits: int,
|
|
63
75
|
printer: Callable[[str], None],
|
|
64
76
|
) -> None:
|
|
65
77
|
for approach, score in performance_sorted:
|
|
66
|
-
printer(f"{approach}: {score
|
|
78
|
+
printer(f"{approach}: {_format_float(score, digits)}")
|
|
67
79
|
|
|
68
80
|
|
|
69
81
|
def _print_scores_csv(
|
|
70
82
|
performance_sorted: Sequence[tuple[ApproachId, float]],
|
|
71
83
|
*,
|
|
84
|
+
digits: int,
|
|
72
85
|
printer: Callable[[str], None],
|
|
73
86
|
) -> None:
|
|
74
87
|
writer = csv.writer(_PrinterIO(printer))
|
|
75
88
|
writer.writerow(("approach", "score"))
|
|
76
89
|
for approach, score in performance_sorted:
|
|
77
|
-
writer.writerow((approach,
|
|
90
|
+
writer.writerow((approach, _format_float(score, digits)))
|
|
78
91
|
|
|
79
92
|
|
|
80
93
|
def _norm_minmax(values: Sequence[float]) -> tuple[float, float]:
|
|
@@ -96,6 +109,7 @@ def _print_scores_latex(
|
|
|
96
109
|
*,
|
|
97
110
|
enable_color: bool = False,
|
|
98
111
|
colormap: str = "viridis",
|
|
112
|
+
digits: int,
|
|
99
113
|
printer: Callable[[str], None],
|
|
100
114
|
) -> None:
|
|
101
115
|
"""LaTeX table (optionally with score cells colored by value)."""
|
|
@@ -110,12 +124,13 @@ def _print_scores_latex(
|
|
|
110
124
|
values = [s for _, s in performance_sorted]
|
|
111
125
|
min_v, max_v = _norm_minmax(values)
|
|
112
126
|
for approach, score in performance_sorted:
|
|
127
|
+
formatted = _format_float(score, digits)
|
|
113
128
|
if enable_color:
|
|
114
129
|
n = _norm_value(score, min_v, max_v)
|
|
115
130
|
hex_color = _colormap_light_hex(n, colormap=colormap)
|
|
116
|
-
printer(rf"{approach} & \cellcolor[HTML]{{{hex_color}}}{{{
|
|
131
|
+
printer(rf"{approach} & \cellcolor[HTML]{{{hex_color}}}{{{formatted}}} \\")
|
|
117
132
|
else:
|
|
118
|
-
printer(f"{approach} & {
|
|
133
|
+
printer(f"{approach} & {formatted} \\\\")
|
|
119
134
|
printer(r"\end{tabular}")
|
|
120
135
|
|
|
121
136
|
|
|
@@ -127,6 +142,7 @@ def print_relcov_corpus_table(
|
|
|
127
142
|
colormap: str = "viridis",
|
|
128
143
|
latex_rotate_headers: Optional[float] = None,
|
|
129
144
|
latex_enable_color: bool = False,
|
|
145
|
+
digits: int = _DEFAULT_DIGITS,
|
|
130
146
|
printer: Callable[[str], None] = print,
|
|
131
147
|
) -> None:
|
|
132
148
|
"""Print a table of relcov performance: rows = approaches, columns = corpus approaches."""
|
|
@@ -138,12 +154,14 @@ def print_relcov_corpus_table(
|
|
|
138
154
|
_print_relcov_corpus_table_plain(
|
|
139
155
|
corpus_approaches,
|
|
140
156
|
table,
|
|
157
|
+
digits=digits,
|
|
141
158
|
printer=printer,
|
|
142
159
|
)
|
|
143
160
|
elif output == "csv":
|
|
144
161
|
_print_relcov_corpus_table_csv(
|
|
145
162
|
corpus_approaches,
|
|
146
163
|
table,
|
|
164
|
+
digits=digits,
|
|
147
165
|
printer=printer,
|
|
148
166
|
)
|
|
149
167
|
elif output == "latex":
|
|
@@ -153,6 +171,7 @@ def print_relcov_corpus_table(
|
|
|
153
171
|
rotate_headers=latex_rotate_headers,
|
|
154
172
|
enable_color=latex_enable_color,
|
|
155
173
|
colormap=colormap,
|
|
174
|
+
digits=digits,
|
|
156
175
|
printer=printer,
|
|
157
176
|
)
|
|
158
177
|
else:
|
|
@@ -163,6 +182,7 @@ def _print_relcov_corpus_table_plain(
|
|
|
163
182
|
corpus_approaches: Sequence[ApproachId],
|
|
164
183
|
table: Mapping[ApproachId, Mapping[ApproachId, float]],
|
|
165
184
|
*,
|
|
185
|
+
digits: int,
|
|
166
186
|
printer: Callable[[str], None],
|
|
167
187
|
) -> None:
|
|
168
188
|
row_labels = sorted(table.keys())
|
|
@@ -171,7 +191,8 @@ def _print_relcov_corpus_table_plain(
|
|
|
171
191
|
(len(str(c)) for c in col_labels + ["approach"]),
|
|
172
192
|
default=7,
|
|
173
193
|
)
|
|
174
|
-
|
|
194
|
+
# Width for aligned numeric columns: sign + integer digit + '.' + fraction + padding.
|
|
195
|
+
num_width = max(10, digits + 4)
|
|
175
196
|
header = "approach".ljust(col_width)
|
|
176
197
|
for c in col_labels:
|
|
177
198
|
header += str(c).rjust(num_width)
|
|
@@ -181,7 +202,7 @@ def _print_relcov_corpus_table_plain(
|
|
|
181
202
|
for c in col_labels:
|
|
182
203
|
val = table[row].get(c)
|
|
183
204
|
if val is not None:
|
|
184
|
-
line += f"{val:>{num_width}.
|
|
205
|
+
line += f"{val:>{num_width}.{digits}f}"
|
|
185
206
|
else:
|
|
186
207
|
line += " " * num_width
|
|
187
208
|
printer(line)
|
|
@@ -191,6 +212,7 @@ def _print_relcov_corpus_table_csv(
|
|
|
191
212
|
corpus_approaches: Sequence[ApproachId],
|
|
192
213
|
table: Mapping[ApproachId, Mapping[ApproachId, float]],
|
|
193
214
|
*,
|
|
215
|
+
digits: int,
|
|
194
216
|
printer: Callable[[str], None],
|
|
195
217
|
) -> None:
|
|
196
218
|
row_labels = sorted(table.keys())
|
|
@@ -201,7 +223,7 @@ def _print_relcov_corpus_table_csv(
|
|
|
201
223
|
cells: list[str] = [str(row)]
|
|
202
224
|
for c in col_labels:
|
|
203
225
|
val = table[row].get(c)
|
|
204
|
-
cells.append(
|
|
226
|
+
cells.append(_format_float(val, digits) if val is not None else "")
|
|
205
227
|
writer.writerow(cells)
|
|
206
228
|
|
|
207
229
|
|
|
@@ -255,6 +277,7 @@ def _print_relcov_corpus_table_latex(
|
|
|
255
277
|
rotate_headers: Optional[float] = None,
|
|
256
278
|
enable_color: bool = False,
|
|
257
279
|
colormap: str = "viridis",
|
|
280
|
+
digits: int,
|
|
258
281
|
printer: Callable[[str], None],
|
|
259
282
|
) -> None:
|
|
260
283
|
try:
|
|
@@ -288,9 +311,11 @@ def _print_relcov_corpus_table_latex(
|
|
|
288
311
|
elif enable_color:
|
|
289
312
|
n = _norm_value(val, min_v, max_v)
|
|
290
313
|
hex_color = _colormap_light_hex(n, colormap=colormap)
|
|
291
|
-
cells.append(
|
|
314
|
+
cells.append(
|
|
315
|
+
rf"\cellcolor[HTML]{{{hex_color}}}{{{_format_float(val, digits)}}}"
|
|
316
|
+
)
|
|
292
317
|
else:
|
|
293
|
-
cells.append(
|
|
318
|
+
cells.append(_format_float(val, digits))
|
|
294
319
|
printer("\t" + " & ".join(cells) + r" \\")
|
|
295
320
|
printer(r"\end{tabular}")
|
|
296
321
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from differential_coverage.readers import afl_showmap, llvm_cov
|
|
1
|
+
from differential_coverage.readers import afl_showmap, libfuzzer_merge, llvm_cov
|
|
2
2
|
from differential_coverage.readers.registry import (
|
|
3
3
|
Granularity,
|
|
4
4
|
GranularityArg,
|
|
@@ -15,4 +15,4 @@ __all__ = [
|
|
|
15
15
|
"resolve_reader",
|
|
16
16
|
]
|
|
17
17
|
|
|
18
|
-
_ = (afl_showmap, llvm_cov)
|
|
18
|
+
_ = (afl_showmap, libfuzzer_merge, llvm_cov)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from differential_coverage.readers.registry import (
|
|
4
|
+
Granularity,
|
|
5
|
+
TrialReader,
|
|
6
|
+
register_reader,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def read(path: Path, *, granularity: Granularity) -> set[str]:
|
|
11
|
+
if granularity != "edge":
|
|
12
|
+
raise ValueError("libfuzzer-merge only supports --granularity edge")
|
|
13
|
+
edges: set[str] = set()
|
|
14
|
+
for line in path.read_text().splitlines():
|
|
15
|
+
parts = line.split()
|
|
16
|
+
if len(parts) < 3 or parts[0] != "COV":
|
|
17
|
+
continue
|
|
18
|
+
edges.update(parts[2:])
|
|
19
|
+
if not edges:
|
|
20
|
+
raise ValueError(f"No covered edges in {path}")
|
|
21
|
+
return edges
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def detect(path: Path) -> bool:
|
|
25
|
+
lines = [line.strip() for line in path.read_text().splitlines() if line.strip()]
|
|
26
|
+
if len(lines) < 3:
|
|
27
|
+
return False
|
|
28
|
+
try:
|
|
29
|
+
num_files = int(lines[0])
|
|
30
|
+
num_first = int(lines[1])
|
|
31
|
+
except ValueError:
|
|
32
|
+
return False
|
|
33
|
+
if num_files <= 0 or num_first > num_files:
|
|
34
|
+
return False
|
|
35
|
+
has_marker = any(
|
|
36
|
+
line.startswith("STARTED ") or line.startswith("COV ") for line in lines
|
|
37
|
+
)
|
|
38
|
+
return has_marker
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
register_reader(TrialReader(name="libfuzzer-merge", read=read, detect=detect))
|
|
@@ -3,18 +3,20 @@ from dataclasses import dataclass
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import Literal, Protocol
|
|
5
5
|
|
|
6
|
-
ReaderName = Literal["afl-showmap", "llvm-cov"]
|
|
7
|
-
InputFormat = Literal["auto", "afl-showmap", "llvm-cov"]
|
|
6
|
+
ReaderName = Literal["afl-showmap", "llvm-cov", "libfuzzer-merge"]
|
|
7
|
+
InputFormat = Literal["auto", "afl-showmap", "llvm-cov", "libfuzzer-merge"]
|
|
8
8
|
Granularity = Literal["branch", "block", "edge"]
|
|
9
9
|
GranularityArg = Literal["auto", "branch", "block", "edge"]
|
|
10
10
|
|
|
11
11
|
_SUPPORTED: dict[ReaderName, frozenset[Granularity]] = {
|
|
12
12
|
"afl-showmap": frozenset({"edge"}),
|
|
13
13
|
"llvm-cov": frozenset({"branch", "block"}),
|
|
14
|
+
"libfuzzer-merge": frozenset({"edge"}),
|
|
14
15
|
}
|
|
15
16
|
_DEFAULT: dict[ReaderName, Granularity] = {
|
|
16
17
|
"afl-showmap": "edge",
|
|
17
18
|
"llvm-cov": "branch",
|
|
19
|
+
"libfuzzer-merge": "edge",
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: differential_coverage
|
|
3
|
-
Version: 1.1
|
|
3
|
+
Version: 1.2.1
|
|
4
4
|
Summary: Compute differential coverage, a better way of comparing testing tools.
|
|
5
5
|
Author-email: Valentin Huber <contact@valentinhuber.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -43,7 +43,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
|
|
|
43
43
|
|
|
44
44
|
## How?
|
|
45
45
|
|
|
46
|
-
Campaign data lives in one directory per approach, with one file per trial.
|
|
46
|
+
Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
|
|
47
|
+
|
|
48
|
+
**All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
|
|
49
|
+
|
|
50
|
+
For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
|
|
47
51
|
|
|
48
52
|
### afl-showmap
|
|
49
53
|
|
|
@@ -88,6 +92,8 @@ llvm-cov export \
|
|
|
88
92
|
|
|
89
93
|
Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
|
|
90
94
|
|
|
95
|
+
On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
|
|
96
|
+
|
|
91
97
|
**All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
|
|
92
98
|
|
|
93
99
|
#### cov-analysis
|
|
@@ -101,11 +107,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
|
|
|
101
107
|
|
|
102
108
|
Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
|
|
103
109
|
|
|
104
|
-
###
|
|
110
|
+
### libFuzzer merge control files
|
|
111
|
+
|
|
112
|
+
Same layout, with merge control files from libFuzzer's `-merge=1` mode:
|
|
105
113
|
|
|
106
|
-
|
|
114
|
+
```
|
|
115
|
+
coverage_data
|
|
116
|
+
|-- approach_1
|
|
117
|
+
| |-- trial_1.merge
|
|
118
|
+
| `-- trial_2.merge
|
|
119
|
+
`-- seeds
|
|
120
|
+
`-- seeds.merge
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
empty=$(mktemp -d)
|
|
127
|
+
./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
|
|
128
|
+
rmdir "$empty"
|
|
129
|
+
```
|
|
107
130
|
|
|
108
|
-
|
|
131
|
+
Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
|
|
109
132
|
|
|
110
133
|
### Installation
|
|
111
134
|
|
|
@@ -123,7 +146,7 @@ pip install differential-coverage[latex]
|
|
|
123
146
|
Generally, the command line interface follows the following structure:
|
|
124
147
|
|
|
125
148
|
```bash
|
|
126
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
149
|
+
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
127
150
|
```
|
|
128
151
|
|
|
129
152
|
Examples:
|
|
@@ -15,12 +15,14 @@ src/differential_coverage.egg-info/requires.txt
|
|
|
15
15
|
src/differential_coverage.egg-info/top_level.txt
|
|
16
16
|
src/differential_coverage/readers/__init__.py
|
|
17
17
|
src/differential_coverage/readers/afl_showmap.py
|
|
18
|
+
src/differential_coverage/readers/libfuzzer_merge.py
|
|
18
19
|
src/differential_coverage/readers/llvm_cov.py
|
|
19
20
|
src/differential_coverage/readers/registry.py
|
|
20
21
|
tests/test_api.py
|
|
21
22
|
tests/test_approach_data.py
|
|
22
23
|
tests/test_cli.py
|
|
23
24
|
tests/test_docs.py
|
|
25
|
+
tests/test_libfuzzer_merge.py
|
|
24
26
|
tests/test_llvm_cov.py
|
|
25
27
|
tests/test_output.py
|
|
26
28
|
tests/test_readers.py
|
|
@@ -275,3 +275,15 @@ def test_cli_latex_color_no_cell_colors() -> None:
|
|
|
275
275
|
assert any(line.startswith(r"\begin{tabular}") for line in lines)
|
|
276
276
|
# Table should not contain any colored cells
|
|
277
277
|
assert not any(r"\cellcolor" in line for line in lines)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def test_cli_digits_relscore() -> None:
|
|
281
|
+
"""CLI --digits controls decimal places in relscore output."""
|
|
282
|
+
code, out, _ = _run_cli(["--digits", "4", "relscore", str(SAMPLE_SHOWMAP_DIR)])
|
|
283
|
+
assert code == 0
|
|
284
|
+
lines = [s.strip() for s in out.strip().splitlines()]
|
|
285
|
+
# Each score should have exactly 4 digits after the decimal point
|
|
286
|
+
for line in lines:
|
|
287
|
+
score = line.split(":", 1)[1].strip()
|
|
288
|
+
assert "." in score
|
|
289
|
+
assert len(score.split(".", 1)[1]) == 4
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Unit tests for libFuzzer merge control file input."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import re
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from differential_coverage.api import DifferentialCoverage
|
|
11
|
+
from differential_coverage.cli import main
|
|
12
|
+
from differential_coverage.fs import read_approach_dir, read_campaign_dir
|
|
13
|
+
from differential_coverage.readers import libfuzzer_merge
|
|
14
|
+
from differential_coverage.readers.registry import detect_reader, resolve_granularity
|
|
15
|
+
|
|
16
|
+
_DONE_COV_RE = re.compile(r"^\#\d+\s+DONE\s+cov:\s+(\d+)", re.MULTILINE)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _run_cli(argv: list[str]) -> tuple[int | str, str]:
|
|
20
|
+
out = io.StringIO()
|
|
21
|
+
old_stdout = sys.stdout
|
|
22
|
+
try:
|
|
23
|
+
sys.stdout = out
|
|
24
|
+
code = main(argv)
|
|
25
|
+
except SystemExit as exc:
|
|
26
|
+
code = int(exc.code) if exc.code is not None else 1
|
|
27
|
+
finally:
|
|
28
|
+
sys.stdout = old_stdout
|
|
29
|
+
return code, out.getvalue()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _calc_merge(calc_merge_dir: Path, *parts: str) -> Path:
|
|
33
|
+
return calc_merge_dir.joinpath(*parts)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _parse_libfuzzer_done_cov(log_text: str) -> int:
|
|
37
|
+
matches = _DONE_COV_RE.findall(log_text)
|
|
38
|
+
if not matches:
|
|
39
|
+
raise ValueError("no DONE cov line in libFuzzer merge output")
|
|
40
|
+
return int(matches[-1])
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_read_cov_matches_libfuzzer_merge_output(
|
|
44
|
+
calc_build_dir: Path, calc_merge_dir: Path
|
|
45
|
+
) -> None:
|
|
46
|
+
merge_logs = calc_build_dir / "build" / "merge_logs"
|
|
47
|
+
for merge_file in sorted(calc_merge_dir.rglob("*.merge")):
|
|
48
|
+
rel = merge_file.relative_to(calc_merge_dir)
|
|
49
|
+
log_file = merge_logs / rel.parent / f"{merge_file.stem}.log"
|
|
50
|
+
assert log_file.is_file(), f"missing libFuzzer log for {merge_file}"
|
|
51
|
+
expected_cov = _parse_libfuzzer_done_cov(log_file.read_text())
|
|
52
|
+
edges = libfuzzer_merge.read(merge_file, granularity="edge")
|
|
53
|
+
assert len(edges) == expected_cov, (
|
|
54
|
+
f"{rel}: reader found {len(edges)} edges, "
|
|
55
|
+
f"libFuzzer reported cov: {expected_cov}"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_read_detect_and_read(calc_merge_dir: Path) -> None:
|
|
60
|
+
merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
|
|
61
|
+
assert libfuzzer_merge.detect(merge_file)
|
|
62
|
+
edges = libfuzzer_merge.read(merge_file, granularity="edge")
|
|
63
|
+
assert edges
|
|
64
|
+
assert all(edge.isdigit() for edge in edges)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_detect_reader(calc_merge_dir: Path) -> None:
|
|
68
|
+
merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
|
|
69
|
+
assert detect_reader(merge_file) == "libfuzzer-merge"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_granularity_rejects_non_edge(calc_merge_dir: Path) -> None:
|
|
73
|
+
merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
|
|
74
|
+
with pytest.raises(ValueError, match="does not support --granularity branch"):
|
|
75
|
+
resolve_granularity("libfuzzer-merge", "branch")
|
|
76
|
+
with pytest.raises(ValueError, match="only supports --granularity edge"):
|
|
77
|
+
libfuzzer_merge.read(merge_file, granularity="block")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_read_raises_without_cov_lines(tmp_path: Path) -> None:
|
|
81
|
+
merge_file = tmp_path / "empty.merge"
|
|
82
|
+
merge_file.write_text("1\n1\n/input\n")
|
|
83
|
+
with pytest.raises(ValueError, match="No covered edges"):
|
|
84
|
+
libfuzzer_merge.read(merge_file, granularity="edge")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_read_approach_dir(calc_merge_dir: Path) -> None:
|
|
88
|
+
trials = read_approach_dir(
|
|
89
|
+
calc_merge_dir / "approach_a",
|
|
90
|
+
input_format="libfuzzer-merge",
|
|
91
|
+
)
|
|
92
|
+
assert trials
|
|
93
|
+
assert all(edges for edges in trials.values())
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_read_campaign_dir(calc_merge_dir: Path) -> None:
|
|
97
|
+
campaign = read_campaign_dir(
|
|
98
|
+
calc_merge_dir,
|
|
99
|
+
input_format="libfuzzer-merge",
|
|
100
|
+
)
|
|
101
|
+
assert {"seeds", "approach_a", "approach_b", "approach_c"} <= set(campaign.keys())
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_relcov_from_merge_campaign(calc_merge_dir: Path) -> None:
|
|
105
|
+
dc = DifferentialCoverage.from_campaign_dir(
|
|
106
|
+
calc_merge_dir,
|
|
107
|
+
input_format="libfuzzer-merge",
|
|
108
|
+
)
|
|
109
|
+
approach_c = dc.approaches["approach_c"]
|
|
110
|
+
for approach in ("approach_a", "approach_b", "seeds"):
|
|
111
|
+
assert approach_c.relcov(dc.approaches[approach]) == 1.0
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_cli_relscore(calc_merge_dir: Path) -> None:
|
|
115
|
+
code, out = _run_cli(
|
|
116
|
+
["--input-format", "libfuzzer-merge", "relscore", str(calc_merge_dir)]
|
|
117
|
+
)
|
|
118
|
+
assert code == 0
|
|
119
|
+
lines = [line.strip() for line in out.strip().splitlines()]
|
|
120
|
+
assert any(line.startswith("approach_c:") for line in lines)
|
|
121
|
+
assert any(line.startswith("approach_a:") for line in lines)
|
|
@@ -30,6 +30,52 @@ def _calc_export(calc_coverage_dir: Path, *parts: str) -> Path:
|
|
|
30
30
|
return calc_coverage_dir.joinpath(*parts)
|
|
31
31
|
|
|
32
32
|
|
|
33
|
+
def _covered_code_region_keys(export: Path) -> list[str]:
|
|
34
|
+
"""Covered CodeRegion location keys (may include duplicates)."""
|
|
35
|
+
data = json.loads(export.read_text())
|
|
36
|
+
keys: list[str] = []
|
|
37
|
+
for item in data.get("data", []):
|
|
38
|
+
for function in item.get("functions", []):
|
|
39
|
+
filenames = function["filenames"]
|
|
40
|
+
for region in function.get("regions", []):
|
|
41
|
+
# CountedRegion: exec=4, file_id=5, kind=7; CodeRegion kind=0
|
|
42
|
+
if region[4] <= 0 or region[7] != 0:
|
|
43
|
+
continue
|
|
44
|
+
file_id = region[5]
|
|
45
|
+
keys.append(
|
|
46
|
+
f"fn:{function['name']}@{filenames[file_id]}:"
|
|
47
|
+
f"{region[0]}:{region[1]}-{region[2]}:{region[3]}"
|
|
48
|
+
)
|
|
49
|
+
return keys
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_read_cov_matches_llvm_cov_summary(
|
|
53
|
+
calc_build_dir: Path, calc_coverage_dir: Path
|
|
54
|
+
) -> None:
|
|
55
|
+
summaries = calc_build_dir / "build" / "llvm_summaries"
|
|
56
|
+
for export in sorted(calc_coverage_dir.rglob("*.json")):
|
|
57
|
+
rel = export.relative_to(calc_coverage_dir)
|
|
58
|
+
summary_file = summaries / rel.parent / f"{export.stem}.json"
|
|
59
|
+
assert summary_file.is_file(), f"missing llvm-cov summary for {export}"
|
|
60
|
+
totals = json.loads(summary_file.read_text())["data"][0]["totals"]
|
|
61
|
+
branch_edges = llvm_cov.read(export, granularity="branch")
|
|
62
|
+
block_edges = llvm_cov.read(export, granularity="block")
|
|
63
|
+
assert len(branch_edges) == totals["branches"]["covered"], (
|
|
64
|
+
f"{rel} branch: reader found {len(branch_edges)}, "
|
|
65
|
+
f"llvm-cov reported {totals['branches']['covered']}"
|
|
66
|
+
)
|
|
67
|
+
# Summary counts CodeRegion hits with multiplicity; the reader dedupes
|
|
68
|
+
# by source location (same as differential-coverage edge identity).
|
|
69
|
+
region_keys = _covered_code_region_keys(export)
|
|
70
|
+
assert len(region_keys) == totals["regions"]["covered"], (
|
|
71
|
+
f"{rel} regions: export has {len(region_keys)} covered CodeRegions, "
|
|
72
|
+
f"llvm-cov reported {totals['regions']['covered']}"
|
|
73
|
+
)
|
|
74
|
+
assert block_edges == set(region_keys), (
|
|
75
|
+
f"{rel} block: reader set does not match unique covered CodeRegions"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
33
79
|
def test_read_llvm_cov_branch_uses_function_scope(calc_coverage_dir: Path) -> None:
|
|
34
80
|
export = _calc_export(calc_coverage_dir, "approach_a", "t1.json")
|
|
35
81
|
edges = llvm_cov.read(export, granularity="branch")
|
|
@@ -75,8 +75,8 @@ def test_print_scores_uses_printer_stdout() -> None:
|
|
|
75
75
|
|
|
76
76
|
# Scores should be sorted by value descending, then by name
|
|
77
77
|
joined = "\n".join(captured)
|
|
78
|
-
assert "a: 1.
|
|
79
|
-
assert "b: 0.
|
|
78
|
+
assert "a: 1.000" in joined
|
|
79
|
+
assert "b: 0.100" in joined
|
|
80
80
|
# Ensure our custom printer was used (no direct writes to sys.stdout here)
|
|
81
81
|
assert joined.count("\n") >= 0
|
|
82
82
|
|
|
@@ -97,3 +97,30 @@ def test_print_scores_uses_printer_csv() -> None:
|
|
|
97
97
|
# Order should be by score descending
|
|
98
98
|
assert rows[1][0] == "a"
|
|
99
99
|
assert rows[2][0] == "b"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_print_scores_respects_digits() -> None:
|
|
103
|
+
"""print_scores formats scores with the requested number of digits."""
|
|
104
|
+
scores = {"a": 1.0 / 3.0}
|
|
105
|
+
captured, printer = _capture_printer()
|
|
106
|
+
|
|
107
|
+
print_scores(scores, output="stdout", digits=4, printer=printer)
|
|
108
|
+
|
|
109
|
+
assert captured == ["a: 0.3333"]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_relcov_table_respects_digits_csv() -> None:
|
|
113
|
+
"""Relcov CSV cells use the requested digit precision."""
|
|
114
|
+
captured, printer = _capture_printer()
|
|
115
|
+
print_relcov_corpus_table(
|
|
116
|
+
["a"],
|
|
117
|
+
{"a": {"a": 1.0 / 3.0}},
|
|
118
|
+
output="csv",
|
|
119
|
+
digits=4,
|
|
120
|
+
printer=printer,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
csv_text = "\n".join(captured)
|
|
124
|
+
reader = csv.reader(io.StringIO(csv_text))
|
|
125
|
+
rows = list(reader)
|
|
126
|
+
assert rows[1][1] == "0.3333"
|
|
@@ -4,7 +4,7 @@ from pathlib import Path
|
|
|
4
4
|
|
|
5
5
|
import pytest
|
|
6
6
|
|
|
7
|
-
from differential_coverage.readers import afl_showmap, llvm_cov
|
|
7
|
+
from differential_coverage.readers import afl_showmap, libfuzzer_merge, llvm_cov
|
|
8
8
|
from differential_coverage.readers.registry import detect_reader, resolve_reader
|
|
9
9
|
|
|
10
10
|
AFL_FIXTURE = Path(__file__).parent / "fixtures" / "afl_showmap" / "t1.txt"
|
|
@@ -14,6 +14,10 @@ def _llvm_export(calc_coverage_dir: Path) -> Path:
|
|
|
14
14
|
return next((calc_coverage_dir / "approach_a").glob("*.json"))
|
|
15
15
|
|
|
16
16
|
|
|
17
|
+
def _merge_export(calc_merge_dir: Path) -> Path:
|
|
18
|
+
return calc_merge_dir / "approach_a" / "t1.merge"
|
|
19
|
+
|
|
20
|
+
|
|
17
21
|
def test_afl_showmap_detect_and_read() -> None:
|
|
18
22
|
assert afl_showmap.detect(AFL_FIXTURE)
|
|
19
23
|
assert not llvm_cov.detect(AFL_FIXTURE)
|
|
@@ -24,27 +28,45 @@ def test_llvm_cov_detect_and_read(calc_coverage_dir: Path) -> None:
|
|
|
24
28
|
export = _llvm_export(calc_coverage_dir)
|
|
25
29
|
assert llvm_cov.detect(export)
|
|
26
30
|
assert not afl_showmap.detect(export)
|
|
31
|
+
assert not libfuzzer_merge.detect(export)
|
|
27
32
|
assert llvm_cov.read(export, granularity="branch")
|
|
28
33
|
|
|
29
34
|
|
|
30
|
-
def
|
|
35
|
+
def test_libfuzzer_merge_detect_and_read(calc_merge_dir: Path) -> None:
|
|
36
|
+
merge_file = _merge_export(calc_merge_dir)
|
|
37
|
+
assert libfuzzer_merge.detect(merge_file)
|
|
38
|
+
assert not afl_showmap.detect(merge_file)
|
|
39
|
+
assert not llvm_cov.detect(merge_file)
|
|
40
|
+
edges = libfuzzer_merge.read(merge_file, granularity="edge")
|
|
41
|
+
assert edges
|
|
42
|
+
assert all(edge.isdigit() for edge in edges)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_detect_reader(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
|
|
31
46
|
export = _llvm_export(calc_coverage_dir)
|
|
47
|
+
merge_file = _merge_export(calc_merge_dir)
|
|
32
48
|
assert detect_reader(AFL_FIXTURE) == "afl-showmap"
|
|
33
49
|
assert detect_reader(export) == "llvm-cov"
|
|
50
|
+
assert detect_reader(merge_file) == "libfuzzer-merge"
|
|
34
51
|
|
|
35
52
|
|
|
36
|
-
def test_resolve_reader_auto(calc_coverage_dir: Path) -> None:
|
|
53
|
+
def test_resolve_reader_auto(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
|
|
37
54
|
export = _llvm_export(calc_coverage_dir)
|
|
55
|
+
merge_file = _merge_export(calc_merge_dir)
|
|
38
56
|
reader = resolve_reader([AFL_FIXTURE], "auto")
|
|
39
57
|
assert reader.name == "afl-showmap"
|
|
40
58
|
reader = resolve_reader([export], "auto")
|
|
41
59
|
assert reader.name == "llvm-cov"
|
|
60
|
+
reader = resolve_reader([merge_file], "auto")
|
|
61
|
+
assert reader.name == "libfuzzer-merge"
|
|
42
62
|
|
|
43
63
|
|
|
44
|
-
def test_resolve_reader_forced(calc_coverage_dir: Path) -> None:
|
|
64
|
+
def test_resolve_reader_forced(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
|
|
45
65
|
export = _llvm_export(calc_coverage_dir)
|
|
66
|
+
merge_file = _merge_export(calc_merge_dir)
|
|
46
67
|
assert resolve_reader([export], "llvm-cov").name == "llvm-cov"
|
|
47
68
|
assert resolve_reader([AFL_FIXTURE], "afl-showmap").name == "afl-showmap"
|
|
69
|
+
assert resolve_reader([merge_file], "libfuzzer-merge").name == "libfuzzer-merge"
|
|
48
70
|
|
|
49
71
|
|
|
50
72
|
def test_afl_showmap_rejects_block_granularity() -> None:
|
|
@@ -59,6 +81,7 @@ def test_resolve_granularity_auto(calc_coverage_dir: Path) -> None:
|
|
|
59
81
|
|
|
60
82
|
assert resolve_granularity("llvm-cov", "auto") == "branch"
|
|
61
83
|
assert resolve_granularity("afl-showmap", "auto") == "edge"
|
|
84
|
+
assert resolve_granularity("libfuzzer-merge", "auto") == "edge"
|
|
62
85
|
assert resolve_granularity("llvm-cov", "block") == "block"
|
|
63
86
|
with pytest.raises(ValueError, match="does not support --granularity edge"):
|
|
64
87
|
resolve_granularity("llvm-cov", "edge")
|
|
File without changes
|
{differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/__init__.py
RENAMED
|
File without changes
|
{differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/api.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/types.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|