differential-coverage 1.1.2__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/PKG-INFO +29 -6
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/README.md +28 -5
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/pyproject.toml +1 -1
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/cli.py +7 -5
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/readers/__init__.py +2 -2
- differential_coverage-1.2.0/src/differential_coverage/readers/libfuzzer_merge.py +41 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/readers/registry.py +4 -2
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage.egg-info/PKG-INFO +29 -6
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage.egg-info/SOURCES.txt +2 -0
- differential_coverage-1.2.0/tests/test_libfuzzer_merge.py +121 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/tests/test_llvm_cov.py +46 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/tests/test_readers.py +27 -4
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/setup.cfg +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/__init__.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/api.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/approach_data.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/fs.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/output.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/readers/afl_showmap.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/readers/llvm_cov.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/types.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage.egg-info/dependency_links.txt +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage.egg-info/entry_points.txt +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage.egg-info/requires.txt +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage.egg-info/top_level.txt +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/tests/test_api.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/tests/test_approach_data.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/tests/test_cli.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/tests/test_docs.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/tests/test_output.py +0 -0
- {differential_coverage-1.1.2 → differential_coverage-1.2.0}/tests/test_reducers.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: differential_coverage
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Compute differential coverage, a better way of comparing testing tools.
|
|
5
5
|
Author-email: Valentin Huber <contact@valentinhuber.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -43,7 +43,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
|
|
|
43
43
|
|
|
44
44
|
## How?
|
|
45
45
|
|
|
46
|
-
Campaign data lives in one directory per approach, with one file per trial.
|
|
46
|
+
Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
|
|
47
|
+
|
|
48
|
+
**All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
|
|
49
|
+
|
|
50
|
+
For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
|
|
47
51
|
|
|
48
52
|
### afl-showmap
|
|
49
53
|
|
|
@@ -88,6 +92,8 @@ llvm-cov export \
|
|
|
88
92
|
|
|
89
93
|
Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
|
|
90
94
|
|
|
95
|
+
On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
|
|
96
|
+
|
|
91
97
|
**All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
|
|
92
98
|
|
|
93
99
|
#### cov-analysis
|
|
@@ -101,11 +107,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
|
|
|
101
107
|
|
|
102
108
|
Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
|
|
103
109
|
|
|
104
|
-
###
|
|
110
|
+
### libFuzzer merge control files
|
|
111
|
+
|
|
112
|
+
Same layout, with merge control files from libFuzzer's `-merge=1` mode:
|
|
105
113
|
|
|
106
|
-
|
|
114
|
+
```
|
|
115
|
+
coverage_data
|
|
116
|
+
|-- approach_1
|
|
117
|
+
| |-- trial_1.merge
|
|
118
|
+
| `-- trial_2.merge
|
|
119
|
+
`-- seeds
|
|
120
|
+
`-- seeds.merge
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
empty=$(mktemp -d)
|
|
127
|
+
./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
|
|
128
|
+
rmdir "$empty"
|
|
129
|
+
```
|
|
107
130
|
|
|
108
|
-
|
|
131
|
+
Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
|
|
109
132
|
|
|
110
133
|
### Installation
|
|
111
134
|
|
|
@@ -123,7 +146,7 @@ pip install differential-coverage[latex]
|
|
|
123
146
|
Generally, the command line interface follows the following structure:
|
|
124
147
|
|
|
125
148
|
```bash
|
|
126
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
149
|
+
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
127
150
|
```
|
|
128
151
|
|
|
129
152
|
Examples:
|
|
@@ -19,7 +19,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
|
|
|
19
19
|
|
|
20
20
|
## How?
|
|
21
21
|
|
|
22
|
-
Campaign data lives in one directory per approach, with one file per trial.
|
|
22
|
+
Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
|
|
23
|
+
|
|
24
|
+
**All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
|
|
25
|
+
|
|
26
|
+
For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
|
|
23
27
|
|
|
24
28
|
### afl-showmap
|
|
25
29
|
|
|
@@ -64,6 +68,8 @@ llvm-cov export \
|
|
|
64
68
|
|
|
65
69
|
Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
|
|
66
70
|
|
|
71
|
+
On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
|
|
72
|
+
|
|
67
73
|
**All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
|
|
68
74
|
|
|
69
75
|
#### cov-analysis
|
|
@@ -77,11 +83,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
|
|
|
77
83
|
|
|
78
84
|
Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
|
|
79
85
|
|
|
80
|
-
###
|
|
86
|
+
### libFuzzer merge control files
|
|
87
|
+
|
|
88
|
+
Same layout, with merge control files from libFuzzer's `-merge=1` mode:
|
|
81
89
|
|
|
82
|
-
|
|
90
|
+
```
|
|
91
|
+
coverage_data
|
|
92
|
+
|-- approach_1
|
|
93
|
+
| |-- trial_1.merge
|
|
94
|
+
| `-- trial_2.merge
|
|
95
|
+
`-- seeds
|
|
96
|
+
`-- seeds.merge
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
empty=$(mktemp -d)
|
|
103
|
+
./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
|
|
104
|
+
rmdir "$empty"
|
|
105
|
+
```
|
|
83
106
|
|
|
84
|
-
|
|
107
|
+
Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
|
|
85
108
|
|
|
86
109
|
### Installation
|
|
87
110
|
|
|
@@ -99,7 +122,7 @@ pip install differential-coverage[latex]
|
|
|
99
122
|
Generally, the command line interface follows the following structure:
|
|
100
123
|
|
|
101
124
|
```bash
|
|
102
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
125
|
+
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
103
126
|
```
|
|
104
127
|
|
|
105
128
|
Examples:
|
{differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/cli.py
RENAMED
|
@@ -12,7 +12,8 @@ from differential_coverage.output import print_relcov_corpus_table, print_scores
|
|
|
12
12
|
# All subcommands expect this directory layout (one campaign dir, no moving files).
|
|
13
13
|
CAMPAIGN_DIR_HELP = (
|
|
14
14
|
"Campaign directory: one subdirectory per approach, each containing "
|
|
15
|
-
"trial coverage files (afl-showmap id:count files
|
|
15
|
+
"trial coverage files (afl-showmap id:count files, llvm-cov export JSON, "
|
|
16
|
+
"or libFuzzer merge control files)."
|
|
16
17
|
)
|
|
17
18
|
|
|
18
19
|
try:
|
|
@@ -133,11 +134,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
133
134
|
|
|
134
135
|
parser.add_argument(
|
|
135
136
|
"--input-format",
|
|
136
|
-
choices=["auto", "afl-showmap", "llvm-cov"],
|
|
137
|
+
choices=["auto", "afl-showmap", "llvm-cov", "libfuzzer-merge"],
|
|
137
138
|
default="auto",
|
|
138
139
|
help=(
|
|
139
140
|
"Input format for trial files: auto (default) detects from file "
|
|
140
|
-
"
|
|
141
|
+
"content; afl-showmap for id:count files; llvm-cov for export JSON; "
|
|
142
|
+
"libfuzzer-merge for -merge=1 control files."
|
|
141
143
|
),
|
|
142
144
|
)
|
|
143
145
|
parser.add_argument(
|
|
@@ -155,8 +157,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
155
157
|
default="auto",
|
|
156
158
|
help=(
|
|
157
159
|
"Coverage granularity: auto (default) uses edge for afl-showmap and "
|
|
158
|
-
"branch for llvm-cov; branch/block for llvm-cov
|
|
159
|
-
"afl-showmap bitmap tuples."
|
|
160
|
+
"libfuzzer-merge, branch for llvm-cov; branch/block for llvm-cov "
|
|
161
|
+
"exports; edge for afl-showmap bitmap tuples and libFuzzer COV lines."
|
|
160
162
|
),
|
|
161
163
|
)
|
|
162
164
|
parser.add_argument(
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from differential_coverage.readers import afl_showmap, llvm_cov
|
|
1
|
+
from differential_coverage.readers import afl_showmap, libfuzzer_merge, llvm_cov
|
|
2
2
|
from differential_coverage.readers.registry import (
|
|
3
3
|
Granularity,
|
|
4
4
|
GranularityArg,
|
|
@@ -15,4 +15,4 @@ __all__ = [
|
|
|
15
15
|
"resolve_reader",
|
|
16
16
|
]
|
|
17
17
|
|
|
18
|
-
_ = (afl_showmap, llvm_cov)
|
|
18
|
+
_ = (afl_showmap, libfuzzer_merge, llvm_cov)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from differential_coverage.readers.registry import (
|
|
4
|
+
Granularity,
|
|
5
|
+
TrialReader,
|
|
6
|
+
register_reader,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def read(path: Path, *, granularity: Granularity) -> set[str]:
|
|
11
|
+
if granularity != "edge":
|
|
12
|
+
raise ValueError("libfuzzer-merge only supports --granularity edge")
|
|
13
|
+
edges: set[str] = set()
|
|
14
|
+
for line in path.read_text().splitlines():
|
|
15
|
+
parts = line.split()
|
|
16
|
+
if len(parts) < 3 or parts[0] != "COV":
|
|
17
|
+
continue
|
|
18
|
+
edges.update(parts[2:])
|
|
19
|
+
if not edges:
|
|
20
|
+
raise ValueError(f"No covered edges in {path}")
|
|
21
|
+
return edges
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def detect(path: Path) -> bool:
|
|
25
|
+
lines = [line.strip() for line in path.read_text().splitlines() if line.strip()]
|
|
26
|
+
if len(lines) < 3:
|
|
27
|
+
return False
|
|
28
|
+
try:
|
|
29
|
+
num_files = int(lines[0])
|
|
30
|
+
num_first = int(lines[1])
|
|
31
|
+
except ValueError:
|
|
32
|
+
return False
|
|
33
|
+
if num_files <= 0 or num_first > num_files:
|
|
34
|
+
return False
|
|
35
|
+
has_marker = any(
|
|
36
|
+
line.startswith("STARTED ") or line.startswith("COV ") for line in lines
|
|
37
|
+
)
|
|
38
|
+
return has_marker
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
register_reader(TrialReader(name="libfuzzer-merge", read=read, detect=detect))
|
|
@@ -3,18 +3,20 @@ from dataclasses import dataclass
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import Literal, Protocol
|
|
5
5
|
|
|
6
|
-
ReaderName = Literal["afl-showmap", "llvm-cov"]
|
|
7
|
-
InputFormat = Literal["auto", "afl-showmap", "llvm-cov"]
|
|
6
|
+
ReaderName = Literal["afl-showmap", "llvm-cov", "libfuzzer-merge"]
|
|
7
|
+
InputFormat = Literal["auto", "afl-showmap", "llvm-cov", "libfuzzer-merge"]
|
|
8
8
|
Granularity = Literal["branch", "block", "edge"]
|
|
9
9
|
GranularityArg = Literal["auto", "branch", "block", "edge"]
|
|
10
10
|
|
|
11
11
|
_SUPPORTED: dict[ReaderName, frozenset[Granularity]] = {
|
|
12
12
|
"afl-showmap": frozenset({"edge"}),
|
|
13
13
|
"llvm-cov": frozenset({"branch", "block"}),
|
|
14
|
+
"libfuzzer-merge": frozenset({"edge"}),
|
|
14
15
|
}
|
|
15
16
|
_DEFAULT: dict[ReaderName, Granularity] = {
|
|
16
17
|
"afl-showmap": "edge",
|
|
17
18
|
"llvm-cov": "branch",
|
|
19
|
+
"libfuzzer-merge": "edge",
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: differential_coverage
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Compute differential coverage, a better way of comparing testing tools.
|
|
5
5
|
Author-email: Valentin Huber <contact@valentinhuber.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -43,7 +43,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
|
|
|
43
43
|
|
|
44
44
|
## How?
|
|
45
45
|
|
|
46
|
-
Campaign data lives in one directory per approach, with one file per trial.
|
|
46
|
+
Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
|
|
47
|
+
|
|
48
|
+
**All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
|
|
49
|
+
|
|
50
|
+
For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
|
|
47
51
|
|
|
48
52
|
### afl-showmap
|
|
49
53
|
|
|
@@ -88,6 +92,8 @@ llvm-cov export \
|
|
|
88
92
|
|
|
89
93
|
Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
|
|
90
94
|
|
|
95
|
+
On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
|
|
96
|
+
|
|
91
97
|
**All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
|
|
92
98
|
|
|
93
99
|
#### cov-analysis
|
|
@@ -101,11 +107,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
|
|
|
101
107
|
|
|
102
108
|
Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
|
|
103
109
|
|
|
104
|
-
###
|
|
110
|
+
### libFuzzer merge control files
|
|
111
|
+
|
|
112
|
+
Same layout, with merge control files from libFuzzer's `-merge=1` mode:
|
|
105
113
|
|
|
106
|
-
|
|
114
|
+
```
|
|
115
|
+
coverage_data
|
|
116
|
+
|-- approach_1
|
|
117
|
+
| |-- trial_1.merge
|
|
118
|
+
| `-- trial_2.merge
|
|
119
|
+
`-- seeds
|
|
120
|
+
`-- seeds.merge
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
empty=$(mktemp -d)
|
|
127
|
+
./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
|
|
128
|
+
rmdir "$empty"
|
|
129
|
+
```
|
|
107
130
|
|
|
108
|
-
|
|
131
|
+
Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
|
|
109
132
|
|
|
110
133
|
### Installation
|
|
111
134
|
|
|
@@ -123,7 +146,7 @@ pip install differential-coverage[latex]
|
|
|
123
146
|
Generally, the command line interface follows the following structure:
|
|
124
147
|
|
|
125
148
|
```bash
|
|
126
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
149
|
+
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
127
150
|
```
|
|
128
151
|
|
|
129
152
|
Examples:
|
|
@@ -15,12 +15,14 @@ src/differential_coverage.egg-info/requires.txt
|
|
|
15
15
|
src/differential_coverage.egg-info/top_level.txt
|
|
16
16
|
src/differential_coverage/readers/__init__.py
|
|
17
17
|
src/differential_coverage/readers/afl_showmap.py
|
|
18
|
+
src/differential_coverage/readers/libfuzzer_merge.py
|
|
18
19
|
src/differential_coverage/readers/llvm_cov.py
|
|
19
20
|
src/differential_coverage/readers/registry.py
|
|
20
21
|
tests/test_api.py
|
|
21
22
|
tests/test_approach_data.py
|
|
22
23
|
tests/test_cli.py
|
|
23
24
|
tests/test_docs.py
|
|
25
|
+
tests/test_libfuzzer_merge.py
|
|
24
26
|
tests/test_llvm_cov.py
|
|
25
27
|
tests/test_output.py
|
|
26
28
|
tests/test_readers.py
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Unit tests for libFuzzer merge control file input."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import re
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from differential_coverage.api import DifferentialCoverage
|
|
11
|
+
from differential_coverage.cli import main
|
|
12
|
+
from differential_coverage.fs import read_approach_dir, read_campaign_dir
|
|
13
|
+
from differential_coverage.readers import libfuzzer_merge
|
|
14
|
+
from differential_coverage.readers.registry import detect_reader, resolve_granularity
|
|
15
|
+
|
|
16
|
+
_DONE_COV_RE = re.compile(r"^\#\d+\s+DONE\s+cov:\s+(\d+)", re.MULTILINE)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _run_cli(argv: list[str]) -> tuple[int | str, str]:
|
|
20
|
+
out = io.StringIO()
|
|
21
|
+
old_stdout = sys.stdout
|
|
22
|
+
try:
|
|
23
|
+
sys.stdout = out
|
|
24
|
+
code = main(argv)
|
|
25
|
+
except SystemExit as exc:
|
|
26
|
+
code = int(exc.code) if exc.code is not None else 1
|
|
27
|
+
finally:
|
|
28
|
+
sys.stdout = old_stdout
|
|
29
|
+
return code, out.getvalue()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _calc_merge(calc_merge_dir: Path, *parts: str) -> Path:
|
|
33
|
+
return calc_merge_dir.joinpath(*parts)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _parse_libfuzzer_done_cov(log_text: str) -> int:
|
|
37
|
+
matches = _DONE_COV_RE.findall(log_text)
|
|
38
|
+
if not matches:
|
|
39
|
+
raise ValueError("no DONE cov line in libFuzzer merge output")
|
|
40
|
+
return int(matches[-1])
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_read_cov_matches_libfuzzer_merge_output(
|
|
44
|
+
calc_build_dir: Path, calc_merge_dir: Path
|
|
45
|
+
) -> None:
|
|
46
|
+
merge_logs = calc_build_dir / "build" / "merge_logs"
|
|
47
|
+
for merge_file in sorted(calc_merge_dir.rglob("*.merge")):
|
|
48
|
+
rel = merge_file.relative_to(calc_merge_dir)
|
|
49
|
+
log_file = merge_logs / rel.parent / f"{merge_file.stem}.log"
|
|
50
|
+
assert log_file.is_file(), f"missing libFuzzer log for {merge_file}"
|
|
51
|
+
expected_cov = _parse_libfuzzer_done_cov(log_file.read_text())
|
|
52
|
+
edges = libfuzzer_merge.read(merge_file, granularity="edge")
|
|
53
|
+
assert len(edges) == expected_cov, (
|
|
54
|
+
f"{rel}: reader found {len(edges)} edges, "
|
|
55
|
+
f"libFuzzer reported cov: {expected_cov}"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_read_detect_and_read(calc_merge_dir: Path) -> None:
|
|
60
|
+
merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
|
|
61
|
+
assert libfuzzer_merge.detect(merge_file)
|
|
62
|
+
edges = libfuzzer_merge.read(merge_file, granularity="edge")
|
|
63
|
+
assert edges
|
|
64
|
+
assert all(edge.isdigit() for edge in edges)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_detect_reader(calc_merge_dir: Path) -> None:
|
|
68
|
+
merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
|
|
69
|
+
assert detect_reader(merge_file) == "libfuzzer-merge"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_granularity_rejects_non_edge(calc_merge_dir: Path) -> None:
|
|
73
|
+
merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
|
|
74
|
+
with pytest.raises(ValueError, match="does not support --granularity branch"):
|
|
75
|
+
resolve_granularity("libfuzzer-merge", "branch")
|
|
76
|
+
with pytest.raises(ValueError, match="only supports --granularity edge"):
|
|
77
|
+
libfuzzer_merge.read(merge_file, granularity="block")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_read_raises_without_cov_lines(tmp_path: Path) -> None:
|
|
81
|
+
merge_file = tmp_path / "empty.merge"
|
|
82
|
+
merge_file.write_text("1\n1\n/input\n")
|
|
83
|
+
with pytest.raises(ValueError, match="No covered edges"):
|
|
84
|
+
libfuzzer_merge.read(merge_file, granularity="edge")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_read_approach_dir(calc_merge_dir: Path) -> None:
|
|
88
|
+
trials = read_approach_dir(
|
|
89
|
+
calc_merge_dir / "approach_a",
|
|
90
|
+
input_format="libfuzzer-merge",
|
|
91
|
+
)
|
|
92
|
+
assert trials
|
|
93
|
+
assert all(edges for edges in trials.values())
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_read_campaign_dir(calc_merge_dir: Path) -> None:
|
|
97
|
+
campaign = read_campaign_dir(
|
|
98
|
+
calc_merge_dir,
|
|
99
|
+
input_format="libfuzzer-merge",
|
|
100
|
+
)
|
|
101
|
+
assert {"seeds", "approach_a", "approach_b", "approach_c"} <= set(campaign.keys())
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_relcov_from_merge_campaign(calc_merge_dir: Path) -> None:
|
|
105
|
+
dc = DifferentialCoverage.from_campaign_dir(
|
|
106
|
+
calc_merge_dir,
|
|
107
|
+
input_format="libfuzzer-merge",
|
|
108
|
+
)
|
|
109
|
+
approach_c = dc.approaches["approach_c"]
|
|
110
|
+
for approach in ("approach_a", "approach_b", "seeds"):
|
|
111
|
+
assert approach_c.relcov(dc.approaches[approach]) == 1.0
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_cli_relscore(calc_merge_dir: Path) -> None:
|
|
115
|
+
code, out = _run_cli(
|
|
116
|
+
["--input-format", "libfuzzer-merge", "relscore", str(calc_merge_dir)]
|
|
117
|
+
)
|
|
118
|
+
assert code == 0
|
|
119
|
+
lines = [line.strip() for line in out.strip().splitlines()]
|
|
120
|
+
assert any(line.startswith("approach_c:") for line in lines)
|
|
121
|
+
assert any(line.startswith("approach_a:") for line in lines)
|
|
@@ -30,6 +30,52 @@ def _calc_export(calc_coverage_dir: Path, *parts: str) -> Path:
|
|
|
30
30
|
return calc_coverage_dir.joinpath(*parts)
|
|
31
31
|
|
|
32
32
|
|
|
33
|
+
def _covered_code_region_keys(export: Path) -> list[str]:
|
|
34
|
+
"""Covered CodeRegion location keys (may include duplicates)."""
|
|
35
|
+
data = json.loads(export.read_text())
|
|
36
|
+
keys: list[str] = []
|
|
37
|
+
for item in data.get("data", []):
|
|
38
|
+
for function in item.get("functions", []):
|
|
39
|
+
filenames = function["filenames"]
|
|
40
|
+
for region in function.get("regions", []):
|
|
41
|
+
# CountedRegion: exec=4, file_id=5, kind=7; CodeRegion kind=0
|
|
42
|
+
if region[4] <= 0 or region[7] != 0:
|
|
43
|
+
continue
|
|
44
|
+
file_id = region[5]
|
|
45
|
+
keys.append(
|
|
46
|
+
f"fn:{function['name']}@{filenames[file_id]}:"
|
|
47
|
+
f"{region[0]}:{region[1]}-{region[2]}:{region[3]}"
|
|
48
|
+
)
|
|
49
|
+
return keys
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_read_cov_matches_llvm_cov_summary(
|
|
53
|
+
calc_build_dir: Path, calc_coverage_dir: Path
|
|
54
|
+
) -> None:
|
|
55
|
+
summaries = calc_build_dir / "build" / "llvm_summaries"
|
|
56
|
+
for export in sorted(calc_coverage_dir.rglob("*.json")):
|
|
57
|
+
rel = export.relative_to(calc_coverage_dir)
|
|
58
|
+
summary_file = summaries / rel.parent / f"{export.stem}.json"
|
|
59
|
+
assert summary_file.is_file(), f"missing llvm-cov summary for {export}"
|
|
60
|
+
totals = json.loads(summary_file.read_text())["data"][0]["totals"]
|
|
61
|
+
branch_edges = llvm_cov.read(export, granularity="branch")
|
|
62
|
+
block_edges = llvm_cov.read(export, granularity="block")
|
|
63
|
+
assert len(branch_edges) == totals["branches"]["covered"], (
|
|
64
|
+
f"{rel} branch: reader found {len(branch_edges)}, "
|
|
65
|
+
f"llvm-cov reported {totals['branches']['covered']}"
|
|
66
|
+
)
|
|
67
|
+
# Summary counts CodeRegion hits with multiplicity; the reader dedupes
|
|
68
|
+
# by source location (same as differential-coverage edge identity).
|
|
69
|
+
region_keys = _covered_code_region_keys(export)
|
|
70
|
+
assert len(region_keys) == totals["regions"]["covered"], (
|
|
71
|
+
f"{rel} regions: export has {len(region_keys)} covered CodeRegions, "
|
|
72
|
+
f"llvm-cov reported {totals['regions']['covered']}"
|
|
73
|
+
)
|
|
74
|
+
assert block_edges == set(region_keys), (
|
|
75
|
+
f"{rel} block: reader set does not match unique covered CodeRegions"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
33
79
|
def test_read_llvm_cov_branch_uses_function_scope(calc_coverage_dir: Path) -> None:
|
|
34
80
|
export = _calc_export(calc_coverage_dir, "approach_a", "t1.json")
|
|
35
81
|
edges = llvm_cov.read(export, granularity="branch")
|
|
@@ -4,7 +4,7 @@ from pathlib import Path
|
|
|
4
4
|
|
|
5
5
|
import pytest
|
|
6
6
|
|
|
7
|
-
from differential_coverage.readers import afl_showmap, llvm_cov
|
|
7
|
+
from differential_coverage.readers import afl_showmap, libfuzzer_merge, llvm_cov
|
|
8
8
|
from differential_coverage.readers.registry import detect_reader, resolve_reader
|
|
9
9
|
|
|
10
10
|
AFL_FIXTURE = Path(__file__).parent / "fixtures" / "afl_showmap" / "t1.txt"
|
|
@@ -14,6 +14,10 @@ def _llvm_export(calc_coverage_dir: Path) -> Path:
|
|
|
14
14
|
return next((calc_coverage_dir / "approach_a").glob("*.json"))
|
|
15
15
|
|
|
16
16
|
|
|
17
|
+
def _merge_export(calc_merge_dir: Path) -> Path:
|
|
18
|
+
return calc_merge_dir / "approach_a" / "t1.merge"
|
|
19
|
+
|
|
20
|
+
|
|
17
21
|
def test_afl_showmap_detect_and_read() -> None:
|
|
18
22
|
assert afl_showmap.detect(AFL_FIXTURE)
|
|
19
23
|
assert not llvm_cov.detect(AFL_FIXTURE)
|
|
@@ -24,27 +28,45 @@ def test_llvm_cov_detect_and_read(calc_coverage_dir: Path) -> None:
|
|
|
24
28
|
export = _llvm_export(calc_coverage_dir)
|
|
25
29
|
assert llvm_cov.detect(export)
|
|
26
30
|
assert not afl_showmap.detect(export)
|
|
31
|
+
assert not libfuzzer_merge.detect(export)
|
|
27
32
|
assert llvm_cov.read(export, granularity="branch")
|
|
28
33
|
|
|
29
34
|
|
|
30
|
-
def
|
|
35
|
+
def test_libfuzzer_merge_detect_and_read(calc_merge_dir: Path) -> None:
|
|
36
|
+
merge_file = _merge_export(calc_merge_dir)
|
|
37
|
+
assert libfuzzer_merge.detect(merge_file)
|
|
38
|
+
assert not afl_showmap.detect(merge_file)
|
|
39
|
+
assert not llvm_cov.detect(merge_file)
|
|
40
|
+
edges = libfuzzer_merge.read(merge_file, granularity="edge")
|
|
41
|
+
assert edges
|
|
42
|
+
assert all(edge.isdigit() for edge in edges)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_detect_reader(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
|
|
31
46
|
export = _llvm_export(calc_coverage_dir)
|
|
47
|
+
merge_file = _merge_export(calc_merge_dir)
|
|
32
48
|
assert detect_reader(AFL_FIXTURE) == "afl-showmap"
|
|
33
49
|
assert detect_reader(export) == "llvm-cov"
|
|
50
|
+
assert detect_reader(merge_file) == "libfuzzer-merge"
|
|
34
51
|
|
|
35
52
|
|
|
36
|
-
def test_resolve_reader_auto(calc_coverage_dir: Path) -> None:
|
|
53
|
+
def test_resolve_reader_auto(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
|
|
37
54
|
export = _llvm_export(calc_coverage_dir)
|
|
55
|
+
merge_file = _merge_export(calc_merge_dir)
|
|
38
56
|
reader = resolve_reader([AFL_FIXTURE], "auto")
|
|
39
57
|
assert reader.name == "afl-showmap"
|
|
40
58
|
reader = resolve_reader([export], "auto")
|
|
41
59
|
assert reader.name == "llvm-cov"
|
|
60
|
+
reader = resolve_reader([merge_file], "auto")
|
|
61
|
+
assert reader.name == "libfuzzer-merge"
|
|
42
62
|
|
|
43
63
|
|
|
44
|
-
def test_resolve_reader_forced(calc_coverage_dir: Path) -> None:
|
|
64
|
+
def test_resolve_reader_forced(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
|
|
45
65
|
export = _llvm_export(calc_coverage_dir)
|
|
66
|
+
merge_file = _merge_export(calc_merge_dir)
|
|
46
67
|
assert resolve_reader([export], "llvm-cov").name == "llvm-cov"
|
|
47
68
|
assert resolve_reader([AFL_FIXTURE], "afl-showmap").name == "afl-showmap"
|
|
69
|
+
assert resolve_reader([merge_file], "libfuzzer-merge").name == "libfuzzer-merge"
|
|
48
70
|
|
|
49
71
|
|
|
50
72
|
def test_afl_showmap_rejects_block_granularity() -> None:
|
|
@@ -59,6 +81,7 @@ def test_resolve_granularity_auto(calc_coverage_dir: Path) -> None:
|
|
|
59
81
|
|
|
60
82
|
assert resolve_granularity("llvm-cov", "auto") == "branch"
|
|
61
83
|
assert resolve_granularity("afl-showmap", "auto") == "edge"
|
|
84
|
+
assert resolve_granularity("libfuzzer-merge", "auto") == "edge"
|
|
62
85
|
assert resolve_granularity("llvm-cov", "block") == "block"
|
|
63
86
|
with pytest.raises(ValueError, match="does not support --granularity edge"):
|
|
64
87
|
resolve_granularity("llvm-cov", "edge")
|
|
File without changes
|
{differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/__init__.py
RENAMED
|
File without changes
|
{differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/api.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/output.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{differential_coverage-1.1.2 → differential_coverage-1.2.0}/src/differential_coverage/types.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|