differential-coverage 1.1.2__tar.gz → 1.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/PKG-INFO +29 -6
  2. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/README.md +28 -5
  3. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/pyproject.toml +1 -1
  4. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/cli.py +24 -16
  5. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/output.py +36 -11
  6. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/readers/__init__.py +2 -2
  7. differential_coverage-1.2.1/src/differential_coverage/readers/libfuzzer_merge.py +41 -0
  8. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/readers/registry.py +4 -2
  9. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/PKG-INFO +29 -6
  10. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/SOURCES.txt +2 -0
  11. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_cli.py +12 -0
  12. differential_coverage-1.2.1/tests/test_libfuzzer_merge.py +121 -0
  13. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_llvm_cov.py +46 -0
  14. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_output.py +29 -2
  15. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_readers.py +27 -4
  16. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/setup.cfg +0 -0
  17. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/__init__.py +0 -0
  18. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/api.py +0 -0
  19. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/approach_data.py +0 -0
  20. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/fs.py +0 -0
  21. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/readers/afl_showmap.py +0 -0
  22. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/readers/llvm_cov.py +0 -0
  23. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage/types.py +0 -0
  24. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/dependency_links.txt +0 -0
  25. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/entry_points.txt +0 -0
  26. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/requires.txt +0 -0
  27. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/src/differential_coverage.egg-info/top_level.txt +0 -0
  28. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_api.py +0 -0
  29. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_approach_data.py +0 -0
  30. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_docs.py +0 -0
  31. {differential_coverage-1.1.2 → differential_coverage-1.2.1}/tests/test_reducers.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: differential_coverage
3
- Version: 1.1.2
3
+ Version: 1.2.1
4
4
  Summary: Compute differential coverage, a better way of comparing testing tools.
5
5
  Author-email: Valentin Huber <contact@valentinhuber.me>
6
6
  License-Expression: MIT
@@ -43,7 +43,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
43
43
 
44
44
  ## How?
45
45
 
46
- Campaign data lives in one directory per approach, with one file per trial. Two input formats are supported:
46
+ Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
47
+
48
+ **All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
49
+
50
+ For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
47
51
 
48
52
  ### afl-showmap
49
53
 
@@ -88,6 +92,8 @@ llvm-cov export \
88
92
 
89
93
  Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
90
94
 
95
+ On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
96
+
91
97
  **All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
92
98
 
93
99
  #### cov-analysis
@@ -101,11 +107,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
101
107
 
102
108
  Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
103
109
 
104
- ### Edge IDs and mixing formats
110
+ ### libFuzzer merge control files
111
+
112
+ Same layout, with merge control files from libFuzzer's `-merge=1` mode:
105
113
 
106
- **All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices) and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign.
114
+ ```
115
+ coverage_data
116
+ |-- approach_1
117
+ | |-- trial_1.merge
118
+ | `-- trial_2.merge
119
+ `-- seeds
120
+ `-- seeds.merge
121
+ ```
122
+
123
+ Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
124
+
125
+ ```bash
126
+ empty=$(mktemp -d)
127
+ ./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
128
+ rmdir "$empty"
129
+ ```
107
130
 
108
- Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
131
+ Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
109
132
 
110
133
  ### Installation
111
134
 
@@ -123,7 +146,7 @@ pip install differential-coverage[latex]
123
146
  Generally, the command line interface follows the following structure:
124
147
 
125
148
  ```bash
126
- differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
149
+ differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
127
150
  ```
128
151
 
129
152
  Examples:
@@ -19,7 +19,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
19
19
 
20
20
  ## How?
21
21
 
22
- Campaign data lives in one directory per approach, with one file per trial. Two input formats are supported:
22
+ Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
23
+
24
+ **All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
25
+
26
+ For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
23
27
 
24
28
  ### afl-showmap
25
29
 
@@ -64,6 +68,8 @@ llvm-cov export \
64
68
 
65
69
  Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
66
70
 
71
+ On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
72
+
67
73
  **All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
68
74
 
69
75
  #### cov-analysis
@@ -77,11 +83,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
77
83
 
78
84
  Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
79
85
 
80
- ### Edge IDs and mixing formats
86
+ ### libFuzzer merge control files
87
+
88
+ Same layout, with merge control files from libFuzzer's `-merge=1` mode:
81
89
 
82
- **All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices) and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign.
90
+ ```
91
+ coverage_data
92
+ |-- approach_1
93
+ | |-- trial_1.merge
94
+ | `-- trial_2.merge
95
+ `-- seeds
96
+ `-- seeds.merge
97
+ ```
98
+
99
+ Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
100
+
101
+ ```bash
102
+ empty=$(mktemp -d)
103
+ ./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
104
+ rmdir "$empty"
105
+ ```
83
106
 
84
- Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
107
+ Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
85
108
 
86
109
  ### Installation
87
110
 
@@ -99,7 +122,7 @@ pip install differential-coverage[latex]
99
122
  Generally, the command line interface follows the following structure:
100
123
 
101
124
  ```bash
102
- differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
125
+ differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
103
126
  ```
104
127
 
105
128
  Examples:
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "differential_coverage"
7
- version = "1.1.2"
7
+ version = "1.2.1"
8
8
  description = "Compute differential coverage, a better way of comparing testing tools."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -12,7 +12,8 @@ from differential_coverage.output import print_relcov_corpus_table, print_scores
12
12
  # All subcommands expect this directory layout (one campaign dir, no moving files).
13
13
  CAMPAIGN_DIR_HELP = (
14
14
  "Campaign directory: one subdirectory per approach, each containing "
15
- "trial coverage files (afl-showmap id:count files or llvm-cov export JSON)."
15
+ "trial coverage files (afl-showmap id:count files, llvm-cov export JSON, "
16
+ "or libFuzzer merge control files)."
16
17
  )
17
18
 
18
19
  try:
@@ -86,8 +87,9 @@ def cmd_relscore(args: argparse.Namespace) -> int:
86
87
  print_scores(
87
88
  scores,
88
89
  output=args.output,
89
- colormap=getattr(args, "colormap", "viridis"),
90
- latex_enable_color=getattr(args, "latex_enable_color", False),
90
+ colormap=args.colormap,
91
+ latex_enable_color=args.latex_enable_color,
92
+ digits=args.digits,
91
93
  )
92
94
  return 0
93
95
 
@@ -96,11 +98,6 @@ def cmd_relcov_performance_over_approach(args: argparse.Namespace) -> int:
96
98
  campaign = _load_campaign(args)
97
99
 
98
100
  dc = DifferentialCoverage(campaign)
99
- output = args.output
100
- colormap = getattr(args, "colormap", "viridis")
101
- latex_rotate_headers = getattr(args, "latex_rotate_headers", None)
102
- latex_enable_color = getattr(args, "latex_enable_color", False)
103
-
104
101
  ref_approaches = list(dc.approaches.keys())
105
102
  table = {
106
103
  f: {ref: dc.approaches[f].relcov(dc.approaches[ref]) for ref in ref_approaches}
@@ -109,10 +106,11 @@ def cmd_relcov_performance_over_approach(args: argparse.Namespace) -> int:
109
106
  print_relcov_corpus_table(
110
107
  ref_approaches,
111
108
  table,
112
- output=output,
113
- colormap=colormap,
114
- latex_rotate_headers=latex_rotate_headers,
115
- latex_enable_color=latex_enable_color,
109
+ output=args.output,
110
+ colormap=args.colormap,
111
+ latex_rotate_headers=args.latex_rotate_headers,
112
+ latex_enable_color=args.latex_enable_color,
113
+ digits=args.digits,
116
114
  )
117
115
  return 0
118
116
 
@@ -133,11 +131,12 @@ def build_parser() -> argparse.ArgumentParser:
133
131
 
134
132
  parser.add_argument(
135
133
  "--input-format",
136
- choices=["auto", "afl-showmap", "llvm-cov"],
134
+ choices=["auto", "afl-showmap", "llvm-cov", "libfuzzer-merge"],
137
135
  default="auto",
138
136
  help=(
139
137
  "Input format for trial files: auto (default) detects from file "
140
- "extensions, afl-showmap for id:count files, llvm-cov for export JSON."
138
+ "content; afl-showmap for id:count files; llvm-cov for export JSON; "
139
+ "libfuzzer-merge for -merge=1 control files."
141
140
  ),
142
141
  )
143
142
  parser.add_argument(
@@ -155,8 +154,8 @@ def build_parser() -> argparse.ArgumentParser:
155
154
  default="auto",
156
155
  help=(
157
156
  "Coverage granularity: auto (default) uses edge for afl-showmap and "
158
- "branch for llvm-cov; branch/block for llvm-cov exports; edge for "
159
- "afl-showmap bitmap tuples."
157
+ "libfuzzer-merge, branch for llvm-cov; branch/block for llvm-cov "
158
+ "exports; edge for afl-showmap bitmap tuples and libFuzzer COV lines."
160
159
  ),
161
160
  )
162
161
  parser.add_argument(
@@ -190,6 +189,15 @@ def build_parser() -> argparse.ArgumentParser:
190
189
  'latex for LaTeX tabular (requires "latex" optional dependencies)'
191
190
  ),
192
191
  )
192
+ parser.add_argument(
193
+ "--digits",
194
+ type=int,
195
+ default=3,
196
+ metavar="N",
197
+ help=(
198
+ "Number of digits after the decimal point for numeric results (default: 3)."
199
+ ),
200
+ )
193
201
  parser.add_argument(
194
202
  "--latex-rotate-headers",
195
203
  type=float,
@@ -30,12 +30,22 @@ class _PrinterIO:
30
30
  return len(s)
31
31
 
32
32
 
33
+ # Default decimal places for numeric output.
34
+ _DEFAULT_DIGITS = 3
35
+
36
+
37
+ def _format_float(value: float, digits: int) -> str:
38
+ """Format a float with a fixed number of digits after the decimal point."""
39
+ return f"{value:.{digits}f}"
40
+
41
+
33
42
  def print_scores(
34
43
  scores: Mapping[ApproachId, float],
35
44
  *,
36
45
  output: OutputFormat = "stdout",
37
46
  colormap: str = "viridis",
38
47
  latex_enable_color: bool = False,
48
+ digits: int = _DEFAULT_DIGITS,
39
49
  printer: Callable[[str], None] = print,
40
50
  ) -> None:
41
51
  """Print one score per approach, in the requested output format."""
@@ -43,14 +53,15 @@ def print_scores(
43
53
  performance_sorted = sorted(name_sorted, key=lambda x: x[1], reverse=True)
44
54
 
45
55
  if output == "stdout":
46
- _print_scores_plain(performance_sorted, printer=printer)
56
+ _print_scores_plain(performance_sorted, digits=digits, printer=printer)
47
57
  elif output == "csv":
48
- _print_scores_csv(performance_sorted, printer=printer)
58
+ _print_scores_csv(performance_sorted, digits=digits, printer=printer)
49
59
  elif output == "latex":
50
60
  _print_scores_latex(
51
61
  performance_sorted,
52
62
  enable_color=latex_enable_color,
53
63
  colormap=colormap,
64
+ digits=digits,
54
65
  printer=printer,
55
66
  )
56
67
  else:
@@ -60,21 +71,23 @@ def print_scores(
60
71
  def _print_scores_plain(
61
72
  performance_sorted: Sequence[tuple[ApproachId, float]],
62
73
  *,
74
+ digits: int,
63
75
  printer: Callable[[str], None],
64
76
  ) -> None:
65
77
  for approach, score in performance_sorted:
66
- printer(f"{approach}: {score:.2f}")
78
+ printer(f"{approach}: {_format_float(score, digits)}")
67
79
 
68
80
 
69
81
  def _print_scores_csv(
70
82
  performance_sorted: Sequence[tuple[ApproachId, float]],
71
83
  *,
84
+ digits: int,
72
85
  printer: Callable[[str], None],
73
86
  ) -> None:
74
87
  writer = csv.writer(_PrinterIO(printer))
75
88
  writer.writerow(("approach", "score"))
76
89
  for approach, score in performance_sorted:
77
- writer.writerow((approach, f"{score:.2f}"))
90
+ writer.writerow((approach, _format_float(score, digits)))
78
91
 
79
92
 
80
93
  def _norm_minmax(values: Sequence[float]) -> tuple[float, float]:
@@ -96,6 +109,7 @@ def _print_scores_latex(
96
109
  *,
97
110
  enable_color: bool = False,
98
111
  colormap: str = "viridis",
112
+ digits: int,
99
113
  printer: Callable[[str], None],
100
114
  ) -> None:
101
115
  """LaTeX table (optionally with score cells colored by value)."""
@@ -110,12 +124,13 @@ def _print_scores_latex(
110
124
  values = [s for _, s in performance_sorted]
111
125
  min_v, max_v = _norm_minmax(values)
112
126
  for approach, score in performance_sorted:
127
+ formatted = _format_float(score, digits)
113
128
  if enable_color:
114
129
  n = _norm_value(score, min_v, max_v)
115
130
  hex_color = _colormap_light_hex(n, colormap=colormap)
116
- printer(rf"{approach} & \cellcolor[HTML]{{{hex_color}}}{{{score:.2f}}} \\")
131
+ printer(rf"{approach} & \cellcolor[HTML]{{{hex_color}}}{{{formatted}}} \\")
117
132
  else:
118
- printer(f"{approach} & {score:.2f} \\\\")
133
+ printer(f"{approach} & {formatted} \\\\")
119
134
  printer(r"\end{tabular}")
120
135
 
121
136
 
@@ -127,6 +142,7 @@ def print_relcov_corpus_table(
127
142
  colormap: str = "viridis",
128
143
  latex_rotate_headers: Optional[float] = None,
129
144
  latex_enable_color: bool = False,
145
+ digits: int = _DEFAULT_DIGITS,
130
146
  printer: Callable[[str], None] = print,
131
147
  ) -> None:
132
148
  """Print a table of relcov performance: rows = approaches, columns = corpus approaches."""
@@ -138,12 +154,14 @@ def print_relcov_corpus_table(
138
154
  _print_relcov_corpus_table_plain(
139
155
  corpus_approaches,
140
156
  table,
157
+ digits=digits,
141
158
  printer=printer,
142
159
  )
143
160
  elif output == "csv":
144
161
  _print_relcov_corpus_table_csv(
145
162
  corpus_approaches,
146
163
  table,
164
+ digits=digits,
147
165
  printer=printer,
148
166
  )
149
167
  elif output == "latex":
@@ -153,6 +171,7 @@ def print_relcov_corpus_table(
153
171
  rotate_headers=latex_rotate_headers,
154
172
  enable_color=latex_enable_color,
155
173
  colormap=colormap,
174
+ digits=digits,
156
175
  printer=printer,
157
176
  )
158
177
  else:
@@ -163,6 +182,7 @@ def _print_relcov_corpus_table_plain(
163
182
  corpus_approaches: Sequence[ApproachId],
164
183
  table: Mapping[ApproachId, Mapping[ApproachId, float]],
165
184
  *,
185
+ digits: int,
166
186
  printer: Callable[[str], None],
167
187
  ) -> None:
168
188
  row_labels = sorted(table.keys())
@@ -171,7 +191,8 @@ def _print_relcov_corpus_table_plain(
171
191
  (len(str(c)) for c in col_labels + ["approach"]),
172
192
  default=7,
173
193
  )
174
- num_width = 10 # e.g. " 1.00"
194
+ # Width for aligned numeric columns: sign + integer digit + '.' + fraction + padding.
195
+ num_width = max(10, digits + 4)
175
196
  header = "approach".ljust(col_width)
176
197
  for c in col_labels:
177
198
  header += str(c).rjust(num_width)
@@ -181,7 +202,7 @@ def _print_relcov_corpus_table_plain(
181
202
  for c in col_labels:
182
203
  val = table[row].get(c)
183
204
  if val is not None:
184
- line += f"{val:>{num_width}.5f}"
205
+ line += f"{val:>{num_width}.{digits}f}"
185
206
  else:
186
207
  line += " " * num_width
187
208
  printer(line)
@@ -191,6 +212,7 @@ def _print_relcov_corpus_table_csv(
191
212
  corpus_approaches: Sequence[ApproachId],
192
213
  table: Mapping[ApproachId, Mapping[ApproachId, float]],
193
214
  *,
215
+ digits: int,
194
216
  printer: Callable[[str], None],
195
217
  ) -> None:
196
218
  row_labels = sorted(table.keys())
@@ -201,7 +223,7 @@ def _print_relcov_corpus_table_csv(
201
223
  cells: list[str] = [str(row)]
202
224
  for c in col_labels:
203
225
  val = table[row].get(c)
204
- cells.append(f"{val:.3f}" if val is not None else "")
226
+ cells.append(_format_float(val, digits) if val is not None else "")
205
227
  writer.writerow(cells)
206
228
 
207
229
 
@@ -255,6 +277,7 @@ def _print_relcov_corpus_table_latex(
255
277
  rotate_headers: Optional[float] = None,
256
278
  enable_color: bool = False,
257
279
  colormap: str = "viridis",
280
+ digits: int,
258
281
  printer: Callable[[str], None],
259
282
  ) -> None:
260
283
  try:
@@ -288,9 +311,11 @@ def _print_relcov_corpus_table_latex(
288
311
  elif enable_color:
289
312
  n = _norm_value(val, min_v, max_v)
290
313
  hex_color = _colormap_light_hex(n, colormap=colormap)
291
- cells.append(rf"\cellcolor[HTML]{{{hex_color}}}{{{val:.3f}}}")
314
+ cells.append(
315
+ rf"\cellcolor[HTML]{{{hex_color}}}{{{_format_float(val, digits)}}}"
316
+ )
292
317
  else:
293
- cells.append(f"{val:.3f}")
318
+ cells.append(_format_float(val, digits))
294
319
  printer("\t" + " & ".join(cells) + r" \\")
295
320
  printer(r"\end{tabular}")
296
321
 
@@ -1,4 +1,4 @@
1
- from differential_coverage.readers import afl_showmap, llvm_cov
1
+ from differential_coverage.readers import afl_showmap, libfuzzer_merge, llvm_cov
2
2
  from differential_coverage.readers.registry import (
3
3
  Granularity,
4
4
  GranularityArg,
@@ -15,4 +15,4 @@ __all__ = [
15
15
  "resolve_reader",
16
16
  ]
17
17
 
18
- _ = (afl_showmap, llvm_cov)
18
+ _ = (afl_showmap, libfuzzer_merge, llvm_cov)
@@ -0,0 +1,41 @@
1
+ from pathlib import Path
2
+
3
+ from differential_coverage.readers.registry import (
4
+ Granularity,
5
+ TrialReader,
6
+ register_reader,
7
+ )
8
+
9
+
10
+ def read(path: Path, *, granularity: Granularity) -> set[str]:
11
+ if granularity != "edge":
12
+ raise ValueError("libfuzzer-merge only supports --granularity edge")
13
+ edges: set[str] = set()
14
+ for line in path.read_text().splitlines():
15
+ parts = line.split()
16
+ if len(parts) < 3 or parts[0] != "COV":
17
+ continue
18
+ edges.update(parts[2:])
19
+ if not edges:
20
+ raise ValueError(f"No covered edges in {path}")
21
+ return edges
22
+
23
+
24
+ def detect(path: Path) -> bool:
25
+ lines = [line.strip() for line in path.read_text().splitlines() if line.strip()]
26
+ if len(lines) < 3:
27
+ return False
28
+ try:
29
+ num_files = int(lines[0])
30
+ num_first = int(lines[1])
31
+ except ValueError:
32
+ return False
33
+ if num_files <= 0 or num_first > num_files:
34
+ return False
35
+ has_marker = any(
36
+ line.startswith("STARTED ") or line.startswith("COV ") for line in lines
37
+ )
38
+ return has_marker
39
+
40
+
41
+ register_reader(TrialReader(name="libfuzzer-merge", read=read, detect=detect))
@@ -3,18 +3,20 @@ from dataclasses import dataclass
3
3
  from pathlib import Path
4
4
  from typing import Literal, Protocol
5
5
 
6
- ReaderName = Literal["afl-showmap", "llvm-cov"]
7
- InputFormat = Literal["auto", "afl-showmap", "llvm-cov"]
6
+ ReaderName = Literal["afl-showmap", "llvm-cov", "libfuzzer-merge"]
7
+ InputFormat = Literal["auto", "afl-showmap", "llvm-cov", "libfuzzer-merge"]
8
8
  Granularity = Literal["branch", "block", "edge"]
9
9
  GranularityArg = Literal["auto", "branch", "block", "edge"]
10
10
 
11
11
  _SUPPORTED: dict[ReaderName, frozenset[Granularity]] = {
12
12
  "afl-showmap": frozenset({"edge"}),
13
13
  "llvm-cov": frozenset({"branch", "block"}),
14
+ "libfuzzer-merge": frozenset({"edge"}),
14
15
  }
15
16
  _DEFAULT: dict[ReaderName, Granularity] = {
16
17
  "afl-showmap": "edge",
17
18
  "llvm-cov": "branch",
19
+ "libfuzzer-merge": "edge",
18
20
  }
19
21
 
20
22
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: differential_coverage
3
- Version: 1.1.2
3
+ Version: 1.2.1
4
4
  Summary: Compute differential coverage, a better way of comparing testing tools.
5
5
  Author-email: Valentin Huber <contact@valentinhuber.me>
6
6
  License-Expression: MIT
@@ -43,7 +43,11 @@ For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEF
43
43
 
44
44
  ## How?
45
45
 
46
- Campaign data lives in one directory per approach, with one file per trial. Two input formats are supported:
46
+ Campaign data lives in one directory per approach, with one file per trial. Three input formats are supported.
47
+
48
+ **All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices), libFuzzer merge COV indices (PC-table indices), and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign. Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
49
+
50
+ For **afl-showmap** and **libfuzzer-merge**, edge IDs are opaque indices into the instrumented binary, so every trial must use the **same binary** (same build/instrumentation). **llvm-cov** uses source locations instead, so the binary may differ as long as source paths are consistent across exports (I think™).
47
51
 
48
52
  ### afl-showmap
49
53
 
@@ -88,6 +92,8 @@ llvm-cov export \
88
92
 
89
93
  Run once per trial (merge profdata first if a trial replays multiple inputs). Use `--granularity branch` or `block` for llvm-cov exports; afl-showmap uses `edge` (selected automatically with `--granularity auto`).
90
94
 
95
+ On macOS, use Homebrew LLVM rather than Apple Clang: `brew install llvm`, then either add `$(brew --prefix llvm)/bin` to your `PATH` or call tools by full path (e.g. `$(brew --prefix llvm)/bin/clang`). Built-in `clang` _may_ work, but is untested.
96
+
91
97
  **All trials in a campaign must use the same input format, granularity, and consistent source paths (llvm-cov).** If paths differ between machines, re-export with `llvm-cov export -path-equivalence=<build-path>,<local-path>`.
92
98
 
93
99
  #### cov-analysis
@@ -101,11 +107,28 @@ cov-analysis -d /path/to/fuzzer-out/ -e "./cov @@"
101
107
 
102
108
  Copy one `coverage.json` per trial into your campaign directory (same layout as above). The format matches `llvm-cov export -format=text`.
103
109
 
104
- ### Edge IDs and mixing formats
110
+ ### libFuzzer merge control files
111
+
112
+ Same layout, with merge control files from libFuzzer's `-merge=1` mode:
105
113
 
106
- **All approaches in one campaign must use the same input format and the same edge ID scheme.** afl-showmap edge IDs (numeric bitmap indices) and llvm-cov edge IDs (source locations) are not comparable — do not mix them in one campaign.
114
+ ```
115
+ coverage_data
116
+ |-- approach_1
117
+ | |-- trial_1.merge
118
+ | `-- trial_2.merge
119
+ `-- seeds
120
+ `-- seeds.merge
121
+ ```
122
+
123
+ Generate each trial file by merging a seeds corpus with trial inputs and keeping the control file:
124
+
125
+ ```bash
126
+ empty=$(mktemp -d)
127
+ ./my_fuzzer -merge=1 -merge_control_file=trial_1.merge "$empty" CORPUS_DIR
128
+ rmdir "$empty"
129
+ ```
107
130
 
108
- Format is detected from file content (`auto`, default). Override with `--input-format` if needed.
131
+ Use an empty first corpus so every input is recorded. The reader unions all `COV` lines (PC-table indices; `--granularity edge`). Same binary across trials; LLVM 10+.
109
132
 
110
133
  ### Installation
111
134
 
@@ -123,7 +146,7 @@ pip install differential-coverage[latex]
123
146
  Generally, the command line interface follows the following structure:
124
147
 
125
148
  ```bash
126
- differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov}] [--granularity {auto,branch,block,edge}] <campaign-dir>
149
+ differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
127
150
  ```
128
151
 
129
152
  Examples:
@@ -15,12 +15,14 @@ src/differential_coverage.egg-info/requires.txt
15
15
  src/differential_coverage.egg-info/top_level.txt
16
16
  src/differential_coverage/readers/__init__.py
17
17
  src/differential_coverage/readers/afl_showmap.py
18
+ src/differential_coverage/readers/libfuzzer_merge.py
18
19
  src/differential_coverage/readers/llvm_cov.py
19
20
  src/differential_coverage/readers/registry.py
20
21
  tests/test_api.py
21
22
  tests/test_approach_data.py
22
23
  tests/test_cli.py
23
24
  tests/test_docs.py
25
+ tests/test_libfuzzer_merge.py
24
26
  tests/test_llvm_cov.py
25
27
  tests/test_output.py
26
28
  tests/test_readers.py
@@ -275,3 +275,15 @@ def test_cli_latex_color_no_cell_colors() -> None:
275
275
  assert any(line.startswith(r"\begin{tabular}") for line in lines)
276
276
  # Table should not contain any colored cells
277
277
  assert not any(r"\cellcolor" in line for line in lines)
278
+
279
+
280
+ def test_cli_digits_relscore() -> None:
281
+ """CLI --digits controls decimal places in relscore output."""
282
+ code, out, _ = _run_cli(["--digits", "4", "relscore", str(SAMPLE_SHOWMAP_DIR)])
283
+ assert code == 0
284
+ lines = [s.strip() for s in out.strip().splitlines()]
285
+ # Each score should have exactly 4 digits after the decimal point
286
+ for line in lines:
287
+ score = line.split(":", 1)[1].strip()
288
+ assert "." in score
289
+ assert len(score.split(".", 1)[1]) == 4
@@ -0,0 +1,121 @@
1
+ """Unit tests for libFuzzer merge control file input."""
2
+
3
+ import io
4
+ import re
5
+ import sys
6
+ from pathlib import Path
7
+
8
+ import pytest
9
+
10
+ from differential_coverage.api import DifferentialCoverage
11
+ from differential_coverage.cli import main
12
+ from differential_coverage.fs import read_approach_dir, read_campaign_dir
13
+ from differential_coverage.readers import libfuzzer_merge
14
+ from differential_coverage.readers.registry import detect_reader, resolve_granularity
15
+
16
+ _DONE_COV_RE = re.compile(r"^\#\d+\s+DONE\s+cov:\s+(\d+)", re.MULTILINE)
17
+
18
+
19
+ def _run_cli(argv: list[str]) -> tuple[int | str, str]:
20
+ out = io.StringIO()
21
+ old_stdout = sys.stdout
22
+ try:
23
+ sys.stdout = out
24
+ code = main(argv)
25
+ except SystemExit as exc:
26
+ code = int(exc.code) if exc.code is not None else 1
27
+ finally:
28
+ sys.stdout = old_stdout
29
+ return code, out.getvalue()
30
+
31
+
32
+ def _calc_merge(calc_merge_dir: Path, *parts: str) -> Path:
33
+ return calc_merge_dir.joinpath(*parts)
34
+
35
+
36
+ def _parse_libfuzzer_done_cov(log_text: str) -> int:
37
+ matches = _DONE_COV_RE.findall(log_text)
38
+ if not matches:
39
+ raise ValueError("no DONE cov line in libFuzzer merge output")
40
+ return int(matches[-1])
41
+
42
+
43
+ def test_read_cov_matches_libfuzzer_merge_output(
44
+ calc_build_dir: Path, calc_merge_dir: Path
45
+ ) -> None:
46
+ merge_logs = calc_build_dir / "build" / "merge_logs"
47
+ for merge_file in sorted(calc_merge_dir.rglob("*.merge")):
48
+ rel = merge_file.relative_to(calc_merge_dir)
49
+ log_file = merge_logs / rel.parent / f"{merge_file.stem}.log"
50
+ assert log_file.is_file(), f"missing libFuzzer log for {merge_file}"
51
+ expected_cov = _parse_libfuzzer_done_cov(log_file.read_text())
52
+ edges = libfuzzer_merge.read(merge_file, granularity="edge")
53
+ assert len(edges) == expected_cov, (
54
+ f"{rel}: reader found {len(edges)} edges, "
55
+ f"libFuzzer reported cov: {expected_cov}"
56
+ )
57
+
58
+
59
+ def test_read_detect_and_read(calc_merge_dir: Path) -> None:
60
+ merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
61
+ assert libfuzzer_merge.detect(merge_file)
62
+ edges = libfuzzer_merge.read(merge_file, granularity="edge")
63
+ assert edges
64
+ assert all(edge.isdigit() for edge in edges)
65
+
66
+
67
+ def test_detect_reader(calc_merge_dir: Path) -> None:
68
+ merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
69
+ assert detect_reader(merge_file) == "libfuzzer-merge"
70
+
71
+
72
+ def test_granularity_rejects_non_edge(calc_merge_dir: Path) -> None:
73
+ merge_file = _calc_merge(calc_merge_dir, "approach_a", "t1.merge")
74
+ with pytest.raises(ValueError, match="does not support --granularity branch"):
75
+ resolve_granularity("libfuzzer-merge", "branch")
76
+ with pytest.raises(ValueError, match="only supports --granularity edge"):
77
+ libfuzzer_merge.read(merge_file, granularity="block")
78
+
79
+
80
+ def test_read_raises_without_cov_lines(tmp_path: Path) -> None:
81
+ merge_file = tmp_path / "empty.merge"
82
+ merge_file.write_text("1\n1\n/input\n")
83
+ with pytest.raises(ValueError, match="No covered edges"):
84
+ libfuzzer_merge.read(merge_file, granularity="edge")
85
+
86
+
87
+ def test_read_approach_dir(calc_merge_dir: Path) -> None:
88
+ trials = read_approach_dir(
89
+ calc_merge_dir / "approach_a",
90
+ input_format="libfuzzer-merge",
91
+ )
92
+ assert trials
93
+ assert all(edges for edges in trials.values())
94
+
95
+
96
+ def test_read_campaign_dir(calc_merge_dir: Path) -> None:
97
+ campaign = read_campaign_dir(
98
+ calc_merge_dir,
99
+ input_format="libfuzzer-merge",
100
+ )
101
+ assert {"seeds", "approach_a", "approach_b", "approach_c"} <= set(campaign.keys())
102
+
103
+
104
+ def test_relcov_from_merge_campaign(calc_merge_dir: Path) -> None:
105
+ dc = DifferentialCoverage.from_campaign_dir(
106
+ calc_merge_dir,
107
+ input_format="libfuzzer-merge",
108
+ )
109
+ approach_c = dc.approaches["approach_c"]
110
+ for approach in ("approach_a", "approach_b", "seeds"):
111
+ assert approach_c.relcov(dc.approaches[approach]) == 1.0
112
+
113
+
114
+ def test_cli_relscore(calc_merge_dir: Path) -> None:
115
+ code, out = _run_cli(
116
+ ["--input-format", "libfuzzer-merge", "relscore", str(calc_merge_dir)]
117
+ )
118
+ assert code == 0
119
+ lines = [line.strip() for line in out.strip().splitlines()]
120
+ assert any(line.startswith("approach_c:") for line in lines)
121
+ assert any(line.startswith("approach_a:") for line in lines)
@@ -30,6 +30,52 @@ def _calc_export(calc_coverage_dir: Path, *parts: str) -> Path:
30
30
  return calc_coverage_dir.joinpath(*parts)
31
31
 
32
32
 
33
+ def _covered_code_region_keys(export: Path) -> list[str]:
34
+ """Covered CodeRegion location keys (may include duplicates)."""
35
+ data = json.loads(export.read_text())
36
+ keys: list[str] = []
37
+ for item in data.get("data", []):
38
+ for function in item.get("functions", []):
39
+ filenames = function["filenames"]
40
+ for region in function.get("regions", []):
41
+ # CountedRegion: exec=4, file_id=5, kind=7; CodeRegion kind=0
42
+ if region[4] <= 0 or region[7] != 0:
43
+ continue
44
+ file_id = region[5]
45
+ keys.append(
46
+ f"fn:{function['name']}@{filenames[file_id]}:"
47
+ f"{region[0]}:{region[1]}-{region[2]}:{region[3]}"
48
+ )
49
+ return keys
50
+
51
+
52
+ def test_read_cov_matches_llvm_cov_summary(
53
+ calc_build_dir: Path, calc_coverage_dir: Path
54
+ ) -> None:
55
+ summaries = calc_build_dir / "build" / "llvm_summaries"
56
+ for export in sorted(calc_coverage_dir.rglob("*.json")):
57
+ rel = export.relative_to(calc_coverage_dir)
58
+ summary_file = summaries / rel.parent / f"{export.stem}.json"
59
+ assert summary_file.is_file(), f"missing llvm-cov summary for {export}"
60
+ totals = json.loads(summary_file.read_text())["data"][0]["totals"]
61
+ branch_edges = llvm_cov.read(export, granularity="branch")
62
+ block_edges = llvm_cov.read(export, granularity="block")
63
+ assert len(branch_edges) == totals["branches"]["covered"], (
64
+ f"{rel} branch: reader found {len(branch_edges)}, "
65
+ f"llvm-cov reported {totals['branches']['covered']}"
66
+ )
67
+ # Summary counts CodeRegion hits with multiplicity; the reader dedupes
68
+ # by source location (same as differential-coverage edge identity).
69
+ region_keys = _covered_code_region_keys(export)
70
+ assert len(region_keys) == totals["regions"]["covered"], (
71
+ f"{rel} regions: export has {len(region_keys)} covered CodeRegions, "
72
+ f"llvm-cov reported {totals['regions']['covered']}"
73
+ )
74
+ assert block_edges == set(region_keys), (
75
+ f"{rel} block: reader set does not match unique covered CodeRegions"
76
+ )
77
+
78
+
33
79
  def test_read_llvm_cov_branch_uses_function_scope(calc_coverage_dir: Path) -> None:
34
80
  export = _calc_export(calc_coverage_dir, "approach_a", "t1.json")
35
81
  edges = llvm_cov.read(export, granularity="branch")
@@ -75,8 +75,8 @@ def test_print_scores_uses_printer_stdout() -> None:
75
75
 
76
76
  # Scores should be sorted by value descending, then by name
77
77
  joined = "\n".join(captured)
78
- assert "a: 1.00" in joined
79
- assert "b: 0.10" in joined
78
+ assert "a: 1.000" in joined
79
+ assert "b: 0.100" in joined
80
80
  # Ensure our custom printer was used (no direct writes to sys.stdout here)
81
81
  assert joined.count("\n") >= 0
82
82
 
@@ -97,3 +97,30 @@ def test_print_scores_uses_printer_csv() -> None:
97
97
  # Order should be by score descending
98
98
  assert rows[1][0] == "a"
99
99
  assert rows[2][0] == "b"
100
+
101
+
102
+ def test_print_scores_respects_digits() -> None:
103
+ """print_scores formats scores with the requested number of digits."""
104
+ scores = {"a": 1.0 / 3.0}
105
+ captured, printer = _capture_printer()
106
+
107
+ print_scores(scores, output="stdout", digits=4, printer=printer)
108
+
109
+ assert captured == ["a: 0.3333"]
110
+
111
+
112
+ def test_relcov_table_respects_digits_csv() -> None:
113
+ """Relcov CSV cells use the requested digit precision."""
114
+ captured, printer = _capture_printer()
115
+ print_relcov_corpus_table(
116
+ ["a"],
117
+ {"a": {"a": 1.0 / 3.0}},
118
+ output="csv",
119
+ digits=4,
120
+ printer=printer,
121
+ )
122
+
123
+ csv_text = "\n".join(captured)
124
+ reader = csv.reader(io.StringIO(csv_text))
125
+ rows = list(reader)
126
+ assert rows[1][1] == "0.3333"
@@ -4,7 +4,7 @@ from pathlib import Path
4
4
 
5
5
  import pytest
6
6
 
7
- from differential_coverage.readers import afl_showmap, llvm_cov
7
+ from differential_coverage.readers import afl_showmap, libfuzzer_merge, llvm_cov
8
8
  from differential_coverage.readers.registry import detect_reader, resolve_reader
9
9
 
10
10
  AFL_FIXTURE = Path(__file__).parent / "fixtures" / "afl_showmap" / "t1.txt"
@@ -14,6 +14,10 @@ def _llvm_export(calc_coverage_dir: Path) -> Path:
14
14
  return next((calc_coverage_dir / "approach_a").glob("*.json"))
15
15
 
16
16
 
17
+ def _merge_export(calc_merge_dir: Path) -> Path:
18
+ return calc_merge_dir / "approach_a" / "t1.merge"
19
+
20
+
17
21
  def test_afl_showmap_detect_and_read() -> None:
18
22
  assert afl_showmap.detect(AFL_FIXTURE)
19
23
  assert not llvm_cov.detect(AFL_FIXTURE)
@@ -24,27 +28,45 @@ def test_llvm_cov_detect_and_read(calc_coverage_dir: Path) -> None:
24
28
  export = _llvm_export(calc_coverage_dir)
25
29
  assert llvm_cov.detect(export)
26
30
  assert not afl_showmap.detect(export)
31
+ assert not libfuzzer_merge.detect(export)
27
32
  assert llvm_cov.read(export, granularity="branch")
28
33
 
29
34
 
30
- def test_detect_reader(calc_coverage_dir: Path) -> None:
35
+ def test_libfuzzer_merge_detect_and_read(calc_merge_dir: Path) -> None:
36
+ merge_file = _merge_export(calc_merge_dir)
37
+ assert libfuzzer_merge.detect(merge_file)
38
+ assert not afl_showmap.detect(merge_file)
39
+ assert not llvm_cov.detect(merge_file)
40
+ edges = libfuzzer_merge.read(merge_file, granularity="edge")
41
+ assert edges
42
+ assert all(edge.isdigit() for edge in edges)
43
+
44
+
45
+ def test_detect_reader(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
31
46
  export = _llvm_export(calc_coverage_dir)
47
+ merge_file = _merge_export(calc_merge_dir)
32
48
  assert detect_reader(AFL_FIXTURE) == "afl-showmap"
33
49
  assert detect_reader(export) == "llvm-cov"
50
+ assert detect_reader(merge_file) == "libfuzzer-merge"
34
51
 
35
52
 
36
- def test_resolve_reader_auto(calc_coverage_dir: Path) -> None:
53
+ def test_resolve_reader_auto(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
37
54
  export = _llvm_export(calc_coverage_dir)
55
+ merge_file = _merge_export(calc_merge_dir)
38
56
  reader = resolve_reader([AFL_FIXTURE], "auto")
39
57
  assert reader.name == "afl-showmap"
40
58
  reader = resolve_reader([export], "auto")
41
59
  assert reader.name == "llvm-cov"
60
+ reader = resolve_reader([merge_file], "auto")
61
+ assert reader.name == "libfuzzer-merge"
42
62
 
43
63
 
44
- def test_resolve_reader_forced(calc_coverage_dir: Path) -> None:
64
+ def test_resolve_reader_forced(calc_coverage_dir: Path, calc_merge_dir: Path) -> None:
45
65
  export = _llvm_export(calc_coverage_dir)
66
+ merge_file = _merge_export(calc_merge_dir)
46
67
  assert resolve_reader([export], "llvm-cov").name == "llvm-cov"
47
68
  assert resolve_reader([AFL_FIXTURE], "afl-showmap").name == "afl-showmap"
69
+ assert resolve_reader([merge_file], "libfuzzer-merge").name == "libfuzzer-merge"
48
70
 
49
71
 
50
72
  def test_afl_showmap_rejects_block_granularity() -> None:
@@ -59,6 +81,7 @@ def test_resolve_granularity_auto(calc_coverage_dir: Path) -> None:
59
81
 
60
82
  assert resolve_granularity("llvm-cov", "auto") == "branch"
61
83
  assert resolve_granularity("afl-showmap", "auto") == "edge"
84
+ assert resolve_granularity("libfuzzer-merge", "auto") == "edge"
62
85
  assert resolve_granularity("llvm-cov", "block") == "block"
63
86
  with pytest.raises(ValueError, match="does not support --granularity edge"):
64
87
  resolve_granularity("llvm-cov", "edge")