differential-coverage 1.2.0__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/PKG-INFO +8 -2
  2. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/README.md +7 -1
  3. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/pyproject.toml +6 -1
  4. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/__init__.py +7 -8
  5. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/api.py +13 -5
  6. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/approach_data.py +3 -1
  7. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/cli.py +47 -14
  8. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/output.py +71 -20
  9. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/readers/libfuzzer_merge.py +1 -3
  10. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/PKG-INFO +8 -2
  11. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_api.py +11 -0
  12. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_approach_data.py +7 -5
  13. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_cli.py +56 -0
  14. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_docs.py +5 -1
  15. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_llvm_cov.py +1 -1
  16. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_output.py +61 -4
  17. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_reducers.py +2 -1
  18. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/setup.cfg +0 -0
  19. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/fs.py +0 -0
  20. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/readers/__init__.py +0 -0
  21. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/readers/afl_showmap.py +0 -0
  22. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/readers/llvm_cov.py +0 -0
  23. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/readers/registry.py +0 -0
  24. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage/types.py +1 -1
  25. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/SOURCES.txt +0 -0
  26. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/dependency_links.txt +0 -0
  27. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/entry_points.txt +0 -0
  28. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/requires.txt +0 -0
  29. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/top_level.txt +0 -0
  30. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_libfuzzer_merge.py +0 -0
  31. {differential_coverage-1.2.0 → differential_coverage-1.3.0}/tests/test_readers.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: differential_coverage
3
- Version: 1.2.0
3
+ Version: 1.3.0
4
4
  Summary: Compute differential coverage, a better way of comparing testing tools.
5
5
  Author-email: Valentin Huber <contact@valentinhuber.me>
6
6
  License-Expression: MIT
@@ -38,6 +38,7 @@ In principle, differential coverage measures how much of approach $a_2$'s covera
38
38
 
39
39
  - `relcov` does exactly this. It is an *asymetrical* measure between coverage of two approaches. If the approach relies on randomness (such as most fuzzers), it can aggregate over multiple trials of these.
40
40
  - `relscore` aggregates all this data to an order of all approaches. This measure, compared to simple coverage numbers, takes into account and values higher approaches that can cover code that no other approach covers. However, compared to `relcov`, this hides a lot of information in this aggregation.
41
+ - `count` just reports absolute coverage counts (union of covered edges/blocks/branches across trials) per approach — useful as a baseline, but without the differential comparison.
41
42
 
42
43
  For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEFINITIONS.md).
43
44
 
@@ -146,13 +147,14 @@ pip install differential-coverage[latex]
146
147
  Generally, the command line interface follows the following structure:
147
148
 
148
149
  ```bash
149
- differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
150
+ differential-coverage {relcov,relscore,count} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
150
151
  ```
151
152
 
152
153
  Examples:
153
154
 
154
155
  ```bash
155
156
  differential-coverage relscore ./coverage_data
157
+ differential-coverage count ./coverage_data
156
158
  differential-coverage --input-format llvm-cov relcov ./coverage_data
157
159
  differential-coverage --output csv relscore ./coverage_data
158
160
  ```
@@ -206,6 +208,10 @@ relscores: dict[str, float] = dc.relscores()
206
208
  for approach_name, relscore in relscores.items():
207
209
  print(f"relscore {approach_name}: {relscore}")
208
210
 
211
+ counts: dict[str, int] = dc.coverage_counts()
212
+ for approach_name, count in counts.items():
213
+ print(f"coverage {approach_name}: {count}")
214
+
209
215
  ```
210
216
 
211
217
  ### Hints
@@ -14,6 +14,7 @@ In principle, differential coverage measures how much of approach $a_2$'s covera
14
14
 
15
15
  - `relcov` does exactly this. It is an *asymetrical* measure between coverage of two approaches. If the approach relies on randomness (such as most fuzzers), it can aggregate over multiple trials of these.
16
16
  - `relscore` aggregates all this data to an order of all approaches. This measure, compared to simple coverage numbers, takes into account and values higher approaches that can cover code that no other approach covers. However, compared to `relcov`, this hides a lot of information in this aggregation.
17
+ - `count` just reports absolute coverage counts (union of covered edges/blocks/branches across trials) per approach — useful as a baseline, but without the differential comparison.
17
18
 
18
19
  For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEFINITIONS.md).
19
20
 
@@ -122,13 +123,14 @@ pip install differential-coverage[latex]
122
123
  Generally, the command line interface follows the following structure:
123
124
 
124
125
  ```bash
125
- differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
126
+ differential-coverage {relcov,relscore,count} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
126
127
  ```
127
128
 
128
129
  Examples:
129
130
 
130
131
  ```bash
131
132
  differential-coverage relscore ./coverage_data
133
+ differential-coverage count ./coverage_data
132
134
  differential-coverage --input-format llvm-cov relcov ./coverage_data
133
135
  differential-coverage --output csv relscore ./coverage_data
134
136
  ```
@@ -182,6 +184,10 @@ relscores: dict[str, float] = dc.relscores()
182
184
  for approach_name, relscore in relscores.items():
183
185
  print(f"relscore {approach_name}: {relscore}")
184
186
 
187
+ counts: dict[str, int] = dc.coverage_counts()
188
+ for approach_name, count in counts.items():
189
+ print(f"coverage {approach_name}: {count}")
190
+
185
191
  ```
186
192
 
187
193
  ### Hints
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "differential_coverage"
7
- version = "1.2.0"
7
+ version = "1.3.0"
8
8
  description = "Compute differential coverage, a better way of comparing testing tools."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -38,5 +38,10 @@ where = ["src"]
38
38
  [tool.pytest.ini_options]
39
39
  pythonpath = ["src"]
40
40
 
41
+ [tool.ruff.lint]
42
+ ignore = [
43
+ "UP045", # Optional[X] -> X | None
44
+ ]
45
+
41
46
  [project.scripts]
42
47
  differential-coverage = "differential_coverage:main"
@@ -1,24 +1,23 @@
1
- #!/usr/bin/env python3
2
- from differential_coverage.cli import main
3
1
  from differential_coverage.api import DifferentialCoverage
4
2
  from differential_coverage.approach_data import ApproachData
3
+ from differential_coverage.cli import main
5
4
  from differential_coverage.types import (
6
- EdgeId,
7
5
  ApproachId,
6
+ CollectionReducer,
7
+ EdgeId,
8
8
  TrialId,
9
9
  ValueReducer,
10
- CollectionReducer,
11
10
  )
12
11
 
13
12
  __all__ = [
14
- "main",
15
- "DifferentialCoverage",
16
13
  "ApproachData",
17
- "EdgeId",
18
14
  "ApproachId",
15
+ "CollectionReducer",
16
+ "DifferentialCoverage",
17
+ "EdgeId",
19
18
  "TrialId",
20
19
  "ValueReducer",
21
- "CollectionReducer",
20
+ "main",
22
21
  ]
23
22
 
24
23
  if __name__ == "__main__":
@@ -1,14 +1,15 @@
1
+ import warnings
2
+ from collections.abc import Collection, Mapping
1
3
  from functools import reduce
2
4
  from pathlib import Path
3
- from typing import Collection, Generic, Mapping
4
- import warnings
5
+ from typing import Generic
5
6
 
6
- from differential_coverage.readers import GranularityArg, InputFormat
7
- from differential_coverage.fs import read_campaign_dir
8
7
  from differential_coverage.approach_data import ApproachData
8
+ from differential_coverage.fs import read_campaign_dir
9
+ from differential_coverage.readers import GranularityArg, InputFormat
9
10
  from differential_coverage.types import (
10
- EdgeId,
11
11
  ApproachId,
12
+ EdgeId,
12
13
  TrialId,
13
14
  )
14
15
 
@@ -76,6 +77,13 @@ class DifferentialCoverage(Generic[ApproachId, TrialId, EdgeId]):
76
77
  def approaches(self) -> Mapping[ApproachId, ApproachData[TrialId, EdgeId]]:
77
78
  return self._approaches
78
79
 
80
+ def coverage_counts(self) -> dict[ApproachId, int]:
81
+ """Absolute coverage count per approach: size of the union of all trial edges."""
82
+ return {
83
+ approach_name: len(approach_data.edges_union)
84
+ for approach_name, approach_data in self._approaches.items()
85
+ }
86
+
79
87
  def relscores(self) -> dict[ApproachId, float]:
80
88
  all_edges = reduce(
81
89
  lambda x, y: x.union(y),
@@ -1,5 +1,7 @@
1
+ from collections.abc import Collection, Mapping
1
2
  from functools import reduce
2
- from typing import Collection, Generic, Mapping
3
+ from typing import Generic
4
+
3
5
  from differential_coverage.types import CollectionReducer, EdgeId, TrialId, ValueReducer
4
6
 
5
7
 
@@ -1,13 +1,17 @@
1
1
  import argparse
2
2
  import re
3
- from collections.abc import Callable
3
+ from collections.abc import Callable, Sequence
4
4
  from importlib import metadata
5
5
  from pathlib import Path
6
- from typing import Optional, Sequence
6
+ from typing import Optional
7
7
 
8
8
  from differential_coverage.api import DifferentialCoverage
9
9
  from differential_coverage.fs import read_campaign_dir
10
- from differential_coverage.output import print_relcov_corpus_table, print_scores
10
+ from differential_coverage.output import (
11
+ print_counts,
12
+ print_relcov_corpus_table,
13
+ print_scores,
14
+ )
11
15
 
12
16
  # All subcommands expect this directory layout (one campaign dir, no moving files).
13
17
  CAMPAIGN_DIR_HELP = (
@@ -87,8 +91,22 @@ def cmd_relscore(args: argparse.Namespace) -> int:
87
91
  print_scores(
88
92
  scores,
89
93
  output=args.output,
90
- colormap=getattr(args, "colormap", "viridis"),
91
- latex_enable_color=getattr(args, "latex_enable_color", False),
94
+ colormap=args.colormap,
95
+ latex_enable_color=args.latex_enable_color,
96
+ digits=args.digits,
97
+ )
98
+ return 0
99
+
100
+
101
+ def cmd_count(args: argparse.Namespace) -> int:
102
+ campaign = _load_campaign(args)
103
+ dc = DifferentialCoverage(campaign)
104
+ counts = dc.coverage_counts()
105
+ print_counts(
106
+ counts,
107
+ output=args.output,
108
+ colormap=args.colormap,
109
+ latex_enable_color=args.latex_enable_color,
92
110
  )
93
111
  return 0
94
112
 
@@ -97,11 +115,6 @@ def cmd_relcov_performance_over_approach(args: argparse.Namespace) -> int:
97
115
  campaign = _load_campaign(args)
98
116
 
99
117
  dc = DifferentialCoverage(campaign)
100
- output = args.output
101
- colormap = getattr(args, "colormap", "viridis")
102
- latex_rotate_headers = getattr(args, "latex_rotate_headers", None)
103
- latex_enable_color = getattr(args, "latex_enable_color", False)
104
-
105
118
  ref_approaches = list(dc.approaches.keys())
106
119
  table = {
107
120
  f: {ref: dc.approaches[f].relcov(dc.approaches[ref]) for ref in ref_approaches}
@@ -110,10 +123,11 @@ def cmd_relcov_performance_over_approach(args: argparse.Namespace) -> int:
110
123
  print_relcov_corpus_table(
111
124
  ref_approaches,
112
125
  table,
113
- output=output,
114
- colormap=colormap,
115
- latex_rotate_headers=latex_rotate_headers,
116
- latex_enable_color=latex_enable_color,
126
+ output=args.output,
127
+ colormap=args.colormap,
128
+ latex_rotate_headers=args.latex_rotate_headers,
129
+ latex_enable_color=args.latex_enable_color,
130
+ digits=args.digits,
117
131
  )
118
132
  return 0
119
133
 
@@ -192,6 +206,15 @@ def build_parser() -> argparse.ArgumentParser:
192
206
  'latex for LaTeX tabular (requires "latex" optional dependencies)'
193
207
  ),
194
208
  )
209
+ parser.add_argument(
210
+ "--digits",
211
+ type=int,
212
+ default=3,
213
+ metavar="N",
214
+ help=(
215
+ "Number of digits after the decimal point for numeric results (default: 3)."
216
+ ),
217
+ )
195
218
  parser.add_argument(
196
219
  "--latex-rotate-headers",
197
220
  type=float,
@@ -246,6 +269,16 @@ def build_parser() -> argparse.ArgumentParser:
246
269
  p_relscore.add_argument("dir", type=Path, help=CAMPAIGN_DIR_HELP)
247
270
  p_relscore.set_defaults(func=cmd_relscore)
248
271
 
272
+ p_count = subparsers.add_parser(
273
+ "count",
274
+ help=(
275
+ "Print absolute coverage counts (union of covered edges/blocks/branches "
276
+ "across trials) for each approach. Does not compute differential coverage."
277
+ ),
278
+ )
279
+ p_count.add_argument("dir", type=Path, help=CAMPAIGN_DIR_HELP)
280
+ p_count.set_defaults(func=cmd_count)
281
+
249
282
  return parser
250
283
 
251
284
 
@@ -6,7 +6,6 @@ import csv
6
6
  from collections.abc import Callable, Mapping, Sequence
7
7
  from typing import Literal, Optional
8
8
 
9
-
10
9
  from differential_coverage.types import ApproachId
11
10
 
12
11
  OutputFormat = Literal["stdout", "csv", "latex"]
@@ -23,58 +22,95 @@ class _PrinterIO:
23
22
  # and avoid printing empty chunks.
24
23
  if not s:
25
24
  return 0
26
- if s.endswith("\n"):
27
- s = s[:-1]
25
+ s = s.removesuffix("\n")
28
26
  if s:
29
27
  self._printer(s)
30
28
  return len(s)
31
29
 
32
30
 
31
+ # Default decimal places for numeric output.
32
+ _DEFAULT_DIGITS = 3
33
+
34
+
35
+ def _format_float(value: float, digits: int) -> str:
36
+ """Format a float with a fixed number of digits after the decimal point."""
37
+ return f"{value:.{digits}f}"
38
+
39
+
33
40
  def print_scores(
34
41
  scores: Mapping[ApproachId, float],
35
42
  *,
36
43
  output: OutputFormat = "stdout",
37
44
  colormap: str = "viridis",
38
45
  latex_enable_color: bool = False,
46
+ digits: int = _DEFAULT_DIGITS,
47
+ value_label: str = "score",
39
48
  printer: Callable[[str], None] = print,
40
49
  ) -> None:
41
- """Print one score per approach, in the requested output format."""
50
+ """Print one numeric value per approach, in the requested output format."""
42
51
  name_sorted = sorted(scores.items(), key=lambda x: x[0])
43
52
  performance_sorted = sorted(name_sorted, key=lambda x: x[1], reverse=True)
44
53
 
45
54
  if output == "stdout":
46
- _print_scores_plain(performance_sorted, printer=printer)
55
+ _print_scores_plain(performance_sorted, digits=digits, printer=printer)
47
56
  elif output == "csv":
48
- _print_scores_csv(performance_sorted, printer=printer)
57
+ _print_scores_csv(
58
+ performance_sorted, digits=digits, value_label=value_label, printer=printer
59
+ )
49
60
  elif output == "latex":
50
61
  _print_scores_latex(
51
62
  performance_sorted,
52
63
  enable_color=latex_enable_color,
53
64
  colormap=colormap,
65
+ digits=digits,
66
+ value_label=value_label,
54
67
  printer=printer,
55
68
  )
56
69
  else:
57
70
  raise ValueError(f"Invalid output format: {output}")
58
71
 
59
72
 
73
+ def print_counts(
74
+ counts: Mapping[ApproachId, int],
75
+ *,
76
+ output: OutputFormat = "stdout",
77
+ colormap: str = "viridis",
78
+ latex_enable_color: bool = False,
79
+ printer: Callable[[str], None] = print,
80
+ ) -> None:
81
+ """Print one absolute coverage count per approach, in the requested output format."""
82
+ print_scores(
83
+ {name: float(count) for name, count in counts.items()},
84
+ output=output,
85
+ colormap=colormap,
86
+ latex_enable_color=latex_enable_color,
87
+ digits=0,
88
+ value_label="coverage",
89
+ printer=printer,
90
+ )
91
+
92
+
60
93
  def _print_scores_plain(
61
94
  performance_sorted: Sequence[tuple[ApproachId, float]],
62
95
  *,
96
+ digits: int,
63
97
  printer: Callable[[str], None],
64
98
  ) -> None:
65
99
  for approach, score in performance_sorted:
66
- printer(f"{approach}: {score:.2f}")
100
+ printer(f"{approach}: {_format_float(score, digits)}")
67
101
 
68
102
 
69
103
  def _print_scores_csv(
70
104
  performance_sorted: Sequence[tuple[ApproachId, float]],
71
105
  *,
106
+ digits: int,
107
+ value_label: str,
72
108
  printer: Callable[[str], None],
73
109
  ) -> None:
74
110
  writer = csv.writer(_PrinterIO(printer))
75
- writer.writerow(("approach", "score"))
111
+ writer.writerow(("approach", value_label))
76
112
  for approach, score in performance_sorted:
77
- writer.writerow((approach, f"{score:.2f}"))
113
+ writer.writerow((approach, _format_float(score, digits)))
78
114
 
79
115
 
80
116
  def _norm_minmax(values: Sequence[float]) -> tuple[float, float]:
@@ -96,11 +132,13 @@ def _print_scores_latex(
96
132
  *,
97
133
  enable_color: bool = False,
98
134
  colormap: str = "viridis",
135
+ digits: int,
136
+ value_label: str,
99
137
  printer: Callable[[str], None],
100
138
  ) -> None:
101
- """LaTeX table (optionally with score cells colored by value)."""
139
+ """LaTeX table (optionally with value cells colored by value)."""
102
140
  printer(r"\begin{tabular}{lr}")
103
- printer(r"approach & score \\")
141
+ printer(rf"approach & {value_label} \\")
104
142
  if not performance_sorted:
105
143
  printer(r"\end{tabular}")
106
144
  return
@@ -110,12 +148,13 @@ def _print_scores_latex(
110
148
  values = [s for _, s in performance_sorted]
111
149
  min_v, max_v = _norm_minmax(values)
112
150
  for approach, score in performance_sorted:
151
+ formatted = _format_float(score, digits)
113
152
  if enable_color:
114
153
  n = _norm_value(score, min_v, max_v)
115
154
  hex_color = _colormap_light_hex(n, colormap=colormap)
116
- printer(rf"{approach} & \cellcolor[HTML]{{{hex_color}}}{{{score:.2f}}} \\")
155
+ printer(rf"{approach} & \cellcolor[HTML]{{{hex_color}}}{{{formatted}}} \\")
117
156
  else:
118
- printer(f"{approach} & {score:.2f} \\\\")
157
+ printer(f"{approach} & {formatted} \\\\")
119
158
  printer(r"\end{tabular}")
120
159
 
121
160
 
@@ -127,6 +166,7 @@ def print_relcov_corpus_table(
127
166
  colormap: str = "viridis",
128
167
  latex_rotate_headers: Optional[float] = None,
129
168
  latex_enable_color: bool = False,
169
+ digits: int = _DEFAULT_DIGITS,
130
170
  printer: Callable[[str], None] = print,
131
171
  ) -> None:
132
172
  """Print a table of relcov performance: rows = approaches, columns = corpus approaches."""
@@ -138,12 +178,14 @@ def print_relcov_corpus_table(
138
178
  _print_relcov_corpus_table_plain(
139
179
  corpus_approaches,
140
180
  table,
181
+ digits=digits,
141
182
  printer=printer,
142
183
  )
143
184
  elif output == "csv":
144
185
  _print_relcov_corpus_table_csv(
145
186
  corpus_approaches,
146
187
  table,
188
+ digits=digits,
147
189
  printer=printer,
148
190
  )
149
191
  elif output == "latex":
@@ -153,6 +195,7 @@ def print_relcov_corpus_table(
153
195
  rotate_headers=latex_rotate_headers,
154
196
  enable_color=latex_enable_color,
155
197
  colormap=colormap,
198
+ digits=digits,
156
199
  printer=printer,
157
200
  )
158
201
  else:
@@ -163,6 +206,7 @@ def _print_relcov_corpus_table_plain(
163
206
  corpus_approaches: Sequence[ApproachId],
164
207
  table: Mapping[ApproachId, Mapping[ApproachId, float]],
165
208
  *,
209
+ digits: int,
166
210
  printer: Callable[[str], None],
167
211
  ) -> None:
168
212
  row_labels = sorted(table.keys())
@@ -171,7 +215,8 @@ def _print_relcov_corpus_table_plain(
171
215
  (len(str(c)) for c in col_labels + ["approach"]),
172
216
  default=7,
173
217
  )
174
- num_width = 10 # e.g. " 1.00"
218
+ # Width for aligned numeric columns: sign + integer digit + '.' + fraction + padding.
219
+ num_width = max(10, digits + 4)
175
220
  header = "approach".ljust(col_width)
176
221
  for c in col_labels:
177
222
  header += str(c).rjust(num_width)
@@ -181,7 +226,7 @@ def _print_relcov_corpus_table_plain(
181
226
  for c in col_labels:
182
227
  val = table[row].get(c)
183
228
  if val is not None:
184
- line += f"{val:>{num_width}.5f}"
229
+ line += f"{val:>{num_width}.{digits}f}"
185
230
  else:
186
231
  line += " " * num_width
187
232
  printer(line)
@@ -191,6 +236,7 @@ def _print_relcov_corpus_table_csv(
191
236
  corpus_approaches: Sequence[ApproachId],
192
237
  table: Mapping[ApproachId, Mapping[ApproachId, float]],
193
238
  *,
239
+ digits: int,
194
240
  printer: Callable[[str], None],
195
241
  ) -> None:
196
242
  row_labels = sorted(table.keys())
@@ -201,7 +247,7 @@ def _print_relcov_corpus_table_csv(
201
247
  cells: list[str] = [str(row)]
202
248
  for c in col_labels:
203
249
  val = table[row].get(c)
204
- cells.append(f"{val:.3f}" if val is not None else "")
250
+ cells.append(_format_float(val, digits) if val is not None else "")
205
251
  writer.writerow(cells)
206
252
 
207
253
 
@@ -255,10 +301,13 @@ def _print_relcov_corpus_table_latex(
255
301
  rotate_headers: Optional[float] = None,
256
302
  enable_color: bool = False,
257
303
  colormap: str = "viridis",
304
+ digits: int,
258
305
  printer: Callable[[str], None],
259
306
  ) -> None:
260
307
  try:
261
- from pylatex.utils import escape_latex # type: ignore[import-untyped] # does not provide types
308
+ from pylatex.utils import ( # type: ignore[import-untyped] # does not provide types
309
+ escape_latex,
310
+ )
262
311
  except ImportError:
263
312
  raise ImportError('latex support requires the "latex" optional dependencies')
264
313
 
@@ -288,9 +337,11 @@ def _print_relcov_corpus_table_latex(
288
337
  elif enable_color:
289
338
  n = _norm_value(val, min_v, max_v)
290
339
  hex_color = _colormap_light_hex(n, colormap=colormap)
291
- cells.append(rf"\cellcolor[HTML]{{{hex_color}}}{{{val:.3f}}}")
340
+ cells.append(
341
+ rf"\cellcolor[HTML]{{{hex_color}}}{{{_format_float(val, digits)}}}"
342
+ )
292
343
  else:
293
- cells.append(f"{val:.3f}")
344
+ cells.append(_format_float(val, digits))
294
345
  printer("\t" + " & ".join(cells) + r" \\")
295
346
  printer(r"\end{tabular}")
296
347
 
@@ -311,7 +362,7 @@ def _colormap_light_hex(t: float, *, colormap: str = "viridis") -> str:
311
362
  if colormap not in colormaps:
312
363
  raise ValueError(f"Invalid colormap: {colormap}")
313
364
  cmap = colormaps[colormap]
314
- [r, g, b, a] = cmap(t)
365
+ [r, g, b, _a] = cmap(t)
315
366
  [r, g, b] = [1 - ((1 - e) * 0.3) for e in [r, g, b]]
316
367
 
317
368
  return mcolors.to_hex((r, g, b), keep_alpha=False)[1:].upper()
@@ -32,9 +32,7 @@ def detect(path: Path) -> bool:
32
32
  return False
33
33
  if num_files <= 0 or num_first > num_files:
34
34
  return False
35
- has_marker = any(
36
- line.startswith("STARTED ") or line.startswith("COV ") for line in lines
37
- )
35
+ has_marker = any(line.startswith(("STARTED ", "COV ")) for line in lines)
38
36
  return has_marker
39
37
 
40
38
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: differential_coverage
3
- Version: 1.2.0
3
+ Version: 1.3.0
4
4
  Summary: Compute differential coverage, a better way of comparing testing tools.
5
5
  Author-email: Valentin Huber <contact@valentinhuber.me>
6
6
  License-Expression: MIT
@@ -38,6 +38,7 @@ In principle, differential coverage measures how much of approach $a_2$'s covera
38
38
 
39
39
  - `relcov` does exactly this. It is an *asymetrical* measure between coverage of two approaches. If the approach relies on randomness (such as most fuzzers), it can aggregate over multiple trials of these.
40
40
  - `relscore` aggregates all this data to an order of all approaches. This measure, compared to simple coverage numbers, takes into account and values higher approaches that can cover code that no other approach covers. However, compared to `relcov`, this hides a lot of information in this aggregation.
41
+ - `count` just reports absolute coverage counts (union of covered edges/blocks/branches across trials) per approach — useful as a baseline, but without the differential comparison.
41
42
 
42
43
  For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEFINITIONS.md).
43
44
 
@@ -146,13 +147,14 @@ pip install differential-coverage[latex]
146
147
  Generally, the command line interface follows the following structure:
147
148
 
148
149
  ```bash
149
- differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
150
+ differential-coverage {relcov,relscore,count} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
150
151
  ```
151
152
 
152
153
  Examples:
153
154
 
154
155
  ```bash
155
156
  differential-coverage relscore ./coverage_data
157
+ differential-coverage count ./coverage_data
156
158
  differential-coverage --input-format llvm-cov relcov ./coverage_data
157
159
  differential-coverage --output csv relscore ./coverage_data
158
160
  ```
@@ -206,6 +208,10 @@ relscores: dict[str, float] = dc.relscores()
206
208
  for approach_name, relscore in relscores.items():
207
209
  print(f"relscore {approach_name}: {relscore}")
208
210
 
211
+ counts: dict[str, int] = dc.coverage_counts()
212
+ for approach_name, count in counts.items():
213
+ print(f"coverage {approach_name}: {count}")
214
+
209
215
  ```
210
216
 
211
217
  ### Hints
@@ -1,4 +1,5 @@
1
1
  from pathlib import Path
2
+
2
3
  import pytest
3
4
 
4
5
  from differential_coverage.api import DifferentialCoverage
@@ -68,3 +69,13 @@ def test_relscores_multiple_approaches() -> None:
68
69
  "approach_c": 0.5, # 3 is not hit by approach_a, but only hit in half of the trials
69
70
  }
70
71
  )
72
+
73
+
74
+ def test_coverage_counts() -> None:
75
+ dc = DifferentialCoverage(SAMPLE_CAMPAIGN_CONTENT)
76
+ assert dc.coverage_counts() == {
77
+ "approach_a": 3, # union of {1,2} and {1,3}
78
+ "approach_b": 2, # union of {1,3}, {1,3}, {1}
79
+ "approach_c": 3, # union of {1,2,3} and {1,2,3}
80
+ "seeds": 1, # {1}
81
+ }
@@ -1,9 +1,11 @@
1
1
  from copy import deepcopy
2
2
  from typing import Any
3
+
4
+ import pytest
5
+ from test_api import SAMPLE_CAMPAIGN_CONTENT
6
+
3
7
  from differential_coverage.approach_data import ApproachData
4
8
  from differential_coverage.types import CollectionReducer, ValueReducer
5
- from test_api import SAMPLE_CAMPAIGN_CONTENT
6
- import pytest
7
9
 
8
10
 
9
11
  def test_constructor() -> None:
@@ -52,7 +54,7 @@ def test_edges_by_trial(trials: dict[str, set[int]]) -> None:
52
54
 
53
55
  def test_eq() -> None:
54
56
  approach_data = ApproachData({"t1": {1, 2}})
55
- assert approach_data == approach_data # eq to self
57
+ assert approach_data == approach_data # eq to self # noqa: PLR0124
56
58
  assert approach_data == deepcopy(approach_data) # eq to deepcopy
57
59
  assert approach_data == ApproachData({"t1": {1, 2}}) # eq to same data
58
60
  assert approach_data != ApproachData({"t1": {1, 2, 3}}) # different data
@@ -162,7 +164,7 @@ def test_relcov_value_reducer() -> None:
162
164
  else:
163
165
  return original_getattr(self, name)
164
166
 
165
- setattr(ValueReducer, "__getattribute__", spy_getattr)
167
+ setattr(ValueReducer, "__getattribute__", spy_getattr) # noqa: B010
166
168
 
167
169
  res = lhs.relcov(rhs, value_reducer=ValueReducer.MEDIAN)
168
170
  assert res == 0.5 # default median from superclass
@@ -189,7 +191,7 @@ def test_relcov_collection_reducer() -> None:
189
191
  else:
190
192
  return original_getattr(self, name)
191
193
 
192
- setattr(CollectionReducer, "__getattribute__", spy_getattr)
194
+ setattr(CollectionReducer, "__getattribute__", spy_getattr) # noqa: B010
193
195
 
194
196
  res = lhs.relcov(rhs, collection_reducer=CollectionReducer.UNION)
195
197
  assert res == 0.5 # default: union of rhs, so half is reached
@@ -3,6 +3,7 @@
3
3
  import io
4
4
  import sys
5
5
  from pathlib import Path
6
+
6
7
  import pytest
7
8
 
8
9
  from differential_coverage.cli import main
@@ -275,3 +276,58 @@ def test_cli_latex_color_no_cell_colors() -> None:
275
276
  assert any(line.startswith(r"\begin{tabular}") for line in lines)
276
277
  # Table should not contain any colored cells
277
278
  assert not any(r"\cellcolor" in line for line in lines)
279
+
280
+
281
+ def test_cli_digits_relscore() -> None:
282
+ """CLI --digits controls decimal places in relscore output."""
283
+ code, out, _ = _run_cli(["--digits", "4", "relscore", str(SAMPLE_SHOWMAP_DIR)])
284
+ assert code == 0
285
+ lines = [s.strip() for s in out.strip().splitlines()]
286
+ # Each score should have exactly 4 digits after the decimal point
287
+ for line in lines:
288
+ score = line.split(":", 1)[1].strip()
289
+ assert "." in score
290
+ assert len(score.split(".", 1)[1]) == 4
291
+
292
+
293
+ def test_cli_count() -> None:
294
+ """CLI count prints absolute coverage counts sorted descending."""
295
+ code, out, _ = _run_cli(["count", str(SAMPLE_SHOWMAP_DIR)])
296
+ assert code == 0
297
+ lines = [s.strip() for s in out.strip().splitlines()]
298
+ assert len(lines) == 4
299
+ # Sample: approach_a=3, approach_c=3, approach_b=2, seeds=1
300
+ counts = {}
301
+ for line in lines:
302
+ name, count = line.split(":", 1)
303
+ counts[name.strip()] = int(count.strip())
304
+ assert counts == {
305
+ "approach_a": 3,
306
+ "approach_b": 2,
307
+ "approach_c": 3,
308
+ "seeds": 1,
309
+ }
310
+ # Sorted by count descending; ties broken by name ascending
311
+ assert lines[0].startswith("approach_a:") or lines[0].startswith("approach_c:")
312
+ assert lines[-1].startswith("seeds:")
313
+
314
+
315
+ def test_cli_csv_count() -> None:
316
+ """CLI --output csv with count outputs CSV with header approach,coverage."""
317
+ code, out, _ = _run_cli(["--output", "csv", "count", str(SAMPLE_SHOWMAP_DIR)])
318
+ assert code == 0
319
+ lines = out.strip().splitlines()
320
+ assert lines[0] == "approach,coverage"
321
+ assert len(lines) == 5 # header + 4 approaches
322
+ assert any(line.startswith("approach_a,3") for line in lines[1:])
323
+ assert any(line.startswith("seeds,1") for line in lines[1:])
324
+
325
+
326
+ def test_cli_latex_count() -> None:
327
+ """CLI --output latex with count outputs LaTeX tabular."""
328
+ code, out, _ = _run_cli(["--output", "latex", "count", str(SAMPLE_SHOWMAP_DIR)])
329
+ assert code == 0
330
+ lines = out.strip().splitlines()
331
+ assert any(line.startswith(r"\begin{tabular}") for line in lines)
332
+ assert any("coverage" in line for line in lines)
333
+ assert lines[-1] == r"\end{tabular}"
@@ -4,9 +4,9 @@ from typing import Any
4
4
 
5
5
  def test_docs() -> None:
6
6
  from differential_coverage import (
7
- DifferentialCoverage,
8
7
  ApproachData,
9
8
  CollectionReducer,
9
+ DifferentialCoverage,
10
10
  ValueReducer,
11
11
  )
12
12
 
@@ -44,3 +44,7 @@ def test_docs() -> None:
44
44
  relscores: dict[str, float] = dc.relscores()
45
45
  for approach_name, relscore in relscores.items():
46
46
  print(f"relscore {approach_name}: {relscore}")
47
+
48
+ counts: dict[str, int] = dc.coverage_counts()
49
+ for approach_name, count in counts.items():
50
+ print(f"coverage {approach_name}: {count}")
@@ -95,7 +95,7 @@ def test_read_llvm_cov_block_reads_calc_export(calc_coverage_dir: Path) -> None:
95
95
  export = _calc_export(calc_coverage_dir, "approach_a", "t1.json")
96
96
  edges = llvm_cov.read(export, granularity="block")
97
97
  assert edges
98
- assert not any(edge.endswith(":true") or edge.endswith(":false") for edge in edges)
98
+ assert not any(edge.endswith((":true", ":false")) for edge in edges)
99
99
 
100
100
 
101
101
  def test_read_llvm_cov_expansion_branches(llvm_exports: dict[str, Path]) -> None:
@@ -2,9 +2,13 @@
2
2
 
3
3
  import csv
4
4
  import io
5
- from typing import Callable
5
+ from collections.abc import Callable
6
6
 
7
- from differential_coverage.output import print_relcov_corpus_table, print_scores
7
+ from differential_coverage.output import (
8
+ print_counts,
9
+ print_relcov_corpus_table,
10
+ print_scores,
11
+ )
8
12
 
9
13
  # Deliberately unsorted order for both corpus (columns) and table (rows)
10
14
  _UNSORTED_CORPUS = ["z_approach", "a_approach", "m_approach"]
@@ -75,8 +79,8 @@ def test_print_scores_uses_printer_stdout() -> None:
75
79
 
76
80
  # Scores should be sorted by value descending, then by name
77
81
  joined = "\n".join(captured)
78
- assert "a: 1.00" in joined
79
- assert "b: 0.10" in joined
82
+ assert "a: 1.000" in joined
83
+ assert "b: 0.100" in joined
80
84
  # Ensure our custom printer was used (no direct writes to sys.stdout here)
81
85
  assert joined.count("\n") >= 0
82
86
 
@@ -97,3 +101,56 @@ def test_print_scores_uses_printer_csv() -> None:
97
101
  # Order should be by score descending
98
102
  assert rows[1][0] == "a"
99
103
  assert rows[2][0] == "b"
104
+
105
+
106
+ def test_print_scores_respects_digits() -> None:
107
+ """print_scores formats scores with the requested number of digits."""
108
+ scores = {"a": 1.0 / 3.0}
109
+ captured, printer = _capture_printer()
110
+
111
+ print_scores(scores, output="stdout", digits=4, printer=printer)
112
+
113
+ assert captured == ["a: 0.3333"]
114
+
115
+
116
+ def test_print_counts_uses_printer_stdout() -> None:
117
+ """print_counts writes integer coverage counts via the provided printer."""
118
+ counts = {"b": 2, "a": 5}
119
+ captured, printer = _capture_printer()
120
+
121
+ print_counts(counts, output="stdout", printer=printer)
122
+
123
+ assert captured == ["a: 5", "b: 2"]
124
+
125
+
126
+ def test_print_counts_uses_printer_csv() -> None:
127
+ """print_counts CSV output is routed through the provided printer."""
128
+ counts = {"b": 2, "a": 5}
129
+ captured, printer = _capture_printer()
130
+
131
+ print_counts(counts, output="csv", printer=printer)
132
+
133
+ csv_text = "\n".join(captured)
134
+ reader = csv.reader(io.StringIO(csv_text))
135
+ rows = list(reader)
136
+
137
+ assert rows[0] == ["approach", "coverage"]
138
+ assert rows[1] == ["a", "5"]
139
+ assert rows[2] == ["b", "2"]
140
+
141
+
142
+ def test_relcov_table_respects_digits_csv() -> None:
143
+ """Relcov CSV cells use the requested digit precision."""
144
+ captured, printer = _capture_printer()
145
+ print_relcov_corpus_table(
146
+ ["a"],
147
+ {"a": {"a": 1.0 / 3.0}},
148
+ output="csv",
149
+ digits=4,
150
+ printer=printer,
151
+ )
152
+
153
+ csv_text = "\n".join(captured)
154
+ reader = csv.reader(io.StringIO(csv_text))
155
+ rows = list(reader)
156
+ assert rows[1][1] == "0.3333"
@@ -1,6 +1,7 @@
1
+ import pytest
2
+
1
3
  from differential_coverage.approach_data import ApproachData
2
4
  from differential_coverage.types import CollectionReducer, ValueReducer
3
- import pytest
4
5
 
5
6
 
6
7
  @pytest.mark.parametrize(
@@ -1,7 +1,7 @@
1
1
  from collections.abc import Callable, Iterable
2
2
  from enum import Enum
3
- from typing import TYPE_CHECKING, Any, Protocol, TypeVar
4
3
  from statistics import mean, median
4
+ from typing import TYPE_CHECKING, Any, Protocol, TypeVar
5
5
 
6
6
  if TYPE_CHECKING:
7
7
  from differential_coverage.approach_data import ApproachData