aver-comparator 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aver_comparator/__init__.py +1 -0
- aver_comparator/happy.py +222 -0
- aver_comparator/pgx.py +125 -0
- aver_comparator/somatic.py +173 -0
- aver_comparator/sv.py +122 -0
- aver_comparator-0.1.0.dist-info/METADATA +7 -0
- aver_comparator-0.1.0.dist-info/RECORD +8 -0
- aver_comparator-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Aver hap.py/vcfeval wrapper and metric normalisation (F-31). Arrives in M0."""
|
aver_comparator/happy.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""hap.py (vcfeval engine) invocation and output normalisation.
|
|
2
|
+
|
|
3
|
+
PRD §13.3 (fixed invocation shape), §13.4 (field mapping), ADR-005 (container
|
|
4
|
+
pinned by digest; never pip-installed). F-31: SNV and INDEL metrics restricted
|
|
5
|
+
to the confident-region BED; the raw bundle is retained.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import csv
|
|
11
|
+
import io
|
|
12
|
+
import math
|
|
13
|
+
import shlex
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import PurePosixPath
|
|
17
|
+
|
|
18
|
+
from aver_core.models import BenchmarkMetric
|
|
19
|
+
|
|
20
|
+
# §13.4 mapping: hap.py column -> Aver field
|
|
21
|
+
FIELD_MAP = {
|
|
22
|
+
"TRUTH.TOTAL": "truth_total",
|
|
23
|
+
"TRUTH.TP": "truth_tp",
|
|
24
|
+
"TRUTH.FN": "truth_fn",
|
|
25
|
+
"QUERY.TOTAL": "query_total",
|
|
26
|
+
"QUERY.FP": "query_fp",
|
|
27
|
+
"METRIC.Recall": "recall",
|
|
28
|
+
"METRIC.Precision": "precision",
|
|
29
|
+
"METRIC.F1_Score": "f_measure",
|
|
30
|
+
}
|
|
31
|
+
REQUIRED = {"Type", "Filter", *FIELD_MAP}
|
|
32
|
+
COUNT_FIELDS = {"truth_total", "truth_tp", "truth_fn", "query_total", "query_fp"}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class HappyOutputError(ValueError):
|
|
36
|
+
"""summary.csv is missing, malformed or internally inconsistent."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def parse_summary_csv(text: str) -> tuple[BenchmarkMetric, ...]:
|
|
40
|
+
"""Parse hap.py `<prefix>.summary.csv` into canonical metric rows.
|
|
41
|
+
|
|
42
|
+
Returns rows for SNP and INDEL, for both PASS and ALL filters, in a fixed
|
|
43
|
+
order (SNP/PASS, SNP/ALL, INDEL/PASS, INDEL/ALL). Headline metrics use PASS;
|
|
44
|
+
ALL rows are kept for completeness (§13.4). Every other column is preserved
|
|
45
|
+
verbatim in `extra`, so nothing hap.py reported is lost.
|
|
46
|
+
"""
|
|
47
|
+
reader = csv.DictReader(io.StringIO(text))
|
|
48
|
+
header = set(reader.fieldnames or [])
|
|
49
|
+
missing = REQUIRED - header
|
|
50
|
+
if missing:
|
|
51
|
+
raise HappyOutputError(f"summary.csv lacks columns: {', '.join(sorted(missing))}")
|
|
52
|
+
rows: dict[tuple[str, str], BenchmarkMetric] = {}
|
|
53
|
+
for raw in reader:
|
|
54
|
+
vclass, filt = raw["Type"], raw["Filter"]
|
|
55
|
+
if vclass not in ("SNP", "INDEL") or filt not in ("PASS", "ALL"):
|
|
56
|
+
raise HappyOutputError(f"unexpected row Type={vclass!r} Filter={filt!r}")
|
|
57
|
+
if (vclass, filt) in rows:
|
|
58
|
+
raise HappyOutputError(f"duplicate row {vclass}/{filt}")
|
|
59
|
+
values: dict[str, object] = {}
|
|
60
|
+
for col, field in FIELD_MAP.items():
|
|
61
|
+
cell = (raw[col] or "").strip()
|
|
62
|
+
if field in COUNT_FIELDS:
|
|
63
|
+
if not cell.isdigit():
|
|
64
|
+
raise HappyOutputError(f"{vclass}/{filt} {col}={cell!r} is not a count")
|
|
65
|
+
values[field] = int(cell)
|
|
66
|
+
else:
|
|
67
|
+
values[field] = _metric(cell, f"{vclass}/{filt} {col}")
|
|
68
|
+
if values["truth_tp"] + values["truth_fn"] != values["truth_total"]: # type: ignore[operator]
|
|
69
|
+
raise HappyOutputError(f"{vclass}/{filt}: TRUTH.TP + TRUTH.FN != TRUTH.TOTAL")
|
|
70
|
+
extra = {k: v for k, v in raw.items() if k not in REQUIRED and k is not None and v is not None}
|
|
71
|
+
rows[(vclass, filt)] = BenchmarkMetric(
|
|
72
|
+
variant_class=vclass,
|
|
73
|
+
filter=filt,
|
|
74
|
+
extra=dict(sorted(extra.items())),
|
|
75
|
+
**values, # type: ignore[arg-type]
|
|
76
|
+
)
|
|
77
|
+
order = [("SNP", "PASS"), ("SNP", "ALL"), ("INDEL", "PASS"), ("INDEL", "ALL")]
|
|
78
|
+
absent = [f"{c}/{f}" for c, f in order if (c, f) not in rows]
|
|
79
|
+
if absent:
|
|
80
|
+
raise HappyOutputError(f"summary.csv lacks rows: {', '.join(absent)}")
|
|
81
|
+
return tuple(rows[k] for k in order)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _count(cell: str, where: str) -> int:
|
|
85
|
+
"""hap.py writes some counts as '123.000000' in extended.csv; accept only integral values."""
|
|
86
|
+
try:
|
|
87
|
+
v = float(cell)
|
|
88
|
+
except ValueError as e:
|
|
89
|
+
raise HappyOutputError(f"{where}={cell!r} is not a count") from e
|
|
90
|
+
if v < 0 or v != int(v):
|
|
91
|
+
raise HappyOutputError(f"{where}={cell!r} is not a count")
|
|
92
|
+
return int(v)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
EXTENDED_REQUIRED = {"Type", "Subtype", "Subset", "Filter", "Genotype", *FIELD_MAP}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def parse_extended_csv(text: str, subsets: Sequence[str]) -> tuple[BenchmarkMetric, ...]:
|
|
99
|
+
"""Per-stratification rows from hap.py `<prefix>.extended.csv` (F-32).
|
|
100
|
+
|
|
101
|
+
Keeps the rows for each requested subset aggregated over subtypes and genotypes
|
|
102
|
+
(`Subtype == Genotype == '*'`), in the order: subset as requested, then SNP/PASS,
|
|
103
|
+
SNP/ALL, INDEL/PASS, INDEL/ALL. A requested subset with no rows is an error, so a
|
|
104
|
+
stratification can never silently disappear from a report.
|
|
105
|
+
"""
|
|
106
|
+
reader = csv.DictReader(io.StringIO(text))
|
|
107
|
+
missing = EXTENDED_REQUIRED - set(reader.fieldnames or [])
|
|
108
|
+
if missing:
|
|
109
|
+
raise HappyOutputError(f"extended.csv lacks columns: {', '.join(sorted(missing))}")
|
|
110
|
+
wanted = set(subsets)
|
|
111
|
+
rows: dict[tuple[str, str, str], BenchmarkMetric] = {}
|
|
112
|
+
for raw in reader:
|
|
113
|
+
sub = raw["Subset"]
|
|
114
|
+
if sub not in wanted or raw["Subtype"] != "*" or raw["Genotype"] != "*":
|
|
115
|
+
continue
|
|
116
|
+
vclass, filt = raw["Type"], raw["Filter"]
|
|
117
|
+
if vclass not in ("SNP", "INDEL") or filt not in ("PASS", "ALL"):
|
|
118
|
+
continue
|
|
119
|
+
where = f"{sub} {vclass}/{filt}"
|
|
120
|
+
values: dict[str, object] = {}
|
|
121
|
+
for col, field in FIELD_MAP.items():
|
|
122
|
+
cell = (raw[col] or "").strip()
|
|
123
|
+
values[field] = _count(cell, f"{where} {col}") if field in COUNT_FIELDS else _metric(cell, f"{where} {col}")
|
|
124
|
+
if values["truth_tp"] + values["truth_fn"] != values["truth_total"]: # type: ignore[operator]
|
|
125
|
+
raise HappyOutputError(f"{where}: TRUTH.TP + TRUTH.FN != TRUTH.TOTAL")
|
|
126
|
+
if (sub, vclass, filt) in rows:
|
|
127
|
+
raise HappyOutputError(f"duplicate row {where}")
|
|
128
|
+
keep = ("Subset.Size", "Subset.IS_CONF.Size")
|
|
129
|
+
rows[(sub, vclass, filt)] = BenchmarkMetric(
|
|
130
|
+
variant_class=vclass,
|
|
131
|
+
filter=filt,
|
|
132
|
+
stratification=sub,
|
|
133
|
+
extra={k: raw[k] for k in keep if raw.get(k)},
|
|
134
|
+
**values, # type: ignore[arg-type]
|
|
135
|
+
)
|
|
136
|
+
out = []
|
|
137
|
+
for sub in subsets:
|
|
138
|
+
for key in (("SNP", "PASS"), ("SNP", "ALL"), ("INDEL", "PASS"), ("INDEL", "ALL")):
|
|
139
|
+
if (sub, *key) not in rows:
|
|
140
|
+
raise HappyOutputError(f"extended.csv lacks {sub} {key[0]}/{key[1]}")
|
|
141
|
+
out.append(rows[(sub, *key)])
|
|
142
|
+
return tuple(out)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _metric(cell: str, where: str) -> float | None:
|
|
146
|
+
if cell == "" or cell.lower() == "nan":
|
|
147
|
+
return None # hap.py writes nan when a denominator is zero
|
|
148
|
+
try:
|
|
149
|
+
v = float(cell)
|
|
150
|
+
except ValueError as e:
|
|
151
|
+
raise HappyOutputError(f"{where}={cell!r} is not a number") from e
|
|
152
|
+
if not (0.0 <= v <= 1.0) or math.isnan(v):
|
|
153
|
+
raise HappyOutputError(f"{where}={cell!r} outside [0, 1]")
|
|
154
|
+
return v
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def headline(metrics: Sequence[BenchmarkMetric]) -> dict[str, BenchmarkMetric]:
|
|
158
|
+
"""Whole-region PASS rows keyed by variant class: the numbers that are reported.
|
|
159
|
+
|
|
160
|
+
Stratified rows (F-32) are never headline numbers.
|
|
161
|
+
"""
|
|
162
|
+
return {m.variant_class: m for m in metrics if m.filter == "PASS" and m.stratification is None}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass(frozen=True)
|
|
166
|
+
class HappyInvocation:
|
|
167
|
+
"""The §13.3 command, run inside the digest-pinned comparator image.
|
|
168
|
+
|
|
169
|
+
Paths are container paths under /data (inputs, read-only) and /out.
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
image: str # repo@sha256:... (ADR-005)
|
|
173
|
+
truth_vcf: str
|
|
174
|
+
query_vcf: str
|
|
175
|
+
confident_bed: str
|
|
176
|
+
reference_fasta: str
|
|
177
|
+
output_prefix: str
|
|
178
|
+
sdf: str | None = None
|
|
179
|
+
target_bed: str | None = None
|
|
180
|
+
stratification_tsv: str | None = None
|
|
181
|
+
threads: int = 4
|
|
182
|
+
locations: str | None = None # e.g. 'chr20'
|
|
183
|
+
|
|
184
|
+
def __post_init__(self) -> None:
|
|
185
|
+
if "@sha256:" not in self.image:
|
|
186
|
+
raise ValueError("comparator image must be pinned by digest (ADR-005)")
|
|
187
|
+
|
|
188
|
+
def happy_args(self) -> list[str]:
|
|
189
|
+
args = ["/opt/hap.py/bin/hap.py", self.truth_vcf, self.query_vcf]
|
|
190
|
+
args += ["-f", self.confident_bed, "-r", self.reference_fasta]
|
|
191
|
+
if self.target_bed:
|
|
192
|
+
args += ["-T", self.target_bed]
|
|
193
|
+
if self.locations:
|
|
194
|
+
args += ["-l", self.locations]
|
|
195
|
+
args += ["--engine=vcfeval"]
|
|
196
|
+
if self.sdf:
|
|
197
|
+
args += ["--engine-vcfeval-template", self.sdf]
|
|
198
|
+
if self.stratification_tsv:
|
|
199
|
+
args += ["--stratification", self.stratification_tsv]
|
|
200
|
+
args += ["--roc", "QUAL", "--threads", str(self.threads), "-o", self.output_prefix]
|
|
201
|
+
return args
|
|
202
|
+
|
|
203
|
+
def command_line(self) -> str:
|
|
204
|
+
"""The exact command recorded on the run (F-30 AC-2)."""
|
|
205
|
+
return shlex.join(self.happy_args())
|
|
206
|
+
|
|
207
|
+
def docker_args(self, data_dir: str, out_dir: str) -> list[str]:
|
|
208
|
+
for p in (self.truth_vcf, self.query_vcf, self.confident_bed, self.reference_fasta):
|
|
209
|
+
if not PurePosixPath(p).is_relative_to("/data"):
|
|
210
|
+
raise ValueError(f"{p} must be under /data")
|
|
211
|
+
return [
|
|
212
|
+
"docker",
|
|
213
|
+
"run",
|
|
214
|
+
"--rm",
|
|
215
|
+
"--network=none",
|
|
216
|
+
"-v",
|
|
217
|
+
f"{data_dir}:/data:ro",
|
|
218
|
+
"-v",
|
|
219
|
+
f"{out_dir}:/out",
|
|
220
|
+
self.image,
|
|
221
|
+
*self.happy_args(),
|
|
222
|
+
]
|
aver_comparator/pgx.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Pharmacogene diplotype concordance (M8.2, docs/plans/M8-platform-deepening.md).
|
|
2
|
+
|
|
3
|
+
A laboratory's star-allele calls for one public GeT-RM Coriell sample are
|
|
4
|
+
compared against the published consensus. The comparison is deliberately
|
|
5
|
+
conservative and fully explained:
|
|
6
|
+
|
|
7
|
+
- a diplotype matches when its '/'-separated alleles are the same set
|
|
8
|
+
(diplotypes are unordered: *1/*2 equals *2/*1), after whitespace
|
|
9
|
+
normalisation and case folding — nothing else is interpreted;
|
|
10
|
+
- a consensus value offering alternatives ("x or y") matches when the call
|
|
11
|
+
matches any alternative;
|
|
12
|
+
- anything the lab calls that the consensus does not characterise is
|
|
13
|
+
`not_characterized`, and a characterised gene the lab did not call is
|
|
14
|
+
`not_called` — neither is a match or a mismatch, and both are reported.
|
|
15
|
+
|
|
16
|
+
Invariant 1: the sample must be a sample in the consensus table (a public
|
|
17
|
+
Coriell line); an unknown sample aborts without echoing the name. Invariant 9
|
|
18
|
+
in spirit: every row carries both strings verbatim, so the judgement is
|
|
19
|
+
checkable by eye.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from collections.abc import Mapping
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class PgxSampleError(ValueError):
|
|
29
|
+
"""The named sample is not in the consensus table (never echoed: it could be an identifier)."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class PgxGeneError(ValueError):
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class PgxRow:
|
|
38
|
+
gene: str # the consensus table's gene name
|
|
39
|
+
called: str | None # the laboratory's verbatim call
|
|
40
|
+
consensus: str | None # the verbatim consensus value
|
|
41
|
+
status: str # match | mismatch | not_characterized | not_called
|
|
42
|
+
|
|
43
|
+
def as_dict(self) -> dict[str, str | None]:
|
|
44
|
+
return {"gene": self.gene, "called": self.called, "consensus": self.consensus, "status": self.status}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class PgxConcordance:
|
|
49
|
+
sample: str
|
|
50
|
+
consensus_set: str
|
|
51
|
+
rows: tuple[PgxRow, ...]
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def counts(self) -> dict[str, int]:
|
|
55
|
+
out = {"match": 0, "mismatch": 0, "not_characterized": 0, "not_called": 0}
|
|
56
|
+
for r in self.rows:
|
|
57
|
+
out[r.status] += 1
|
|
58
|
+
return out
|
|
59
|
+
|
|
60
|
+
def as_dict(self) -> dict[str, object]:
|
|
61
|
+
return {
|
|
62
|
+
"sample": self.sample,
|
|
63
|
+
"consensus_set": self.consensus_set,
|
|
64
|
+
"rows": [r.as_dict() for r in self.rows],
|
|
65
|
+
"counts": self.counts,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _norm(diplotype: str) -> tuple[str, ...]:
|
|
70
|
+
parts = [p.strip().lower() for p in " ".join(diplotype.split()).split("/")]
|
|
71
|
+
return tuple(sorted(p for p in parts if p))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def diplotypes_match(called: str, consensus: str) -> bool:
|
|
75
|
+
alternatives = [a for a in consensus.split(" or ") if a.strip()] or [consensus]
|
|
76
|
+
return any(_norm(called) == _norm(a) for a in alternatives)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def resolve_gene(lab_name: str, genes: tuple[str, ...]) -> str:
|
|
80
|
+
"""The consensus gene for a laboratory's gene name: exact (case-insensitive),
|
|
81
|
+
else the unique gene whose name starts with `lab_name` + space (long locus
|
|
82
|
+
headers such as the CYP2C cluster). Ambiguity is an error, never a guess."""
|
|
83
|
+
wanted = " ".join(lab_name.split()).lower()
|
|
84
|
+
exact = [g for g in genes if g.lower() == wanted]
|
|
85
|
+
if exact:
|
|
86
|
+
return exact[0]
|
|
87
|
+
prefixed = [g for g in genes if g.lower().startswith(wanted + " ")]
|
|
88
|
+
if len(prefixed) == 1:
|
|
89
|
+
return prefixed[0]
|
|
90
|
+
if prefixed:
|
|
91
|
+
raise PgxGeneError(f"gene name {lab_name!r} is ambiguous in the consensus table: {', '.join(sorted(prefixed))}")
|
|
92
|
+
raise PgxGeneError(f"gene {lab_name!r} is not in the consensus table")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def concordance(
|
|
96
|
+
sample: str,
|
|
97
|
+
calls: Mapping[str, str],
|
|
98
|
+
*,
|
|
99
|
+
consensus_set: str,
|
|
100
|
+
genes: tuple[str, ...],
|
|
101
|
+
consensus_for_sample: Mapping[str, str],
|
|
102
|
+
) -> PgxConcordance:
|
|
103
|
+
"""Every called gene judged against the consensus, plus every characterised
|
|
104
|
+
gene the laboratory did not call. `consensus_for_sample` maps the consensus
|
|
105
|
+
table's gene names to verbatim values for this sample."""
|
|
106
|
+
if not calls:
|
|
107
|
+
raise PgxGeneError("no gene calls supplied")
|
|
108
|
+
rows: list[PgxRow] = []
|
|
109
|
+
called_genes: set[str] = set()
|
|
110
|
+
for lab_name in sorted(calls):
|
|
111
|
+
gene = resolve_gene(lab_name, genes)
|
|
112
|
+
if gene in called_genes:
|
|
113
|
+
raise PgxGeneError(f"gene {gene!r} is called more than once")
|
|
114
|
+
called_genes.add(gene)
|
|
115
|
+
called = " ".join(str(calls[lab_name]).split())
|
|
116
|
+
cons = consensus_for_sample.get(gene)
|
|
117
|
+
if cons is None:
|
|
118
|
+
rows.append(PgxRow(gene, called, None, "not_characterized"))
|
|
119
|
+
else:
|
|
120
|
+
rows.append(PgxRow(gene, called, cons, "match" if diplotypes_match(called, cons) else "mismatch"))
|
|
121
|
+
for gene in sorted(consensus_for_sample):
|
|
122
|
+
if gene not in called_genes:
|
|
123
|
+
rows.append(PgxRow(gene, None, consensus_for_sample[gene], "not_called"))
|
|
124
|
+
rows.sort(key=lambda r: r.gene)
|
|
125
|
+
return PgxConcordance(sample=sample, consensus_set=consensus_set, rows=tuple(rows))
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""som.py invocation and output normalisation for somatic truth sets (M7, ADR-014).
|
|
2
|
+
|
|
3
|
+
som.py ships in the same digest-pinned comparator image as hap.py (ADR-005) and is
|
|
4
|
+
the comparison tool the SEQC2 publications themselves use. It compares a somatic
|
|
5
|
+
query against a truth call set without genotype matching and, by default, counts
|
|
6
|
+
PASS query records only — so rows are recorded with filter="PASS", honestly, and
|
|
7
|
+
there is no ALL split unless a future need adds a second -P run.
|
|
8
|
+
|
|
9
|
+
The stats.csv layout was verified against the pinned image's real output
|
|
10
|
+
(2026-10-12): an unnamed index column, then type/total.truth/total.query/tp/fp/
|
|
11
|
+
fn/unk/ambi/recall/.../sompyversion/sompycmd, with rows 'SNVs', 'indels' and the
|
|
12
|
+
aggregate 'records'.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import csv
|
|
18
|
+
import io
|
|
19
|
+
import math
|
|
20
|
+
import shlex
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import PurePosixPath
|
|
23
|
+
|
|
24
|
+
from aver_core.models import BenchmarkMetric
|
|
25
|
+
|
|
26
|
+
TYPE_MAP = {"SNVs": "SNP", "indels": "INDEL", "records": "ALL"}
|
|
27
|
+
COUNTS = {
|
|
28
|
+
"total.truth": "truth_total",
|
|
29
|
+
"tp": "truth_tp",
|
|
30
|
+
"fn": "truth_fn",
|
|
31
|
+
"total.query": "query_total",
|
|
32
|
+
"fp": "query_fp",
|
|
33
|
+
}
|
|
34
|
+
RATES = {"recall": "recall", "precision": "precision"}
|
|
35
|
+
REQUIRED = {"type", *COUNTS, *RATES}
|
|
36
|
+
# Kept verbatim in `extra` so nothing som.py reported is lost; sompycmd is excluded
|
|
37
|
+
# (the executed command is recorded on the run itself, F-30 AC-2).
|
|
38
|
+
EXTRA = (
|
|
39
|
+
"unk",
|
|
40
|
+
"ambi",
|
|
41
|
+
"recall_lower",
|
|
42
|
+
"recall_upper",
|
|
43
|
+
"precision_lower",
|
|
44
|
+
"precision_upper",
|
|
45
|
+
"fp.region.size",
|
|
46
|
+
"fp.rate",
|
|
47
|
+
"sompyversion",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class SompyOutputError(ValueError):
|
|
52
|
+
"""stats.csv is missing, malformed or internally inconsistent."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_sompy_stats(text: str) -> tuple[BenchmarkMetric, ...]:
|
|
56
|
+
"""Canonical metric rows from som.py `<prefix>.stats.csv`.
|
|
57
|
+
|
|
58
|
+
Returns the rows som.py wrote, ordered SNP, INDEL, ALL (filter="PASS";
|
|
59
|
+
som.py's default comparison counts PASS query records only). Against a
|
|
60
|
+
per-class truth set som.py omits the other class's row entirely — verified
|
|
61
|
+
on a real SEQC2 sSNV run 2026-10-13 — so only the aggregate 'records' row
|
|
62
|
+
is required; the caller decides which class rows it needs. f_measure is
|
|
63
|
+
computed from recall and precision, since som.py does not report it.
|
|
64
|
+
"""
|
|
65
|
+
reader = csv.DictReader(io.StringIO(text))
|
|
66
|
+
header = set(reader.fieldnames or [])
|
|
67
|
+
missing = REQUIRED - header
|
|
68
|
+
if missing:
|
|
69
|
+
raise SompyOutputError(f"stats.csv lacks columns: {', '.join(sorted(missing))}")
|
|
70
|
+
rows: dict[str, BenchmarkMetric] = {}
|
|
71
|
+
for raw in reader:
|
|
72
|
+
kind = (raw.get("type") or "").strip()
|
|
73
|
+
vclass = TYPE_MAP.get(kind)
|
|
74
|
+
if vclass is None:
|
|
75
|
+
continue # future som.py row kinds are ignored, never guessed at
|
|
76
|
+
if vclass in rows:
|
|
77
|
+
raise SompyOutputError(f"duplicate row for {kind}")
|
|
78
|
+
where = f"{kind}"
|
|
79
|
+
values: dict[str, object] = {}
|
|
80
|
+
for col, field in COUNTS.items():
|
|
81
|
+
values[field] = _count((raw[col] or "").strip(), f"{where} {col}")
|
|
82
|
+
for col, field in RATES.items():
|
|
83
|
+
values[field] = _rate((raw[col] or "").strip(), f"{where} {col}")
|
|
84
|
+
if values["truth_tp"] + values["truth_fn"] != values["truth_total"]: # type: ignore[operator]
|
|
85
|
+
raise SompyOutputError(f"{where}: tp + fn != total.truth")
|
|
86
|
+
r, p = values["recall"], values["precision"]
|
|
87
|
+
f1 = (2 * r * p / (r + p)) if isinstance(r, float) and isinstance(p, float) and (r + p) > 0 else None # type: ignore[operator]
|
|
88
|
+
rows[vclass] = BenchmarkMetric(
|
|
89
|
+
variant_class=vclass,
|
|
90
|
+
filter="PASS",
|
|
91
|
+
f_measure=f1,
|
|
92
|
+
extra={k: raw[k].strip() for k in EXTRA if raw.get(k, "").strip()},
|
|
93
|
+
**values, # type: ignore[arg-type]
|
|
94
|
+
)
|
|
95
|
+
if "ALL" not in rows:
|
|
96
|
+
raise SompyOutputError("stats.csv lacks the aggregate 'records' row")
|
|
97
|
+
return tuple(rows[c] for c in ("SNP", "INDEL", "ALL") if c in rows)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _count(cell: str, where: str) -> int:
|
|
101
|
+
try:
|
|
102
|
+
v = float(cell)
|
|
103
|
+
except ValueError as e:
|
|
104
|
+
raise SompyOutputError(f"{where}={cell!r} is not a count") from e
|
|
105
|
+
if v < 0 or v != int(v):
|
|
106
|
+
raise SompyOutputError(f"{where}={cell!r} is not a count")
|
|
107
|
+
return int(v)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _rate(cell: str, where: str) -> float | None:
|
|
111
|
+
if cell == "" or cell.lower() == "nan":
|
|
112
|
+
return None # zero denominator
|
|
113
|
+
try:
|
|
114
|
+
v = float(cell)
|
|
115
|
+
except ValueError as e:
|
|
116
|
+
raise SompyOutputError(f"{where}={cell!r} is not a number") from e
|
|
117
|
+
if not (0.0 <= v <= 1.0) or math.isnan(v):
|
|
118
|
+
raise SompyOutputError(f"{where}={cell!r} outside [0, 1]")
|
|
119
|
+
return v
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class SompyInvocation:
|
|
124
|
+
"""The som.py command, run inside the digest-pinned comparator image.
|
|
125
|
+
|
|
126
|
+
`truth_vcf` is one per-class SEQC2 call set; a somatic comparison runs som.py
|
|
127
|
+
once per class VCF and keeps that class's row. Paths are container paths under
|
|
128
|
+
/data (read-only) and /out.
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
image: str # repo@sha256:... (ADR-005)
|
|
132
|
+
truth_vcf: str
|
|
133
|
+
query_vcf: str
|
|
134
|
+
fp_regions_bed: str # the high-confidence BED: calls outside it are UNK, not FP
|
|
135
|
+
reference_fasta: str
|
|
136
|
+
output_prefix: str
|
|
137
|
+
target_bed: str | None = None
|
|
138
|
+
location: str | None = None # e.g. 'chr20'
|
|
139
|
+
|
|
140
|
+
def __post_init__(self) -> None:
|
|
141
|
+
if "@sha256:" not in self.image:
|
|
142
|
+
raise ValueError("comparator image must be pinned by digest (ADR-005)")
|
|
143
|
+
|
|
144
|
+
def sompy_args(self) -> list[str]:
|
|
145
|
+
args = ["/opt/hap.py/bin/som.py", self.truth_vcf, self.query_vcf]
|
|
146
|
+
args += ["-r", self.reference_fasta, "-f", self.fp_regions_bed]
|
|
147
|
+
if self.target_bed:
|
|
148
|
+
args += ["-T", self.target_bed]
|
|
149
|
+
if self.location:
|
|
150
|
+
args += ["-l", self.location]
|
|
151
|
+
args += ["-o", self.output_prefix]
|
|
152
|
+
return args
|
|
153
|
+
|
|
154
|
+
def command_line(self) -> str:
|
|
155
|
+
"""The exact command recorded on the run (F-30 AC-2)."""
|
|
156
|
+
return shlex.join(self.sompy_args())
|
|
157
|
+
|
|
158
|
+
def docker_args(self, data_dir: str, out_dir: str) -> list[str]:
|
|
159
|
+
for p in (self.truth_vcf, self.query_vcf, self.fp_regions_bed, self.reference_fasta):
|
|
160
|
+
if not PurePosixPath(p).is_relative_to("/data"):
|
|
161
|
+
raise ValueError(f"{p} must be under /data")
|
|
162
|
+
return [
|
|
163
|
+
"docker",
|
|
164
|
+
"run",
|
|
165
|
+
"--rm",
|
|
166
|
+
"--network=none",
|
|
167
|
+
"-v",
|
|
168
|
+
f"{data_dir}:/data:ro",
|
|
169
|
+
"-v",
|
|
170
|
+
f"{out_dir}:/out",
|
|
171
|
+
self.image,
|
|
172
|
+
*self.sompy_args(),
|
|
173
|
+
]
|
aver_comparator/sv.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""truvari invocation and output normalisation for SV truth sets (M8.3, docs/plans/M8-platform-deepening.md).
|
|
2
|
+
|
|
3
|
+
truvari is the tool the GIAB SV README itself recommends (with `--passonly
|
|
4
|
+
--includebed <Tier1 bed> -r 2000`); it runs from its own digest-pinned
|
|
5
|
+
container (ADR-005). Its `summary.json` reports base/comp counts, TP/FP/FN
|
|
6
|
+
and precision/recall/F1 directly; rows map onto BenchmarkMetric with
|
|
7
|
+
variant_class='SV' and filter='PASS' (truvari compares PASS records under
|
|
8
|
+
--passonly). Every other summary field is preserved stringified in `extra`.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import shlex
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from pathlib import PurePosixPath
|
|
18
|
+
|
|
19
|
+
from aver_core.models import BenchmarkMetric
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TruvariOutputError(ValueError):
|
|
23
|
+
"""summary.json is missing, malformed or internally inconsistent."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse_truvari_summary(text: str) -> BenchmarkMetric:
|
|
27
|
+
try:
|
|
28
|
+
data = json.loads(text)
|
|
29
|
+
except ValueError as e:
|
|
30
|
+
raise TruvariOutputError(f"summary.json is not valid JSON: {e}") from e
|
|
31
|
+
if not isinstance(data, dict):
|
|
32
|
+
raise TruvariOutputError("summary.json is not an object")
|
|
33
|
+
|
|
34
|
+
def count(key: str) -> int:
|
|
35
|
+
v = data.get(key)
|
|
36
|
+
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v != int(v):
|
|
37
|
+
raise TruvariOutputError(f"summary.json {key}={v!r} is not a count")
|
|
38
|
+
return int(v)
|
|
39
|
+
|
|
40
|
+
def rate(key: str) -> float | None:
|
|
41
|
+
v = data.get(key)
|
|
42
|
+
if v is None:
|
|
43
|
+
return None
|
|
44
|
+
if not isinstance(v, (int, float)) or isinstance(v, bool) or math.isnan(v) or not 0.0 <= v <= 1.0:
|
|
45
|
+
raise TruvariOutputError(f"summary.json {key}={v!r} is not a rate in [0, 1]")
|
|
46
|
+
return float(v)
|
|
47
|
+
|
|
48
|
+
base, tp_base, fn = count("base cnt"), count("TP-base"), count("FN")
|
|
49
|
+
comp, fp = count("comp cnt"), count("FP")
|
|
50
|
+
if tp_base + fn != base:
|
|
51
|
+
raise TruvariOutputError("summary.json: TP-base + FN != base cnt")
|
|
52
|
+
used = {"base cnt", "TP-base", "FN", "comp cnt", "FP", "precision", "recall", "f1"}
|
|
53
|
+
extra = {
|
|
54
|
+
k: json.dumps(v, sort_keys=True) if isinstance(v, (dict, list)) else str(v)
|
|
55
|
+
for k, v in sorted(data.items())
|
|
56
|
+
if k not in used
|
|
57
|
+
}
|
|
58
|
+
return BenchmarkMetric(
|
|
59
|
+
variant_class="SV",
|
|
60
|
+
filter="PASS",
|
|
61
|
+
truth_total=base,
|
|
62
|
+
truth_tp=tp_base,
|
|
63
|
+
truth_fn=fn,
|
|
64
|
+
query_total=comp,
|
|
65
|
+
query_fp=fp,
|
|
66
|
+
recall=rate("recall"),
|
|
67
|
+
precision=rate("precision"),
|
|
68
|
+
f_measure=rate("f1"),
|
|
69
|
+
extra=extra,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class TruvariInvocation:
|
|
75
|
+
"""`truvari bench` exactly as the GIAB SV README recommends, in the pinned container.
|
|
76
|
+
|
|
77
|
+
Paths are container paths under /data (read-only); truvari creates
|
|
78
|
+
`/out/<output_name>` itself and refuses to overwrite it.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
image: str # repo@sha256:... (ADR-005)
|
|
82
|
+
truth_vcf: str
|
|
83
|
+
query_vcf: str
|
|
84
|
+
include_bed: str
|
|
85
|
+
output_name: str = "truvari"
|
|
86
|
+
refdist: int = 2000 # the README's -r 2000
|
|
87
|
+
sequence_comparison: bool = True # False adds --pctseq 0 (size/type matching only, per the README)
|
|
88
|
+
|
|
89
|
+
def __post_init__(self) -> None:
|
|
90
|
+
if "@sha256:" not in self.image:
|
|
91
|
+
raise ValueError("comparator image must be pinned by digest (ADR-005)")
|
|
92
|
+
|
|
93
|
+
def truvari_args(self) -> list[str]:
|
|
94
|
+
args = ["truvari", "bench", "-b", self.truth_vcf, "-c", self.query_vcf]
|
|
95
|
+
args += ["--includebed", self.include_bed, "--passonly", "-r", str(self.refdist)]
|
|
96
|
+
# truvari 4 requires chunksize >= refdist (its default chunksize is 1000).
|
|
97
|
+
args += ["--chunksize", str(max(self.refdist, 1000))]
|
|
98
|
+
if not self.sequence_comparison:
|
|
99
|
+
args += ["--pctseq", "0"]
|
|
100
|
+
args += ["-o", f"/out/{self.output_name}"]
|
|
101
|
+
return args
|
|
102
|
+
|
|
103
|
+
def command_line(self) -> str:
|
|
104
|
+
"""The exact command recorded on the run (F-30 AC-2)."""
|
|
105
|
+
return shlex.join(self.truvari_args())
|
|
106
|
+
|
|
107
|
+
def docker_args(self, data_dir: str, out_dir: str) -> list[str]:
|
|
108
|
+
for p in (self.truth_vcf, self.query_vcf, self.include_bed):
|
|
109
|
+
if not PurePosixPath(p).is_relative_to("/data"):
|
|
110
|
+
raise ValueError(f"{p} must be under /data")
|
|
111
|
+
return [
|
|
112
|
+
"docker",
|
|
113
|
+
"run",
|
|
114
|
+
"--rm",
|
|
115
|
+
"--network=none",
|
|
116
|
+
"-v",
|
|
117
|
+
f"{data_dir}:/data:ro",
|
|
118
|
+
"-v",
|
|
119
|
+
f"{out_dir}:/out",
|
|
120
|
+
self.image,
|
|
121
|
+
*self.truvari_args(),
|
|
122
|
+
]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
aver_comparator/__init__.py,sha256=X_Z7c_j4h1qcfkzF4wTFeloq6oq4jRwPsSLVpG69Io4,82
|
|
2
|
+
aver_comparator/happy.py,sha256=KZW_MWb7pyx7QtX1T76f0Tc3Na8A4Ht_1T2hMAUBYeA,8967
|
|
3
|
+
aver_comparator/pgx.py,sha256=CcsY-FXcA0Z_MR0oGMVuoGf11YiWyQt0JygTJzRbWOE,4885
|
|
4
|
+
aver_comparator/somatic.py,sha256=SBfXVYskHRvGj0qyGy2pVzek300o7v3H9XVP1PH5jnA,6534
|
|
5
|
+
aver_comparator/sv.py,sha256=sRSmsDPOYHmKtNHyUy5hTEv0TfMOD0vTkl-sZYpeQYc,4487
|
|
6
|
+
aver_comparator-0.1.0.dist-info/WHEEL,sha256=7hzKWg-J8I3Buqyw5tBii5z_MmAVsDYXXijd_QAtNZ8,81
|
|
7
|
+
aver_comparator-0.1.0.dist-info/METADATA,sha256=hbT0n_X2VP1CHIqNSX0JdMAUEFlrke78l8-wKBz9ZkA,224
|
|
8
|
+
aver_comparator-0.1.0.dist-info/RECORD,,
|