aver-drift 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aver_drift/__init__.py +22 -0
- aver_drift/classify.py +441 -0
- aver_drift/diff.py +114 -0
- aver_drift/policy.py +91 -0
- aver_drift-0.1.0.dist-info/METADATA +7 -0
- aver_drift-0.1.0.dist-info/RECORD +7 -0
- aver_drift-0.1.0.dist-info/WHEEL +4 -0
aver_drift/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Aver drift: snapshot diff, rule-derived severity and the policy engine (F-40–F-42)."""
|
|
2
|
+
|
|
3
|
+
from .classify import DEFAULT_POLICY, Classification, Fired, MetricDelta, classify, metric_deltas, semver_bump
|
|
4
|
+
from .diff import Change, diff_snapshots, unpinned
|
|
5
|
+
from .policy import HistoricalEvent, PolicyError, simulate, validate_policy
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"DEFAULT_POLICY",
|
|
9
|
+
"Change",
|
|
10
|
+
"Classification",
|
|
11
|
+
"Fired",
|
|
12
|
+
"HistoricalEvent",
|
|
13
|
+
"MetricDelta",
|
|
14
|
+
"PolicyError",
|
|
15
|
+
"classify",
|
|
16
|
+
"diff_snapshots",
|
|
17
|
+
"metric_deltas",
|
|
18
|
+
"semver_bump",
|
|
19
|
+
"simulate",
|
|
20
|
+
"unpinned",
|
|
21
|
+
"validate_policy",
|
|
22
|
+
]
|
aver_drift/classify.py
ADDED
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
"""Rule-derived drift severity (F-41, F-42, PRD §14.2–14.4, Appendix C, ADR-011; invariant 9).
|
|
2
|
+
|
|
3
|
+
No model decides severity. Every classification lists each rule that fired,
|
|
4
|
+
with its source and reason, so a laboratory can defend it to an inspector.
|
|
5
|
+
|
|
6
|
+
1. diff baseline snapshot vs new snapshot
|
|
7
|
+
2. class rules, semver rules, digest-only rules per change
|
|
8
|
+
3. re-run needed if anything is above `documentation`
|
|
9
|
+
4. metric deltas (pp) from the re-run vs baseline
|
|
10
|
+
5. policy rules on deltas and dependency classes
|
|
11
|
+
6. final severity = max(dependency-derived, metric-derived)
|
|
12
|
+
7. rationale = every rule that fired
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from .diff import Change
|
|
23
|
+
|
|
24
|
+
SEVERITIES = ("informational", "documentation", "review_required", "revalidation_required", "critical")
|
|
25
|
+
RANK = {s: i for i, s in enumerate(SEVERITIES)}
|
|
26
|
+
|
|
27
|
+
# Parameters that cannot change results. Labs extend this list in their policy.
|
|
28
|
+
DEFAULT_SAFE_PARAMS = frozenset(
|
|
29
|
+
{"publish_dir_mode", "max_multiqc_email_size", "multiqc_title", "email", "validate_params"}
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class Fired:
|
|
35
|
+
rule_id: str
|
|
36
|
+
source: str # class_rule | semver_rule | digest_rule | policy | integrity
|
|
37
|
+
severity: str
|
|
38
|
+
reason: str
|
|
39
|
+
change: Mapping[str, Any] | None = None
|
|
40
|
+
|
|
41
|
+
def as_dict(self) -> dict[str, Any]:
|
|
42
|
+
d = {"rule_id": self.rule_id, "source": self.source, "severity": self.severity, "reason": self.reason}
|
|
43
|
+
if self.change is not None:
|
|
44
|
+
d["change"] = dict(self.change)
|
|
45
|
+
return d
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class MetricDelta:
|
|
50
|
+
variant_class: str # SNP | INDEL
|
|
51
|
+
metric: str # recall | precision | f_measure
|
|
52
|
+
stratum: str | None
|
|
53
|
+
baseline: float
|
|
54
|
+
current: float
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def delta_pp(self) -> float:
|
|
58
|
+
return round((self.current - self.baseline) * 100, 6)
|
|
59
|
+
|
|
60
|
+
def as_dict(self) -> dict[str, Any]:
|
|
61
|
+
return {
|
|
62
|
+
"variant_class": self.variant_class,
|
|
63
|
+
"metric": self.metric,
|
|
64
|
+
"stratum": self.stratum,
|
|
65
|
+
"baseline": self.baseline,
|
|
66
|
+
"current": self.current,
|
|
67
|
+
"delta_pp": self.delta_pp,
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class Classification:
|
|
73
|
+
severity: str
|
|
74
|
+
rationale: tuple[Fired, ...]
|
|
75
|
+
rerun_required: bool
|
|
76
|
+
blocks_baseline: bool
|
|
77
|
+
changes: tuple[Change, ...] = ()
|
|
78
|
+
deltas: tuple[MetricDelta, ...] = field(default_factory=tuple)
|
|
79
|
+
|
|
80
|
+
def as_dict(self) -> dict[str, Any]:
|
|
81
|
+
return {
|
|
82
|
+
"severity": self.severity,
|
|
83
|
+
"rerun_required": self.rerun_required,
|
|
84
|
+
"blocks_baseline": self.blocks_baseline,
|
|
85
|
+
"rationale": [f.as_dict() for f in self.rationale],
|
|
86
|
+
"changes": [c.as_dict() for c in self.changes],
|
|
87
|
+
"metric_deltas": [d.as_dict() for d in self.deltas],
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
_SEMVER = re.compile(r"^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def semver_bump(old: str | None, new: str | None) -> str | None:
|
|
95
|
+
"""'major' | 'minor' | 'patch' | 'other' for two version strings, None if equal."""
|
|
96
|
+
if old == new:
|
|
97
|
+
return None
|
|
98
|
+
mo, mn = _SEMVER.match(old or ""), _SEMVER.match(new or "")
|
|
99
|
+
if not (mo and mn):
|
|
100
|
+
return "other"
|
|
101
|
+
a = [int(x) if x else 0 for x in mo.groups()]
|
|
102
|
+
b = [int(x) if x else 0 for x in mn.groups()]
|
|
103
|
+
if a[0] != b[0]:
|
|
104
|
+
return "major"
|
|
105
|
+
if a[1] != b[1]:
|
|
106
|
+
return "minor"
|
|
107
|
+
if a[2] != b[2]:
|
|
108
|
+
return "patch"
|
|
109
|
+
return "other" # e.g. build suffix only
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def classify_changes(
|
|
113
|
+
changes: Sequence[Change], *, clinical: bool, safe_params: Iterable[str] = DEFAULT_SAFE_PARAMS
|
|
114
|
+
) -> list[Fired]:
|
|
115
|
+
"""Steps 2a–2c: dependency-derived rules (Appendix C)."""
|
|
116
|
+
safe = set(safe_params)
|
|
117
|
+
fired: list[Fired] = []
|
|
118
|
+
tool_version_changed_procs = {p for c in changes if c.kind == "tool" and c.field == "version" for p in c.processes}
|
|
119
|
+
conda_envs_fired: set[str] = set()
|
|
120
|
+
for c in changes:
|
|
121
|
+
ch = c.as_dict()
|
|
122
|
+
if c.kind == "reference_genome":
|
|
123
|
+
fired.append(
|
|
124
|
+
Fired(
|
|
125
|
+
"reference-genome-change",
|
|
126
|
+
"class_rule",
|
|
127
|
+
"revalidation_required",
|
|
128
|
+
f"Reference genome {c.name} changed ({c.field}: {c.old} → {c.new}).",
|
|
129
|
+
ch,
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
elif c.kind == "annotation_db":
|
|
133
|
+
fired.append(
|
|
134
|
+
Fired(
|
|
135
|
+
"annotation-db-release",
|
|
136
|
+
"class_rule",
|
|
137
|
+
"review_required",
|
|
138
|
+
f"Annotation database {c.name} changed ({c.old} → {c.new}).",
|
|
139
|
+
ch,
|
|
140
|
+
)
|
|
141
|
+
)
|
|
142
|
+
elif c.kind == "tool" and c.field == "version":
|
|
143
|
+
bump = semver_bump(c.old, c.new)
|
|
144
|
+
weight = "elevated weight" if bump in ("major", "minor") else "standard weight"
|
|
145
|
+
fired.append(
|
|
146
|
+
Fired(
|
|
147
|
+
f"semver-{bump}",
|
|
148
|
+
"semver_rule",
|
|
149
|
+
"review_required",
|
|
150
|
+
f"{c.name} {bump} version change {c.old} → {c.new} ({weight}); benchmark re-run required.",
|
|
151
|
+
ch,
|
|
152
|
+
)
|
|
153
|
+
)
|
|
154
|
+
elif c.kind == "container" and c.field == "digest":
|
|
155
|
+
if set(c.processes) & tool_version_changed_procs or c.old is None or c.new is None:
|
|
156
|
+
continue # covered by the tool rule, or by the pinned rule below
|
|
157
|
+
fired.append(
|
|
158
|
+
Fired(
|
|
159
|
+
"container-rebuilt-same-versions",
|
|
160
|
+
"digest_rule",
|
|
161
|
+
"documentation",
|
|
162
|
+
f"{c.name} digest changed with identical tool versions; results provably unchanged "
|
|
163
|
+
"only if tool versions match, so the manifest is updated without a re-run.",
|
|
164
|
+
ch,
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
elif c.kind == "container" and c.field == "version":
|
|
168
|
+
if not set(c.processes) & tool_version_changed_procs:
|
|
169
|
+
fired.append(
|
|
170
|
+
Fired(
|
|
171
|
+
"container-tag-change",
|
|
172
|
+
"class_rule",
|
|
173
|
+
"review_required",
|
|
174
|
+
f"{c.name} tag changed {c.old} → {c.new} without a readable tool version.",
|
|
175
|
+
ch,
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
elif c.kind == "conda_env" and c.field in ("digest", "version"):
|
|
179
|
+
if c.old is None or c.new is None:
|
|
180
|
+
continue # env file became (un)readable; the unpinned rules cover it
|
|
181
|
+
if set(c.processes) & tool_version_changed_procs:
|
|
182
|
+
continue # an exactly-declared tool version moved; the semver rule covers it
|
|
183
|
+
if c.name in conda_envs_fired:
|
|
184
|
+
continue # one firing per environment when digest and wrapper release both changed
|
|
185
|
+
conda_envs_fired.add(c.name)
|
|
186
|
+
what = "wrapper release" if c.field == "version" else "file"
|
|
187
|
+
detail = f" {c.old} → {c.new}" if c.field == "version" else ""
|
|
188
|
+
if clinical:
|
|
189
|
+
fired.append(
|
|
190
|
+
Fired(
|
|
191
|
+
"conda-env-changed-clinical",
|
|
192
|
+
"class_rule",
|
|
193
|
+
"review_required",
|
|
194
|
+
f"Conda environment {c.name} {what} changed{detail} with no exactly-declared tool "
|
|
195
|
+
"version change; what conda installs may move, so the benchmark re-runs (ADR-011).",
|
|
196
|
+
ch,
|
|
197
|
+
)
|
|
198
|
+
)
|
|
199
|
+
else:
|
|
200
|
+
fired.append(
|
|
201
|
+
Fired(
|
|
202
|
+
"conda-env-changed",
|
|
203
|
+
"class_rule",
|
|
204
|
+
"documentation",
|
|
205
|
+
f"Conda environment {c.name} {what} changed{detail} with no exactly-declared tool "
|
|
206
|
+
"version change; recorded without a re-run (ADR-011; pipeline not marked clinical).",
|
|
207
|
+
ch,
|
|
208
|
+
)
|
|
209
|
+
)
|
|
210
|
+
elif c.kind == "engine" and c.field == "version":
|
|
211
|
+
fired.append(
|
|
212
|
+
Fired(
|
|
213
|
+
"engine-version-change",
|
|
214
|
+
"class_rule",
|
|
215
|
+
"review_required",
|
|
216
|
+
f"Workflow engine {c.name} changed {c.old} → {c.new}.",
|
|
217
|
+
ch,
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
elif c.kind == "parameter":
|
|
221
|
+
if c.name in safe:
|
|
222
|
+
fired.append(
|
|
223
|
+
Fired(
|
|
224
|
+
"parameter-declared-safe",
|
|
225
|
+
"class_rule",
|
|
226
|
+
"informational",
|
|
227
|
+
f"Parameter {c.name} changed; it is on the declared-safe list.",
|
|
228
|
+
ch,
|
|
229
|
+
)
|
|
230
|
+
)
|
|
231
|
+
else:
|
|
232
|
+
fired.append(
|
|
233
|
+
Fired(
|
|
234
|
+
"parameter-change",
|
|
235
|
+
"class_rule",
|
|
236
|
+
"review_required",
|
|
237
|
+
f"Parameter {c.name} changed {c.old} → {c.new} (not on the declared-safe list).",
|
|
238
|
+
ch,
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
elif c.field in ("added", "removed"):
|
|
242
|
+
fired.append(
|
|
243
|
+
Fired(
|
|
244
|
+
f"dependency-{c.field}",
|
|
245
|
+
"class_rule",
|
|
246
|
+
"review_required",
|
|
247
|
+
f"{c.kind} {c.name} {c.field} ({c.old or c.new}).",
|
|
248
|
+
ch,
|
|
249
|
+
)
|
|
250
|
+
)
|
|
251
|
+
return fired
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _unpinned_rule(kind: str, name: str, clinical: bool, change: Mapping[str, Any] | None) -> Fired:
|
|
255
|
+
if clinical:
|
|
256
|
+
return Fired(
|
|
257
|
+
"unpinned-dependency-clinical",
|
|
258
|
+
"class_rule",
|
|
259
|
+
"critical",
|
|
260
|
+
f"{kind} {name} is not pinned to an immutable version in a pipeline marked clinical.",
|
|
261
|
+
change,
|
|
262
|
+
)
|
|
263
|
+
return Fired(
|
|
264
|
+
"unpinned-dependency",
|
|
265
|
+
"class_rule",
|
|
266
|
+
"informational",
|
|
267
|
+
f"{kind} {name} is not pinned to an immutable version (finding; pipeline not marked clinical).",
|
|
268
|
+
change,
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def unpinned_rules(unpinned_deps: Iterable[Mapping[str, Any]], *, clinical: bool) -> list[Fired]:
|
|
273
|
+
"""F-20 AC-4: unpinned dependencies present in the new snapshot."""
|
|
274
|
+
return [_unpinned_rule(d["kind"], d["name"], clinical, None) for d in unpinned_deps]
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def evaluate_policy(
|
|
278
|
+
rules: Sequence[Mapping[str, Any]],
|
|
279
|
+
*,
|
|
280
|
+
deltas: Sequence[MetricDelta],
|
|
281
|
+
changes: Sequence[Change],
|
|
282
|
+
clinical: bool,
|
|
283
|
+
unpinned_count: int,
|
|
284
|
+
policy_version: int | None = None,
|
|
285
|
+
) -> list[Fired]:
|
|
286
|
+
"""Step 5: policy rules in the §14.4 schema."""
|
|
287
|
+
fired: list[Fired] = []
|
|
288
|
+
tag = f" (policy v{policy_version})" if policy_version else ""
|
|
289
|
+
for r in rules:
|
|
290
|
+
when, cond, sev = r.get("when", {}), r.get("condition", {}), r["severity"]
|
|
291
|
+
if sev not in RANK:
|
|
292
|
+
raise ValueError(f"rule {r.get('id')}: unknown severity {sev!r}")
|
|
293
|
+
if "metric" in when:
|
|
294
|
+
for d in deltas:
|
|
295
|
+
if d.metric != when["metric"] or d.variant_class != when.get("variant_class", d.variant_class):
|
|
296
|
+
continue
|
|
297
|
+
if "stratum" in when and d.stratum != when["stratum"]:
|
|
298
|
+
continue
|
|
299
|
+
if "stratum" not in when and d.stratum is not None:
|
|
300
|
+
continue
|
|
301
|
+
limit = cond.get("delta_pp_below")
|
|
302
|
+
if limit is not None and d.delta_pp < float(limit):
|
|
303
|
+
fired.append(
|
|
304
|
+
Fired(
|
|
305
|
+
r["id"],
|
|
306
|
+
"policy",
|
|
307
|
+
sev,
|
|
308
|
+
f"{_label(d)} delta {_pp(d.delta_pp)} is below the {_pp(float(limit))} threshold{tag}.",
|
|
309
|
+
d.as_dict(),
|
|
310
|
+
)
|
|
311
|
+
)
|
|
312
|
+
elif "dependency_kind" in when:
|
|
313
|
+
hit = [c for c in changes if c.kind == when["dependency_kind"]]
|
|
314
|
+
if hit and cond.get("changed", True):
|
|
315
|
+
fired.append(
|
|
316
|
+
Fired(
|
|
317
|
+
r["id"],
|
|
318
|
+
"policy",
|
|
319
|
+
sev,
|
|
320
|
+
f"Any change to {when['dependency_kind']} triggers {sev}{tag}.",
|
|
321
|
+
hit[0].as_dict(),
|
|
322
|
+
)
|
|
323
|
+
)
|
|
324
|
+
elif when.get("dependency_pinned") is False:
|
|
325
|
+
applies = when.get("pipeline_clinical") is None or when.get("pipeline_clinical") == clinical
|
|
326
|
+
if applies and unpinned_count > 0 and cond.get("present", True):
|
|
327
|
+
fired.append(Fired(r["id"], "policy", sev, f"{unpinned_count} unpinned dependencies present{tag}."))
|
|
328
|
+
return fired
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _label(d: MetricDelta) -> str:
|
|
332
|
+
cls = "SNV" if d.variant_class == "SNP" else d.variant_class
|
|
333
|
+
metric = {"f_measure": "F1"}.get(d.metric, d.metric)
|
|
334
|
+
return f"{cls} {metric}" + (f" ({d.stratum})" if d.stratum else "")
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _pp(v: float) -> str:
|
|
338
|
+
sign = "−" if v < 0 else "+"
|
|
339
|
+
return f"{sign}{abs(v):.3f} pp"
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def classify(
|
|
343
|
+
changes: Sequence[Change],
|
|
344
|
+
*,
|
|
345
|
+
clinical: bool,
|
|
346
|
+
deltas: Sequence[MetricDelta] = (),
|
|
347
|
+
policy_rules: Sequence[Mapping[str, Any]] = (),
|
|
348
|
+
policy_version: int | None = None,
|
|
349
|
+
unpinned_deps: Sequence[Mapping[str, Any]] = (),
|
|
350
|
+
safe_params: Iterable[str] = DEFAULT_SAFE_PARAMS,
|
|
351
|
+
integrity: Sequence[Fired] = (),
|
|
352
|
+
) -> Classification:
|
|
353
|
+
"""Final severity = max(dependency-derived, metric-derived); rationale lists every rule."""
|
|
354
|
+
fired = [*integrity, *classify_changes(changes, clinical=clinical, safe_params=safe_params)]
|
|
355
|
+
fired += unpinned_rules(unpinned_deps, clinical=clinical)
|
|
356
|
+
fired += evaluate_policy(
|
|
357
|
+
policy_rules,
|
|
358
|
+
deltas=deltas,
|
|
359
|
+
changes=changes,
|
|
360
|
+
clinical=clinical,
|
|
361
|
+
unpinned_count=len(unpinned_deps),
|
|
362
|
+
policy_version=policy_version,
|
|
363
|
+
)
|
|
364
|
+
if not fired and changes:
|
|
365
|
+
fired.append(Fired("no-rule-matched", "class_rule", "informational", "Changes detected; no rule raised them."))
|
|
366
|
+
if not changes and not fired:
|
|
367
|
+
fired.append(
|
|
368
|
+
Fired(
|
|
369
|
+
"no-dependency-change",
|
|
370
|
+
"class_rule",
|
|
371
|
+
"informational",
|
|
372
|
+
"Commit changed but the dependency snapshot is identical (e.g. documentation only).",
|
|
373
|
+
)
|
|
374
|
+
)
|
|
375
|
+
fired.sort(key=lambda f: (-RANK[f.severity], f.source, f.rule_id))
|
|
376
|
+
severity = fired[0].severity
|
|
377
|
+
dep_severity = max((RANK[f.severity] for f in fired if f.source != "policy"), default=0)
|
|
378
|
+
return Classification(
|
|
379
|
+
severity=severity,
|
|
380
|
+
rationale=tuple(fired),
|
|
381
|
+
rerun_required=dep_severity > RANK["documentation"] and not deltas,
|
|
382
|
+
blocks_baseline=RANK[severity] >= RANK["revalidation_required"],
|
|
383
|
+
changes=tuple(changes),
|
|
384
|
+
deltas=tuple(deltas),
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def metric_deltas(baseline: Sequence[Mapping[str, Any]], current: Sequence[Mapping[str, Any]]) -> list[MetricDelta]:
|
|
389
|
+
"""Per class and stratum deltas from two sets of PASS metric rows."""
|
|
390
|
+
|
|
391
|
+
def key(m):
|
|
392
|
+
return (m["variant_class"], m.get("stratification"))
|
|
393
|
+
|
|
394
|
+
base = {key(m): m for m in baseline if m.get("filter", "PASS") == "PASS"}
|
|
395
|
+
out: list[MetricDelta] = []
|
|
396
|
+
for m in current:
|
|
397
|
+
if m.get("filter", "PASS") != "PASS" or key(m) not in base:
|
|
398
|
+
continue
|
|
399
|
+
b = base[key(m)]
|
|
400
|
+
for metric in ("recall", "precision", "f_measure"):
|
|
401
|
+
if m.get(metric) is not None and b.get(metric) is not None:
|
|
402
|
+
out.append(MetricDelta(m["variant_class"], metric, m.get("stratification"), b[metric], m[metric]))
|
|
403
|
+
return sorted(out, key=lambda d: (d.variant_class, d.stratum or "", d.metric))
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
DEFAULT_POLICY = {
|
|
407
|
+
"version": 1,
|
|
408
|
+
"rules": [
|
|
409
|
+
{
|
|
410
|
+
"id": "snv-recall-guard",
|
|
411
|
+
"when": {"metric": "recall", "variant_class": "SNP", "stratum": None},
|
|
412
|
+
"condition": {"delta_pp_below": -0.5},
|
|
413
|
+
"severity": "revalidation_required",
|
|
414
|
+
},
|
|
415
|
+
{
|
|
416
|
+
"id": "indel-recall-guard",
|
|
417
|
+
"when": {"metric": "recall", "variant_class": "INDEL"},
|
|
418
|
+
"condition": {"delta_pp_below": -1.0},
|
|
419
|
+
"severity": "revalidation_required",
|
|
420
|
+
},
|
|
421
|
+
{
|
|
422
|
+
"id": "indel-precision-guard",
|
|
423
|
+
"when": {"metric": "precision", "variant_class": "INDEL"},
|
|
424
|
+
"condition": {"delta_pp_below": -1.0},
|
|
425
|
+
"severity": "revalidation_required",
|
|
426
|
+
},
|
|
427
|
+
{
|
|
428
|
+
"id": "reference-genome-change",
|
|
429
|
+
"when": {"dependency_kind": "reference_genome"},
|
|
430
|
+
"condition": {"changed": True},
|
|
431
|
+
"severity": "revalidation_required",
|
|
432
|
+
},
|
|
433
|
+
{
|
|
434
|
+
"id": "unpinned-in-clinical",
|
|
435
|
+
"when": {"pipeline_clinical": True, "dependency_pinned": False},
|
|
436
|
+
"condition": {"present": True},
|
|
437
|
+
"severity": "critical",
|
|
438
|
+
},
|
|
439
|
+
],
|
|
440
|
+
}
|
|
441
|
+
"""PRD §14.4 example policy, used when a lab has not configured its own."""
|
aver_drift/diff.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Structured diff between two dependency snapshots (F-21, F-40 AC-3).
|
|
2
|
+
|
|
3
|
+
Every change names the field, the old value, the new value and the workflow
|
|
4
|
+
processes it affects. Output order is deterministic.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
ChangeField = str # 'version' | 'digest' | 'added' | 'removed' | 'pinned' | 'value'
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class Change:
|
|
18
|
+
kind: str
|
|
19
|
+
name: str
|
|
20
|
+
field: ChangeField
|
|
21
|
+
old: str | None
|
|
22
|
+
new: str | None
|
|
23
|
+
processes: tuple[str, ...] = ()
|
|
24
|
+
meta: Mapping[str, Any] = field(default_factory=dict)
|
|
25
|
+
|
|
26
|
+
def as_dict(self) -> dict[str, Any]:
|
|
27
|
+
return {
|
|
28
|
+
"kind": self.kind,
|
|
29
|
+
"name": self.name,
|
|
30
|
+
"field": self.field,
|
|
31
|
+
"from": self.old,
|
|
32
|
+
"to": self.new,
|
|
33
|
+
"processes": list(self.processes),
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _deps(snapshot: Mapping[str, Any] | Any) -> list[Mapping[str, Any]]:
|
|
38
|
+
if hasattr(snapshot, "model_dump"):
|
|
39
|
+
snapshot = snapshot.model_dump(mode="json")
|
|
40
|
+
return list(snapshot["dependencies"])
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _key(d: Mapping[str, Any]) -> tuple[str, str, tuple[str, ...]]:
|
|
44
|
+
# A container repository can be used at several tags by different processes,
|
|
45
|
+
# so containers are keyed by the processes that use them as well.
|
|
46
|
+
procs = tuple(d.get("process_names") or ()) if d["kind"] in ("container", "tool") else ()
|
|
47
|
+
return (d["kind"], d["name"], procs)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _pair(old: Sequence[Mapping[str, Any]], new: Sequence[Mapping[str, Any]]):
|
|
51
|
+
"""Match dependencies by (kind, name, processes), then by (kind, name) for the rest."""
|
|
52
|
+
old_by = {_key(d): d for d in old}
|
|
53
|
+
new_by = {_key(d): d for d in new}
|
|
54
|
+
pairs = [(old_by[k], new_by[k]) for k in sorted(old_by.keys() & new_by.keys())]
|
|
55
|
+
rest_old = [old_by[k] for k in sorted(old_by.keys() - new_by.keys())]
|
|
56
|
+
rest_new = [new_by[k] for k in sorted(new_by.keys() - old_by.keys())]
|
|
57
|
+
# second pass: same kind and name, processes changed
|
|
58
|
+
unmatched_new = list(rest_new)
|
|
59
|
+
still_old = []
|
|
60
|
+
for o in rest_old:
|
|
61
|
+
m = next((n for n in unmatched_new if n["kind"] == o["kind"] and n["name"] == o["name"]), None)
|
|
62
|
+
if m is None:
|
|
63
|
+
still_old.append(o)
|
|
64
|
+
else:
|
|
65
|
+
unmatched_new.remove(m)
|
|
66
|
+
pairs.append((o, m))
|
|
67
|
+
return pairs, still_old, unmatched_new
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def diff_snapshots(old: Any, new: Any) -> list[Change]:
|
|
71
|
+
"""Changes from `old` to `new`, sorted by (kind, name, field)."""
|
|
72
|
+
pairs, removed, added = _pair(_deps(old), _deps(new))
|
|
73
|
+
changes: list[Change] = []
|
|
74
|
+
for o, n in pairs:
|
|
75
|
+
procs = tuple(sorted(set(o.get("process_names") or ()) | set(n.get("process_names") or ())))
|
|
76
|
+
if o["kind"] == "parameter":
|
|
77
|
+
if o.get("version") != n.get("version"):
|
|
78
|
+
changes.append(Change("parameter", o["name"], "value", o.get("version"), n.get("version")))
|
|
79
|
+
continue
|
|
80
|
+
if o.get("version") != n.get("version"):
|
|
81
|
+
changes.append(Change(o["kind"], o["name"], "version", o.get("version"), n.get("version"), procs))
|
|
82
|
+
if o.get("digest") != n.get("digest"):
|
|
83
|
+
changes.append(Change(o["kind"], o["name"], "digest", o.get("digest"), n.get("digest"), procs))
|
|
84
|
+
if bool(o.get("is_pinned", True)) != bool(n.get("is_pinned", True)):
|
|
85
|
+
changes.append(
|
|
86
|
+
Change(
|
|
87
|
+
o["kind"],
|
|
88
|
+
o["name"],
|
|
89
|
+
"pinned",
|
|
90
|
+
str(o.get("is_pinned")).lower(),
|
|
91
|
+
str(n.get("is_pinned")).lower(),
|
|
92
|
+
procs,
|
|
93
|
+
)
|
|
94
|
+
)
|
|
95
|
+
for d in removed:
|
|
96
|
+
changes.append(Change(d["kind"], d["name"], "removed", _label(d), None, tuple(d.get("process_names") or ())))
|
|
97
|
+
for d in added:
|
|
98
|
+
changes.append(Change(d["kind"], d["name"], "added", None, _label(d), tuple(d.get("process_names") or ())))
|
|
99
|
+
return sorted(changes, key=lambda c: (c.kind, c.name, c.field, c.processes))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _label(d: Mapping[str, Any]) -> str | None:
|
|
103
|
+
return d.get("version") or d.get("digest")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def unpinned(snapshot: Any) -> list[Mapping[str, Any]]:
|
|
107
|
+
return [d for d in _deps(snapshot) if not d.get("is_pinned", True)]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def summarise(changes: Iterable[Change]) -> dict[str, int]:
|
|
111
|
+
out: dict[str, int] = {}
|
|
112
|
+
for c in changes:
|
|
113
|
+
out[c.kind] = out.get(c.kind, 0) + 1
|
|
114
|
+
return dict(sorted(out.items()))
|
aver_drift/policy.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Revalidation policy engine: validation and simulation (F-42, PRD §14.4).
|
|
2
|
+
|
|
3
|
+
`simulate` answers "what would this threshold have flagged over our history?"
|
|
4
|
+
before a lab adopts it (AC-1..3). Policies are data: versioned, audited rows.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Mapping, Sequence
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from .classify import RANK, MetricDelta, evaluate_policy
|
|
14
|
+
from .diff import Change
|
|
15
|
+
|
|
16
|
+
KNOWN_WHEN = {"metric", "variant_class", "stratum", "dependency_kind", "pipeline_clinical", "dependency_pinned"}
|
|
17
|
+
KNOWN_COND = {"delta_pp_below", "changed", "present"}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class PolicyError(ValueError):
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def validate_policy(policy: Mapping[str, Any]) -> None:
|
|
25
|
+
"""Reject rules the engine would silently ignore."""
|
|
26
|
+
rules = policy.get("rules")
|
|
27
|
+
if not isinstance(policy.get("version"), int) or not isinstance(rules, list) or not rules:
|
|
28
|
+
raise PolicyError("policy needs an integer version and a non-empty rules list")
|
|
29
|
+
ids = [r.get("id") for r in rules]
|
|
30
|
+
if len(set(ids)) != len(ids) or not all(isinstance(i, str) and i for i in ids):
|
|
31
|
+
raise PolicyError("every rule needs a unique, non-empty id")
|
|
32
|
+
for r in rules:
|
|
33
|
+
if r.get("severity") not in RANK:
|
|
34
|
+
raise PolicyError(f"{r['id']}: severity must be one of {', '.join(RANK)}")
|
|
35
|
+
unknown = (set(r.get("when", {})) - KNOWN_WHEN) | (set(r.get("condition", {})) - KNOWN_COND)
|
|
36
|
+
if unknown:
|
|
37
|
+
raise PolicyError(f"{r['id']}: unknown keys {', '.join(sorted(unknown))}")
|
|
38
|
+
if "metric" in r.get("when", {}):
|
|
39
|
+
if r["when"]["metric"] not in ("recall", "precision", "f_measure"):
|
|
40
|
+
raise PolicyError(f"{r['id']}: metric must be recall, precision or f_measure")
|
|
41
|
+
if not isinstance(r.get("condition", {}).get("delta_pp_below"), (int, float)):
|
|
42
|
+
raise PolicyError(f"{r['id']}: metric rules need a numeric condition.delta_pp_below")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class HistoricalEvent:
|
|
47
|
+
event_id: str
|
|
48
|
+
pipeline: str
|
|
49
|
+
title: str
|
|
50
|
+
detected_at: str
|
|
51
|
+
clinical: bool
|
|
52
|
+
changes: tuple[Change, ...]
|
|
53
|
+
deltas: tuple[MetricDelta, ...]
|
|
54
|
+
unpinned_count: int = 0
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def simulate(policy: Mapping[str, Any], history: Sequence[HistoricalEvent]) -> dict[str, Any]:
|
|
58
|
+
"""Evaluate `policy` against past events; deterministic output."""
|
|
59
|
+
validate_policy(policy)
|
|
60
|
+
flagged = []
|
|
61
|
+
for e in sorted(history, key=lambda e: (e.detected_at, e.event_id)):
|
|
62
|
+
fired = evaluate_policy(
|
|
63
|
+
policy["rules"],
|
|
64
|
+
deltas=e.deltas,
|
|
65
|
+
changes=e.changes,
|
|
66
|
+
clinical=e.clinical,
|
|
67
|
+
unpinned_count=e.unpinned_count,
|
|
68
|
+
policy_version=policy["version"],
|
|
69
|
+
)
|
|
70
|
+
if fired:
|
|
71
|
+
worst = max(fired, key=lambda f: RANK[f.severity])
|
|
72
|
+
snv = next(
|
|
73
|
+
(
|
|
74
|
+
d.delta_pp
|
|
75
|
+
for d in e.deltas
|
|
76
|
+
if d.variant_class == "SNP" and d.metric == "recall" and d.stratum is None
|
|
77
|
+
),
|
|
78
|
+
None,
|
|
79
|
+
)
|
|
80
|
+
flagged.append(
|
|
81
|
+
{
|
|
82
|
+
"event_id": e.event_id,
|
|
83
|
+
"pipeline": e.pipeline,
|
|
84
|
+
"title": e.title,
|
|
85
|
+
"detected_at": e.detected_at,
|
|
86
|
+
"severity": worst.severity,
|
|
87
|
+
"rules": sorted({f.rule_id for f in fired}),
|
|
88
|
+
"snv_recall_delta_pp": snv,
|
|
89
|
+
}
|
|
90
|
+
)
|
|
91
|
+
return {"policy_version": policy["version"], "evaluated": len(history), "flagged": flagged}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
aver_drift/__init__.py,sha256=I6AWBE7GtqX7dV3CLoGxv_pVv7vmvYHcuIfgFM_mSjE,611
|
|
2
|
+
aver_drift/classify.py,sha256=iGdiHyeVfHpMPr_qTPPGtUGZ04tWGSXYf2zJnVetRjI,17062
|
|
3
|
+
aver_drift/diff.py,sha256=JMxJp_gf8n_qLZKUfe6t8wE0-U6Pp85knbNuQcaCEaQ,4448
|
|
4
|
+
aver_drift/policy.py,sha256=hC0uuyHVenl1WzFZ0B7UZf_gmzGMmVdGSq0ZuX2OyH8,3591
|
|
5
|
+
aver_drift-0.1.0.dist-info/WHEEL,sha256=7hzKWg-J8I3Buqyw5tBii5z_MmAVsDYXXijd_QAtNZ8,81
|
|
6
|
+
aver_drift-0.1.0.dist-info/METADATA,sha256=2tqiTTVn9PAj9onM1WHzAlhJif8JqsywdlaJb6lm_bc,219
|
|
7
|
+
aver_drift-0.1.0.dist-info/RECORD,,
|