patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
patchsight/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """PatchSight: symbol-level vulnerability triage for Python dependencies."""
2
+
3
+ from importlib.metadata import version
4
+
5
+ __version__ = version("patchsight")
patchsight/cli.py ADDED
@@ -0,0 +1,130 @@
1
+ """User-facing command-line interface.
2
+
3
+ This module is on the hot path: it must never import ``patchsight.pipeline``,
4
+ ``patchsight.eval`` or any LLM client. ``tests/test_import_boundary.py`` enforces this.
5
+ """
6
+
7
+ from pathlib import Path
8
+ from typing import Annotated
9
+
10
+ import typer
11
+
12
+ from patchsight import __version__
13
+
14
+ app = typer.Typer(
15
+ name="patchsight",
16
+ help="Symbol-level vulnerability triage for Python dependencies.",
17
+ no_args_is_help=True,
18
+ add_completion=False,
19
+ )
20
+
21
+
22
+ def _print_version(value: bool) -> None:
23
+ if value:
24
+ typer.echo(f"patchsight {__version__}")
25
+ raise typer.Exit
26
+
27
+
28
+ @app.callback()
29
+ def main(
30
+ version: Annotated[
31
+ bool,
32
+ typer.Option(
33
+ "--version",
34
+ callback=_print_version,
35
+ is_eager=True,
36
+ help="Show the version and exit.",
37
+ ),
38
+ ] = False,
39
+ ) -> None:
40
+ """Symbol-level vulnerability triage for Python dependencies."""
41
+
42
+
43
+ db_app = typer.Typer(help="Download and inspect the symbol database.", no_args_is_help=True)
44
+ app.add_typer(db_app, name="db")
45
+
46
+
47
+ @db_app.command("update")
48
+ def db_update() -> None:
49
+ """Download the latest published database if it is newer, verifying its SHA-256."""
50
+ from patchsight.db import DatabaseError
51
+ from patchsight.scan import dbcache
52
+
53
+ try:
54
+ status, pointer = dbcache.update()
55
+ except (DatabaseError, OSError) as error:
56
+ typer.echo(f"error: {error}", err=True)
57
+ raise typer.Exit(code=2) from None
58
+ verb = "Downloaded" if status == "updated" else "Already have"
59
+ typer.echo(
60
+ f"{verb} database {pointer.db_version} ({pointer.records:,} records) "
61
+ f"at {dbcache.default_db_path()}"
62
+ )
63
+
64
+
65
+ @db_app.command("info")
66
+ def db_info(
67
+ database: Annotated[Path | None, typer.Option("--db", help="Database to describe.")] = None,
68
+ ) -> None:
69
+ """Show which database a scan would use."""
70
+ from patchsight.db import DatabaseError, SymbolDatabase
71
+ from patchsight.scan import dbcache
72
+ from patchsight.scan.report import stale_warning
73
+
74
+ path = database or dbcache.default_db_path()
75
+ try:
76
+ opened = SymbolDatabase(path)
77
+ except DatabaseError as error:
78
+ typer.echo(f"error: {error}; run 'patchsight db update'", err=True)
79
+ raise typer.Exit(code=2) from None
80
+ meta = opened.meta
81
+ opened.close()
82
+ typer.echo(f"{path}")
83
+ for key in ("db_version", "schema_version", "builder", "built_at", "snapshot_sha256"):
84
+ typer.echo(f" {key}: {meta.get(key, '?')}")
85
+ if warning := stale_warning(meta):
86
+ typer.echo(f" warning: {warning}")
87
+
88
+
89
+ @app.command()
90
+ def scan(
91
+ path: Annotated[Path, typer.Argument(help="Project directory to scan.")] = Path(),
92
+ requirements_file: Annotated[
93
+ Path | None, typer.Option("--requirements", help="Lockfile (default: requirements.txt).")
94
+ ] = None,
95
+ database: Annotated[
96
+ Path | None,
97
+ typer.Option("--db", help="Symbol database (default: the one 'db update' downloaded)."),
98
+ ] = None,
99
+ output: Annotated[str, typer.Option("--format", help="text or json.")] = "text",
100
+ fail_on: Annotated[
101
+ str, typer.Option(help="Exit 1 when an alert is: called, unknown (or called), or never.")
102
+ ] = "never",
103
+ ) -> None:
104
+ """Label every dependency alert as called, imported but not called, not imported, or unknown."""
105
+ from patchsight.db import DatabaseError
106
+ from patchsight.scan.dbcache import default_db_path
107
+ from patchsight.scan.osv_client import OsvError
108
+ from patchsight.scan.report import render_json, render_text
109
+ from patchsight.scan.run import run_scan
110
+
111
+ lockfile = requirements_file or path / "requirements.txt"
112
+ if not lockfile.exists():
113
+ typer.echo(f"error: no lockfile at {lockfile}; pass --requirements", err=True)
114
+ raise typer.Exit(code=2)
115
+ try:
116
+ alerts, dependencies, meta = run_scan(path, lockfile, database or default_db_path())
117
+ except DatabaseError as error:
118
+ typer.echo(f"error: {error}; run 'patchsight db update' or pass --db", err=True)
119
+ raise typer.Exit(code=2) from None
120
+ except OsvError as error:
121
+ typer.echo(f"error: {error}", err=True)
122
+ raise typer.Exit(code=2) from None
123
+ shown = lockfile.as_posix()
124
+ if output == "json":
125
+ typer.echo(render_json(alerts, dependencies, shown, meta))
126
+ else:
127
+ typer.echo(render_text(alerts, dependencies, shown, meta))
128
+ failing = {"called": {"CALLED"}, "unknown": {"CALLED", "UNKNOWN"}}.get(fail_on, set())
129
+ if any(a.verdict in failing for a in alerts):
130
+ raise typer.Exit(code=1)
patchsight/db.py ADDED
@@ -0,0 +1,221 @@
1
+ """The symbol database: the one artefact that crosses from the offline pipeline to the scan.
2
+
3
+ A record maps one vulnerability (an alias group) and one package to the symbols that carry
4
+ the bug, with confidence, evidence and provenance. Every advisory ID and alias points at
5
+ its vulnerability, so the scanner can look up whatever ID OSV returns.
6
+
7
+ * ``high`` confidence: LLM extraction that passed validation and agreement checks.
8
+ * ``low`` confidence: the fallback (every changed function) or an unconfirmed extraction.
9
+ Low-confidence symbols can make an alert ``CALLED`` but never ``IMPORTED_NOT_CALLED``.
10
+
11
+ Stdlib ``sqlite3`` only, so the scan path can read it without any pipeline dependency.
12
+ """
13
+
14
+ import json
15
+ import sqlite3
16
+ from collections.abc import Iterable
17
+ from datetime import datetime
18
+ from pathlib import Path
19
+ from typing import Literal
20
+
21
+ from pydantic import BaseModel, ConfigDict
22
+
23
+ SCHEMA_VERSION = 1
24
+
25
+ Confidence = Literal["high", "low"]
26
+ Source = Literal["llm", "fallback", "override"]
27
+
28
+ _SCHEMA = """
29
+ CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
30
+ CREATE TABLE aliases (identifier TEXT PRIMARY KEY, vulnerability TEXT NOT NULL);
31
+ CREATE TABLE records (
32
+ vulnerability TEXT NOT NULL,
33
+ package TEXT NOT NULL,
34
+ confidence TEXT NOT NULL CHECK (confidence IN ('high', 'low')),
35
+ source TEXT NOT NULL CHECK (source IN ('llm', 'fallback', 'override')),
36
+ import_roots TEXT NOT NULL,
37
+ entry_points TEXT NOT NULL,
38
+ fix_commits TEXT NOT NULL,
39
+ model TEXT,
40
+ prompt_version TEXT,
41
+ input_hash TEXT NOT NULL,
42
+ advisory_modified TEXT NOT NULL,
43
+ truncated INTEGER NOT NULL,
44
+ extracted_at TEXT NOT NULL,
45
+ PRIMARY KEY (vulnerability, package)
46
+ );
47
+ CREATE TABLE symbols (
48
+ vulnerability TEXT NOT NULL,
49
+ package TEXT NOT NULL,
50
+ qualified_name TEXT NOT NULL,
51
+ module TEXT,
52
+ name TEXT NOT NULL,
53
+ kind TEXT NOT NULL,
54
+ path TEXT NOT NULL,
55
+ FOREIGN KEY (vulnerability, package) REFERENCES records (vulnerability, package)
56
+ );
57
+ CREATE INDEX symbols_by_record ON symbols (vulnerability, package);
58
+ """
59
+
60
+
61
+ class DatabaseError(RuntimeError):
62
+ """The database is missing, of an unsupported schema version, or malformed."""
63
+
64
+
65
+ class FixCommitRef(BaseModel):
66
+ model_config = ConfigDict(frozen=True)
67
+
68
+ repo: str
69
+ sha: str
70
+
71
+
72
+ class Symbol(BaseModel):
73
+ """A vulnerable definition. ``qualified_name`` is ``module.qualname`` when the module
74
+ is known, otherwise just the qualname."""
75
+
76
+ model_config = ConfigDict(frozen=True)
77
+
78
+ qualified_name: str
79
+ module: str | None
80
+ name: str
81
+ kind: str
82
+ path: str
83
+
84
+
85
+ class Record(BaseModel):
86
+ model_config = ConfigDict(frozen=True)
87
+
88
+ vulnerability: str
89
+ package: str
90
+ confidence: Confidence
91
+ source: Source
92
+ import_roots: tuple[str, ...]
93
+ entry_points: tuple[str, ...]
94
+ symbols: tuple[Symbol, ...]
95
+ fix_commits: tuple[FixCommitRef, ...]
96
+ model: str | None
97
+ prompt_version: str | None
98
+ input_hash: str
99
+ advisory_modified: datetime
100
+ truncated: bool
101
+ extracted_at: datetime
102
+
103
+
104
+ def write(
105
+ path: Path, records: Iterable[Record], aliases: dict[str, str], meta: dict[str, str]
106
+ ) -> int:
107
+ """Write a fresh database file. Returns the number of records written."""
108
+ if path.exists():
109
+ path.unlink()
110
+ path.parent.mkdir(parents=True, exist_ok=True)
111
+ connection = sqlite3.connect(path)
112
+ count = 0
113
+ try:
114
+ connection.executescript(_SCHEMA)
115
+ connection.executemany(
116
+ "INSERT INTO meta VALUES (?, ?)",
117
+ sorted({**meta, "schema_version": str(SCHEMA_VERSION)}.items()),
118
+ )
119
+ connection.executemany("INSERT INTO aliases VALUES (?, ?)", sorted(aliases.items()))
120
+ for record in records:
121
+ count += 1
122
+ connection.execute(
123
+ "INSERT INTO records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
124
+ (
125
+ record.vulnerability,
126
+ record.package,
127
+ record.confidence,
128
+ record.source,
129
+ json.dumps(record.import_roots),
130
+ json.dumps(record.entry_points),
131
+ json.dumps([c.model_dump() for c in record.fix_commits]),
132
+ record.model,
133
+ record.prompt_version,
134
+ record.input_hash,
135
+ record.advisory_modified.isoformat(),
136
+ int(record.truncated),
137
+ record.extracted_at.isoformat(),
138
+ ),
139
+ )
140
+ connection.executemany(
141
+ "INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?, ?)",
142
+ [
143
+ (
144
+ record.vulnerability,
145
+ record.package,
146
+ s.qualified_name,
147
+ s.module,
148
+ s.name,
149
+ s.kind,
150
+ s.path,
151
+ )
152
+ for s in record.symbols
153
+ ],
154
+ )
155
+ connection.commit()
156
+ finally:
157
+ connection.close()
158
+ return count
159
+
160
+
161
+ class SymbolDatabase:
162
+ """Read-only access for the scanner."""
163
+
164
+ def __init__(self, path: Path) -> None:
165
+ if not path.exists():
166
+ raise DatabaseError(f"no symbol database at {path}")
167
+ self._connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
168
+ try:
169
+ self.meta = dict(self._connection.execute("SELECT key, value FROM meta").fetchall())
170
+ except sqlite3.DatabaseError as error:
171
+ raise DatabaseError(f"{path} is not a PatchSight database: {error}") from None
172
+ version = self.meta.get("schema_version")
173
+ if version != str(SCHEMA_VERSION):
174
+ raise DatabaseError(
175
+ f"{path} has schema version {version}; this version of PatchSight reads "
176
+ f"version {SCHEMA_VERSION}. Update PatchSight or download a matching database."
177
+ )
178
+
179
+ def close(self) -> None:
180
+ self._connection.close()
181
+
182
+ def record_count(self) -> int:
183
+ return int(self._connection.execute("SELECT COUNT(*) FROM records").fetchone()[0])
184
+
185
+ def vulnerability_for(self, identifier: str) -> str | None:
186
+ row = self._connection.execute(
187
+ "SELECT vulnerability FROM aliases WHERE identifier = ?", (identifier,)
188
+ ).fetchone()
189
+ return str(row[0]) if row else None
190
+
191
+ def record(self, vulnerability: str, package: str) -> Record | None:
192
+ row = self._connection.execute(
193
+ "SELECT * FROM records WHERE vulnerability = ? AND package = ?",
194
+ (vulnerability, package),
195
+ ).fetchone()
196
+ if row is None:
197
+ return None
198
+ symbols = self._connection.execute(
199
+ "SELECT qualified_name, module, name, kind, path FROM symbols "
200
+ "WHERE vulnerability = ? AND package = ? ORDER BY qualified_name",
201
+ (vulnerability, package),
202
+ ).fetchall()
203
+ return Record(
204
+ vulnerability=row[0],
205
+ package=row[1],
206
+ confidence=row[2],
207
+ source=row[3],
208
+ import_roots=tuple(json.loads(row[4])),
209
+ entry_points=tuple(json.loads(row[5])),
210
+ fix_commits=tuple(FixCommitRef(**c) for c in json.loads(row[6])),
211
+ model=row[7],
212
+ prompt_version=row[8],
213
+ input_hash=row[9],
214
+ advisory_modified=datetime.fromisoformat(row[10]),
215
+ truncated=bool(row[11]),
216
+ extracted_at=datetime.fromisoformat(row[12]),
217
+ symbols=tuple(
218
+ Symbol(qualified_name=s[0], module=s[1], name=s[2], kind=s[3], path=s[4])
219
+ for s in symbols
220
+ ),
221
+ )
@@ -0,0 +1 @@
1
+ """Evaluation harness: metrics, benchmark scoring and the release gate. Offline only."""
@@ -0,0 +1,169 @@
1
+ """Confidence tiers from two-model agreement, and the fallback union (proposal §6, step 5).
2
+
3
+ For each report, the extractor (primary) and a model of a different family (secondary)
4
+ answer independently. A report is **high** confidence only when all of these hold:
5
+
6
+ * both answers passed validation;
7
+ * the prompt was not truncated to the token budget;
8
+ * the symbol set is not empty, because "nothing is vulnerable" must never turn into
9
+ "not called" for every user;
10
+ * the two answers agree under the chosen rule;
11
+ * at least one agreed function or entry point is **public** (an exported Go name, or a
12
+ Python name without a leading underscore). A record whose symbols are all private
13
+ helpers cannot support "not called": users reach a helper only through public APIs
14
+ the record does not name.
15
+
16
+ Everything else is **low** confidence and, with the fallback union, predicts every changed
17
+ function. Uncertainty can therefore only add symbols, never remove them.
18
+
19
+ Agreement rules (open question 1). ``subset`` with the public-symbol condition was chosen on
20
+ the dev split (ADR 0004) and is the default:
21
+
22
+ * ``identical``: the same set; predicts it.
23
+ * ``subset``: one set contains the other; predicts their union.
24
+ * ``overlap``: Jaccard similarity of at least 0.5; predicts their union.
25
+ """
26
+
27
+ from collections import Counter
28
+ from collections.abc import Mapping, Set
29
+ from typing import Literal
30
+
31
+ from pydantic import BaseModel, ConfigDict
32
+
33
+ from patchsight.eval.metrics import SetScores, jaccard, score
34
+ from patchsight.eval.safety import SafetyScores, safety
35
+
36
+ AgreementRule = Literal["identical", "subset", "overlap"]
37
+ RULES: tuple[AgreementRule, ...] = ("identical", "subset", "overlap")
38
+ Tier = Literal["high", "low"]
39
+ VALID = frozenset({"ok", "repaired"})
40
+ CHOSEN_RULE: AgreementRule = "subset"
41
+ Language = Literal["go", "python"]
42
+
43
+
44
+ def is_public(name: str, language: Language) -> bool:
45
+ """Whether users can call this name directly: an exported Go identifier, or a Python
46
+ name without a leading underscore (dunder methods such as ``__init__`` count)."""
47
+ leaf = name.split(".")[-1]
48
+ if language == "go":
49
+ return leaf[:1].isupper()
50
+ return not leaf.startswith("_") or (leaf.startswith("__") and leaf.endswith("__"))
51
+
52
+
53
+ class Answer(BaseModel):
54
+ """One model's validated outcome for one report."""
55
+
56
+ model_config = ConfigDict(frozen=True)
57
+
58
+ status: str
59
+ vulnerable: frozenset[str]
60
+ truncated: bool = False
61
+ entry_points: frozenset[str] = frozenset()
62
+
63
+
64
+ class Decision(BaseModel):
65
+ model_config = ConfigDict(frozen=True)
66
+
67
+ tier: Tier
68
+ reason: str
69
+ agreed: frozenset[str]
70
+ # Agreed functions plus both models' entry points: what the scanner would match.
71
+ symbols: frozenset[str] = frozenset()
72
+
73
+
74
+ def decide(
75
+ primary: Answer,
76
+ secondary: Answer | None,
77
+ rule: AgreementRule = CHOSEN_RULE,
78
+ require_public: bool = True,
79
+ language: Language = "go",
80
+ ) -> Decision:
81
+ empty: frozenset[str] = frozenset()
82
+ if primary.status not in VALID:
83
+ return Decision(tier="low", reason="extractor answer failed validation", agreed=empty)
84
+ if secondary is None or secondary.status not in VALID:
85
+ return Decision(tier="low", reason="no valid second opinion", agreed=empty)
86
+ if primary.truncated or secondary.truncated:
87
+ return Decision(tier="low", reason="prompt truncated to the token budget", agreed=empty)
88
+ a, b = primary.vulnerable, secondary.vulnerable
89
+ if not a or not b:
90
+ return Decision(tier="low", reason="an empty symbol set", agreed=empty)
91
+ if rule == "identical":
92
+ agrees, agreed = a == b, a
93
+ elif rule == "subset":
94
+ agrees, agreed = a <= b or b <= a, a | b
95
+ else:
96
+ agrees, agreed = jaccard(a, b) >= 0.5, a | b
97
+ if not agrees:
98
+ return Decision(tier="low", reason="the two models disagree", agreed=empty)
99
+ symbols = agreed | primary.entry_points | secondary.entry_points
100
+ if require_public and not any(is_public(name, language) for name in symbols):
101
+ return Decision(
102
+ tier="low", reason="no public symbol among the agreed answers", agreed=empty
103
+ )
104
+ return Decision(
105
+ tier="high", reason=f"both models agree ({rule})", agreed=agreed, symbols=symbols
106
+ )
107
+
108
+
109
+ class TierScores(BaseModel):
110
+ rule: AgreementRule
111
+ require_public: bool
112
+ fallback: bool
113
+ overall: SetScores
114
+ high_reports: int
115
+ high_share: float
116
+ high: SetScores
117
+ safety_overall: SafetyScores
118
+ safety_high: SafetyScores
119
+ low_reasons: dict[str, int]
120
+
121
+
122
+ def evaluate(
123
+ primary: Mapping[str, Answer],
124
+ secondary: Mapping[str, Answer],
125
+ changed: Mapping[str, Set[str]],
126
+ truth: Mapping[str, Set[str]],
127
+ rule: AgreementRule,
128
+ fallback: bool,
129
+ require_public: bool = False,
130
+ language: Language = "go",
131
+ ) -> TierScores:
132
+ """Score one configuration in two views.
133
+
134
+ * **Benchmark view** (``overall``, ``high``): functions only, exact names. Without the
135
+ fallback, low-confidence reports keep the extractor's own set.
136
+ * **Safety view** (``safety_*``): functions plus entry points, lenient names. High
137
+ confidence keeps the agreed symbols; low confidence keeps every changed function plus
138
+ both models' entry points.
139
+ """
140
+ predicted: dict[str, set[str]] = {}
141
+ high: dict[str, set[str]] = {}
142
+ product: dict[str, set[str]] = {}
143
+ product_high: dict[str, set[str]] = {}
144
+ reasons: Counter[str] = Counter()
145
+ for report, answer in primary.items():
146
+ other = secondary.get(report)
147
+ decision = decide(answer, other, rule, require_public, language)
148
+ entry = set(answer.entry_points) | (set(other.entry_points) if other else set())
149
+ if decision.tier == "high":
150
+ predicted[report] = high[report] = set(decision.agreed)
151
+ product[report] = product_high[report] = set(decision.symbols)
152
+ else:
153
+ reasons[decision.reason] += 1
154
+ predicted[report] = set(changed[report]) if fallback else set(answer.vulnerable)
155
+ product[report] = (set(changed[report]) if fallback else set(answer.vulnerable)) | entry
156
+ reports = list(primary)
157
+ truth_of = {r: set(truth[r]) for r in reports}
158
+ return TierScores(
159
+ rule=rule,
160
+ require_public=require_public,
161
+ fallback=fallback,
162
+ overall=score(predicted, truth_of),
163
+ high_reports=len(high),
164
+ high_share=len(high) / len(reports) if reports else 0.0,
165
+ high=score(high, {r: truth_of[r] for r in high}),
166
+ safety_overall=safety(product, truth_of),
167
+ safety_high=safety(product_high, {r: truth_of[r] for r in product_high}),
168
+ low_reasons=dict(reasons.most_common()),
169
+ )
@@ -0,0 +1,134 @@
1
+ """Set-based extraction metrics over advisories (proposal §10).
2
+
3
+ The unit is a (report, symbol name) pair. For each report, predicted symbols are compared
4
+ with the true vulnerable symbols:
5
+
6
+ * true positive: predicted and true;
7
+ * false positive: predicted, not true;
8
+ * false negative: true, not predicted. This includes reports with no prediction at all.
9
+
10
+ Precision, recall and F1 are micro-averaged over all pairs. Confidence intervals come
11
+ from a bootstrap that resamples whole reports, not pairs, because symbols within one report
12
+ are correlated.
13
+ """
14
+
15
+ import random
16
+ from collections.abc import Mapping, Set
17
+ from dataclasses import dataclass
18
+
19
+ from pydantic import BaseModel, ConfigDict
20
+
21
+ Predictions = Mapping[str, Set[str]]
22
+
23
+
24
+ class Interval(BaseModel):
25
+ model_config = ConfigDict(frozen=True)
26
+
27
+ low: float
28
+ high: float
29
+
30
+
31
+ class SetScores(BaseModel):
32
+ model_config = ConfigDict(frozen=True)
33
+
34
+ reports: int
35
+ true_positives: int
36
+ false_positives: int
37
+ false_negatives: int
38
+ precision: float
39
+ recall: float
40
+ f1: float
41
+ miss_rate: float
42
+ exact_match: float
43
+ precision_ci95: Interval | None = None
44
+ recall_ci95: Interval | None = None
45
+ f1_ci95: Interval | None = None
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class _Counts:
50
+ tp: int
51
+ fp: int
52
+ fn: int
53
+ missed: bool
54
+ exact: bool
55
+
56
+
57
+ def _counts(predicted: Set[str], truth: Set[str]) -> _Counts:
58
+ tp = len(predicted & truth)
59
+ return _Counts(
60
+ tp=tp,
61
+ fp=len(predicted - truth),
62
+ fn=len(truth - predicted),
63
+ missed=bool(truth) and tp == 0,
64
+ exact=predicted == truth,
65
+ )
66
+
67
+
68
+ def _prf(tp: int, fp: int, fn: int) -> tuple[float, float, float]:
69
+ precision = tp / (tp + fp) if tp + fp else 0.0
70
+ recall = tp / (tp + fn) if tp + fn else 0.0
71
+ f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
72
+ return precision, recall, f1
73
+
74
+
75
+ def _percentile(values: list[float], q: float) -> float:
76
+ ordered = sorted(values)
77
+ position = q * (len(ordered) - 1)
78
+ low = int(position)
79
+ high = min(low + 1, len(ordered) - 1)
80
+ return ordered[low] + (ordered[high] - ordered[low]) * (position - low)
81
+
82
+
83
+ def score(
84
+ predicted: Predictions,
85
+ truth: Predictions,
86
+ bootstrap: int = 0,
87
+ seed: int = 0,
88
+ ) -> SetScores:
89
+ """Score predictions for every report in ``truth``. Extra predicted reports are ignored."""
90
+ reports = sorted(truth)
91
+ per_report = [_counts(predicted.get(r, frozenset()), truth[r]) for r in reports]
92
+ tp = sum(c.tp for c in per_report)
93
+ fp = sum(c.fp for c in per_report)
94
+ fn = sum(c.fn for c in per_report)
95
+ precision, recall, f1 = _prf(tp, fp, fn)
96
+ with_truth = [c for c, r in zip(per_report, reports, strict=True) if truth[r]]
97
+
98
+ intervals: dict[str, Interval | None] = {
99
+ "precision_ci95": None,
100
+ "recall_ci95": None,
101
+ "f1_ci95": None,
102
+ }
103
+ if bootstrap and per_report:
104
+ rng = random.Random(seed) # noqa: S311 - statistical resampling, not security
105
+ samples: list[tuple[float, float, float]] = []
106
+ for _ in range(bootstrap):
107
+ draw = [per_report[rng.randrange(len(per_report))] for _ in per_report]
108
+ samples.append(
109
+ _prf(sum(c.tp for c in draw), sum(c.fp for c in draw), sum(c.fn for c in draw))
110
+ )
111
+ for index, key in enumerate(("precision_ci95", "recall_ci95", "f1_ci95")):
112
+ values = [s[index] for s in samples]
113
+ intervals[key] = Interval(
114
+ low=_percentile(values, 0.025), high=_percentile(values, 0.975)
115
+ )
116
+
117
+ return SetScores(
118
+ reports=len(reports),
119
+ true_positives=tp,
120
+ false_positives=fp,
121
+ false_negatives=fn,
122
+ precision=precision,
123
+ recall=recall,
124
+ f1=f1,
125
+ miss_rate=sum(c.missed for c in with_truth) / len(with_truth) if with_truth else 0.0,
126
+ exact_match=sum(c.exact for c in per_report) / len(per_report) if per_report else 0.0,
127
+ **intervals,
128
+ )
129
+
130
+
131
+ def jaccard(a: Set[str], b: Set[str]) -> float:
132
+ """Overlap of two sets; 1.0 when both are empty."""
133
+ union = a | b
134
+ return len(a & b) / len(union) if union else 1.0