vexpath-core 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. vexpath_core/__init__.py +20 -0
  2. vexpath_core/advisory/__init__.py +37 -0
  3. vexpath_core/advisory/coverage.py +260 -0
  4. vexpath_core/advisory/enrich.py +377 -0
  5. vexpath_core/advisory/osv.py +401 -0
  6. vexpath_core/advisory/source.py +220 -0
  7. vexpath_core/advisory/sync.py +294 -0
  8. vexpath_core/attest.py +630 -0
  9. vexpath_core/cdxvex.py +342 -0
  10. vexpath_core/cvss.py +179 -0
  11. vexpath_core/enums.py +428 -0
  12. vexpath_core/ids.py +79 -0
  13. vexpath_core/keyformat.py +56 -0
  14. vexpath_core/models/__init__.py +67 -0
  15. vexpath_core/models/advisory.py +295 -0
  16. vexpath_core/models/common.py +50 -0
  17. vexpath_core/models/finding.py +198 -0
  18. vexpath_core/models/results.py +147 -0
  19. vexpath_core/models/sbom.py +90 -0
  20. vexpath_core/patchdiff/__init__.py +73 -0
  21. vexpath_core/patchdiff/extract.py +332 -0
  22. vexpath_core/patchdiff/fetch.py +432 -0
  23. vexpath_core/patchdiff/layout.py +282 -0
  24. vexpath_core/patchdiff/pipeline.py +211 -0
  25. vexpath_core/patchdiff/references.py +216 -0
  26. vexpath_core/patchdiff/unidiff.py +190 -0
  27. vexpath_core/policy.py +435 -0
  28. vexpath_core/protocols.py +96 -0
  29. vexpath_core/purl.py +123 -0
  30. vexpath_core/py.typed +0 -0
  31. vexpath_core/reach/__init__.py +63 -0
  32. vexpath_core/reach/archives.py +365 -0
  33. vexpath_core/reach/astutil.py +55 -0
  34. vexpath_core/reach/cache.py +230 -0
  35. vexpath_core/reach/dependencies.py +359 -0
  36. vexpath_core/reach/entrypoints.py +341 -0
  37. vexpath_core/reach/graph.py +476 -0
  38. vexpath_core/reach/pybuild.py +998 -0
  39. vexpath_core/reach/verdict.py +497 -0
  40. vexpath_core/report.py +567 -0
  41. vexpath_core/sbom/__init__.py +106 -0
  42. vexpath_core/sbom/cyclonedx.py +439 -0
  43. vexpath_core/sbom/generate.py +306 -0
  44. vexpath_core/sbom/lockfiles.py +747 -0
  45. vexpath_core/sbom/minimum_elements.py +104 -0
  46. vexpath_core/sbom/spdx.py +470 -0
  47. vexpath_core/scan.py +391 -0
  48. vexpath_core/snapshot.py +238 -0
  49. vexpath_core/triage.py +373 -0
  50. vexpath_core/versions.py +280 -0
  51. vexpath_core/vex.py +377 -0
  52. vexpath_core/vulnmatch.py +288 -0
  53. vexpath_core-0.1.0.dist-info/METADATA +56 -0
  54. vexpath_core-0.1.0.dist-info/RECORD +56 -0
  55. vexpath_core-0.1.0.dist-info/WHEEL +4 -0
  56. vexpath_core-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,20 @@
1
+ """Analysis core for the software supply-chain security platform.
2
+
3
+ This package decides verdicts. It is used unchanged by the CLI, the GitHub
4
+ Action and the platform workers, so that where a scan runs never affects what
5
+ it concludes.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from importlib.metadata import PackageNotFoundError, version
11
+
12
+ try: # pragma: no cover - depends on install mode
13
+ __version__ = version("vexpath-core")
14
+ except PackageNotFoundError: # pragma: no cover - running from a source tree
15
+ __version__ = "0.0.0+unknown"
16
+
17
+ #: Version of the ``vexpath.results`` JSON schema this build emits.
18
+ RESULTS_SCHEMA = "vexpath.results/1"
19
+
20
+ __all__ = ["RESULTS_SCHEMA", "__version__"]
@@ -0,0 +1,37 @@
1
+ """Advisory ingest, normalisation and access.
2
+
3
+ The corpus is the product's factual foundation. Everything here exists to make
4
+ it trustworthy: normalised on the way in so matching cannot silently miss,
5
+ indexed so a scan is fast, and versioned as snapshots so a verdict can be
6
+ re-derived rather than merely believed.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from vexpath_core.advisory.coverage import (
12
+ CoverageReport,
13
+ EcosystemCoverage,
14
+ FixReferenceTier,
15
+ classify_fix_reference,
16
+ measure,
17
+ )
18
+ from vexpath_core.advisory.osv import OsvRecordError, normalise, normalise_all
19
+ from vexpath_core.advisory.source import AdvisorySourceError, OsvApiSource, SnapshotSource
20
+ from vexpath_core.advisory.sync import AdvisorySyncError, SyncReport, sync
21
+
22
+ __all__ = [
23
+ "AdvisorySourceError",
24
+ "AdvisorySyncError",
25
+ "CoverageReport",
26
+ "EcosystemCoverage",
27
+ "FixReferenceTier",
28
+ "OsvApiSource",
29
+ "OsvRecordError",
30
+ "SnapshotSource",
31
+ "SyncReport",
32
+ "classify_fix_reference",
33
+ "measure",
34
+ "normalise",
35
+ "normalise_all",
36
+ "sync",
37
+ ]
@@ -0,0 +1,260 @@
1
+ """Measure whether reachability analysis is possible at all.
2
+
3
+ Reachability needs to know which functions an advisory's fix changed. For one
4
+ ecosystem, Go, OSV publishes those symbols outright. For every other ecosystem,
5
+ including PyPI, it does not: they have to be recovered by diffing the commit
6
+ that fixed the issue.
7
+
8
+ So there are two different numbers, and conflating them would be misleading:
9
+
10
+ **Symbol coverage** is the share of advisories that already name their
11
+ vulnerable symbols. For PyPI this is zero, and will stay zero, because OSV
12
+ carries no such field for it. A zero here means "nothing was handed to us", not
13
+ "nothing is possible".
14
+
15
+ **Fix-reference coverage** is the share of advisories from which a fix commit
16
+ can be recovered. This is the number that decides whether the call-graph engine
17
+ is worth building, because it caps how many findings reachability can ever
18
+ speak to. It is reported by tier, because the tiers differ in how much trust
19
+ the resulting symbols deserve:
20
+
21
+ | Tier | What it means | Symbol confidence it supports |
22
+ |---|---|---|
23
+ | `commit` | A reference points directly at a fix commit | high |
24
+ | `git_range` | A GIT range names the repository and a fixed commit | high |
25
+ | `pull_request` | A reference points at a merge request, resolvable to a commit | medium |
26
+ | `tag_derivable` | Repository and fixed version are known, so release tags can be diffed | low |
27
+ | `none` | Nothing to work from | none |
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import re
33
+ from collections.abc import Sequence
34
+ from dataclasses import dataclass, field
35
+ from enum import StrEnum
36
+
37
+ from vexpath_core.models.advisory import Advisory, RangeType
38
+
39
+ #: Forge URLs that identify a single commit.
40
+ _COMMIT_URL = re.compile(
41
+ r"https?://(?:www\.)?(?:github\.com|gitlab\.com|bitbucket\.org|codeberg\.org)"
42
+ r"/[^/]+/[^/]+/(?:-/)?commits?/[0-9a-f]{7,40}",
43
+ re.IGNORECASE,
44
+ )
45
+
46
+ #: Forge URLs that identify a merge request, which resolves to a commit with
47
+ #: one extra API call.
48
+ _PULL_REQUEST_URL = re.compile(
49
+ r"https?://(?:www\.)?(?:github\.com|gitlab\.com|bitbucket\.org|codeberg\.org)"
50
+ r"/[^/]+/[^/]+/(?:pull/\d+|-/merge_requests/\d+|pull-requests/\d+)",
51
+ re.IGNORECASE,
52
+ )
53
+
54
+
55
+ class FixReferenceTier(StrEnum):
56
+ """How a fix commit can be recovered for an advisory, best first."""
57
+
58
+ COMMIT = "commit"
59
+ GIT_RANGE = "git_range"
60
+ PULL_REQUEST = "pull_request"
61
+ TAG_DERIVABLE = "tag_derivable"
62
+ NONE = "none"
63
+
64
+ @property
65
+ def is_usable(self) -> bool:
66
+ """Whether symbol extraction can start from this tier at all."""
67
+ return self is not FixReferenceTier.NONE
68
+
69
+ @property
70
+ def rank(self) -> int:
71
+ return _TIER_RANK[self]
72
+
73
+
74
+ _TIER_RANK: dict[FixReferenceTier, int] = {
75
+ FixReferenceTier.COMMIT: 4,
76
+ FixReferenceTier.GIT_RANGE: 3,
77
+ FixReferenceTier.PULL_REQUEST: 2,
78
+ FixReferenceTier.TAG_DERIVABLE: 1,
79
+ FixReferenceTier.NONE: 0,
80
+ }
81
+
82
+
83
+ def classify_fix_reference(advisory: Advisory) -> FixReferenceTier:
84
+ """Return the best tier from which this advisory's fix can be recovered.
85
+
86
+ Checked in descending order of trustworthiness, so an advisory with both a
87
+ commit reference and a derivable tag reports the commit.
88
+ """
89
+ urls = [reference.url for reference in advisory.references]
90
+
91
+ if any(_COMMIT_URL.search(url) for url in urls):
92
+ return FixReferenceTier.COMMIT
93
+
94
+ for package in advisory.affected:
95
+ for affected_range in package.ranges:
96
+ # A GIT range's fixed event is itself a commit identifier, so a
97
+ # range that names both a repository and a fix is as good as a
98
+ # direct commit reference.
99
+ if (
100
+ affected_range.type is RangeType.GIT
101
+ and affected_range.repo
102
+ and any(event.fixed for event in affected_range.events)
103
+ ):
104
+ return FixReferenceTier.GIT_RANGE
105
+
106
+ if any(_PULL_REQUEST_URL.search(url) for url in urls):
107
+ return FixReferenceTier.PULL_REQUEST
108
+
109
+ # Diffing the fixed release tag against the last affected one needs both a
110
+ # repository and a fixed version. It is the weakest tier because a release
111
+ # diff contains everything else that shipped in it.
112
+ has_fixed_version = any(package.fixed_versions for package in advisory.affected)
113
+ if advisory.repo_url and has_fixed_version:
114
+ return FixReferenceTier.TAG_DERIVABLE
115
+
116
+ return FixReferenceTier.NONE
117
+
118
+
119
+ @dataclass(frozen=True, slots=True)
120
+ class EcosystemCoverage:
121
+ """What can be analysed for one ecosystem.
122
+
123
+ ``advisories`` counts only the records reachability analysis could apply
124
+ to. Two kinds are excluded and counted separately, because including
125
+ either would misreport what the analysis can do:
126
+
127
+ *Withdrawn* records never produce a finding at all.
128
+
129
+ *Malicious packages* have no fix commit because there is no fix, and
130
+ reachability is the wrong question for them: installing malware runs it.
131
+ They dominate the PyPI corpus by count, so leaving them in the denominator
132
+ understates fix-commit coverage by a factor of nearly three.
133
+ """
134
+
135
+ ecosystem: str
136
+ advisories: int
137
+ withdrawn: int
138
+ malicious: int
139
+ with_symbols: int
140
+ tiers: dict[FixReferenceTier, int] = field(default_factory=dict)
141
+
142
+ @property
143
+ def total_records(self) -> int:
144
+ """Every record naming this ecosystem, including the excluded kinds."""
145
+ return self.advisories + self.withdrawn + self.malicious
146
+
147
+ @property
148
+ def symbol_coverage(self) -> float:
149
+ """Share of advisories that already name their vulnerable symbols."""
150
+ return self.with_symbols / self.advisories if self.advisories else 0.0
151
+
152
+ @property
153
+ def usable_fix_references(self) -> int:
154
+ return sum(count for tier, count in self.tiers.items() if tier.is_usable)
155
+
156
+ @property
157
+ def fix_reference_coverage(self) -> float:
158
+ """Share of advisories whose fix commit can be recovered.
159
+
160
+ The ceiling on what reachability analysis can ever decide for this
161
+ ecosystem.
162
+ """
163
+ return self.usable_fix_references / self.advisories if self.advisories else 0.0
164
+
165
+ @property
166
+ def high_confidence_fix_references(self) -> int:
167
+ """Advisories whose fix resolves to an exact commit."""
168
+ return self.tiers.get(FixReferenceTier.COMMIT, 0) + self.tiers.get(
169
+ FixReferenceTier.GIT_RANGE, 0
170
+ )
171
+
172
+ @property
173
+ def high_confidence_coverage(self) -> float:
174
+ return self.high_confidence_fix_references / self.advisories if self.advisories else 0.0
175
+
176
+
177
+ @dataclass(frozen=True, slots=True)
178
+ class CoverageReport:
179
+ """What a whole corpus can support."""
180
+
181
+ snapshot_id: str | None
182
+ advisories: int
183
+ ecosystems: tuple[EcosystemCoverage, ...]
184
+ #: Records excluded from the reachability denominators, and why.
185
+ withdrawn: int = 0
186
+ malicious: int = 0
187
+
188
+ def for_ecosystem(self, ecosystem: str) -> EcosystemCoverage | None:
189
+ return next((entry for entry in self.ecosystems if entry.ecosystem == ecosystem), None)
190
+
191
+
192
+ def measure(advisories: Sequence[Advisory], *, snapshot_id: str | None = None) -> CoverageReport:
193
+ """Measure what a corpus can support, per ecosystem.
194
+
195
+ Withdrawn advisories are counted separately and excluded from the
196
+ denominators, because they never produce a finding and including them would
197
+ flatter every percentage.
198
+ """
199
+ totals: dict[str, dict[str, int]] = {}
200
+ tiers: dict[str, dict[FixReferenceTier, int]] = {}
201
+
202
+ for advisory in advisories:
203
+ ecosystems = {package.ecosystem for package in advisory.affected}
204
+ for ecosystem in ecosystems:
205
+ bucket = totals.setdefault(
206
+ ecosystem,
207
+ {"advisories": 0, "withdrawn": 0, "malicious": 0, "with_symbols": 0},
208
+ )
209
+ if advisory.is_withdrawn:
210
+ bucket["withdrawn"] += 1
211
+ continue
212
+ if advisory.is_malicious_package:
213
+ # No fix exists to diff, and reachability is the wrong
214
+ # question: installing malware runs it.
215
+ bucket["malicious"] += 1
216
+ continue
217
+ bucket["advisories"] += 1
218
+ if advisory.symbols:
219
+ bucket["with_symbols"] += 1
220
+ per_tier = tiers.setdefault(ecosystem, {})
221
+ tier = classify_fix_reference(advisory)
222
+ per_tier[tier] = per_tier.get(tier, 0) + 1
223
+
224
+ entries = tuple(
225
+ EcosystemCoverage(
226
+ ecosystem=ecosystem,
227
+ advisories=counts["advisories"],
228
+ withdrawn=counts["withdrawn"],
229
+ malicious=counts["malicious"],
230
+ with_symbols=counts["with_symbols"],
231
+ tiers=dict(tiers.get(ecosystem, {})),
232
+ )
233
+ for ecosystem, counts in sorted(totals.items())
234
+ )
235
+
236
+ analysable = sum(
237
+ 1
238
+ for advisory in advisories
239
+ if not advisory.is_withdrawn and not advisory.is_malicious_package
240
+ )
241
+ return CoverageReport(
242
+ snapshot_id=snapshot_id,
243
+ advisories=analysable,
244
+ ecosystems=entries,
245
+ withdrawn=sum(1 for advisory in advisories if advisory.is_withdrawn),
246
+ malicious=sum(
247
+ 1
248
+ for advisory in advisories
249
+ if advisory.is_malicious_package and not advisory.is_withdrawn
250
+ ),
251
+ )
252
+
253
+
254
+ __all__ = [
255
+ "CoverageReport",
256
+ "EcosystemCoverage",
257
+ "FixReferenceTier",
258
+ "classify_fix_reference",
259
+ "measure",
260
+ ]
@@ -0,0 +1,377 @@
1
+ """Exploitation signals: EPSS probabilities and the CISA KEV catalogue.
2
+
3
+ These change what to fix first, never whether something is reported. A
4
+ reachable critical with an EPSS score of 0.002 and a reachable medium that
5
+ appears in the Known Exploited Vulnerabilities catalogue are not the same
6
+ problem, and severity alone cannot tell them apart.
7
+
8
+ Both feeds are keyed by CVE identifier, which is why an advisory's aliases
9
+ matter: a GHSA-primary record carries its CVE as an alias, and looking up only
10
+ the primary identifier would miss most of the corpus.
11
+
12
+ Two properties shape the implementation.
13
+
14
+ **A missing feed is not a failed sync.** Enrichment is advisory data about
15
+ advisory data. If EPSS is unreachable, the corpus is still correct and still
16
+ usable, so a failure is reported and the corpus is written without it. The
17
+ alternative, failing the whole sync, would mean one third party's outage stops
18
+ a security tool from updating.
19
+
20
+ **A stale score is worse than none for the KEV flag.** EPSS is a probability
21
+ that ages gracefully; KEV membership is a statement that something is being
22
+ exploited right now. Both are stamped with the date they were read so a reader
23
+ can see how old the answer is.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import csv
29
+ import gzip
30
+ import io
31
+ import json
32
+ from collections.abc import Iterable, Mapping
33
+ from dataclasses import dataclass, field
34
+ from datetime import UTC, date, datetime
35
+
36
+ import httpx
37
+
38
+ from vexpath_core.models.advisory import Advisory
39
+
40
+ #: Daily EPSS export. Gzipped CSV with a comment line, then a header, then one
41
+ #: row per CVE.
42
+ EPSS_URL = "https://epss.cyentia.com/epss_scores-current.csv.gz"
43
+
44
+ #: The CISA Known Exploited Vulnerabilities catalogue.
45
+ KEV_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
46
+
47
+ #: Refuse a download larger than this. EPSS is a few megabytes compressed and
48
+ #: KEV is under one; an order of magnitude more means something is wrong
49
+ #: upstream, and parsing it would exhaust a small worker.
50
+ MAX_FEED_BYTES = 128 * 1024 * 1024
51
+
52
+ _DEFAULT_TIMEOUT = 60.0
53
+
54
+
55
+ class EnrichmentError(Exception):
56
+ """Raised when a feed cannot be downloaded or parsed.
57
+
58
+ Callers are expected to carry on without the signal rather than abandon a
59
+ corpus over it.
60
+ """
61
+
62
+
63
+ @dataclass(frozen=True, slots=True)
64
+ class EpssScore:
65
+ """One CVE's probability of exploitation in the next 30 days."""
66
+
67
+ score: float
68
+ percentile: float
69
+
70
+
71
+ @dataclass(slots=True)
72
+ class ExploitSignals:
73
+ """Everything the two feeds know, keyed by CVE identifier."""
74
+
75
+ epss: dict[str, EpssScore] = field(default_factory=dict)
76
+ #: CVE identifier to the date CISA added it to the catalogue.
77
+ kev: dict[str, date] = field(default_factory=dict)
78
+ #: When these were read. Stamped because KEV is a statement about now.
79
+ read_at: datetime = field(default_factory=lambda: datetime.now(UTC))
80
+ #: Feeds that could not be read, and why. Never fatal.
81
+ failures: tuple[str, ...] = ()
82
+
83
+ @property
84
+ def is_empty(self) -> bool:
85
+ return not self.epss and not self.kev
86
+
87
+ def summary_line(self) -> str:
88
+ parts = [f"{len(self.epss)} EPSS scores", f"{len(self.kev)} KEV entries"]
89
+ if self.failures:
90
+ parts.append(f"{len(self.failures)} feed(s) unavailable")
91
+ return ", ".join(parts)
92
+
93
+
94
+ def fetch_signals(
95
+ *,
96
+ client: httpx.Client | None = None,
97
+ epss_url: str = EPSS_URL,
98
+ kev_url: str = KEV_URL,
99
+ ) -> ExploitSignals:
100
+ """Download both feeds, tolerating the failure of either.
101
+
102
+ Args:
103
+ client: HTTP client to use. Injected so tests run offline.
104
+ epss_url: Override for the EPSS export.
105
+ kev_url: Override for the KEV catalogue.
106
+
107
+ Returns:
108
+ What could be read, and a note for whatever could not.
109
+ """
110
+ http = client or httpx.Client(timeout=_DEFAULT_TIMEOUT, follow_redirects=True)
111
+ failures: list[str] = []
112
+ epss: dict[str, EpssScore] = {}
113
+ kev: dict[str, date] = {}
114
+
115
+ try:
116
+ epss = parse_epss(_download(http, epss_url))
117
+ except EnrichmentError as exc:
118
+ failures.append(f"EPSS: {exc}")
119
+
120
+ try:
121
+ kev = parse_kev(_download(http, kev_url))
122
+ except EnrichmentError as exc:
123
+ failures.append(f"KEV: {exc}")
124
+
125
+ if client is None:
126
+ http.close()
127
+
128
+ return ExploitSignals(epss=epss, kev=kev, failures=tuple(failures))
129
+
130
+
131
+ def _download(client: httpx.Client, url: str) -> bytes:
132
+ try:
133
+ response = client.get(url)
134
+ response.raise_for_status()
135
+ except httpx.HTTPError as exc:
136
+ msg = f"could not download {url}: {exc}"
137
+ raise EnrichmentError(msg) from exc
138
+
139
+ if len(response.content) > MAX_FEED_BYTES:
140
+ msg = f"{url} returned {len(response.content)} bytes, above the {MAX_FEED_BYTES} limit"
141
+ raise EnrichmentError(msg)
142
+ return response.content
143
+
144
+
145
+ def parse_epss(payload: bytes) -> dict[str, EpssScore]:
146
+ """Parse the EPSS export, gzipped or not.
147
+
148
+ The file begins with a comment line naming the model version and date,
149
+ then a header row. Rows whose score is unparseable are skipped rather than
150
+ failing the file: a prioritisation signal missing for one CVE is a far
151
+ smaller problem than no signal for any.
152
+ """
153
+ text = _maybe_gunzip(payload).decode("utf-8", errors="replace")
154
+ lines = [line for line in text.splitlines() if line and not line.startswith("#")]
155
+ if not lines:
156
+ msg = "the EPSS export has no rows"
157
+ raise EnrichmentError(msg)
158
+
159
+ scores: dict[str, EpssScore] = {}
160
+ for row in csv.DictReader(lines):
161
+ identifier = (row.get("cve") or "").strip().upper()
162
+ if not identifier:
163
+ continue
164
+ try:
165
+ scores[identifier] = EpssScore(
166
+ score=float(row.get("epss") or 0.0),
167
+ percentile=float(row.get("percentile") or 0.0),
168
+ )
169
+ except (TypeError, ValueError):
170
+ continue
171
+
172
+ if not scores:
173
+ msg = "the EPSS export parsed to no scores, so its format has probably changed"
174
+ raise EnrichmentError(msg)
175
+ return scores
176
+
177
+
178
+ def parse_kev(payload: bytes) -> dict[str, date]:
179
+ """Parse the KEV catalogue into CVE identifier and date added."""
180
+ try:
181
+ document = json.loads(_maybe_gunzip(payload).decode("utf-8", errors="replace"))
182
+ except (json.JSONDecodeError, UnicodeDecodeError) as exc:
183
+ msg = f"the KEV catalogue is not readable JSON: {exc}"
184
+ raise EnrichmentError(msg) from exc
185
+
186
+ if not isinstance(document, dict):
187
+ msg = "the KEV catalogue is not a JSON object"
188
+ raise EnrichmentError(msg)
189
+
190
+ entries = document.get("vulnerabilities")
191
+ if not isinstance(entries, list):
192
+ msg = "the KEV catalogue has no 'vulnerabilities' array"
193
+ raise EnrichmentError(msg)
194
+
195
+ found: dict[str, date] = {}
196
+ for entry in entries:
197
+ if not isinstance(entry, dict):
198
+ continue
199
+ identifier = str(entry.get("cveID") or "").strip().upper()
200
+ if not identifier:
201
+ continue
202
+ found[identifier] = _parse_date(entry.get("dateAdded"))
203
+
204
+ if not found:
205
+ msg = "the KEV catalogue parsed to no entries, so its format has probably changed"
206
+ raise EnrichmentError(msg)
207
+ return found
208
+
209
+
210
+ def _parse_date(raw: object) -> date:
211
+ """Read a KEV date, falling back to today when it is unreadable.
212
+
213
+ The date is presentational. Refusing a catalogue entry over an unparseable
214
+ date would drop a statement that something is actively exploited, which is
215
+ the most important thing either feed says.
216
+ """
217
+ try:
218
+ return date.fromisoformat(str(raw))
219
+ except (TypeError, ValueError):
220
+ return datetime.now(UTC).date()
221
+
222
+
223
+ def _maybe_gunzip(payload: bytes) -> bytes:
224
+ """Decompress if the payload is gzipped, otherwise return it unchanged."""
225
+ if payload[:2] != b"\x1f\x8b":
226
+ return payload
227
+ try:
228
+ with gzip.GzipFile(fileobj=io.BytesIO(payload)) as handle:
229
+ return handle.read(MAX_FEED_BYTES + 1)
230
+ except OSError as exc:
231
+ msg = f"the feed is gzipped but could not be decompressed: {exc}"
232
+ raise EnrichmentError(msg) from exc
233
+
234
+
235
+ def enrich(advisory: Advisory, signals: ExploitSignals) -> Advisory:
236
+ """Attach exploitation signals to one advisory.
237
+
238
+ Looked up by every identifier the advisory carries, not just the primary
239
+ one. A GHSA-primary record holds its CVE as an alias, and both feeds are
240
+ keyed by CVE, so matching only the primary identifier would miss most of
241
+ the corpus.
242
+
243
+ Returns the advisory unchanged when neither feed knows it, so an unenriched
244
+ record is distinguishable from one scored zero. A real EPSS score of 0.0 is
245
+ meaningful: it says exploitation is very unlikely.
246
+ """
247
+ update: dict[str, object] = {}
248
+
249
+ for identifier in advisory.all_ids:
250
+ key = identifier.upper()
251
+ if "epss_score" not in update and (score := signals.epss.get(key)) is not None:
252
+ update["epss_score"] = score.score
253
+ update["epss_percentile"] = score.percentile
254
+ if "kev" not in update and (added := signals.kev.get(key)) is not None:
255
+ update["kev"] = True
256
+ update["kev_added_at"] = added
257
+
258
+ return advisory.model_copy(update=update) if update else advisory
259
+
260
+
261
+ def enrich_all(
262
+ advisories: Iterable[Advisory], signals: ExploitSignals
263
+ ) -> tuple[tuple[Advisory, ...], EnrichmentCounts]:
264
+ """Attach signals to a whole corpus, and count what landed."""
265
+ enriched: list[Advisory] = []
266
+ with_epss = 0
267
+ with_kev = 0
268
+ scorable = 0
269
+ for advisory in advisories:
270
+ result = enrich(advisory, signals)
271
+ enriched.append(result)
272
+ with_epss += 1 if result.epss_score is not None else 0
273
+ with_kev += 1 if result.kev else 0
274
+ scorable += 1 if has_cve_identifier(result) else 0
275
+ return tuple(enriched), EnrichmentCounts(
276
+ advisories=len(enriched),
277
+ with_epss=with_epss,
278
+ with_kev=with_kev,
279
+ scorable=scorable,
280
+ )
281
+
282
+
283
+ def has_cve_identifier(advisory: Advisory) -> bool:
284
+ """Whether either feed could possibly know this advisory.
285
+
286
+ Both are keyed by CVE. A malicious-package record has no CVE and never
287
+ will, so counting it as an enrichment miss understates coverage.
288
+ """
289
+ return any(identifier.upper().startswith("CVE-") for identifier in advisory.all_ids)
290
+
291
+
292
+ @dataclass(frozen=True, slots=True)
293
+ class EnrichmentCounts:
294
+ """How much of a corpus carries each signal."""
295
+
296
+ advisories: int
297
+ with_epss: int
298
+ with_kev: int
299
+ #: Advisories that could have a score at all. Both feeds are keyed by CVE,
300
+ #: so a record with no CVE identifier anywhere is not a miss, it is out of
301
+ #: scope. On the real corpus that is 12,003 of 19,552 records, and dividing
302
+ #: by the wrong one of those turns 89 per cent coverage into 34 per cent.
303
+ scorable: int = 0
304
+
305
+ @property
306
+ def epss_coverage(self) -> float:
307
+ """Share of the advisories that could be scored, and were."""
308
+ denominator = self.scorable or self.advisories
309
+ return self.with_epss / denominator if denominator else 0.0
310
+
311
+ @property
312
+ def out_of_scope(self) -> int:
313
+ return max(self.advisories - (self.scorable or self.advisories), 0)
314
+
315
+ def summary_line(self) -> str:
316
+ line = (
317
+ f"{self.with_epss} of {self.scorable or self.advisories} advisories "
318
+ f"with a CVE identifier have an EPSS score ({self.epss_coverage:.1%}), "
319
+ f"{self.with_kev} are known exploited"
320
+ )
321
+ if self.out_of_scope:
322
+ line += (
323
+ f". {self.out_of_scope} record(s) carry no CVE identifier, so no "
324
+ f"feed can score them"
325
+ )
326
+ return line
327
+
328
+
329
+ def highest_priority(advisories: Iterable[Advisory], *, limit: int = 10) -> tuple[Advisory, ...]:
330
+ """The advisories to look at first, worst plausible outcome soonest.
331
+
332
+ Known exploited comes before everything: it is not a prediction. After
333
+ that, EPSS descending, then severity, so a medium being actively exploited
334
+ outranks a critical that nobody has worked out how to use.
335
+ """
336
+ ordered = sorted(
337
+ advisories,
338
+ key=lambda advisory: (
339
+ not advisory.kev,
340
+ -(advisory.epss_score or 0.0),
341
+ -advisory.severity.rank,
342
+ advisory.id,
343
+ ),
344
+ )
345
+ return tuple(ordered[:limit])
346
+
347
+
348
+ def signals_from_mapping(
349
+ epss: Mapping[str, tuple[float, float]], kev: Mapping[str, date]
350
+ ) -> ExploitSignals:
351
+ """Build signals from plain data, for tests and for a cached feed."""
352
+ return ExploitSignals(
353
+ epss={
354
+ identifier.upper(): EpssScore(score=score, percentile=percentile)
355
+ for identifier, (score, percentile) in epss.items()
356
+ },
357
+ kev={identifier.upper(): added for identifier, added in kev.items()},
358
+ )
359
+
360
+
361
+ __all__ = [
362
+ "EPSS_URL",
363
+ "KEV_URL",
364
+ "MAX_FEED_BYTES",
365
+ "EnrichmentCounts",
366
+ "EnrichmentError",
367
+ "EpssScore",
368
+ "ExploitSignals",
369
+ "enrich",
370
+ "enrich_all",
371
+ "fetch_signals",
372
+ "has_cve_identifier",
373
+ "highest_priority",
374
+ "parse_epss",
375
+ "parse_kev",
376
+ "signals_from_mapping",
377
+ ]