vexpath-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vexpath_core/__init__.py +20 -0
- vexpath_core/advisory/__init__.py +37 -0
- vexpath_core/advisory/coverage.py +260 -0
- vexpath_core/advisory/enrich.py +377 -0
- vexpath_core/advisory/osv.py +401 -0
- vexpath_core/advisory/source.py +220 -0
- vexpath_core/advisory/sync.py +294 -0
- vexpath_core/attest.py +630 -0
- vexpath_core/cdxvex.py +342 -0
- vexpath_core/cvss.py +179 -0
- vexpath_core/enums.py +428 -0
- vexpath_core/ids.py +79 -0
- vexpath_core/keyformat.py +56 -0
- vexpath_core/models/__init__.py +67 -0
- vexpath_core/models/advisory.py +295 -0
- vexpath_core/models/common.py +50 -0
- vexpath_core/models/finding.py +198 -0
- vexpath_core/models/results.py +147 -0
- vexpath_core/models/sbom.py +90 -0
- vexpath_core/patchdiff/__init__.py +73 -0
- vexpath_core/patchdiff/extract.py +332 -0
- vexpath_core/patchdiff/fetch.py +432 -0
- vexpath_core/patchdiff/layout.py +282 -0
- vexpath_core/patchdiff/pipeline.py +211 -0
- vexpath_core/patchdiff/references.py +216 -0
- vexpath_core/patchdiff/unidiff.py +190 -0
- vexpath_core/policy.py +435 -0
- vexpath_core/protocols.py +96 -0
- vexpath_core/purl.py +123 -0
- vexpath_core/py.typed +0 -0
- vexpath_core/reach/__init__.py +63 -0
- vexpath_core/reach/archives.py +365 -0
- vexpath_core/reach/astutil.py +55 -0
- vexpath_core/reach/cache.py +230 -0
- vexpath_core/reach/dependencies.py +359 -0
- vexpath_core/reach/entrypoints.py +341 -0
- vexpath_core/reach/graph.py +476 -0
- vexpath_core/reach/pybuild.py +998 -0
- vexpath_core/reach/verdict.py +497 -0
- vexpath_core/report.py +567 -0
- vexpath_core/sbom/__init__.py +106 -0
- vexpath_core/sbom/cyclonedx.py +439 -0
- vexpath_core/sbom/generate.py +306 -0
- vexpath_core/sbom/lockfiles.py +747 -0
- vexpath_core/sbom/minimum_elements.py +104 -0
- vexpath_core/sbom/spdx.py +470 -0
- vexpath_core/scan.py +391 -0
- vexpath_core/snapshot.py +238 -0
- vexpath_core/triage.py +373 -0
- vexpath_core/versions.py +280 -0
- vexpath_core/vex.py +377 -0
- vexpath_core/vulnmatch.py +288 -0
- vexpath_core-0.1.0.dist-info/METADATA +56 -0
- vexpath_core-0.1.0.dist-info/RECORD +56 -0
- vexpath_core-0.1.0.dist-info/WHEEL +4 -0
- vexpath_core-0.1.0.dist-info/licenses/LICENSE +201 -0
vexpath_core/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Analysis core for the software supply-chain security platform.
|
|
2
|
+
|
|
3
|
+
This package decides verdicts. It is used unchanged by the CLI, the GitHub
|
|
4
|
+
Action and the platform workers, so that where a scan runs never affects what
|
|
5
|
+
it concludes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
11
|
+
|
|
12
|
+
try: # pragma: no cover - depends on install mode
|
|
13
|
+
__version__ = version("vexpath-core")
|
|
14
|
+
except PackageNotFoundError: # pragma: no cover - running from a source tree
|
|
15
|
+
__version__ = "0.0.0+unknown"
|
|
16
|
+
|
|
17
|
+
#: Version of the ``vexpath.results`` JSON schema this build emits.
|
|
18
|
+
RESULTS_SCHEMA = "vexpath.results/1"
|
|
19
|
+
|
|
20
|
+
__all__ = ["RESULTS_SCHEMA", "__version__"]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Advisory ingest, normalisation and access.
|
|
2
|
+
|
|
3
|
+
The corpus is the product's factual foundation. Everything here exists to make
|
|
4
|
+
it trustworthy: normalised on the way in so matching cannot silently miss,
|
|
5
|
+
indexed so a scan is fast, and versioned as snapshots so a verdict can be
|
|
6
|
+
re-derived rather than merely believed.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from vexpath_core.advisory.coverage import (
|
|
12
|
+
CoverageReport,
|
|
13
|
+
EcosystemCoverage,
|
|
14
|
+
FixReferenceTier,
|
|
15
|
+
classify_fix_reference,
|
|
16
|
+
measure,
|
|
17
|
+
)
|
|
18
|
+
from vexpath_core.advisory.osv import OsvRecordError, normalise, normalise_all
|
|
19
|
+
from vexpath_core.advisory.source import AdvisorySourceError, OsvApiSource, SnapshotSource
|
|
20
|
+
from vexpath_core.advisory.sync import AdvisorySyncError, SyncReport, sync
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"AdvisorySourceError",
|
|
24
|
+
"AdvisorySyncError",
|
|
25
|
+
"CoverageReport",
|
|
26
|
+
"EcosystemCoverage",
|
|
27
|
+
"FixReferenceTier",
|
|
28
|
+
"OsvApiSource",
|
|
29
|
+
"OsvRecordError",
|
|
30
|
+
"SnapshotSource",
|
|
31
|
+
"SyncReport",
|
|
32
|
+
"classify_fix_reference",
|
|
33
|
+
"measure",
|
|
34
|
+
"normalise",
|
|
35
|
+
"normalise_all",
|
|
36
|
+
"sync",
|
|
37
|
+
]
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Measure whether reachability analysis is possible at all.
|
|
2
|
+
|
|
3
|
+
Reachability needs to know which functions an advisory's fix changed. For one
|
|
4
|
+
ecosystem, Go, OSV publishes those symbols outright. For every other ecosystem,
|
|
5
|
+
including PyPI, it does not: they have to be recovered by diffing the commit
|
|
6
|
+
that fixed the issue.
|
|
7
|
+
|
|
8
|
+
So there are two different numbers, and conflating them would be misleading:
|
|
9
|
+
|
|
10
|
+
**Symbol coverage** is the share of advisories that already name their
|
|
11
|
+
vulnerable symbols. For PyPI this is zero, and will stay zero, because OSV
|
|
12
|
+
carries no such field for it. A zero here means "nothing was handed to us", not
|
|
13
|
+
"nothing is possible".
|
|
14
|
+
|
|
15
|
+
**Fix-reference coverage** is the share of advisories from which a fix commit
|
|
16
|
+
can be recovered. This is the number that decides whether the call-graph engine
|
|
17
|
+
is worth building, because it caps how many findings reachability can ever
|
|
18
|
+
speak to. It is reported by tier, because the tiers differ in how much trust
|
|
19
|
+
the resulting symbols deserve:
|
|
20
|
+
|
|
21
|
+
| Tier | What it means | Symbol confidence it supports |
|
|
22
|
+
|---|---|---|
|
|
23
|
+
| `commit` | A reference points directly at a fix commit | high |
|
|
24
|
+
| `git_range` | A GIT range names the repository and a fixed commit | high |
|
|
25
|
+
| `pull_request` | A reference points at a merge request, resolvable to a commit | medium |
|
|
26
|
+
| `tag_derivable` | Repository and fixed version are known, so release tags can be diffed | low |
|
|
27
|
+
| `none` | Nothing to work from | none |
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import re
|
|
33
|
+
from collections.abc import Sequence
|
|
34
|
+
from dataclasses import dataclass, field
|
|
35
|
+
from enum import StrEnum
|
|
36
|
+
|
|
37
|
+
from vexpath_core.models.advisory import Advisory, RangeType
|
|
38
|
+
|
|
39
|
+
#: Forge URLs that identify a single commit.
|
|
40
|
+
_COMMIT_URL = re.compile(
|
|
41
|
+
r"https?://(?:www\.)?(?:github\.com|gitlab\.com|bitbucket\.org|codeberg\.org)"
|
|
42
|
+
r"/[^/]+/[^/]+/(?:-/)?commits?/[0-9a-f]{7,40}",
|
|
43
|
+
re.IGNORECASE,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
#: Forge URLs that identify a merge request, which resolves to a commit with
|
|
47
|
+
#: one extra API call.
|
|
48
|
+
_PULL_REQUEST_URL = re.compile(
|
|
49
|
+
r"https?://(?:www\.)?(?:github\.com|gitlab\.com|bitbucket\.org|codeberg\.org)"
|
|
50
|
+
r"/[^/]+/[^/]+/(?:pull/\d+|-/merge_requests/\d+|pull-requests/\d+)",
|
|
51
|
+
re.IGNORECASE,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class FixReferenceTier(StrEnum):
|
|
56
|
+
"""How a fix commit can be recovered for an advisory, best first."""
|
|
57
|
+
|
|
58
|
+
COMMIT = "commit"
|
|
59
|
+
GIT_RANGE = "git_range"
|
|
60
|
+
PULL_REQUEST = "pull_request"
|
|
61
|
+
TAG_DERIVABLE = "tag_derivable"
|
|
62
|
+
NONE = "none"
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def is_usable(self) -> bool:
|
|
66
|
+
"""Whether symbol extraction can start from this tier at all."""
|
|
67
|
+
return self is not FixReferenceTier.NONE
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def rank(self) -> int:
|
|
71
|
+
return _TIER_RANK[self]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
_TIER_RANK: dict[FixReferenceTier, int] = {
|
|
75
|
+
FixReferenceTier.COMMIT: 4,
|
|
76
|
+
FixReferenceTier.GIT_RANGE: 3,
|
|
77
|
+
FixReferenceTier.PULL_REQUEST: 2,
|
|
78
|
+
FixReferenceTier.TAG_DERIVABLE: 1,
|
|
79
|
+
FixReferenceTier.NONE: 0,
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def classify_fix_reference(advisory: Advisory) -> FixReferenceTier:
|
|
84
|
+
"""Return the best tier from which this advisory's fix can be recovered.
|
|
85
|
+
|
|
86
|
+
Checked in descending order of trustworthiness, so an advisory with both a
|
|
87
|
+
commit reference and a derivable tag reports the commit.
|
|
88
|
+
"""
|
|
89
|
+
urls = [reference.url for reference in advisory.references]
|
|
90
|
+
|
|
91
|
+
if any(_COMMIT_URL.search(url) for url in urls):
|
|
92
|
+
return FixReferenceTier.COMMIT
|
|
93
|
+
|
|
94
|
+
for package in advisory.affected:
|
|
95
|
+
for affected_range in package.ranges:
|
|
96
|
+
# A GIT range's fixed event is itself a commit identifier, so a
|
|
97
|
+
# range that names both a repository and a fix is as good as a
|
|
98
|
+
# direct commit reference.
|
|
99
|
+
if (
|
|
100
|
+
affected_range.type is RangeType.GIT
|
|
101
|
+
and affected_range.repo
|
|
102
|
+
and any(event.fixed for event in affected_range.events)
|
|
103
|
+
):
|
|
104
|
+
return FixReferenceTier.GIT_RANGE
|
|
105
|
+
|
|
106
|
+
if any(_PULL_REQUEST_URL.search(url) for url in urls):
|
|
107
|
+
return FixReferenceTier.PULL_REQUEST
|
|
108
|
+
|
|
109
|
+
# Diffing the fixed release tag against the last affected one needs both a
|
|
110
|
+
# repository and a fixed version. It is the weakest tier because a release
|
|
111
|
+
# diff contains everything else that shipped in it.
|
|
112
|
+
has_fixed_version = any(package.fixed_versions for package in advisory.affected)
|
|
113
|
+
if advisory.repo_url and has_fixed_version:
|
|
114
|
+
return FixReferenceTier.TAG_DERIVABLE
|
|
115
|
+
|
|
116
|
+
return FixReferenceTier.NONE
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass(frozen=True, slots=True)
|
|
120
|
+
class EcosystemCoverage:
|
|
121
|
+
"""What can be analysed for one ecosystem.
|
|
122
|
+
|
|
123
|
+
``advisories`` counts only the records reachability analysis could apply
|
|
124
|
+
to. Two kinds are excluded and counted separately, because including
|
|
125
|
+
either would misreport what the analysis can do:
|
|
126
|
+
|
|
127
|
+
*Withdrawn* records never produce a finding at all.
|
|
128
|
+
|
|
129
|
+
*Malicious packages* have no fix commit because there is no fix, and
|
|
130
|
+
reachability is the wrong question for them: installing malware runs it.
|
|
131
|
+
They dominate the PyPI corpus by count, so leaving them in the denominator
|
|
132
|
+
understates fix-commit coverage by a factor of nearly three.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
ecosystem: str
|
|
136
|
+
advisories: int
|
|
137
|
+
withdrawn: int
|
|
138
|
+
malicious: int
|
|
139
|
+
with_symbols: int
|
|
140
|
+
tiers: dict[FixReferenceTier, int] = field(default_factory=dict)
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def total_records(self) -> int:
|
|
144
|
+
"""Every record naming this ecosystem, including the excluded kinds."""
|
|
145
|
+
return self.advisories + self.withdrawn + self.malicious
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def symbol_coverage(self) -> float:
|
|
149
|
+
"""Share of advisories that already name their vulnerable symbols."""
|
|
150
|
+
return self.with_symbols / self.advisories if self.advisories else 0.0
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def usable_fix_references(self) -> int:
|
|
154
|
+
return sum(count for tier, count in self.tiers.items() if tier.is_usable)
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def fix_reference_coverage(self) -> float:
|
|
158
|
+
"""Share of advisories whose fix commit can be recovered.
|
|
159
|
+
|
|
160
|
+
The ceiling on what reachability analysis can ever decide for this
|
|
161
|
+
ecosystem.
|
|
162
|
+
"""
|
|
163
|
+
return self.usable_fix_references / self.advisories if self.advisories else 0.0
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def high_confidence_fix_references(self) -> int:
|
|
167
|
+
"""Advisories whose fix resolves to an exact commit."""
|
|
168
|
+
return self.tiers.get(FixReferenceTier.COMMIT, 0) + self.tiers.get(
|
|
169
|
+
FixReferenceTier.GIT_RANGE, 0
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def high_confidence_coverage(self) -> float:
|
|
174
|
+
return self.high_confidence_fix_references / self.advisories if self.advisories else 0.0
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass(frozen=True, slots=True)
|
|
178
|
+
class CoverageReport:
|
|
179
|
+
"""What a whole corpus can support."""
|
|
180
|
+
|
|
181
|
+
snapshot_id: str | None
|
|
182
|
+
advisories: int
|
|
183
|
+
ecosystems: tuple[EcosystemCoverage, ...]
|
|
184
|
+
#: Records excluded from the reachability denominators, and why.
|
|
185
|
+
withdrawn: int = 0
|
|
186
|
+
malicious: int = 0
|
|
187
|
+
|
|
188
|
+
def for_ecosystem(self, ecosystem: str) -> EcosystemCoverage | None:
|
|
189
|
+
return next((entry for entry in self.ecosystems if entry.ecosystem == ecosystem), None)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def measure(advisories: Sequence[Advisory], *, snapshot_id: str | None = None) -> CoverageReport:
|
|
193
|
+
"""Measure what a corpus can support, per ecosystem.
|
|
194
|
+
|
|
195
|
+
Withdrawn advisories are counted separately and excluded from the
|
|
196
|
+
denominators, because they never produce a finding and including them would
|
|
197
|
+
flatter every percentage.
|
|
198
|
+
"""
|
|
199
|
+
totals: dict[str, dict[str, int]] = {}
|
|
200
|
+
tiers: dict[str, dict[FixReferenceTier, int]] = {}
|
|
201
|
+
|
|
202
|
+
for advisory in advisories:
|
|
203
|
+
ecosystems = {package.ecosystem for package in advisory.affected}
|
|
204
|
+
for ecosystem in ecosystems:
|
|
205
|
+
bucket = totals.setdefault(
|
|
206
|
+
ecosystem,
|
|
207
|
+
{"advisories": 0, "withdrawn": 0, "malicious": 0, "with_symbols": 0},
|
|
208
|
+
)
|
|
209
|
+
if advisory.is_withdrawn:
|
|
210
|
+
bucket["withdrawn"] += 1
|
|
211
|
+
continue
|
|
212
|
+
if advisory.is_malicious_package:
|
|
213
|
+
# No fix exists to diff, and reachability is the wrong
|
|
214
|
+
# question: installing malware runs it.
|
|
215
|
+
bucket["malicious"] += 1
|
|
216
|
+
continue
|
|
217
|
+
bucket["advisories"] += 1
|
|
218
|
+
if advisory.symbols:
|
|
219
|
+
bucket["with_symbols"] += 1
|
|
220
|
+
per_tier = tiers.setdefault(ecosystem, {})
|
|
221
|
+
tier = classify_fix_reference(advisory)
|
|
222
|
+
per_tier[tier] = per_tier.get(tier, 0) + 1
|
|
223
|
+
|
|
224
|
+
entries = tuple(
|
|
225
|
+
EcosystemCoverage(
|
|
226
|
+
ecosystem=ecosystem,
|
|
227
|
+
advisories=counts["advisories"],
|
|
228
|
+
withdrawn=counts["withdrawn"],
|
|
229
|
+
malicious=counts["malicious"],
|
|
230
|
+
with_symbols=counts["with_symbols"],
|
|
231
|
+
tiers=dict(tiers.get(ecosystem, {})),
|
|
232
|
+
)
|
|
233
|
+
for ecosystem, counts in sorted(totals.items())
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
analysable = sum(
|
|
237
|
+
1
|
|
238
|
+
for advisory in advisories
|
|
239
|
+
if not advisory.is_withdrawn and not advisory.is_malicious_package
|
|
240
|
+
)
|
|
241
|
+
return CoverageReport(
|
|
242
|
+
snapshot_id=snapshot_id,
|
|
243
|
+
advisories=analysable,
|
|
244
|
+
ecosystems=entries,
|
|
245
|
+
withdrawn=sum(1 for advisory in advisories if advisory.is_withdrawn),
|
|
246
|
+
malicious=sum(
|
|
247
|
+
1
|
|
248
|
+
for advisory in advisories
|
|
249
|
+
if advisory.is_malicious_package and not advisory.is_withdrawn
|
|
250
|
+
),
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
__all__ = [
|
|
255
|
+
"CoverageReport",
|
|
256
|
+
"EcosystemCoverage",
|
|
257
|
+
"FixReferenceTier",
|
|
258
|
+
"classify_fix_reference",
|
|
259
|
+
"measure",
|
|
260
|
+
]
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
"""Exploitation signals: EPSS probabilities and the CISA KEV catalogue.
|
|
2
|
+
|
|
3
|
+
These change what to fix first, never whether something is reported. A
|
|
4
|
+
reachable critical with an EPSS score of 0.002 and a reachable medium that
|
|
5
|
+
appears in the Known Exploited Vulnerabilities catalogue are not the same
|
|
6
|
+
problem, and severity alone cannot tell them apart.
|
|
7
|
+
|
|
8
|
+
Both feeds are keyed by CVE identifier, which is why an advisory's aliases
|
|
9
|
+
matter: a GHSA-primary record carries its CVE as an alias, and looking up only
|
|
10
|
+
the primary identifier would miss most of the corpus.
|
|
11
|
+
|
|
12
|
+
Two properties shape the implementation.
|
|
13
|
+
|
|
14
|
+
**A missing feed is not a failed sync.** Enrichment is advisory data about
|
|
15
|
+
advisory data. If EPSS is unreachable, the corpus is still correct and still
|
|
16
|
+
usable, so a failure is reported and the corpus is written without it. The
|
|
17
|
+
alternative, failing the whole sync, would mean one third party's outage stops
|
|
18
|
+
a security tool from updating.
|
|
19
|
+
|
|
20
|
+
**A stale score is worse than none for the KEV flag.** EPSS is a probability
|
|
21
|
+
that ages gracefully; KEV membership is a statement that something is being
|
|
22
|
+
exploited right now. Both are stamped with the date they were read so a reader
|
|
23
|
+
can see how old the answer is.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import csv
|
|
29
|
+
import gzip
|
|
30
|
+
import io
|
|
31
|
+
import json
|
|
32
|
+
from collections.abc import Iterable, Mapping
|
|
33
|
+
from dataclasses import dataclass, field
|
|
34
|
+
from datetime import UTC, date, datetime
|
|
35
|
+
|
|
36
|
+
import httpx
|
|
37
|
+
|
|
38
|
+
from vexpath_core.models.advisory import Advisory
|
|
39
|
+
|
|
40
|
+
#: Daily EPSS export. Gzipped CSV with a comment line, then a header, then one
|
|
41
|
+
#: row per CVE.
|
|
42
|
+
EPSS_URL = "https://epss.cyentia.com/epss_scores-current.csv.gz"
|
|
43
|
+
|
|
44
|
+
#: The CISA Known Exploited Vulnerabilities catalogue.
|
|
45
|
+
KEV_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
|
|
46
|
+
|
|
47
|
+
#: Refuse a download larger than this. EPSS is a few megabytes compressed and
|
|
48
|
+
#: KEV is under one; an order of magnitude more means something is wrong
|
|
49
|
+
#: upstream, and parsing it would exhaust a small worker.
|
|
50
|
+
MAX_FEED_BYTES = 128 * 1024 * 1024
|
|
51
|
+
|
|
52
|
+
_DEFAULT_TIMEOUT = 60.0
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class EnrichmentError(Exception):
|
|
56
|
+
"""Raised when a feed cannot be downloaded or parsed.
|
|
57
|
+
|
|
58
|
+
Callers are expected to carry on without the signal rather than abandon a
|
|
59
|
+
corpus over it.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True, slots=True)
|
|
64
|
+
class EpssScore:
|
|
65
|
+
"""One CVE's probability of exploitation in the next 30 days."""
|
|
66
|
+
|
|
67
|
+
score: float
|
|
68
|
+
percentile: float
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(slots=True)
|
|
72
|
+
class ExploitSignals:
|
|
73
|
+
"""Everything the two feeds know, keyed by CVE identifier."""
|
|
74
|
+
|
|
75
|
+
epss: dict[str, EpssScore] = field(default_factory=dict)
|
|
76
|
+
#: CVE identifier to the date CISA added it to the catalogue.
|
|
77
|
+
kev: dict[str, date] = field(default_factory=dict)
|
|
78
|
+
#: When these were read. Stamped because KEV is a statement about now.
|
|
79
|
+
read_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
|
80
|
+
#: Feeds that could not be read, and why. Never fatal.
|
|
81
|
+
failures: tuple[str, ...] = ()
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def is_empty(self) -> bool:
|
|
85
|
+
return not self.epss and not self.kev
|
|
86
|
+
|
|
87
|
+
def summary_line(self) -> str:
|
|
88
|
+
parts = [f"{len(self.epss)} EPSS scores", f"{len(self.kev)} KEV entries"]
|
|
89
|
+
if self.failures:
|
|
90
|
+
parts.append(f"{len(self.failures)} feed(s) unavailable")
|
|
91
|
+
return ", ".join(parts)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def fetch_signals(
|
|
95
|
+
*,
|
|
96
|
+
client: httpx.Client | None = None,
|
|
97
|
+
epss_url: str = EPSS_URL,
|
|
98
|
+
kev_url: str = KEV_URL,
|
|
99
|
+
) -> ExploitSignals:
|
|
100
|
+
"""Download both feeds, tolerating the failure of either.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
client: HTTP client to use. Injected so tests run offline.
|
|
104
|
+
epss_url: Override for the EPSS export.
|
|
105
|
+
kev_url: Override for the KEV catalogue.
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
What could be read, and a note for whatever could not.
|
|
109
|
+
"""
|
|
110
|
+
http = client or httpx.Client(timeout=_DEFAULT_TIMEOUT, follow_redirects=True)
|
|
111
|
+
failures: list[str] = []
|
|
112
|
+
epss: dict[str, EpssScore] = {}
|
|
113
|
+
kev: dict[str, date] = {}
|
|
114
|
+
|
|
115
|
+
try:
|
|
116
|
+
epss = parse_epss(_download(http, epss_url))
|
|
117
|
+
except EnrichmentError as exc:
|
|
118
|
+
failures.append(f"EPSS: {exc}")
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
kev = parse_kev(_download(http, kev_url))
|
|
122
|
+
except EnrichmentError as exc:
|
|
123
|
+
failures.append(f"KEV: {exc}")
|
|
124
|
+
|
|
125
|
+
if client is None:
|
|
126
|
+
http.close()
|
|
127
|
+
|
|
128
|
+
return ExploitSignals(epss=epss, kev=kev, failures=tuple(failures))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _download(client: httpx.Client, url: str) -> bytes:
|
|
132
|
+
try:
|
|
133
|
+
response = client.get(url)
|
|
134
|
+
response.raise_for_status()
|
|
135
|
+
except httpx.HTTPError as exc:
|
|
136
|
+
msg = f"could not download {url}: {exc}"
|
|
137
|
+
raise EnrichmentError(msg) from exc
|
|
138
|
+
|
|
139
|
+
if len(response.content) > MAX_FEED_BYTES:
|
|
140
|
+
msg = f"{url} returned {len(response.content)} bytes, above the {MAX_FEED_BYTES} limit"
|
|
141
|
+
raise EnrichmentError(msg)
|
|
142
|
+
return response.content
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def parse_epss(payload: bytes) -> dict[str, EpssScore]:
|
|
146
|
+
"""Parse the EPSS export, gzipped or not.
|
|
147
|
+
|
|
148
|
+
The file begins with a comment line naming the model version and date,
|
|
149
|
+
then a header row. Rows whose score is unparseable are skipped rather than
|
|
150
|
+
failing the file: a prioritisation signal missing for one CVE is a far
|
|
151
|
+
smaller problem than no signal for any.
|
|
152
|
+
"""
|
|
153
|
+
text = _maybe_gunzip(payload).decode("utf-8", errors="replace")
|
|
154
|
+
lines = [line for line in text.splitlines() if line and not line.startswith("#")]
|
|
155
|
+
if not lines:
|
|
156
|
+
msg = "the EPSS export has no rows"
|
|
157
|
+
raise EnrichmentError(msg)
|
|
158
|
+
|
|
159
|
+
scores: dict[str, EpssScore] = {}
|
|
160
|
+
for row in csv.DictReader(lines):
|
|
161
|
+
identifier = (row.get("cve") or "").strip().upper()
|
|
162
|
+
if not identifier:
|
|
163
|
+
continue
|
|
164
|
+
try:
|
|
165
|
+
scores[identifier] = EpssScore(
|
|
166
|
+
score=float(row.get("epss") or 0.0),
|
|
167
|
+
percentile=float(row.get("percentile") or 0.0),
|
|
168
|
+
)
|
|
169
|
+
except (TypeError, ValueError):
|
|
170
|
+
continue
|
|
171
|
+
|
|
172
|
+
if not scores:
|
|
173
|
+
msg = "the EPSS export parsed to no scores, so its format has probably changed"
|
|
174
|
+
raise EnrichmentError(msg)
|
|
175
|
+
return scores
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def parse_kev(payload: bytes) -> dict[str, date]:
|
|
179
|
+
"""Parse the KEV catalogue into CVE identifier and date added."""
|
|
180
|
+
try:
|
|
181
|
+
document = json.loads(_maybe_gunzip(payload).decode("utf-8", errors="replace"))
|
|
182
|
+
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
|
183
|
+
msg = f"the KEV catalogue is not readable JSON: {exc}"
|
|
184
|
+
raise EnrichmentError(msg) from exc
|
|
185
|
+
|
|
186
|
+
if not isinstance(document, dict):
|
|
187
|
+
msg = "the KEV catalogue is not a JSON object"
|
|
188
|
+
raise EnrichmentError(msg)
|
|
189
|
+
|
|
190
|
+
entries = document.get("vulnerabilities")
|
|
191
|
+
if not isinstance(entries, list):
|
|
192
|
+
msg = "the KEV catalogue has no 'vulnerabilities' array"
|
|
193
|
+
raise EnrichmentError(msg)
|
|
194
|
+
|
|
195
|
+
found: dict[str, date] = {}
|
|
196
|
+
for entry in entries:
|
|
197
|
+
if not isinstance(entry, dict):
|
|
198
|
+
continue
|
|
199
|
+
identifier = str(entry.get("cveID") or "").strip().upper()
|
|
200
|
+
if not identifier:
|
|
201
|
+
continue
|
|
202
|
+
found[identifier] = _parse_date(entry.get("dateAdded"))
|
|
203
|
+
|
|
204
|
+
if not found:
|
|
205
|
+
msg = "the KEV catalogue parsed to no entries, so its format has probably changed"
|
|
206
|
+
raise EnrichmentError(msg)
|
|
207
|
+
return found
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _parse_date(raw: object) -> date:
|
|
211
|
+
"""Read a KEV date, falling back to today when it is unreadable.
|
|
212
|
+
|
|
213
|
+
The date is presentational. Refusing a catalogue entry over an unparseable
|
|
214
|
+
date would drop a statement that something is actively exploited, which is
|
|
215
|
+
the most important thing either feed says.
|
|
216
|
+
"""
|
|
217
|
+
try:
|
|
218
|
+
return date.fromisoformat(str(raw))
|
|
219
|
+
except (TypeError, ValueError):
|
|
220
|
+
return datetime.now(UTC).date()
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _maybe_gunzip(payload: bytes) -> bytes:
|
|
224
|
+
"""Decompress if the payload is gzipped, otherwise return it unchanged."""
|
|
225
|
+
if payload[:2] != b"\x1f\x8b":
|
|
226
|
+
return payload
|
|
227
|
+
try:
|
|
228
|
+
with gzip.GzipFile(fileobj=io.BytesIO(payload)) as handle:
|
|
229
|
+
return handle.read(MAX_FEED_BYTES + 1)
|
|
230
|
+
except OSError as exc:
|
|
231
|
+
msg = f"the feed is gzipped but could not be decompressed: {exc}"
|
|
232
|
+
raise EnrichmentError(msg) from exc
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def enrich(advisory: Advisory, signals: ExploitSignals) -> Advisory:
|
|
236
|
+
"""Attach exploitation signals to one advisory.
|
|
237
|
+
|
|
238
|
+
Looked up by every identifier the advisory carries, not just the primary
|
|
239
|
+
one. A GHSA-primary record holds its CVE as an alias, and both feeds are
|
|
240
|
+
keyed by CVE, so matching only the primary identifier would miss most of
|
|
241
|
+
the corpus.
|
|
242
|
+
|
|
243
|
+
Returns the advisory unchanged when neither feed knows it, so an unenriched
|
|
244
|
+
record is distinguishable from one scored zero. A real EPSS score of 0.0 is
|
|
245
|
+
meaningful: it says exploitation is very unlikely.
|
|
246
|
+
"""
|
|
247
|
+
update: dict[str, object] = {}
|
|
248
|
+
|
|
249
|
+
for identifier in advisory.all_ids:
|
|
250
|
+
key = identifier.upper()
|
|
251
|
+
if "epss_score" not in update and (score := signals.epss.get(key)) is not None:
|
|
252
|
+
update["epss_score"] = score.score
|
|
253
|
+
update["epss_percentile"] = score.percentile
|
|
254
|
+
if "kev" not in update and (added := signals.kev.get(key)) is not None:
|
|
255
|
+
update["kev"] = True
|
|
256
|
+
update["kev_added_at"] = added
|
|
257
|
+
|
|
258
|
+
return advisory.model_copy(update=update) if update else advisory
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def enrich_all(
|
|
262
|
+
advisories: Iterable[Advisory], signals: ExploitSignals
|
|
263
|
+
) -> tuple[tuple[Advisory, ...], EnrichmentCounts]:
|
|
264
|
+
"""Attach signals to a whole corpus, and count what landed."""
|
|
265
|
+
enriched: list[Advisory] = []
|
|
266
|
+
with_epss = 0
|
|
267
|
+
with_kev = 0
|
|
268
|
+
scorable = 0
|
|
269
|
+
for advisory in advisories:
|
|
270
|
+
result = enrich(advisory, signals)
|
|
271
|
+
enriched.append(result)
|
|
272
|
+
with_epss += 1 if result.epss_score is not None else 0
|
|
273
|
+
with_kev += 1 if result.kev else 0
|
|
274
|
+
scorable += 1 if has_cve_identifier(result) else 0
|
|
275
|
+
return tuple(enriched), EnrichmentCounts(
|
|
276
|
+
advisories=len(enriched),
|
|
277
|
+
with_epss=with_epss,
|
|
278
|
+
with_kev=with_kev,
|
|
279
|
+
scorable=scorable,
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def has_cve_identifier(advisory: Advisory) -> bool:
|
|
284
|
+
"""Whether either feed could possibly know this advisory.
|
|
285
|
+
|
|
286
|
+
Both are keyed by CVE. A malicious-package record has no CVE and never
|
|
287
|
+
will, so counting it as an enrichment miss understates coverage.
|
|
288
|
+
"""
|
|
289
|
+
return any(identifier.upper().startswith("CVE-") for identifier in advisory.all_ids)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
@dataclass(frozen=True, slots=True)
|
|
293
|
+
class EnrichmentCounts:
|
|
294
|
+
"""How much of a corpus carries each signal."""
|
|
295
|
+
|
|
296
|
+
advisories: int
|
|
297
|
+
with_epss: int
|
|
298
|
+
with_kev: int
|
|
299
|
+
#: Advisories that could have a score at all. Both feeds are keyed by CVE,
|
|
300
|
+
#: so a record with no CVE identifier anywhere is not a miss, it is out of
|
|
301
|
+
#: scope. On the real corpus that is 12,003 of 19,552 records, and dividing
|
|
302
|
+
#: by the wrong one of those turns 89 per cent coverage into 34 per cent.
|
|
303
|
+
scorable: int = 0
|
|
304
|
+
|
|
305
|
+
@property
|
|
306
|
+
def epss_coverage(self) -> float:
|
|
307
|
+
"""Share of the advisories that could be scored, and were."""
|
|
308
|
+
denominator = self.scorable or self.advisories
|
|
309
|
+
return self.with_epss / denominator if denominator else 0.0
|
|
310
|
+
|
|
311
|
+
@property
|
|
312
|
+
def out_of_scope(self) -> int:
|
|
313
|
+
return max(self.advisories - (self.scorable or self.advisories), 0)
|
|
314
|
+
|
|
315
|
+
def summary_line(self) -> str:
|
|
316
|
+
line = (
|
|
317
|
+
f"{self.with_epss} of {self.scorable or self.advisories} advisories "
|
|
318
|
+
f"with a CVE identifier have an EPSS score ({self.epss_coverage:.1%}), "
|
|
319
|
+
f"{self.with_kev} are known exploited"
|
|
320
|
+
)
|
|
321
|
+
if self.out_of_scope:
|
|
322
|
+
line += (
|
|
323
|
+
f". {self.out_of_scope} record(s) carry no CVE identifier, so no "
|
|
324
|
+
f"feed can score them"
|
|
325
|
+
)
|
|
326
|
+
return line
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def highest_priority(advisories: Iterable[Advisory], *, limit: int = 10) -> tuple[Advisory, ...]:
|
|
330
|
+
"""The advisories to look at first, worst plausible outcome soonest.
|
|
331
|
+
|
|
332
|
+
Known exploited comes before everything: it is not a prediction. After
|
|
333
|
+
that, EPSS descending, then severity, so a medium being actively exploited
|
|
334
|
+
outranks a critical that nobody has worked out how to use.
|
|
335
|
+
"""
|
|
336
|
+
ordered = sorted(
|
|
337
|
+
advisories,
|
|
338
|
+
key=lambda advisory: (
|
|
339
|
+
not advisory.kev,
|
|
340
|
+
-(advisory.epss_score or 0.0),
|
|
341
|
+
-advisory.severity.rank,
|
|
342
|
+
advisory.id,
|
|
343
|
+
),
|
|
344
|
+
)
|
|
345
|
+
return tuple(ordered[:limit])
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def signals_from_mapping(
|
|
349
|
+
epss: Mapping[str, tuple[float, float]], kev: Mapping[str, date]
|
|
350
|
+
) -> ExploitSignals:
|
|
351
|
+
"""Build signals from plain data, for tests and for a cached feed."""
|
|
352
|
+
return ExploitSignals(
|
|
353
|
+
epss={
|
|
354
|
+
identifier.upper(): EpssScore(score=score, percentile=percentile)
|
|
355
|
+
for identifier, (score, percentile) in epss.items()
|
|
356
|
+
},
|
|
357
|
+
kev={identifier.upper(): added for identifier, added in kev.items()},
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
__all__ = [
|
|
362
|
+
"EPSS_URL",
|
|
363
|
+
"KEV_URL",
|
|
364
|
+
"MAX_FEED_BYTES",
|
|
365
|
+
"EnrichmentCounts",
|
|
366
|
+
"EnrichmentError",
|
|
367
|
+
"EpssScore",
|
|
368
|
+
"ExploitSignals",
|
|
369
|
+
"enrich",
|
|
370
|
+
"enrich_all",
|
|
371
|
+
"fetch_signals",
|
|
372
|
+
"has_cve_identifier",
|
|
373
|
+
"highest_priority",
|
|
374
|
+
"parse_epss",
|
|
375
|
+
"parse_kev",
|
|
376
|
+
"signals_from_mapping",
|
|
377
|
+
]
|