vulnctl 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vulnctl/__init__.py +5 -0
- vulnctl/adapters/__init__.py +5 -0
- vulnctl/adapters/base.py +150 -0
- vulnctl/adapters/epss.py +158 -0
- vulnctl/adapters/exploits.py +83 -0
- vulnctl/adapters/ghsa.py +220 -0
- vulnctl/adapters/kev.py +139 -0
- vulnctl/adapters/nvd.py +218 -0
- vulnctl/adapters/osv.py +437 -0
- vulnctl/cache.py +133 -0
- vulnctl/cli.py +160 -0
- vulnctl/context.py +140 -0
- vulnctl/data/__init__.py +9 -0
- vulnctl/data/epss_snapshot.csv.gz +0 -0
- vulnctl/data/exploit_index.json.gz +0 -0
- vulnctl/data/kev_snapshot.json.gz +0 -0
- vulnctl/ingest/__init__.py +5 -0
- vulnctl/ingest/cve_list.py +33 -0
- vulnctl/ingest/cyclonedx.py +135 -0
- vulnctl/ingest/grype.py +166 -0
- vulnctl/models.py +263 -0
- vulnctl/output/__init__.py +32 -0
- vulnctl/output/json_out.py +52 -0
- vulnctl/output/markdown.py +189 -0
- vulnctl/output/render.py +59 -0
- vulnctl/output/sarif.py +174 -0
- vulnctl/output/table.py +167 -0
- vulnctl/pipeline.py +334 -0
- vulnctl/py.typed +0 -0
- vulnctl/ssvc/__init__.py +1 -0
- vulnctl/ssvc/engine.py +100 -0
- vulnctl/ssvc/tree.py +341 -0
- vulnctl/ssvc/trees/cisa-deployer-v1.yaml +164 -0
- vulnctl-0.1.0.dist-info/METADATA +175 -0
- vulnctl-0.1.0.dist-info/RECORD +38 -0
- vulnctl-0.1.0.dist-info/WHEEL +4 -0
- vulnctl-0.1.0.dist-info/entry_points.txt +2 -0
- vulnctl-0.1.0.dist-info/licenses/LICENSE +202 -0
vulnctl/__init__.py
ADDED
vulnctl/adapters/base.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Source-adapter contract, registry, and shared fetch helpers (FRAMEWORK.md §3.2).
|
|
2
|
+
|
|
3
|
+
Isolation rule (CLAUDE.md architecture rule 1): adapter modules may import
|
|
4
|
+
only from here, ``vulnctl.cache``, and ``vulnctl.models`` — never each other,
|
|
5
|
+
never the SSVC engine.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import time
|
|
12
|
+
from abc import ABC, abstractmethod
|
|
13
|
+
from collections import deque
|
|
14
|
+
from collections.abc import Awaitable, Iterable
|
|
15
|
+
from datetime import timedelta
|
|
16
|
+
from typing import ClassVar, TypeVar
|
|
17
|
+
|
|
18
|
+
import httpx
|
|
19
|
+
from pydantic import BaseModel, ConfigDict
|
|
20
|
+
|
|
21
|
+
from vulnctl.cache import Cache
|
|
22
|
+
from vulnctl.models import (
|
|
23
|
+
EpssData,
|
|
24
|
+
ExploitData,
|
|
25
|
+
GhsaData,
|
|
26
|
+
KevData,
|
|
27
|
+
NvdData,
|
|
28
|
+
SourceMeta,
|
|
29
|
+
Unavailable,
|
|
30
|
+
VersionData,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
T = TypeVar("T")
|
|
34
|
+
|
|
35
|
+
#: Passed as TTL when running --offline: any cached row beats no data at all.
|
|
36
|
+
OFFLINE_TTL = timedelta(days=36500)
|
|
37
|
+
|
|
38
|
+
#: Refuse to parse response bodies larger than this — a compromised or
|
|
39
|
+
#: misbehaving feed must not be able to exhaust memory during JSON parsing.
|
|
40
|
+
#: (Largest legitimate body today is the ~3 MiB KEV catalog.)
|
|
41
|
+
MAX_RESPONSE_BYTES = 32 * 1024 * 1024
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def body_too_large(response: httpx.Response) -> bool:
|
|
45
|
+
"""True if the response body exceeds :data:`MAX_RESPONSE_BYTES`."""
|
|
46
|
+
return len(response.content) > MAX_RESPONSE_BYTES
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
AdapterData = EpssData | ExploitData | GhsaData | KevData | NvdData | VersionData
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class SourceResult(BaseModel):
|
|
53
|
+
"""One adapter's answer for one CVE: the payload (or why it's missing) plus provenance."""
|
|
54
|
+
|
|
55
|
+
model_config = ConfigDict(strict=True, frozen=True, extra="forbid")
|
|
56
|
+
|
|
57
|
+
data: AdapterData | Unavailable
|
|
58
|
+
meta: SourceMeta
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SourceAdapter(ABC):
|
|
62
|
+
"""Base class for intel-source adapters.
|
|
63
|
+
|
|
64
|
+
Lifecycle per FRAMEWORK.md §3.2: check cache → fetch misses (bounded
|
|
65
|
+
concurrency, per-source rate limit) → validate strictly → write cache.
|
|
66
|
+
The ``httpx.AsyncClient`` is injected, never created per call.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
name: ClassVar[str]
|
|
70
|
+
ttl: ClassVar[timedelta]
|
|
71
|
+
supports_offline: ClassVar[bool]
|
|
72
|
+
|
|
73
|
+
def __init__(self, client: httpx.AsyncClient, cache: Cache, *, offline: bool = False) -> None:
|
|
74
|
+
self._client = client
|
|
75
|
+
self._cache = cache
|
|
76
|
+
self._offline = offline
|
|
77
|
+
|
|
78
|
+
@abstractmethod
|
|
79
|
+
async def fetch(self, cve_ids: list[str]) -> dict[str, SourceResult]:
|
|
80
|
+
"""Return a :class:`SourceResult` for every requested CVE ID."""
|
|
81
|
+
|
|
82
|
+
def _cache_ttl(self) -> timedelta:
|
|
83
|
+
return OFFLINE_TTL if self._offline else self.ttl
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
_REGISTRY: dict[str, type[SourceAdapter]] = {}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def register(cls: type[SourceAdapter]) -> type[SourceAdapter]:
|
|
90
|
+
"""Class decorator adding an adapter to the registry (keyed by ``name``)."""
|
|
91
|
+
if cls.name in _REGISTRY:
|
|
92
|
+
raise ValueError(f"duplicate adapter name {cls.name!r}")
|
|
93
|
+
_REGISTRY[cls.name] = cls
|
|
94
|
+
return cls
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def get_adapter(name: str) -> type[SourceAdapter]:
|
|
98
|
+
"""Look up a registered adapter class by name."""
|
|
99
|
+
try:
|
|
100
|
+
return _REGISTRY[name]
|
|
101
|
+
except KeyError:
|
|
102
|
+
raise KeyError(f"no adapter registered under {name!r}") from None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def all_adapters() -> list[type[SourceAdapter]]:
|
|
106
|
+
"""All registered adapter classes, in registration order."""
|
|
107
|
+
return list(_REGISTRY.values())
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class RateLimit(BaseModel):
|
|
111
|
+
"""At most ``requests`` calls per sliding ``window_seconds`` window."""
|
|
112
|
+
|
|
113
|
+
model_config = ConfigDict(strict=True, frozen=True, extra="forbid")
|
|
114
|
+
|
|
115
|
+
requests: int
|
|
116
|
+
window_seconds: float
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class RateLimiter:
|
|
120
|
+
"""Async sliding-window rate limiter shared by an adapter's in-flight requests."""
|
|
121
|
+
|
|
122
|
+
def __init__(self, limit: RateLimit) -> None:
|
|
123
|
+
self._limit = limit
|
|
124
|
+
self._timestamps: deque[float] = deque()
|
|
125
|
+
self._lock = asyncio.Lock()
|
|
126
|
+
|
|
127
|
+
async def acquire(self) -> None:
|
|
128
|
+
"""Block until a request slot is available inside the window."""
|
|
129
|
+
while True:
|
|
130
|
+
async with self._lock:
|
|
131
|
+
now = time.monotonic()
|
|
132
|
+
window = self._limit.window_seconds
|
|
133
|
+
while self._timestamps and now - self._timestamps[0] >= window:
|
|
134
|
+
self._timestamps.popleft()
|
|
135
|
+
if len(self._timestamps) < self._limit.requests:
|
|
136
|
+
self._timestamps.append(now)
|
|
137
|
+
return
|
|
138
|
+
wait = window - (now - self._timestamps[0])
|
|
139
|
+
await asyncio.sleep(wait)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
async def bounded_gather(coros: Iterable[Awaitable[T]], *, limit: int) -> list[T]:
|
|
143
|
+
"""``asyncio.gather`` capped at ``limit`` concurrent awaitables, order preserved."""
|
|
144
|
+
semaphore = asyncio.Semaphore(limit)
|
|
145
|
+
|
|
146
|
+
async def _run(coro: Awaitable[T]) -> T:
|
|
147
|
+
async with semaphore:
|
|
148
|
+
return await coro
|
|
149
|
+
|
|
150
|
+
return await asyncio.gather(*(_run(c) for c in coros))
|
vulnctl/adapters/epss.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""FIRST EPSS adapter: exploitation probability + percentile per CVE.
|
|
2
|
+
|
|
3
|
+
API: https://api.first.org/data/v1/epss (batched via comma-separated ``cve``
|
|
4
|
+
param; numeric fields arrive as *strings* and are converted explicitly before
|
|
5
|
+
strict model construction). TTL 24h.
|
|
6
|
+
|
|
7
|
+
Offline: ``vulnctl/data/epss_snapshot.csv.gz`` — a dated capture of ~50
|
|
8
|
+
well-known CVEs in ``cve,epss,percentile,date`` form (header comment records
|
|
9
|
+
the snapshot date). A malformed response row degrades that CVE to
|
|
10
|
+
``Unavailable(not_found)`` with detail; it never crashes the run.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import csv
|
|
16
|
+
import gzip
|
|
17
|
+
from datetime import UTC, date, datetime, time, timedelta
|
|
18
|
+
from functools import cache as memoize
|
|
19
|
+
from importlib import resources
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
import httpx
|
|
23
|
+
from pydantic import ValidationError
|
|
24
|
+
|
|
25
|
+
from vulnctl.adapters.base import SourceAdapter, SourceResult, body_too_large, register
|
|
26
|
+
from vulnctl.models import EpssData, SourceMeta, Unavailable, UnavailableReason
|
|
27
|
+
|
|
28
|
+
API_URL = "https://api.first.org/data/v1/epss"
|
|
29
|
+
_BATCH_SIZE = 100 # the API caps rows per response at its `limit` param
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@memoize
|
|
33
|
+
def _load_snapshot() -> dict[str, EpssData]:
|
|
34
|
+
path = resources.files("vulnctl.data").joinpath("epss_snapshot.csv.gz")
|
|
35
|
+
with path.open("rb") as f:
|
|
36
|
+
text = gzip.decompress(f.read()).decode("utf-8")
|
|
37
|
+
lines = [line for line in text.splitlines() if not line.startswith("#")]
|
|
38
|
+
snapshot: dict[str, EpssData] = {}
|
|
39
|
+
for row in csv.DictReader(lines):
|
|
40
|
+
parsed = _parse_row(row)
|
|
41
|
+
if parsed is not None:
|
|
42
|
+
snapshot[parsed[0]] = parsed[1]
|
|
43
|
+
return snapshot
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _parse_row(row: Any) -> tuple[str, EpssData] | None:
|
|
47
|
+
"""Convert one API/CSV row into (cve_id, EpssData); None if malformed."""
|
|
48
|
+
if not isinstance(row, dict):
|
|
49
|
+
return None
|
|
50
|
+
try:
|
|
51
|
+
cve_id = str(row["cve"]).upper()
|
|
52
|
+
data = EpssData(
|
|
53
|
+
score=float(row["epss"]),
|
|
54
|
+
percentile=float(row["percentile"]),
|
|
55
|
+
date=date.fromisoformat(str(row["date"])),
|
|
56
|
+
)
|
|
57
|
+
except (KeyError, TypeError, ValueError, ValidationError):
|
|
58
|
+
return None
|
|
59
|
+
return cve_id, data
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@register
|
|
63
|
+
class EpssAdapter(SourceAdapter):
|
|
64
|
+
"""Cache-through EPSS lookups, batched ``_BATCH_SIZE`` CVEs per request."""
|
|
65
|
+
|
|
66
|
+
name = "epss"
|
|
67
|
+
ttl = timedelta(hours=24)
|
|
68
|
+
supports_offline = True
|
|
69
|
+
|
|
70
|
+
async def fetch(self, cve_ids: list[str]) -> dict[str, SourceResult]:
|
|
71
|
+
results: dict[str, SourceResult] = {}
|
|
72
|
+
misses: list[str] = []
|
|
73
|
+
for cve_id in cve_ids:
|
|
74
|
+
cached = self._cached_result(cve_id)
|
|
75
|
+
if cached is not None:
|
|
76
|
+
results[cve_id] = cached
|
|
77
|
+
else:
|
|
78
|
+
misses.append(cve_id)
|
|
79
|
+
|
|
80
|
+
if self._offline:
|
|
81
|
+
results.update(self._from_snapshot(misses))
|
|
82
|
+
else:
|
|
83
|
+
for start in range(0, len(misses), _BATCH_SIZE):
|
|
84
|
+
batch = misses[start : start + _BATCH_SIZE]
|
|
85
|
+
results.update(await self._fetch_batch(batch))
|
|
86
|
+
return results
|
|
87
|
+
|
|
88
|
+
def _cached_result(self, cve_id: str) -> SourceResult | None:
|
|
89
|
+
entry = self._cache.get_entry(self.name, cve_id, self._cache_ttl())
|
|
90
|
+
if entry is None:
|
|
91
|
+
return None
|
|
92
|
+
try:
|
|
93
|
+
data = EpssData.model_validate_json(entry.payload)
|
|
94
|
+
except ValidationError:
|
|
95
|
+
return None # cache row written by an incompatible version: refetch
|
|
96
|
+
return SourceResult(data=data, meta=self._meta(entry.fetched_at, cache_hit=True))
|
|
97
|
+
|
|
98
|
+
def _from_snapshot(self, cve_ids: list[str]) -> dict[str, SourceResult]:
|
|
99
|
+
snapshot = _load_snapshot()
|
|
100
|
+
results: dict[str, SourceResult] = {}
|
|
101
|
+
for cve_id in cve_ids:
|
|
102
|
+
data = snapshot.get(cve_id)
|
|
103
|
+
if data is not None:
|
|
104
|
+
# Snapshot rows carry their own score date; use it as fetched_at.
|
|
105
|
+
fetched_at = datetime.combine(data.date, time.min, tzinfo=UTC)
|
|
106
|
+
results[cve_id] = SourceResult(
|
|
107
|
+
data=data, meta=self._meta(fetched_at, cache_hit=False)
|
|
108
|
+
)
|
|
109
|
+
else:
|
|
110
|
+
results[cve_id] = self._unavailable(
|
|
111
|
+
UnavailableReason.OFFLINE, "not in cache or bundled EPSS snapshot"
|
|
112
|
+
)
|
|
113
|
+
return results
|
|
114
|
+
|
|
115
|
+
async def _fetch_batch(self, batch: list[str]) -> dict[str, SourceResult]:
|
|
116
|
+
try:
|
|
117
|
+
response = await self._client.get(
|
|
118
|
+
API_URL, params={"cve": ",".join(batch), "limit": str(len(batch))}
|
|
119
|
+
)
|
|
120
|
+
response.raise_for_status()
|
|
121
|
+
if body_too_large(response):
|
|
122
|
+
failure = self._unavailable(
|
|
123
|
+
UnavailableReason.SOURCE_DOWN, "response exceeds size limit"
|
|
124
|
+
)
|
|
125
|
+
return dict.fromkeys(batch, failure)
|
|
126
|
+
payload = response.json()
|
|
127
|
+
except (httpx.HTTPError, ValueError) as exc:
|
|
128
|
+
failure = self._unavailable(UnavailableReason.SOURCE_DOWN, str(exc))
|
|
129
|
+
return dict.fromkeys(batch, failure)
|
|
130
|
+
|
|
131
|
+
results: dict[str, SourceResult] = {}
|
|
132
|
+
rows = payload.get("data") if isinstance(payload, dict) else None
|
|
133
|
+
for row in rows if isinstance(rows, list) else []:
|
|
134
|
+
parsed = _parse_row(row)
|
|
135
|
+
if parsed is None:
|
|
136
|
+
continue # malformed row: the CVE degrades to not_found below
|
|
137
|
+
cve_id, data = parsed
|
|
138
|
+
if cve_id not in batch:
|
|
139
|
+
continue
|
|
140
|
+
self._cache.set(self.name, cve_id, data.model_dump_json())
|
|
141
|
+
results[cve_id] = SourceResult(
|
|
142
|
+
data=data, meta=self._meta(datetime.now(UTC), cache_hit=False)
|
|
143
|
+
)
|
|
144
|
+
for cve_id in batch:
|
|
145
|
+
if cve_id not in results:
|
|
146
|
+
results[cve_id] = self._unavailable(
|
|
147
|
+
UnavailableReason.NOT_FOUND, "no usable EPSS row for this CVE"
|
|
148
|
+
)
|
|
149
|
+
return results
|
|
150
|
+
|
|
151
|
+
def _meta(self, fetched_at: datetime, *, cache_hit: bool) -> SourceMeta:
|
|
152
|
+
return SourceMeta(source=self.name, fetched_at=fetched_at, cache_hit=cache_hit)
|
|
153
|
+
|
|
154
|
+
def _unavailable(self, reason: UnavailableReason, detail: str) -> SourceResult:
|
|
155
|
+
return SourceResult(
|
|
156
|
+
data=Unavailable(reason=reason, detail=detail),
|
|
157
|
+
meta=self._meta(datetime.now(UTC), cache_hit=False),
|
|
158
|
+
)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Exploit-presence adapter: public exploit indicators per CVE.
|
|
2
|
+
|
|
3
|
+
Snapshot-only for v0.1. Exploit-DB, Metasploit, and nuclei publish bulk
|
|
4
|
+
indexes, not per-CVE query APIs, so the adapter answers from the bundled
|
|
5
|
+
``vulnctl/data/exploit_index.json.gz`` (built by
|
|
6
|
+
``scripts/build_exploit_index.py``) in both online and offline modes — a live
|
|
7
|
+
refresh is v0.2 work. ``supports_offline`` is therefore ``True`` and no
|
|
8
|
+
network call is ever made.
|
|
9
|
+
|
|
10
|
+
Matching is exact CVE ID only (CLAUDE.md / ROADMAP #19): fuzzy title matching
|
|
11
|
+
would produce false positives, and a false "exploit present" flips the
|
|
12
|
+
``exploitation`` resolver from ``none`` to ``poc``, poisoning the verdict.
|
|
13
|
+
|
|
14
|
+
A CVE absent from the index returns an *empty* ``ExploitData()`` — a real
|
|
15
|
+
"no known public exploit as of the snapshot date" answer, not
|
|
16
|
+
``Unavailable``. That lets the resolver conclude ``exploitation = none`` (when
|
|
17
|
+
KEV also has it unlisted) instead of degrading to a default. The trade-off is
|
|
18
|
+
snapshot staleness: a brand-new exploit is invisible until the index is
|
|
19
|
+
rebuilt.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import gzip
|
|
25
|
+
import json
|
|
26
|
+
from datetime import UTC, date, datetime, time, timedelta
|
|
27
|
+
from functools import cache as memoize
|
|
28
|
+
from importlib import resources
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
from vulnctl.adapters.base import SourceAdapter, SourceResult, register
|
|
32
|
+
from vulnctl.models import ExploitData, SourceMeta
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@memoize
|
|
36
|
+
def _load_index() -> tuple[date, dict[str, ExploitData]]:
|
|
37
|
+
"""Return (snapshot date, CVE→ExploitData) from the bundled gzip. Memoized."""
|
|
38
|
+
path = resources.files("vulnctl.data").joinpath("exploit_index.json.gz")
|
|
39
|
+
with path.open("rb") as f:
|
|
40
|
+
payload = json.loads(gzip.decompress(f.read()).decode("utf-8"))
|
|
41
|
+
generated = _parse_date(payload.get("generated"))
|
|
42
|
+
index: dict[str, ExploitData] = {}
|
|
43
|
+
raw = payload.get("cves")
|
|
44
|
+
for cve_id, entry in raw.items() if isinstance(raw, dict) else []:
|
|
45
|
+
if isinstance(entry, dict):
|
|
46
|
+
index[cve_id.upper()] = ExploitData(
|
|
47
|
+
edb_ids=_str_list(entry.get("edb_ids")),
|
|
48
|
+
msf_modules=_str_list(entry.get("msf_modules")),
|
|
49
|
+
nuclei_templates=_str_list(entry.get("nuclei_templates")),
|
|
50
|
+
)
|
|
51
|
+
return generated, index
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _parse_date(value: Any) -> date:
|
|
55
|
+
if isinstance(value, str):
|
|
56
|
+
try:
|
|
57
|
+
return date.fromisoformat(value)
|
|
58
|
+
except ValueError:
|
|
59
|
+
pass
|
|
60
|
+
return datetime.now(UTC).date()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _str_list(value: Any) -> list[str]:
|
|
64
|
+
return [v for v in value if isinstance(v, str)] if isinstance(value, list) else []
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@register
|
|
68
|
+
class ExploitsAdapter(SourceAdapter):
|
|
69
|
+
"""Exact-match exploit-presence lookups from the bundled index."""
|
|
70
|
+
|
|
71
|
+
name = "exploits"
|
|
72
|
+
ttl = timedelta(days=1) # unused: snapshot-only, no cache/network for v0.1
|
|
73
|
+
supports_offline = True
|
|
74
|
+
|
|
75
|
+
async def fetch(self, cve_ids: list[str]) -> dict[str, SourceResult]:
|
|
76
|
+
generated, index = _load_index()
|
|
77
|
+
fetched_at = datetime.combine(generated, time.min, tzinfo=UTC)
|
|
78
|
+
meta = SourceMeta(source=self.name, fetched_at=fetched_at, cache_hit=False)
|
|
79
|
+
# An absent CVE is a real empty answer, never Unavailable (see module docstring).
|
|
80
|
+
return {
|
|
81
|
+
cve_id: SourceResult(data=index.get(cve_id.upper(), ExploitData()), meta=meta)
|
|
82
|
+
for cve_id in cve_ids
|
|
83
|
+
}
|
vulnctl/adapters/ghsa.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""GitHub Security Advisories adapter (REST Global Advisories API).
|
|
2
|
+
|
|
3
|
+
REST rather than GraphQL: the GraphQL endpoint rejects anonymous requests
|
|
4
|
+
outright, while ``GET /advisories`` works unauthenticated (60 req/h) — and
|
|
5
|
+
SPEC.md NFR-2 requires zero mandatory credentials. ``VULNCTL_GITHUB_TOKEN``
|
|
6
|
+
(env only, CLAUDE.md rule 6) raises the limit to 5,000 req/h. There is no
|
|
7
|
+
client-side rate limiter — pacing 60 req/h would stall runs; instead a
|
|
8
|
+
403/429 answer degrades that CVE to ``Unavailable(rate_limited)`` with no
|
|
9
|
+
retries, and the 24h cache keeps repeat runs cheap.
|
|
10
|
+
|
|
11
|
+
Lookup by ID kind: CVE IDs via ``?cve_id=`` (a JSON *list*; empty →
|
|
12
|
+
``not_found``; the first GitHub-reviewed entry is used — the API orders by
|
|
13
|
+
relevance); GHSA IDs via ``/advisories/{ghsa_id}`` (404 → ``not_found``).
|
|
14
|
+
Any other ID kind (PYSEC-…, GO-…) is ``not_found`` without a request —
|
|
15
|
+
GitHub cannot answer for those.
|
|
16
|
+
|
|
17
|
+
Version ranges are normalized to the exact format the OSV adapter emits
|
|
18
|
+
(purl-style label + ``<4.17.21``-style ranges) so the pipeline's keep-both
|
|
19
|
+
merge can compare the two sources without false conflicts.
|
|
20
|
+
|
|
21
|
+
No bundled snapshot: offline mode answers from cache only
|
|
22
|
+
(``supports_offline = False``); misses degrade to ``Unavailable(offline)``.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
from datetime import UTC, datetime, timedelta
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
import httpx
|
|
33
|
+
from pydantic import ValidationError
|
|
34
|
+
|
|
35
|
+
from vulnctl.adapters.base import (
|
|
36
|
+
SourceAdapter,
|
|
37
|
+
SourceResult,
|
|
38
|
+
body_too_large,
|
|
39
|
+
bounded_gather,
|
|
40
|
+
register,
|
|
41
|
+
)
|
|
42
|
+
from vulnctl.cache import Cache
|
|
43
|
+
from vulnctl.models import GhsaData, SourceMeta, Unavailable, UnavailableReason, VersionData
|
|
44
|
+
|
|
45
|
+
API_URL = "https://api.github.com/advisories"
|
|
46
|
+
TOKEN_ENV = "VULNCTL_GITHUB_TOKEN"
|
|
47
|
+
|
|
48
|
+
_CVE_RE = re.compile(r"CVE-\d{4}-\d{4,}", re.IGNORECASE)
|
|
49
|
+
_GHSA_RE = re.compile(r"GHSA(-[23456789cfghjmpqrvwx]{4}){3}", re.IGNORECASE)
|
|
50
|
+
_CONCURRENCY = 8
|
|
51
|
+
|
|
52
|
+
#: GHSA ecosystem names → purl types, matching the labels OSV records carry.
|
|
53
|
+
_PURL_TYPES = {
|
|
54
|
+
"npm": "npm",
|
|
55
|
+
"pip": "pypi",
|
|
56
|
+
"rubygems": "gem",
|
|
57
|
+
"maven": "maven",
|
|
58
|
+
"nuget": "nuget",
|
|
59
|
+
"composer": "composer",
|
|
60
|
+
"go": "golang",
|
|
61
|
+
"rust": "cargo",
|
|
62
|
+
"erlang": "hex",
|
|
63
|
+
"pub": "pub",
|
|
64
|
+
"swift": "swift",
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _purl_label(package: Any) -> str | None:
|
|
69
|
+
if not isinstance(package, dict):
|
|
70
|
+
return None
|
|
71
|
+
name = package.get("name")
|
|
72
|
+
if not isinstance(name, str) or not name:
|
|
73
|
+
return None
|
|
74
|
+
ecosystem = str(package.get("ecosystem") or "generic").lower()
|
|
75
|
+
purl_type = _PURL_TYPES.get(ecosystem, ecosystem)
|
|
76
|
+
if purl_type == "maven":
|
|
77
|
+
name = name.replace(":", "/") # group:artifact → group/artifact purl form
|
|
78
|
+
return f"pkg:{purl_type}/{name}"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _normalize_range(range_text: str) -> str:
|
|
82
|
+
"""``">= 3.0.0, < 3.9.2"`` → ``">=3.0.0 <3.9.2"`` (the OSV adapter's format)."""
|
|
83
|
+
clauses = [clause.strip().replace(" ", "") for clause in range_text.split(",")]
|
|
84
|
+
return " ".join(clause for clause in clauses if clause)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _parse_versions(vulnerabilities: Any) -> VersionData:
|
|
88
|
+
affected: list[str] = []
|
|
89
|
+
fixed: list[str] = []
|
|
90
|
+
for entry in vulnerabilities if isinstance(vulnerabilities, list) else []:
|
|
91
|
+
if not isinstance(entry, dict):
|
|
92
|
+
continue
|
|
93
|
+
label = _purl_label(entry.get("package"))
|
|
94
|
+
range_text = entry.get("vulnerable_version_range")
|
|
95
|
+
if isinstance(range_text, str) and range_text:
|
|
96
|
+
normalized = _normalize_range(range_text)
|
|
97
|
+
affected.append(f"{label} {normalized}" if label else normalized)
|
|
98
|
+
patched = entry.get("first_patched_version")
|
|
99
|
+
if isinstance(patched, str) and patched:
|
|
100
|
+
fixed.append(f"{label} {patched}" if label else patched)
|
|
101
|
+
return VersionData(affected=list(dict.fromkeys(affected)), fixed=list(dict.fromkeys(fixed)))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _parse_advisory(advisory: Any) -> GhsaData | None:
|
|
105
|
+
"""One advisory object → GhsaData; None if the shape is unusable."""
|
|
106
|
+
if not isinstance(advisory, dict) or not isinstance(advisory.get("ghsa_id"), str):
|
|
107
|
+
return None
|
|
108
|
+
severity = advisory.get("severity")
|
|
109
|
+
summary = advisory.get("summary")
|
|
110
|
+
return GhsaData(
|
|
111
|
+
ghsa_id=advisory["ghsa_id"],
|
|
112
|
+
severity=severity if isinstance(severity, str) and severity else "unknown",
|
|
113
|
+
summary=summary if isinstance(summary, str) else "",
|
|
114
|
+
versions=_parse_versions(advisory.get("vulnerabilities")),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _pick_advisory(payload: list[Any]) -> Any:
|
|
119
|
+
"""Prefer the first GitHub-reviewed advisory; fall back to the first entry."""
|
|
120
|
+
for entry in payload:
|
|
121
|
+
if isinstance(entry, dict) and entry.get("github_reviewed_at") is not None:
|
|
122
|
+
return entry
|
|
123
|
+
return payload[0]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@register
|
|
127
|
+
class GhsaAdapter(SourceAdapter):
|
|
128
|
+
"""Cache-through GitHub advisory lookups, anonymous or token-keyed."""
|
|
129
|
+
|
|
130
|
+
name = "ghsa"
|
|
131
|
+
ttl = timedelta(hours=24)
|
|
132
|
+
supports_offline = False
|
|
133
|
+
|
|
134
|
+
def __init__(self, client: httpx.AsyncClient, cache: Cache, *, offline: bool = False) -> None:
|
|
135
|
+
super().__init__(client, cache, offline=offline)
|
|
136
|
+
token = os.environ.get(TOKEN_ENV)
|
|
137
|
+
self._headers = {
|
|
138
|
+
"Accept": "application/vnd.github+json",
|
|
139
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
140
|
+
}
|
|
141
|
+
if token:
|
|
142
|
+
self._headers["Authorization"] = f"Bearer {token}"
|
|
143
|
+
|
|
144
|
+
async def fetch(self, cve_ids: list[str]) -> dict[str, SourceResult]:
|
|
145
|
+
results: dict[str, SourceResult] = {}
|
|
146
|
+
misses: list[str] = []
|
|
147
|
+
for cve_id in cve_ids:
|
|
148
|
+
if not (_CVE_RE.fullmatch(cve_id) or _GHSA_RE.fullmatch(cve_id)):
|
|
149
|
+
results[cve_id] = self._unavailable(
|
|
150
|
+
UnavailableReason.NOT_FOUND, "GitHub answers only CVE or GHSA IDs"
|
|
151
|
+
)
|
|
152
|
+
continue
|
|
153
|
+
cached = self._cached_result(cve_id)
|
|
154
|
+
if cached is not None:
|
|
155
|
+
results[cve_id] = cached
|
|
156
|
+
else:
|
|
157
|
+
misses.append(cve_id)
|
|
158
|
+
|
|
159
|
+
if self._offline:
|
|
160
|
+
for cve_id in misses:
|
|
161
|
+
results[cve_id] = self._unavailable(
|
|
162
|
+
UnavailableReason.OFFLINE, "GHSA has no bundled snapshot; cache miss"
|
|
163
|
+
)
|
|
164
|
+
elif misses:
|
|
165
|
+
fetched = await bounded_gather(
|
|
166
|
+
(self._fetch_one(cve_id) for cve_id in misses), limit=_CONCURRENCY
|
|
167
|
+
)
|
|
168
|
+
results.update(zip(misses, fetched, strict=True))
|
|
169
|
+
return results
|
|
170
|
+
|
|
171
|
+
def _cached_result(self, cve_id: str) -> SourceResult | None:
|
|
172
|
+
entry = self._cache.get_entry(self.name, cve_id, self._cache_ttl())
|
|
173
|
+
if entry is None:
|
|
174
|
+
return None
|
|
175
|
+
try:
|
|
176
|
+
data = GhsaData.model_validate_json(entry.payload)
|
|
177
|
+
except ValidationError:
|
|
178
|
+
return None # cache row written by an incompatible version: refetch
|
|
179
|
+
return SourceResult(data=data, meta=self._meta(entry.fetched_at, cache_hit=True))
|
|
180
|
+
|
|
181
|
+
async def _fetch_one(self, cve_id: str) -> SourceResult:
|
|
182
|
+
if _CVE_RE.fullmatch(cve_id):
|
|
183
|
+
url, params = API_URL, {"cve_id": cve_id}
|
|
184
|
+
else:
|
|
185
|
+
url, params = f"{API_URL}/{cve_id}", {}
|
|
186
|
+
try:
|
|
187
|
+
response = await self._client.get(url, params=params, headers=self._headers)
|
|
188
|
+
except httpx.HTTPError as exc:
|
|
189
|
+
return self._unavailable(UnavailableReason.SOURCE_DOWN, str(exc))
|
|
190
|
+
if response.status_code == 404:
|
|
191
|
+
return self._unavailable(UnavailableReason.NOT_FOUND, "no GitHub advisory")
|
|
192
|
+
if response.status_code in (403, 429):
|
|
193
|
+
return self._unavailable(UnavailableReason.RATE_LIMITED, f"HTTP {response.status_code}")
|
|
194
|
+
if response.status_code != 200:
|
|
195
|
+
return self._unavailable(UnavailableReason.SOURCE_DOWN, f"HTTP {response.status_code}")
|
|
196
|
+
if body_too_large(response):
|
|
197
|
+
return self._unavailable(UnavailableReason.SOURCE_DOWN, "response exceeds size limit")
|
|
198
|
+
try:
|
|
199
|
+
payload = response.json()
|
|
200
|
+
except ValueError as exc:
|
|
201
|
+
return self._unavailable(UnavailableReason.SOURCE_DOWN, f"invalid JSON: {exc}")
|
|
202
|
+
|
|
203
|
+
if isinstance(payload, list): # the ?cve_id= endpoint
|
|
204
|
+
if not payload:
|
|
205
|
+
return self._unavailable(UnavailableReason.NOT_FOUND, "no GitHub advisory")
|
|
206
|
+
payload = _pick_advisory(payload)
|
|
207
|
+
data = _parse_advisory(payload)
|
|
208
|
+
if data is None:
|
|
209
|
+
return self._unavailable(UnavailableReason.SOURCE_DOWN, "unrecognized advisory shape")
|
|
210
|
+
self._cache.set(self.name, cve_id, data.model_dump_json())
|
|
211
|
+
return SourceResult(data=data, meta=self._meta(datetime.now(UTC), cache_hit=False))
|
|
212
|
+
|
|
213
|
+
def _meta(self, fetched_at: datetime, *, cache_hit: bool) -> SourceMeta:
|
|
214
|
+
return SourceMeta(source=self.name, fetched_at=fetched_at, cache_hit=cache_hit)
|
|
215
|
+
|
|
216
|
+
def _unavailable(self, reason: UnavailableReason, detail: str) -> SourceResult:
|
|
217
|
+
return SourceResult(
|
|
218
|
+
data=Unavailable(reason=reason, detail=detail),
|
|
219
|
+
meta=self._meta(datetime.now(UTC), cache_hit=False),
|
|
220
|
+
)
|