advisory-rss 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- advisory_rss/__init__.py +1 -0
- advisory_rss/auth/pat.py +31 -0
- advisory_rss/cache/models.py +106 -0
- advisory_rss/cache/store.py +269 -0
- advisory_rss/cli.py +561 -0
- advisory_rss/config/constants.py +34 -0
- advisory_rss/config/settings.py +279 -0
- advisory_rss/github/client.py +702 -0
- advisory_rss/github/filter.py +77 -0
- advisory_rss/github/normalize.py +227 -0
- advisory_rss/github/paginate.py +51 -0
- advisory_rss/logging_config.py +345 -0
- advisory_rss/rss/builder.py +355 -0
- advisory_rss/rss/xml_util.py +31 -0
- advisory_rss/server/app.py +308 -0
- advisory_rss/server/bind.py +35 -0
- advisory_rss/server/headers.py +24 -0
- advisory_rss-1.0.0.dist-info/METADATA +306 -0
- advisory_rss-1.0.0.dist-info/RECORD +22 -0
- advisory_rss-1.0.0.dist-info/WHEEL +4 -0
- advisory_rss-1.0.0.dist-info/entry_points.txt +3 -0
- advisory_rss-1.0.0.dist-info/licenses/LICENSE +674 -0
advisory_rss/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.0.0"
|
advisory_rss/auth/pat.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from advisory_rss.config.settings import Settings
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def load_token(settings: Settings) -> str | None:
|
|
11
|
+
tok = settings.token
|
|
12
|
+
if tok:
|
|
13
|
+
logger.debug(
|
|
14
|
+
"token loaded preview=%s", tok[:4] + "..." + tok[-4:] if len(tok) > 8 else "***"
|
|
15
|
+
)
|
|
16
|
+
else:
|
|
17
|
+
logger.debug("no token configured")
|
|
18
|
+
return tok
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def token_preview(token: str | None) -> str:
|
|
22
|
+
if not token:
|
|
23
|
+
return "not set"
|
|
24
|
+
t = token.strip()
|
|
25
|
+
if len(t) <= 8:
|
|
26
|
+
return "***"
|
|
27
|
+
return f"{t[:4]}...{t[-4:]}"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def is_token_set(settings: Settings) -> bool:
|
|
31
|
+
return load_token(settings) is not None
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Normalized advisory dataclass and cache meta."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import asdict, dataclass, field
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class NormalizedAdvisory:
|
|
12
|
+
ghsa_id: str
|
|
13
|
+
cve_id: str | None = None
|
|
14
|
+
summary: str = ""
|
|
15
|
+
description: str = ""
|
|
16
|
+
severity: str | None = None
|
|
17
|
+
package_ecosystem: str | None = None
|
|
18
|
+
package_name: str | None = None
|
|
19
|
+
vulnerable_version_range: str | None = None
|
|
20
|
+
patched_versions: str | None = None
|
|
21
|
+
affected_versions: str | None = None # alias of vulnerable_version_range for legacy
|
|
22
|
+
published_at: datetime | None = None
|
|
23
|
+
updated_at: datetime | None = None
|
|
24
|
+
withdrawn_at: datetime | None = None
|
|
25
|
+
created_at: datetime | None = None
|
|
26
|
+
state: str = "published"
|
|
27
|
+
html_url: str = ""
|
|
28
|
+
repository_url: str | None = None
|
|
29
|
+
repository_full_name: str | None = None
|
|
30
|
+
references: list[str] = field(default_factory=list)
|
|
31
|
+
author_login: str | None = None
|
|
32
|
+
publisher_login: str | None = None
|
|
33
|
+
identifiers: list[dict[str, str]] = field(default_factory=list)
|
|
34
|
+
raw: dict[str, Any] = field(default_factory=dict, repr=False)
|
|
35
|
+
|
|
36
|
+
def sort_key(self) -> datetime:
|
|
37
|
+
# Use updated_at > published_at > created_at > distant past
|
|
38
|
+
for dt in (self.updated_at, self.published_at, self.created_at):
|
|
39
|
+
if dt is not None:
|
|
40
|
+
# ensure tz aware
|
|
41
|
+
if dt.tzinfo is None:
|
|
42
|
+
dt = dt.replace(tzinfo=UTC)
|
|
43
|
+
return dt
|
|
44
|
+
return datetime(1970, 1, 1, tzinfo=UTC)
|
|
45
|
+
|
|
46
|
+
def to_dict(self) -> dict[str, Any]:
|
|
47
|
+
d = asdict(self)
|
|
48
|
+
# serialize datetimes as ISO
|
|
49
|
+
for k in ("published_at", "updated_at", "withdrawn_at", "created_at"):
|
|
50
|
+
v = d[k]
|
|
51
|
+
if isinstance(v, datetime):
|
|
52
|
+
d[k] = v.isoformat()
|
|
53
|
+
return d
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def from_dict(cls, data: dict[str, Any]) -> NormalizedAdvisory:
|
|
57
|
+
def _parse(v: str | None) -> datetime | None:
|
|
58
|
+
if not v:
|
|
59
|
+
return None
|
|
60
|
+
try:
|
|
61
|
+
# handle Z suffix
|
|
62
|
+
if v.endswith("Z"):
|
|
63
|
+
v = v[:-1] + "+00:00"
|
|
64
|
+
dt = datetime.fromisoformat(v)
|
|
65
|
+
if dt.tzinfo is None:
|
|
66
|
+
dt = dt.replace(tzinfo=UTC)
|
|
67
|
+
return dt
|
|
68
|
+
except (ValueError, TypeError, AttributeError):
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
adv = cls(
|
|
72
|
+
ghsa_id=data.get("ghsa_id", ""),
|
|
73
|
+
cve_id=data.get("cve_id"),
|
|
74
|
+
summary=data.get("summary", ""),
|
|
75
|
+
description=data.get("description", ""),
|
|
76
|
+
severity=data.get("severity"),
|
|
77
|
+
package_ecosystem=data.get("package_ecosystem"),
|
|
78
|
+
package_name=data.get("package_name"),
|
|
79
|
+
vulnerable_version_range=data.get("vulnerable_version_range"),
|
|
80
|
+
patched_versions=data.get("patched_versions"),
|
|
81
|
+
affected_versions=data.get("affected_versions"),
|
|
82
|
+
published_at=_parse(data.get("published_at")),
|
|
83
|
+
updated_at=_parse(data.get("updated_at")),
|
|
84
|
+
withdrawn_at=_parse(data.get("withdrawn_at")),
|
|
85
|
+
created_at=_parse(data.get("created_at")),
|
|
86
|
+
state=data.get("state", "published"),
|
|
87
|
+
html_url=data.get("html_url", ""),
|
|
88
|
+
repository_url=data.get("repository_url"),
|
|
89
|
+
repository_full_name=data.get("repository_full_name"),
|
|
90
|
+
references=list(data.get("references") or []),
|
|
91
|
+
author_login=data.get("author_login"),
|
|
92
|
+
publisher_login=data.get("publisher_login"),
|
|
93
|
+
identifiers=list(data.get("identifiers") or []),
|
|
94
|
+
raw=data.get("raw") or {},
|
|
95
|
+
)
|
|
96
|
+
return adv
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass
|
|
100
|
+
class CacheMeta:
|
|
101
|
+
last_successful_sync: str | None = None # ISO
|
|
102
|
+
next_scheduled_sync: str | None = None
|
|
103
|
+
advisories_count: int = 0
|
|
104
|
+
rate_limited_until: str | None = None
|
|
105
|
+
last_error: str | None = None
|
|
106
|
+
authenticated_user: str | None = None
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""Persistent cache SQLite with WAL, parameterized queries, ETag map, stale fallback."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import sqlite3
|
|
8
|
+
import threading
|
|
9
|
+
from datetime import UTC, datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from advisory_rss.cache.models import CacheMeta, NormalizedAdvisory
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
_lock = threading.Lock()
|
|
17
|
+
|
|
18
|
+
# Never log token - this store never touches token
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _connect(db_path: Path) -> sqlite3.Connection:
|
|
22
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
23
|
+
conn = sqlite3.connect(
|
|
24
|
+
str(db_path), timeout=30.0, check_same_thread=False, isolation_level=None
|
|
25
|
+
)
|
|
26
|
+
# Safety pragmas
|
|
27
|
+
try:
|
|
28
|
+
conn.execute("PRAGMA journal_mode=WAL;")
|
|
29
|
+
conn.execute("PRAGMA foreign_keys=ON;")
|
|
30
|
+
conn.execute("PRAGMA synchronous=NORMAL;")
|
|
31
|
+
conn.execute("PRAGMA temp_store=MEMORY;")
|
|
32
|
+
except sqlite3.Error:
|
|
33
|
+
pass
|
|
34
|
+
return conn
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
SCHEMA = """
|
|
38
|
+
CREATE TABLE IF NOT EXISTS advisories(
|
|
39
|
+
ghsa_id TEXT PRIMARY KEY,
|
|
40
|
+
raw_json TEXT NOT NULL,
|
|
41
|
+
normalized_json TEXT NOT NULL,
|
|
42
|
+
updated_at TEXT,
|
|
43
|
+
state TEXT,
|
|
44
|
+
html_url TEXT,
|
|
45
|
+
author_login TEXT
|
|
46
|
+
);
|
|
47
|
+
CREATE TABLE IF NOT EXISTS meta(
|
|
48
|
+
key TEXT PRIMARY KEY,
|
|
49
|
+
value TEXT
|
|
50
|
+
);
|
|
51
|
+
CREATE TABLE IF NOT EXISTS etags(
|
|
52
|
+
url TEXT PRIMARY KEY,
|
|
53
|
+
etag TEXT,
|
|
54
|
+
updated_at TEXT
|
|
55
|
+
);
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class CacheStore:
|
|
60
|
+
def __init__(self, db_path: str | Path):
|
|
61
|
+
self.db_path = Path(db_path)
|
|
62
|
+
self._conn = _connect(self.db_path)
|
|
63
|
+
self._init_schema()
|
|
64
|
+
|
|
65
|
+
def _init_schema(self) -> None:
|
|
66
|
+
logger.debug("init schema db=%s", self.db_path)
|
|
67
|
+
try:
|
|
68
|
+
self._conn.executescript(SCHEMA)
|
|
69
|
+
logger.debug("cache schema ready db=%s", self.db_path)
|
|
70
|
+
except sqlite3.Error as e:
|
|
71
|
+
logger.error("Cache schema init failed: %s", e, exc_info=True)
|
|
72
|
+
|
|
73
|
+
def upsert_advisories(self, advisories: list[NormalizedAdvisory]) -> int:
|
|
74
|
+
"""Upsert list; returns count. Uses parameterized queries."""
|
|
75
|
+
if not advisories:
|
|
76
|
+
logger.debug("upsert_advisories empty - no-op")
|
|
77
|
+
return 0
|
|
78
|
+
logger.info("upsert_advisories start count=%d", len(advisories))
|
|
79
|
+
count = 0
|
|
80
|
+
with _lock:
|
|
81
|
+
try:
|
|
82
|
+
cur = self._conn.cursor()
|
|
83
|
+
cur.execute("BEGIN IMMEDIATE")
|
|
84
|
+
for adv in advisories:
|
|
85
|
+
try:
|
|
86
|
+
normalized_json = json.dumps(adv.to_dict(), ensure_ascii=False)
|
|
87
|
+
raw_json = json.dumps(adv.raw or {}, ensure_ascii=False)
|
|
88
|
+
updated_iso = adv.updated_at.isoformat() if adv.updated_at else None
|
|
89
|
+
cur.execute(
|
|
90
|
+
"""
|
|
91
|
+
INSERT INTO advisories(ghsa_id, raw_json, normalized_json, updated_at, state, html_url, author_login)
|
|
92
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
93
|
+
ON CONFLICT(ghsa_id) DO UPDATE SET
|
|
94
|
+
raw_json=excluded.raw_json,
|
|
95
|
+
normalized_json=excluded.normalized_json,
|
|
96
|
+
updated_at=excluded.updated_at,
|
|
97
|
+
state=excluded.state,
|
|
98
|
+
html_url=excluded.html_url,
|
|
99
|
+
author_login=excluded.author_login
|
|
100
|
+
""",
|
|
101
|
+
(
|
|
102
|
+
adv.ghsa_id,
|
|
103
|
+
raw_json,
|
|
104
|
+
normalized_json,
|
|
105
|
+
updated_iso,
|
|
106
|
+
adv.state,
|
|
107
|
+
adv.html_url,
|
|
108
|
+
adv.author_login,
|
|
109
|
+
),
|
|
110
|
+
)
|
|
111
|
+
count += 1
|
|
112
|
+
except (sqlite3.Error, ValueError, TypeError, OSError) as e:
|
|
113
|
+
logger.warning("Failed to upsert %s: %s", adv.ghsa_id, e)
|
|
114
|
+
cur.execute("COMMIT")
|
|
115
|
+
logger.info("upsert_advisories committed count=%d", count)
|
|
116
|
+
except sqlite3.Error as e:
|
|
117
|
+
logger.error("Cache upsert transaction failed: %s", e, exc_info=True)
|
|
118
|
+
try:
|
|
119
|
+
self._conn.execute("ROLLBACK")
|
|
120
|
+
except sqlite3.Error:
|
|
121
|
+
pass
|
|
122
|
+
return count
|
|
123
|
+
|
|
124
|
+
def load_all(self) -> list[NormalizedAdvisory]:
|
|
125
|
+
with _lock:
|
|
126
|
+
try:
|
|
127
|
+
cur = self._conn.execute("SELECT normalized_json FROM advisories")
|
|
128
|
+
rows = cur.fetchall()
|
|
129
|
+
out: list[NormalizedAdvisory] = []
|
|
130
|
+
for (j,) in rows:
|
|
131
|
+
try:
|
|
132
|
+
data = json.loads(j)
|
|
133
|
+
out.append(NormalizedAdvisory.from_dict(data))
|
|
134
|
+
except (json.JSONDecodeError, ValueError, TypeError) as e:
|
|
135
|
+
logger.warning("Corrupt cache row skipped: %s", e)
|
|
136
|
+
logger.debug("load_all fetched %d rows -> %d advisories", len(rows), len(out))
|
|
137
|
+
return out
|
|
138
|
+
except sqlite3.Error as e:
|
|
139
|
+
logger.error("Cache load failed: %s", e, exc_info=True)
|
|
140
|
+
return []
|
|
141
|
+
|
|
142
|
+
def load_sorted(self, limit: int | None = None) -> list[NormalizedAdvisory]:
|
|
143
|
+
# load_all already locks; sort outside lock
|
|
144
|
+
all_adv = self.load_all()
|
|
145
|
+
# sort updated_at DESC
|
|
146
|
+
all_adv.sort(key=lambda a: a.sort_key(), reverse=True)
|
|
147
|
+
logger.debug(
|
|
148
|
+
"load_sorted total=%d limit=%s -> %d",
|
|
149
|
+
len(all_adv),
|
|
150
|
+
limit,
|
|
151
|
+
len(all_adv[:limit] if limit is not None else all_adv),
|
|
152
|
+
)
|
|
153
|
+
if limit is not None:
|
|
154
|
+
return all_adv[:limit]
|
|
155
|
+
return all_adv
|
|
156
|
+
|
|
157
|
+
def count(self) -> int:
|
|
158
|
+
with _lock:
|
|
159
|
+
try:
|
|
160
|
+
cur = self._conn.execute("SELECT COUNT(*) FROM advisories")
|
|
161
|
+
return int(cur.fetchone()[0])
|
|
162
|
+
except sqlite3.Error:
|
|
163
|
+
return 0
|
|
164
|
+
|
|
165
|
+
def clear(self) -> None:
|
|
166
|
+
logger.info("cache clear initiated")
|
|
167
|
+
with _lock:
|
|
168
|
+
try:
|
|
169
|
+
self._conn.execute("BEGIN IMMEDIATE")
|
|
170
|
+
self._conn.execute("DELETE FROM advisories")
|
|
171
|
+
self._conn.execute("DELETE FROM etags")
|
|
172
|
+
self._conn.execute("COMMIT")
|
|
173
|
+
logger.info("cache cleared")
|
|
174
|
+
except sqlite3.Error as e:
|
|
175
|
+
logger.error("Cache clear failed: %s", e, exc_info=True)
|
|
176
|
+
try:
|
|
177
|
+
self._conn.execute("ROLLBACK")
|
|
178
|
+
except sqlite3.Error:
|
|
179
|
+
pass
|
|
180
|
+
|
|
181
|
+
# --- meta ---
|
|
182
|
+
def get_meta(self, key: str) -> str | None:
|
|
183
|
+
with _lock:
|
|
184
|
+
try:
|
|
185
|
+
cur = self._conn.execute("SELECT value FROM meta WHERE key=?", (key,))
|
|
186
|
+
row = cur.fetchone()
|
|
187
|
+
return row[0] if row else None
|
|
188
|
+
except sqlite3.Error:
|
|
189
|
+
return None
|
|
190
|
+
|
|
191
|
+
def set_meta(self, key: str, value: str | None) -> None:
|
|
192
|
+
with _lock:
|
|
193
|
+
try:
|
|
194
|
+
if value is None:
|
|
195
|
+
self._conn.execute("DELETE FROM meta WHERE key=?", (key,))
|
|
196
|
+
else:
|
|
197
|
+
self._conn.execute(
|
|
198
|
+
"INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
199
|
+
(key, value),
|
|
200
|
+
)
|
|
201
|
+
except sqlite3.Error as e:
|
|
202
|
+
logger.warning("set_meta failed %s: %s", key, e)
|
|
203
|
+
|
|
204
|
+
def get_cache_meta(self) -> CacheMeta:
|
|
205
|
+
return CacheMeta(
|
|
206
|
+
last_successful_sync=self.get_meta("last_successful_sync"),
|
|
207
|
+
next_scheduled_sync=self.get_meta("next_scheduled_sync"),
|
|
208
|
+
advisories_count=self.count(),
|
|
209
|
+
rate_limited_until=self.get_meta("rate_limited_until"),
|
|
210
|
+
last_error=self.get_meta("last_error"),
|
|
211
|
+
authenticated_user=self.get_meta("authenticated_user"),
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
def set_cache_meta(self, meta: CacheMeta) -> None:
|
|
215
|
+
self.set_meta("last_successful_sync", meta.last_successful_sync)
|
|
216
|
+
self.set_meta("next_scheduled_sync", meta.next_scheduled_sync)
|
|
217
|
+
self.set_meta("rate_limited_until", meta.rate_limited_until)
|
|
218
|
+
self.set_meta("last_error", meta.last_error)
|
|
219
|
+
self.set_meta("authenticated_user", meta.authenticated_user)
|
|
220
|
+
|
|
221
|
+
# --- etags ---
|
|
222
|
+
def get_etag(self, url: str) -> str | None:
|
|
223
|
+
with _lock:
|
|
224
|
+
try:
|
|
225
|
+
cur = self._conn.execute("SELECT etag FROM etags WHERE url=?", (url,))
|
|
226
|
+
row = cur.fetchone()
|
|
227
|
+
return row[0] if row else None
|
|
228
|
+
except sqlite3.Error:
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
def set_etag(self, url: str, etag: str | None) -> None:
|
|
232
|
+
with _lock:
|
|
233
|
+
try:
|
|
234
|
+
if etag is None:
|
|
235
|
+
self._conn.execute("DELETE FROM etags WHERE url=?", (url,))
|
|
236
|
+
logger.debug("etag deleted url=%s", url[:120])
|
|
237
|
+
else:
|
|
238
|
+
now = datetime.now(UTC).isoformat()
|
|
239
|
+
self._conn.execute(
|
|
240
|
+
"INSERT INTO etags(url, etag, updated_at) VALUES (?, ?, ?) ON CONFLICT(url) DO UPDATE SET etag=excluded.etag, updated_at=excluded.updated_at",
|
|
241
|
+
(url, etag, now),
|
|
242
|
+
)
|
|
243
|
+
logger.debug("etag set url=%s etag=%s", url[:120], etag[:80])
|
|
244
|
+
except sqlite3.Error as e:
|
|
245
|
+
logger.warning("set_etag failed: %s", e)
|
|
246
|
+
|
|
247
|
+
def close(self) -> None:
|
|
248
|
+
try:
|
|
249
|
+
self._conn.close()
|
|
250
|
+
except sqlite3.Error:
|
|
251
|
+
pass
|
|
252
|
+
|
|
253
|
+
# Helpers for status
|
|
254
|
+
def mark_success(self, user: str | None = None) -> None:
|
|
255
|
+
now = datetime.now(UTC).isoformat()
|
|
256
|
+
self.set_meta("last_successful_sync", now)
|
|
257
|
+
self.set_meta("last_error", None)
|
|
258
|
+
self.set_meta("rate_limited_until", None)
|
|
259
|
+
if user:
|
|
260
|
+
self.set_meta("authenticated_user", user)
|
|
261
|
+
# compute next
|
|
262
|
+
# caller sets next_scheduled via interval; but also set here roughly
|
|
263
|
+
self.set_meta("advisories_count", str(self.count()))
|
|
264
|
+
logger.info("cache mark_success user=%s time=%s", user or "unknown", now)
|
|
265
|
+
|
|
266
|
+
def mark_error(self, err: str) -> None:
|
|
267
|
+
# truncate error
|
|
268
|
+
self.set_meta("last_error", err[:500])
|
|
269
|
+
logger.warning("cache mark_error: %s", err[:200])
|