predisclose 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- predisclose/__init__.py +3 -0
- predisclose/__main__.py +3 -0
- predisclose/checks/__init__.py +1 -0
- predisclose/checks/activity.py +66 -0
- predisclose/checks/fixes.py +215 -0
- predisclose/checks/known.py +79 -0
- predisclose/checks/route.py +238 -0
- predisclose/checks/similarity.py +63 -0
- predisclose/cli.py +267 -0
- predisclose/core.py +78 -0
- predisclose/ecosystems.py +101 -0
- predisclose/http.py +171 -0
- predisclose/ledger.py +170 -0
- predisclose/models.py +82 -0
- predisclose/output.py +92 -0
- predisclose/sources/__init__.py +1 -0
- predisclose/sources/advisory.py +69 -0
- predisclose/sources/ghsa.py +76 -0
- predisclose/sources/github.py +79 -0
- predisclose/sources/osv.py +69 -0
- predisclose/sources/registry.py +53 -0
- predisclose-0.1.0.dist-info/METADATA +163 -0
- predisclose-0.1.0.dist-info/RECORD +26 -0
- predisclose-0.1.0.dist-info/WHEEL +4 -0
- predisclose-0.1.0.dist-info/entry_points.txt +2 -0
- predisclose-0.1.0.dist-info/licenses/LICENSE +202 -0
predisclose/cli.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Command line: predisclose check | route | ledger."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
import tomllib
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Annotated
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
|
|
13
|
+
from predisclose import __version__
|
|
14
|
+
from predisclose import ledger as ledger_mod
|
|
15
|
+
from predisclose.core import ALL_CHECKS, run_check
|
|
16
|
+
from predisclose.ecosystems import normalize_cwe, parse_package, parse_repo
|
|
17
|
+
from predisclose.http import Http, github_token_from_env
|
|
18
|
+
from predisclose.models import Finding
|
|
19
|
+
from predisclose.output import to_json, to_markdown, to_terminal
|
|
20
|
+
|
|
21
|
+
app = typer.Typer(
|
|
22
|
+
add_completion=False,
|
|
23
|
+
no_args_is_help=True,
|
|
24
|
+
help="Before you disclose a vulnerability: check public records, public fixes, "
|
|
25
|
+
"and the project's official reporting channel. Uses public data only.",
|
|
26
|
+
)
|
|
27
|
+
ledger_app = typer.Typer(no_args_is_help=True, help="Track the reports you have sent (stored locally).")
|
|
28
|
+
app.add_typer(ledger_app, name="ledger")
|
|
29
|
+
|
|
30
|
+
out = Console()
|
|
31
|
+
err = Console(stderr=True)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _version_callback(value: bool) -> None:
|
|
35
|
+
if value:
|
|
36
|
+
out.print(f"predisclose {__version__}")
|
|
37
|
+
raise typer.Exit()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@app.callback()
|
|
41
|
+
def main(
|
|
42
|
+
show_version: Annotated[
|
|
43
|
+
bool,
|
|
44
|
+
typer.Option("--about", callback=_version_callback, is_eager=True, help="Show the version."),
|
|
45
|
+
] = False,
|
|
46
|
+
) -> None:
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _fail(message: str) -> None:
|
|
51
|
+
err.print(f"[red]error:[/] {message}")
|
|
52
|
+
raise typer.Exit(2)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _load_finding_file(path: Path) -> dict:
|
|
56
|
+
try:
|
|
57
|
+
with path.open("rb") as f:
|
|
58
|
+
data = tomllib.load(f)
|
|
59
|
+
except (OSError, tomllib.TOMLDecodeError) as exc:
|
|
60
|
+
_fail(f"cannot read {path}: {exc}")
|
|
61
|
+
allowed = {"package", "version", "repo", "cwe", "files", "functions", "keywords", "notes"}
|
|
62
|
+
unknown = set(data) - allowed
|
|
63
|
+
if unknown:
|
|
64
|
+
_fail(f"unknown keys in {path}: {', '.join(sorted(unknown))}")
|
|
65
|
+
return data
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _build_finding(
|
|
69
|
+
pkg: str | None,
|
|
70
|
+
version: str | None,
|
|
71
|
+
repo: str | None,
|
|
72
|
+
cwe: list[str],
|
|
73
|
+
files: list[str],
|
|
74
|
+
functions: list[str],
|
|
75
|
+
keywords: list[str],
|
|
76
|
+
from_file: Path | None,
|
|
77
|
+
) -> Finding:
|
|
78
|
+
base: dict = _load_finding_file(from_file) if from_file else {}
|
|
79
|
+
pkg = pkg or base.get("package")
|
|
80
|
+
finding = Finding(
|
|
81
|
+
version=version or base.get("version"),
|
|
82
|
+
files=list(files or base.get("files") or []),
|
|
83
|
+
functions=list(functions or base.get("functions") or []),
|
|
84
|
+
keywords=list(keywords or base.get("keywords") or []),
|
|
85
|
+
)
|
|
86
|
+
try:
|
|
87
|
+
if pkg:
|
|
88
|
+
finding.ecosystem, finding.package = parse_package(str(pkg))
|
|
89
|
+
finding.cwe = [normalize_cwe(str(c)) for c in (cwe or base.get("cwe") or [])]
|
|
90
|
+
except ValueError as exc:
|
|
91
|
+
_fail(str(exc))
|
|
92
|
+
repo_text = repo or base.get("repo")
|
|
93
|
+
if repo_text:
|
|
94
|
+
finding.repo = parse_repo(str(repo_text))
|
|
95
|
+
if not finding.repo:
|
|
96
|
+
_fail(f"cannot understand repository {repo_text!r}; use owner/name or a GitHub URL")
|
|
97
|
+
if not (finding.package or finding.repo):
|
|
98
|
+
_fail("give a package (--pkg pypi:NAME) or a repository (--repo owner/name)")
|
|
99
|
+
return finding
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@app.command()
|
|
103
|
+
def check(
|
|
104
|
+
pkg: Annotated[str | None, typer.Option("--pkg", "-p", help="ECOSYSTEM:NAME, e.g. pypi:jinja2, npm:lodash")] = None,
|
|
105
|
+
version: Annotated[str | None, typer.Option("--version", "-v", help="Version, tag, or commit you tested.")] = None,
|
|
106
|
+
repo: Annotated[str | None, typer.Option("--repo", "-r", help="GitHub repository, owner/name or URL.")] = None,
|
|
107
|
+
cwe: Annotated[list[str], typer.Option("--cwe", help="Weakness type, e.g. CWE-79. Repeatable.")] = [], # noqa: B006
|
|
108
|
+
file: Annotated[list[str], typer.Option("--file", "-f", help="Affected file path. Repeatable.")] = [], # noqa: B006
|
|
109
|
+
function: Annotated[list[str], typer.Option("--function", help="Affected function. Repeatable.")] = [], # noqa: B006
|
|
110
|
+
keyword: Annotated[list[str], typer.Option("--keyword", "-k", help="Extra word to look for. Repeatable.")] = [], # noqa: B006
|
|
111
|
+
from_file: Annotated[Path | None, typer.Option("--from", help="Read the finding from a TOML file.")] = None,
|
|
112
|
+
only: Annotated[str | None, typer.Option("--only", help="Comma list of checks: known,fixes,activity,route")] = None,
|
|
113
|
+
json_out: Annotated[bool, typer.Option("--json", help="Print JSON instead of a table.")] = False,
|
|
114
|
+
markdown: Annotated[Path | None, typer.Option("--markdown", help="Also write a Markdown report here.")] = None,
|
|
115
|
+
show_queries: Annotated[bool, typer.Option("--show-queries", help="List every request that was sent.")] = False,
|
|
116
|
+
dry_run: Annotated[bool, typer.Option("--dry-run", help="Show the first requests; send nothing.")] = False,
|
|
117
|
+
no_cache: Annotated[bool, typer.Option("--no-cache", help="Do not read or write the local cache.")] = False,
|
|
118
|
+
) -> None:
|
|
119
|
+
"""Check public records, public fixes, recent activity, and where to report."""
|
|
120
|
+
finding = _build_finding(pkg, version, repo, cwe, file, function, keyword, from_file)
|
|
121
|
+
checks = ALL_CHECKS
|
|
122
|
+
if only:
|
|
123
|
+
checks = tuple(c.strip() for c in only.split(",") if c.strip())
|
|
124
|
+
bad = [c for c in checks if c not in ALL_CHECKS]
|
|
125
|
+
if bad:
|
|
126
|
+
_fail(f"unknown check(s): {', '.join(bad)}; choose from {', '.join(ALL_CHECKS)}")
|
|
127
|
+
|
|
128
|
+
http = Http(dry_run=dry_run, use_cache=not no_cache, token=github_token_from_env())
|
|
129
|
+
try:
|
|
130
|
+
report = run_check(http, finding, checks)
|
|
131
|
+
finally:
|
|
132
|
+
http.close()
|
|
133
|
+
|
|
134
|
+
if json_out:
|
|
135
|
+
sys.stdout.write(to_json(report) + "\n")
|
|
136
|
+
else:
|
|
137
|
+
to_terminal(report, out)
|
|
138
|
+
if markdown:
|
|
139
|
+
markdown.write_text(to_markdown(report), encoding="utf-8")
|
|
140
|
+
err.print(f"[dim]Markdown report written to {markdown}[/]")
|
|
141
|
+
if show_queries or dry_run:
|
|
142
|
+
err.print("\n[bold]Requests[/]" + (" (dry run: none were sent)" if dry_run else ""))
|
|
143
|
+
for q in report.queries:
|
|
144
|
+
err.print(f" {q}", markup=False, highlight=False)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@app.command()
|
|
148
|
+
def route(
|
|
149
|
+
repo: Annotated[str, typer.Argument(help="GitHub repository, owner/name or URL.")],
|
|
150
|
+
json_out: Annotated[bool, typer.Option("--json", help="Print JSON instead of a table.")] = False,
|
|
151
|
+
no_cache: Annotated[bool, typer.Option("--no-cache", help="Do not use the local cache.")] = False,
|
|
152
|
+
) -> None:
|
|
153
|
+
"""Only find where the project wants security reports."""
|
|
154
|
+
target = parse_repo(repo)
|
|
155
|
+
if not target:
|
|
156
|
+
_fail(f"cannot understand repository {repo!r}; use owner/name or a GitHub URL")
|
|
157
|
+
http = Http(use_cache=not no_cache, token=github_token_from_env())
|
|
158
|
+
try:
|
|
159
|
+
report = run_check(http, Finding(repo=target), ("route",))
|
|
160
|
+
finally:
|
|
161
|
+
http.close()
|
|
162
|
+
if json_out:
|
|
163
|
+
sys.stdout.write(to_json(report) + "\n")
|
|
164
|
+
else:
|
|
165
|
+
to_terminal(report, out)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# -- ledger --------------------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _ledger() -> ledger_mod.Ledger:
|
|
172
|
+
return ledger_mod.Ledger()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@ledger_app.command("add")
|
|
176
|
+
def ledger_add(
|
|
177
|
+
project: Annotated[str, typer.Argument(help="Project, e.g. owner/name.")],
|
|
178
|
+
title: Annotated[str, typer.Argument(help="Short title. Stays on this machine.")],
|
|
179
|
+
channel: Annotated[str, typer.Option(help="e.g. github-pvr, email, hackerone")] = "",
|
|
180
|
+
sent: Annotated[str | None, typer.Option(help="Date sent, YYYY-MM-DD (default: today).")] = None,
|
|
181
|
+
deadline_days: Annotated[int, typer.Option(help="Disclosure deadline in days.")] = 90,
|
|
182
|
+
status: Annotated[str, typer.Option(help="One of: " + ", ".join(ledger_mod.STATUSES))] = "sent",
|
|
183
|
+
) -> None:
|
|
184
|
+
"""Record a report you sent."""
|
|
185
|
+
lg = _ledger()
|
|
186
|
+
try:
|
|
187
|
+
rid = lg.add(project, title, channel=channel, sent_on=sent, deadline_days=deadline_days, status=status)
|
|
188
|
+
except ValueError as exc:
|
|
189
|
+
_fail(str(exc))
|
|
190
|
+
finally:
|
|
191
|
+
lg.close()
|
|
192
|
+
out.print(f"Added report #{rid}")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@ledger_app.command("update")
|
|
196
|
+
def ledger_update(
|
|
197
|
+
report_id: Annotated[int, typer.Argument(help="Report number.")],
|
|
198
|
+
status: Annotated[str | None, typer.Option(help="New status.")] = None,
|
|
199
|
+
ids: Annotated[str | None, typer.Option("--ids", help="CVE / GHSA ids, comma separated.")] = None,
|
|
200
|
+
url: Annotated[str | None, typer.Option(help="Public advisory URL.")] = None,
|
|
201
|
+
public: Annotated[bool | None, typer.Option("--public/--private", help="Mark as public or not.")] = None,
|
|
202
|
+
notes: Annotated[str | None, typer.Option(help="Private notes.")] = None,
|
|
203
|
+
) -> None:
|
|
204
|
+
"""Change the status or details of a report."""
|
|
205
|
+
lg = _ledger()
|
|
206
|
+
try:
|
|
207
|
+
ok = lg.update(report_id, status=status, ids=ids, url=url, public=public, notes=notes)
|
|
208
|
+
except ValueError as exc:
|
|
209
|
+
_fail(str(exc))
|
|
210
|
+
finally:
|
|
211
|
+
lg.close()
|
|
212
|
+
if not ok:
|
|
213
|
+
_fail(f"report #{report_id} not found or nothing to change")
|
|
214
|
+
out.print(f"Updated report #{report_id}")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@ledger_app.command("list")
|
|
218
|
+
def ledger_list(
|
|
219
|
+
status: Annotated[str | None, typer.Option(help="Only this status.")] = None,
|
|
220
|
+
) -> None:
|
|
221
|
+
"""Show every report (full details, local only)."""
|
|
222
|
+
from rich.table import Table
|
|
223
|
+
|
|
224
|
+
lg = _ledger()
|
|
225
|
+
rows = lg.all(status)
|
|
226
|
+
lg.close()
|
|
227
|
+
table = Table("#", "Project", "Title", "Channel", "Sent", "Deadline", "Status", "IDs", "Public")
|
|
228
|
+
for r in rows:
|
|
229
|
+
table.add_row(
|
|
230
|
+
str(r["id"]), r["project"], r["title"], r["channel"], r["sent_on"], r["deadline"],
|
|
231
|
+
r["status"], r["ids"], "yes" if r["public"] else "no",
|
|
232
|
+
)
|
|
233
|
+
out.print(table)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@ledger_app.command("due")
|
|
237
|
+
def ledger_due(
|
|
238
|
+
within: Annotated[int, typer.Option(help="Days ahead to look.")] = 14,
|
|
239
|
+
) -> None:
|
|
240
|
+
"""Reports whose disclosure deadline is near or past."""
|
|
241
|
+
lg = _ledger()
|
|
242
|
+
rows = lg.due(within)
|
|
243
|
+
lg.close()
|
|
244
|
+
if not rows:
|
|
245
|
+
out.print(f"Nothing due in the next {within} days.")
|
|
246
|
+
return
|
|
247
|
+
for r in rows:
|
|
248
|
+
out.print(f"#{r['id']} {r['deadline']} {r['status']:<12} {r['project']} {r['title']}")
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@ledger_app.command("export")
|
|
252
|
+
def ledger_export(
|
|
253
|
+
fmt: Annotated[str, typer.Option("--format", help="md or json")] = "md",
|
|
254
|
+
show_projects: Annotated[bool, typer.Option(help="Keep project names of reports that are not public yet.")] = False,
|
|
255
|
+
output: Annotated[Path | None, typer.Option("--output", "-o", help="Write to a file.")] = None,
|
|
256
|
+
) -> None:
|
|
257
|
+
"""Export for a portfolio. Reports that are not public are reduced to a placeholder."""
|
|
258
|
+
if fmt not in ("md", "json"):
|
|
259
|
+
_fail("format must be md or json")
|
|
260
|
+
lg = _ledger()
|
|
261
|
+
text = ledger_mod.export(lg.all(), fmt, show_projects=show_projects)
|
|
262
|
+
lg.close()
|
|
263
|
+
if output:
|
|
264
|
+
output.write_text(text, encoding="utf-8")
|
|
265
|
+
err.print(f"[dim]Written to {output}[/]")
|
|
266
|
+
else:
|
|
267
|
+
sys.stdout.write(text)
|
predisclose/core.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Run the checks for one finding and build a Report."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
|
|
8
|
+
from predisclose import __version__
|
|
9
|
+
from predisclose.checks import activity, fixes, known, route
|
|
10
|
+
from predisclose.http import Http, NetworkError, RateLimited
|
|
11
|
+
from predisclose.models import CheckResult, Finding, Report
|
|
12
|
+
from predisclose.sources import registry
|
|
13
|
+
|
|
14
|
+
ALL_CHECKS = ("known", "fixes", "activity", "route")
|
|
15
|
+
CHECK_NAMES = {
|
|
16
|
+
"known": known.NAME,
|
|
17
|
+
"fixes": fixes.NAME,
|
|
18
|
+
"activity": activity.NAME,
|
|
19
|
+
"route": route.NAME,
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _guard(name: str, fn) -> CheckResult:
|
|
24
|
+
try:
|
|
25
|
+
return fn()
|
|
26
|
+
except RateLimited as exc:
|
|
27
|
+
return CheckResult(name, "error", "none", str(exc))
|
|
28
|
+
except NetworkError as exc:
|
|
29
|
+
return CheckResult(name, "error", "none", f"Network error: {exc}")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def run_check(http: Http, finding: Finding, checks: Iterable[str] = ALL_CHECKS) -> Report:
|
|
33
|
+
wanted = [c for c in ALL_CHECKS if c in set(checks)]
|
|
34
|
+
results: list[CheckResult] = []
|
|
35
|
+
|
|
36
|
+
# Map a package to its source repository when the user did not give one.
|
|
37
|
+
repo_note = None
|
|
38
|
+
if not finding.repo and finding.package and finding.ecosystem:
|
|
39
|
+
try:
|
|
40
|
+
finding.repo = registry.infer_repo(http, finding.ecosystem, finding.package)
|
|
41
|
+
if finding.repo:
|
|
42
|
+
repo_note = f"Repository {finding.repo} taken from {finding.ecosystem} metadata."
|
|
43
|
+
except (RateLimited, NetworkError):
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
advisories, gather_notes = [], []
|
|
47
|
+
if "known" in wanted or not finding.repo:
|
|
48
|
+
try:
|
|
49
|
+
advisories, gather_notes = known.gather(http, finding)
|
|
50
|
+
except (RateLimited, NetworkError) as exc:
|
|
51
|
+
results.append(CheckResult(known.NAME, "error", "none", str(exc)))
|
|
52
|
+
wanted = [c for c in wanted if c != "known"]
|
|
53
|
+
|
|
54
|
+
if not finding.repo:
|
|
55
|
+
hint = next((a.repo_hint for a in advisories if a.repo_hint), None)
|
|
56
|
+
if hint:
|
|
57
|
+
finding.repo = hint
|
|
58
|
+
repo_note = f"Repository {hint} taken from advisory data."
|
|
59
|
+
|
|
60
|
+
if "known" in wanted:
|
|
61
|
+
result = known.check(http, finding, advisories, gather_notes)
|
|
62
|
+
if repo_note:
|
|
63
|
+
result.notes.append(repo_note)
|
|
64
|
+
results.append(result)
|
|
65
|
+
if "fixes" in wanted:
|
|
66
|
+
results.append(_guard(fixes.NAME, lambda: fixes.check(http, finding, advisories)))
|
|
67
|
+
if "activity" in wanted:
|
|
68
|
+
results.append(_guard(activity.NAME, lambda: activity.check(http, finding)))
|
|
69
|
+
if "route" in wanted:
|
|
70
|
+
results.append(_guard(route.NAME, lambda: route.check(http, finding.repo)))
|
|
71
|
+
|
|
72
|
+
return Report(
|
|
73
|
+
tool_version=__version__,
|
|
74
|
+
generated_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
75
|
+
finding=finding,
|
|
76
|
+
results=results,
|
|
77
|
+
queries=list(http.queries),
|
|
78
|
+
)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Input parsing: ecosystems, package specs, repositories, CWE ids."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
# User-friendly names -> OSV ecosystem names.
|
|
8
|
+
ALIASES = {
|
|
9
|
+
"pypi": "PyPI",
|
|
10
|
+
"pip": "PyPI",
|
|
11
|
+
"python": "PyPI",
|
|
12
|
+
"npm": "npm",
|
|
13
|
+
"node": "npm",
|
|
14
|
+
"crates.io": "crates.io",
|
|
15
|
+
"crates": "crates.io",
|
|
16
|
+
"cargo": "crates.io",
|
|
17
|
+
"rust": "crates.io",
|
|
18
|
+
"go": "Go",
|
|
19
|
+
"golang": "Go",
|
|
20
|
+
"maven": "Maven",
|
|
21
|
+
"nuget": "NuGet",
|
|
22
|
+
"rubygems": "RubyGems",
|
|
23
|
+
"gem": "RubyGems",
|
|
24
|
+
"packagist": "Packagist",
|
|
25
|
+
"composer": "Packagist",
|
|
26
|
+
"pub": "Pub",
|
|
27
|
+
"hex": "Hex",
|
|
28
|
+
"actions": "GitHub Actions",
|
|
29
|
+
"github-actions": "GitHub Actions",
|
|
30
|
+
"swift": "SwiftURL",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
# OSV ecosystem names -> GitHub Advisory Database ecosystem names.
|
|
34
|
+
GHSA_ECOSYSTEM = {
|
|
35
|
+
"PyPI": "pip",
|
|
36
|
+
"npm": "npm",
|
|
37
|
+
"crates.io": "rust",
|
|
38
|
+
"Go": "go",
|
|
39
|
+
"Maven": "maven",
|
|
40
|
+
"NuGet": "nuget",
|
|
41
|
+
"RubyGems": "rubygems",
|
|
42
|
+
"Packagist": "composer",
|
|
43
|
+
"Pub": "pub",
|
|
44
|
+
"Hex": "erlang",
|
|
45
|
+
"GitHub Actions": "actions",
|
|
46
|
+
"SwiftURL": "swift",
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def parse_package(spec: str) -> tuple[str, str]:
|
|
51
|
+
"""'pypi:jinja2' -> ('PyPI', 'jinja2'). Maven keeps 'group:artifact' after the first colon."""
|
|
52
|
+
if ":" not in spec:
|
|
53
|
+
raise ValueError(f"package must look like ECOSYSTEM:NAME (e.g. pypi:jinja2), got {spec!r}")
|
|
54
|
+
eco, name = spec.split(":", 1)
|
|
55
|
+
key = eco.strip().lower()
|
|
56
|
+
if key not in ALIASES:
|
|
57
|
+
known = ", ".join(sorted(set(ALIASES)))
|
|
58
|
+
raise ValueError(f"unknown ecosystem {eco!r}; known: {known}")
|
|
59
|
+
name = name.strip()
|
|
60
|
+
if not name:
|
|
61
|
+
raise ValueError("package name is empty")
|
|
62
|
+
return ALIASES[key], name
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
_GITHUB_URL = re.compile(r"github\.com[/:]([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)")
|
|
66
|
+
_OWNER_NAME = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def parse_repo(text: str | None) -> str | None:
|
|
70
|
+
"""Accept 'owner/name', a GitHub URL, or a git URL. Returns 'owner/name' or None."""
|
|
71
|
+
if not text:
|
|
72
|
+
return None
|
|
73
|
+
t = text.strip()
|
|
74
|
+
m = _GITHUB_URL.search(t)
|
|
75
|
+
if m:
|
|
76
|
+
owner, name = m.group(1), m.group(2)
|
|
77
|
+
elif _OWNER_NAME.fullmatch(t):
|
|
78
|
+
owner, name = t.split("/", 1)
|
|
79
|
+
else:
|
|
80
|
+
return None
|
|
81
|
+
name = name.removesuffix(".git")
|
|
82
|
+
if not owner or not name or owner in {".", ".."} or name in {".", ".."}:
|
|
83
|
+
return None
|
|
84
|
+
return f"{owner}/{name}"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def normalize_cwe(value: str) -> str:
|
|
88
|
+
"""'79', 'cwe-79', 'CWE-79' -> 'CWE-79'."""
|
|
89
|
+
v = value.strip().upper()
|
|
90
|
+
if v.startswith("CWE-"):
|
|
91
|
+
v = v[4:]
|
|
92
|
+
if not v.isdigit():
|
|
93
|
+
raise ValueError(f"CWE must be a number like 79 or CWE-79, got {value!r}")
|
|
94
|
+
return f"CWE-{int(v)}"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
_SHA = re.compile(r"[0-9a-f]{7,40}")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def looks_like_sha(value: str) -> bool:
|
|
101
|
+
return bool(_SHA.fullmatch(value.lower()))
|
predisclose/http.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""One HTTP client for every source: records each outgoing query, caches answers, never leaks tokens."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sqlite3
|
|
9
|
+
import time
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
from urllib.parse import urlencode
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from predisclose import __version__
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class RateLimited(Exception):
|
|
20
|
+
"""GitHub refused the request because the hourly or per-minute quota is used up."""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class NetworkError(Exception):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def default_cache_dir() -> Path:
|
|
28
|
+
base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
|
|
29
|
+
return Path(base) / "predisclose"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def github_token_from_env() -> str | None:
|
|
33
|
+
return os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Http:
|
|
37
|
+
"""
|
|
38
|
+
dry_run: record the queries that would be sent, send nothing, return (0, None).
|
|
39
|
+
use_cache: answers (200/404 only) are kept in a local sqlite file for `ttl` seconds.
|
|
40
|
+
The GitHub token is attached only to api.github.com requests and never written to
|
|
41
|
+
the query log or the cache key.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
*,
|
|
47
|
+
cache_dir: Path | None = None,
|
|
48
|
+
ttl: int = 6 * 3600,
|
|
49
|
+
use_cache: bool = True,
|
|
50
|
+
dry_run: bool = False,
|
|
51
|
+
token: str | None = None,
|
|
52
|
+
transport: httpx.BaseTransport | None = None,
|
|
53
|
+
) -> None:
|
|
54
|
+
self.queries: list[str] = []
|
|
55
|
+
self.dry_run = dry_run
|
|
56
|
+
self.ttl = ttl
|
|
57
|
+
self.token = token
|
|
58
|
+
self._db: sqlite3.Connection | None = None
|
|
59
|
+
if use_cache and not dry_run:
|
|
60
|
+
directory = cache_dir or default_cache_dir()
|
|
61
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
self._db = sqlite3.connect(directory / "http.sqlite")
|
|
63
|
+
self._db.execute(
|
|
64
|
+
"CREATE TABLE IF NOT EXISTS cache "
|
|
65
|
+
"(k TEXT PRIMARY KEY, t REAL NOT NULL, status INTEGER NOT NULL, body TEXT NOT NULL)"
|
|
66
|
+
)
|
|
67
|
+
self._client = httpx.Client(
|
|
68
|
+
timeout=20.0,
|
|
69
|
+
follow_redirects=True,
|
|
70
|
+
headers={"User-Agent": f"predisclose/{__version__}"},
|
|
71
|
+
transport=transport,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def close(self) -> None:
|
|
75
|
+
self._client.close()
|
|
76
|
+
if self._db is not None:
|
|
77
|
+
self._db.close()
|
|
78
|
+
|
|
79
|
+
# -- public helpers ---------------------------------------------------------------
|
|
80
|
+
def get_json(self, url: str, params: dict[str, Any] | None = None) -> tuple[int, Any]:
|
|
81
|
+
return self._request("GET", url, params=params, body=None, as_json=True)
|
|
82
|
+
|
|
83
|
+
def post_json(self, url: str, body: dict[str, Any]) -> tuple[int, Any]:
|
|
84
|
+
return self._request("POST", url, params=None, body=body, as_json=True)
|
|
85
|
+
|
|
86
|
+
def get_text(self, url: str) -> tuple[int, str | None]:
|
|
87
|
+
return self._request("GET", url, params=None, body=None, as_json=False)
|
|
88
|
+
|
|
89
|
+
# -- internals --------------------------------------------------------------------
|
|
90
|
+
def _request(
|
|
91
|
+
self,
|
|
92
|
+
method: str,
|
|
93
|
+
url: str,
|
|
94
|
+
*,
|
|
95
|
+
params: dict[str, Any] | None,
|
|
96
|
+
body: dict[str, Any] | None,
|
|
97
|
+
as_json: bool,
|
|
98
|
+
) -> tuple[int, Any]:
|
|
99
|
+
shown = url + ("?" + urlencode(params) if params else "")
|
|
100
|
+
line = f"{method} {shown}"
|
|
101
|
+
if body is not None:
|
|
102
|
+
line += " body=" + json.dumps(body, sort_keys=True)
|
|
103
|
+
self.queries.append(line)
|
|
104
|
+
if self.dry_run:
|
|
105
|
+
return 0, None
|
|
106
|
+
|
|
107
|
+
key = hashlib.sha256(
|
|
108
|
+
json.dumps([method, url, params or {}, body], sort_keys=True).encode()
|
|
109
|
+
).hexdigest()
|
|
110
|
+
cached = self._cache_get(key)
|
|
111
|
+
if cached is not None:
|
|
112
|
+
status, text = cached
|
|
113
|
+
return status, self._decode(text, as_json)
|
|
114
|
+
|
|
115
|
+
headers: dict[str, str] = {}
|
|
116
|
+
is_github_api = url.startswith("https://api.github.com/")
|
|
117
|
+
if is_github_api:
|
|
118
|
+
headers["Accept"] = "application/vnd.github+json"
|
|
119
|
+
headers["X-GitHub-Api-Version"] = "2022-11-28"
|
|
120
|
+
if self.token:
|
|
121
|
+
headers["Authorization"] = f"Bearer {self.token}"
|
|
122
|
+
try:
|
|
123
|
+
response = self._client.request(method, url, params=params, json=body, headers=headers)
|
|
124
|
+
except httpx.HTTPError as exc:
|
|
125
|
+
raise NetworkError(f"{method} {url}: {exc}") from exc
|
|
126
|
+
|
|
127
|
+
if is_github_api and response.status_code in (403, 429):
|
|
128
|
+
remaining = response.headers.get("x-ratelimit-remaining")
|
|
129
|
+
if response.status_code == 429 or remaining == "0" or "rate limit" in response.text.lower():
|
|
130
|
+
reset = response.headers.get("x-ratelimit-reset")
|
|
131
|
+
when = ""
|
|
132
|
+
if reset and reset.isdigit():
|
|
133
|
+
when = f" (resets at {time.strftime('%H:%M:%S', time.localtime(int(reset)))})"
|
|
134
|
+
raise RateLimited(
|
|
135
|
+
"GitHub API rate limit reached"
|
|
136
|
+
+ when
|
|
137
|
+
+ ". Set GITHUB_TOKEN to raise the limit, e.g. export GITHUB_TOKEN=$(gh auth token)"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
text = response.text
|
|
141
|
+
if response.status_code in (200, 404):
|
|
142
|
+
self._cache_put(key, response.status_code, text)
|
|
143
|
+
return response.status_code, self._decode(text, as_json)
|
|
144
|
+
|
|
145
|
+
@staticmethod
|
|
146
|
+
def _decode(text: str, as_json: bool) -> Any:
|
|
147
|
+
if not as_json:
|
|
148
|
+
return text
|
|
149
|
+
if not text:
|
|
150
|
+
return None
|
|
151
|
+
try:
|
|
152
|
+
return json.loads(text)
|
|
153
|
+
except json.JSONDecodeError:
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
def _cache_get(self, key: str) -> tuple[int, str] | None:
|
|
157
|
+
if self._db is None:
|
|
158
|
+
return None
|
|
159
|
+
row = self._db.execute("SELECT t, status, body FROM cache WHERE k = ?", (key,)).fetchone()
|
|
160
|
+
if row is None or time.time() - row[0] > self.ttl:
|
|
161
|
+
return None
|
|
162
|
+
return int(row[1]), str(row[2])
|
|
163
|
+
|
|
164
|
+
def _cache_put(self, key: str, status: int, text: str) -> None:
|
|
165
|
+
if self._db is None:
|
|
166
|
+
return
|
|
167
|
+
self._db.execute(
|
|
168
|
+
"INSERT OR REPLACE INTO cache (k, t, status, body) VALUES (?, ?, ?, ?)",
|
|
169
|
+
(key, time.time(), status, text),
|
|
170
|
+
)
|
|
171
|
+
self._db.commit()
|