crowbar-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crowbar/__init__.py +3 -0
- crowbar/catalog.py +174 -0
- crowbar/cli.py +1417 -0
- crowbar/datasets.py +584 -0
- crowbar/db.py +47 -0
- crowbar/indexer.py +985 -0
- crowbar/intel.py +562 -0
- crowbar/loaders.py +382 -0
- crowbar/mcp.py +387 -0
- crowbar/paths.py +54 -0
- crowbar/query.py +551 -0
- crowbar_cli-0.1.0.dist-info/METADATA +287 -0
- crowbar_cli-0.1.0.dist-info/RECORD +16 -0
- crowbar_cli-0.1.0.dist-info/WHEEL +4 -0
- crowbar_cli-0.1.0.dist-info/entry_points.txt +4 -0
- crowbar_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
crowbar/cli.py
ADDED
|
@@ -0,0 +1,1417 @@
|
|
|
1
|
+
"""Command line interface. Terminal only - no web UI, no TUI dependency."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import pathlib
|
|
8
|
+
import shutil
|
|
9
|
+
import sqlite3
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import textwrap
|
|
13
|
+
import time
|
|
14
|
+
from typing import NoReturn
|
|
15
|
+
|
|
16
|
+
from . import catalog, datasets, db, paths, query
|
|
17
|
+
|
|
18
|
+
TTY = sys.stdout.isatty() and os.environ.get("NO_COLOR") is None
|
|
19
|
+
DIM, BOLD, RED, YEL, GRN, CYN, MAG, RST = (
|
|
20
|
+
("\033[2m", "\033[1m", "\033[31m", "\033[33m", "\033[32m", "\033[36m", "\033[35m", "\033[0m")
|
|
21
|
+
if TTY else ("",) * 8)
|
|
22
|
+
SEV_COLOR = {"CRITICAL": RED, "HIGH": YEL, "MEDIUM": YEL, "LOW": GRN, "NONE": DIM}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# ---------------------------------------------------------------- formatting
|
|
26
|
+
def out_json(obj) -> None:
|
|
27
|
+
print(json.dumps(obj, indent=2, ensure_ascii=False, default=str))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def die(msg: str, code: int = 1) -> NoReturn:
|
|
31
|
+
print(msg, file=sys.stderr)
|
|
32
|
+
sys.exit(code)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def money(v) -> str:
|
|
36
|
+
if v is None:
|
|
37
|
+
return "-"
|
|
38
|
+
if v == 0:
|
|
39
|
+
return "free"
|
|
40
|
+
return f"{v:.2f}" if v >= 0.01 else f"{v:.4f}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def table(headers: list[str], rows: list[list], wrap_last: int = 0) -> None:
|
|
44
|
+
if not rows:
|
|
45
|
+
print(f"{DIM}no results{RST}")
|
|
46
|
+
return
|
|
47
|
+
cells = [[("" if c is None else str(c)) for c in r] for r in rows]
|
|
48
|
+
if wrap_last:
|
|
49
|
+
cells = [r[:-1] + [textwrap.shorten(r[-1], wrap_last, placeholder="…")] for r in cells]
|
|
50
|
+
widths = [max(len(h), *(len(r[i]) for r in cells)) for i, h in enumerate(headers)]
|
|
51
|
+
print(" ".join(f"{BOLD}{h:<{w}}{RST}" for h, w in zip(headers, widths)))
|
|
52
|
+
print(f"{DIM}{' '.join('─' * w for w in widths)}{RST}")
|
|
53
|
+
for r in cells:
|
|
54
|
+
print(" ".join(f"{c:<{w}}" for c, w in zip(r, widths)))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def sev_badge(sev: str | None, cvss: float | None) -> str:
|
|
58
|
+
s = (sev or "").upper()
|
|
59
|
+
label = {"CRITICAL": "CRIT", "HIGH": "HIGH", "MEDIUM": "MED", "LOW": "LOW"}.get(s, s[:4] or "?")
|
|
60
|
+
return f"{SEV_COLOR.get(s, DIM)}{label:<4}{RST} {('%.1f' % cvss) if cvss is not None else ' - '}"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def risk_bits(r) -> str:
|
|
64
|
+
bits = []
|
|
65
|
+
if r["kev"]:
|
|
66
|
+
bits.append(f"{RED}KEV{RST}")
|
|
67
|
+
if r["epss"] is not None:
|
|
68
|
+
bits.append(f"epss {r['epss']:.2f}")
|
|
69
|
+
return " ".join(bits)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def connect_or_die():
|
|
73
|
+
try:
|
|
74
|
+
con = db.connect()
|
|
75
|
+
except db.NotIndexed as exc:
|
|
76
|
+
die(f"{exc}")
|
|
77
|
+
if db.schema_version(con) != paths.SCHEMA_VERSION:
|
|
78
|
+
die(f"index schema v{db.schema_version(con)} != v{paths.SCHEMA_VERSION} - run: crowbar build")
|
|
79
|
+
if datasets.refresh_in_background():
|
|
80
|
+
print(f"{DIM}index is over a day old - refreshing in the background{RST}", file=sys.stderr)
|
|
81
|
+
return con
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# ---------------------------------------------------------------- read commands
|
|
85
|
+
def cmd_search(args) -> None:
|
|
86
|
+
con = connect_or_die()
|
|
87
|
+
rows = query.search(
|
|
88
|
+
con, args.query or "", limit=args.limit, offset=args.offset, sort=args.sort,
|
|
89
|
+
severity=args.severity, year=args.year, since=args.since, kev=args.kev,
|
|
90
|
+
exploit=args.exploit, epss=args.epss, cvss=args.cvss, cwe=args.cwe,
|
|
91
|
+
product=args.product, vendor=args.vendor, ssvc=args.ssvc, poc=args.poc)
|
|
92
|
+
if args.json:
|
|
93
|
+
out_json(rows)
|
|
94
|
+
return
|
|
95
|
+
for r in rows:
|
|
96
|
+
tag = f" {CYN}exp×{r['exploits']}{RST}" if r["exploits"] else ""
|
|
97
|
+
tag += f" {DIM}nuc×{r['nuclei_count']}{RST}" if r["nuclei_count"] else ""
|
|
98
|
+
tag += f" {MAG}poc×{r['poc_count']}{RST}" if r.get("poc_count") else ""
|
|
99
|
+
print(f"{BOLD}{r['id']}{RST} {sev_badge(r['severity'], r['cvss'])} {risk_bits(r)}{tag}".rstrip())
|
|
100
|
+
print(f" {DIM}{(r['published'] or '')[:10]}{RST} "
|
|
101
|
+
f"{textwrap.shorten(r['description'] or '', args.width - 4, placeholder='…')}")
|
|
102
|
+
print(f"\n{DIM}{len(rows)} shown · next page: --offset {args.offset + args.limit}{RST}")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def cmd_browse(args) -> None:
|
|
106
|
+
"""Interactive fzf browser: type to filter, preview the CVE, enter to open."""
|
|
107
|
+
con = connect_or_die()
|
|
108
|
+
fzf = shutil.which("fzf")
|
|
109
|
+
rows = query.search(con, args.query or "", limit=args.limit, kev=args.kev,
|
|
110
|
+
exploit=args.exploit, sort="relevance" if args.query else "newest")
|
|
111
|
+
if not fzf or not (sys.stdin.isatty() and sys.stdout.isatty()):
|
|
112
|
+
for r in rows:
|
|
113
|
+
print(f"{r['id']} {sev_badge(r['severity'], r['cvss'])} {risk_bits(r)} "
|
|
114
|
+
f"{(r['description'] or '')[:80]}")
|
|
115
|
+
if not fzf:
|
|
116
|
+
print("\n(fzf not installed - showing plain list)", file=sys.stderr)
|
|
117
|
+
return
|
|
118
|
+
lines = "\n".join(
|
|
119
|
+
f"{r['id']}\t{r['severity'] or '-'}\t{r['cvss'] if r['cvss'] is not None else '-'}\t"
|
|
120
|
+
f"{'KEV' if r['kev'] else ''}\t{(r['description'] or '')[:110]}"
|
|
121
|
+
for r in rows)
|
|
122
|
+
hv = shutil.which("crowbar") or str(paths.ROOT / "bin" / "crowbar")
|
|
123
|
+
pick = subprocess.run(
|
|
124
|
+
[fzf, "--ansi", "--delimiter=\t", "--with-nth=1,2,3,4,5",
|
|
125
|
+
"--header", f"{len(rows)} CVEs - enter to open, esc to quit",
|
|
126
|
+
"--preview", f"{hv} cve {{1}} --width 100",
|
|
127
|
+
"--preview-window", "right:62%:wrap"],
|
|
128
|
+
input=lines, text=True, capture_output=True)
|
|
129
|
+
cid = (pick.stdout or "").split("\t")[0].strip()
|
|
130
|
+
if cid:
|
|
131
|
+
subprocess.run([hv, "cve", cid])
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def cmd_cve(args) -> None:
|
|
135
|
+
con = connect_or_die()
|
|
136
|
+
try:
|
|
137
|
+
d = query.get_cve(con, args.id)
|
|
138
|
+
except KeyError as exc:
|
|
139
|
+
msg = exc.args[0]
|
|
140
|
+
die(msg if " " in msg else f"{msg} not in index")
|
|
141
|
+
if args.json:
|
|
142
|
+
out_json(d)
|
|
143
|
+
return
|
|
144
|
+
print(f"{BOLD}{d['id']}{RST} {sev_badge(d['severity'], d['cvss'])} {risk_bits(d)}")
|
|
145
|
+
print(f"{BOLD}{textwrap.shorten(d['description'] or '', args.width, placeholder='…')}{RST}\n")
|
|
146
|
+
print(f" {DIM}published{RST} {d['published']} {DIM}modified{RST} {d['modified']}"
|
|
147
|
+
f" {DIM}status{RST} {d['status']}")
|
|
148
|
+
if d["cvss_vector"]:
|
|
149
|
+
print(f" {DIM}vector{RST} {d['cvss_vector']}")
|
|
150
|
+
if d["epss"] is not None:
|
|
151
|
+
print(f" {DIM}epss{RST} {d['epss']:.4f} (percentile {d['epss_pct']:.2f})")
|
|
152
|
+
if d["cwe"]:
|
|
153
|
+
print(f" {DIM}weakness{RST} {', '.join(d['cwe'])}")
|
|
154
|
+
if d["affects"]:
|
|
155
|
+
labels = [a["label"] for a in d["affects"]]
|
|
156
|
+
print(f" {DIM}affects{RST} {textwrap.shorten(', '.join(labels[:10]), args.width - 14, placeholder=f'… (+{len(labels) - 10} ranges)')}")
|
|
157
|
+
if d["ssvc"]:
|
|
158
|
+
s = d["ssvc"]
|
|
159
|
+
print(f" {DIM}ssvc{RST} exploitation={s['exploitation']} automatable={s['automatable']}"
|
|
160
|
+
f" impact={s['technical_impact']}")
|
|
161
|
+
print()
|
|
162
|
+
if d["kev_detail"]:
|
|
163
|
+
k = d["kev_detail"]
|
|
164
|
+
print(f"{RED}▌ CISA KEV{RST} added {k['date_added']} due {k['due']} ransomware={k['ransomware']}")
|
|
165
|
+
print(f" {k['vendor']} {k['product']} - {k['name']}")
|
|
166
|
+
if k["action"]:
|
|
167
|
+
print(f" {DIM}action:{RST} {textwrap.shorten(k['action'], args.width - 10, placeholder='…')}")
|
|
168
|
+
print()
|
|
169
|
+
if d["exploits"]:
|
|
170
|
+
print(f"{MAG}▌ Exploit-DB ({len(d['exploits'])}){RST}")
|
|
171
|
+
for e in d["exploits"][:8]:
|
|
172
|
+
print(f" {e['id']:>6} {(e['platform'] or '-'):<10} {(e['type'] or '-'):<8} "
|
|
173
|
+
f"{textwrap.shorten(e['description'] or '', args.width - 34, placeholder='…')}")
|
|
174
|
+
print()
|
|
175
|
+
if d["nuclei"]:
|
|
176
|
+
print(f"{CYN}▌ Nuclei ({len(d['nuclei'])}){RST}")
|
|
177
|
+
for n in d["nuclei"][:8]:
|
|
178
|
+
print(f" {(n['severity'] or '?'):<8} {n['id']} {DIM}{n['path']}{RST}")
|
|
179
|
+
print()
|
|
180
|
+
if d["metasploit"]:
|
|
181
|
+
print(f"{CYN}▌ Metasploit ({len(d['metasploit'])}){RST}")
|
|
182
|
+
for m in d["metasploit"][:8]:
|
|
183
|
+
print(f" {m['type']:<9} {m['name']}")
|
|
184
|
+
print()
|
|
185
|
+
if d.get("pocs"):
|
|
186
|
+
print(f"{MAG}▌ Public PoCs ({d['poc_count']}){RST}")
|
|
187
|
+
for p in d["pocs"][:10]:
|
|
188
|
+
print(f" ★{p['stars']:<6} {p['repo']} {DIM}{p['url']}{RST}")
|
|
189
|
+
print()
|
|
190
|
+
if d.get("cna") and d["cna"].get("description") and d["cna"]["description"] != d["description"]:
|
|
191
|
+
print(f"{DIM}CNA record ({d['cna']['assigner'] or 'unknown'}):{RST} "
|
|
192
|
+
f"{textwrap.shorten(d['cna']['description'], args.width - 4, placeholder='…')}\n")
|
|
193
|
+
if d.get("aliases"):
|
|
194
|
+
print(f"{DIM}aliases{RST} {', '.join(d['aliases'][:20])}")
|
|
195
|
+
if d["references"]:
|
|
196
|
+
print(f"{DIM}references ({len(d['references'])}){RST}")
|
|
197
|
+
for ref in d["references"][:10]:
|
|
198
|
+
print(f" {DIM}·{RST} {ref['url']} {DIM}{ref['tags']}{RST}")
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def cmd_product(args) -> None:
|
|
202
|
+
con = connect_or_die()
|
|
203
|
+
rows = query.products(con, args.name, version=args.version, limit=args.limit)
|
|
204
|
+
if args.json:
|
|
205
|
+
out_json(rows)
|
|
206
|
+
return
|
|
207
|
+
head = f"CVEs affecting {args.name!r}"
|
|
208
|
+
if args.version:
|
|
209
|
+
head += f" version {args.version}"
|
|
210
|
+
print(f"{DIM}{head}{RST}\n")
|
|
211
|
+
for r in rows:
|
|
212
|
+
print(f"{BOLD}{r['id']}{RST} {sev_badge(r['severity'], r['cvss'])} {risk_bits(r)} "
|
|
213
|
+
f"{DIM}{(r['published'] or '')[:10]}{RST}")
|
|
214
|
+
if r["matched"]:
|
|
215
|
+
print(f" {DIM}matched{RST} {', '.join(r['matched'])}")
|
|
216
|
+
print(f" {textwrap.shorten(r['description'] or '', args.width - 2, placeholder='…')}")
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def cmd_kev(args) -> None:
|
|
220
|
+
con = connect_or_die()
|
|
221
|
+
rows = query.kev_list(con, args.query, args.limit)
|
|
222
|
+
if args.json:
|
|
223
|
+
out_json(rows)
|
|
224
|
+
return
|
|
225
|
+
for r in rows:
|
|
226
|
+
rw = f"{RED}ransom{RST}" if (r["ransomware"] or "").lower() == "known" else ""
|
|
227
|
+
print(f"{BOLD}{r['cve']}{RST} {DIM}{r['date_added']}{RST} {rw}")
|
|
228
|
+
print(f" {r['vendor']} {r['product']} - {r['name']}")
|
|
229
|
+
print(f"\n{DIM}{len(rows)} entries{RST}")
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def cmd_exploit(args) -> None:
|
|
233
|
+
con = connect_or_die()
|
|
234
|
+
if args.query.isdigit():
|
|
235
|
+
try:
|
|
236
|
+
e = query.exploit_fetch(con, args.query)
|
|
237
|
+
except KeyError:
|
|
238
|
+
die(f"exploit {args.query} not found")
|
|
239
|
+
if args.json:
|
|
240
|
+
out_json(e)
|
|
241
|
+
else:
|
|
242
|
+
print(f"{BOLD}{e['id']}{RST} {e['platform']} {e['type']} {e['description']}")
|
|
243
|
+
print(f"{DIM}local:{RST} {e['local_path']}")
|
|
244
|
+
return
|
|
245
|
+
rows = query.exploit_search(con, args.query, args.limit)
|
|
246
|
+
if args.json:
|
|
247
|
+
out_json([{k: v for k, v in r.items() if k != "rank"} for r in rows])
|
|
248
|
+
return
|
|
249
|
+
for r in rows:
|
|
250
|
+
v = f"{GRN}✓{RST}" if r["verified"] else " "
|
|
251
|
+
print(f"{v} {BOLD}{r['id']:>6}{RST} {DIM}{(r['date'] or '')[:10]:<10}{RST} "
|
|
252
|
+
f"{(r['platform'] or '-'):<12} "
|
|
253
|
+
f"{textwrap.shorten(r['description'] or '', args.width - 40, placeholder='…')}")
|
|
254
|
+
if r["codes"]:
|
|
255
|
+
print(f" {DIM}{r['codes']}{RST}")
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def cmd_nuclei(args) -> None:
|
|
259
|
+
con = connect_or_die()
|
|
260
|
+
rows = query.nuclei_search(con, args.query, args.limit, args.domain)
|
|
261
|
+
if args.json:
|
|
262
|
+
out_json(rows)
|
|
263
|
+
return
|
|
264
|
+
for r in rows:
|
|
265
|
+
print(f"{BOLD}{r['id']}{RST} [{r['severity'] or '?'}] {r['domain']:<9} "
|
|
266
|
+
f"{textwrap.shorten(r['name'] or '', args.width - 40, placeholder='…')}")
|
|
267
|
+
print(f" {DIM}{r['path']} {r['cve']}{RST}")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def cmd_msf(args) -> None:
|
|
271
|
+
con = connect_or_die()
|
|
272
|
+
rows = query.msf_search(con, args.query, args.limit)
|
|
273
|
+
if args.json:
|
|
274
|
+
out_json(rows)
|
|
275
|
+
return
|
|
276
|
+
for r in rows:
|
|
277
|
+
print(f"{r['type']:<10} {BOLD}{r['name']}{RST}")
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def cmd_models(args) -> None:
|
|
281
|
+
con = connect_or_die()
|
|
282
|
+
rows = query.models_list(con, args.query, args.provider, args.limit)
|
|
283
|
+
if args.json:
|
|
284
|
+
out_json(rows)
|
|
285
|
+
return
|
|
286
|
+
table(["provider", "model", "ctx", "out", "$in/M", "$out/M", "cap"],
|
|
287
|
+
[[r["provider"], r["name"], f"{(r['context'] or 0) // 1000}k",
|
|
288
|
+
f"{(r['output_limit'] or 0) // 1000}k", money(r["input_cost"]),
|
|
289
|
+
money(r["output_cost"]),
|
|
290
|
+
"".join(x for x, on in (("R", r["reasoning"]), ("T", r["tool_call"]), ("V", r["vision"])) if on)]
|
|
291
|
+
for r in rows])
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def cmd_attack(args) -> None:
|
|
295
|
+
con = connect_or_die()
|
|
296
|
+
rows = query.attack_lookup(con, args.query, args.limit, args.domain)
|
|
297
|
+
if args.json:
|
|
298
|
+
out_json(rows)
|
|
299
|
+
return
|
|
300
|
+
for r in rows:
|
|
301
|
+
print(f"{BOLD}{r['id']}{RST} {r['name']} {DIM}{r['domain']} · {r['tactics']}{RST}")
|
|
302
|
+
print(f" {textwrap.shorten(r['description'] or '', args.width - 2, placeholder='…')}")
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def cmd_cwe(args) -> None:
|
|
306
|
+
con = connect_or_die()
|
|
307
|
+
try:
|
|
308
|
+
res = query.cwe_get(con, args.query, args.limit)
|
|
309
|
+
except KeyError as exc:
|
|
310
|
+
die(f"{exc.args[0]} not found")
|
|
311
|
+
if args.json:
|
|
312
|
+
out_json(res)
|
|
313
|
+
return
|
|
314
|
+
if isinstance(res, dict):
|
|
315
|
+
print(f"{BOLD}{res['id']}{RST} {res['name']} {DIM}{res['abstraction']} {res['status']}{RST}\n")
|
|
316
|
+
print(textwrap.fill(res["description"] or "", args.width))
|
|
317
|
+
return
|
|
318
|
+
for r in res:
|
|
319
|
+
print(f"{r['id']:<9} {r['name']}")
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def cmd_capec(args) -> None:
|
|
323
|
+
con = connect_or_die()
|
|
324
|
+
rows = query.capec_search(con, args.query, args.limit)
|
|
325
|
+
if args.json:
|
|
326
|
+
out_json(rows)
|
|
327
|
+
return
|
|
328
|
+
for r in rows:
|
|
329
|
+
print(f"{r['id']:<12} {r['name']} {DIM}{r['severity'] or ''} {r['cwe']}{RST}")
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def cmd_os(args) -> None:
|
|
333
|
+
con = connect_or_die()
|
|
334
|
+
rows = query.os_search(con, args.query, args.platform, args.impact,
|
|
335
|
+
args.exploited, args.since, args.limit)
|
|
336
|
+
if args.json:
|
|
337
|
+
out_json(rows)
|
|
338
|
+
return
|
|
339
|
+
head = "OS advisories"
|
|
340
|
+
if args.platform:
|
|
341
|
+
head += f" · {args.platform}"
|
|
342
|
+
if args.impact:
|
|
343
|
+
head += f" · {args.impact}"
|
|
344
|
+
if args.exploited:
|
|
345
|
+
head += " · exploited"
|
|
346
|
+
print(f"{DIM}{head}{RST}\n")
|
|
347
|
+
for r in rows:
|
|
348
|
+
cve = r["cve"] or " - "
|
|
349
|
+
print(f"{BOLD}{cve:<16}{RST} {r['platform']:<6} {r['severity'] or '-':<9} "
|
|
350
|
+
f"{(r['impact'] or '-'):<14} {DIM}{(r['released'] or '')[:10]}{RST} "
|
|
351
|
+
f"{textwrap.shorten(r['product'] or '', args.width - 70, placeholder='…')}")
|
|
352
|
+
if r["version"] and r["version"] not in (r["product"] or ""):
|
|
353
|
+
print(f" {DIM}{r['version']}{RST}")
|
|
354
|
+
print(f"\n{DIM}{len(rows)} rows{RST}")
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def cmd_advisory(args) -> None:
|
|
358
|
+
con = connect_or_die()
|
|
359
|
+
rows = query.advisory_search(con, args.query, args.ecosystem, args.package, args.limit)
|
|
360
|
+
if args.json:
|
|
361
|
+
out_json(rows)
|
|
362
|
+
return
|
|
363
|
+
for r in rows:
|
|
364
|
+
cve = f" {DIM}{r['cve']}{RST}" if r["cve"] else ""
|
|
365
|
+
print(f"{BOLD}{r['id']}{RST} [{r['ecosystem'] or '?'}] {r['package']}{cve}")
|
|
366
|
+
if r["summary"]:
|
|
367
|
+
print(f" {textwrap.shorten(r['summary'], args.width - 2, placeholder='…')}")
|
|
368
|
+
for x in r.get("ranges", [])[:4]:
|
|
369
|
+
bits = []
|
|
370
|
+
if x["introduced"]:
|
|
371
|
+
bits.append(f">={x['introduced']}")
|
|
372
|
+
if x["fixed"]:
|
|
373
|
+
bits.append(f"fixed {x['fixed']}")
|
|
374
|
+
if x["last_affected"]:
|
|
375
|
+
bits.append(f"<={x['last_affected']}")
|
|
376
|
+
if x["versions"]:
|
|
377
|
+
bits.append(x["versions"][:80])
|
|
378
|
+
print(f" {DIM}{x['package']} {' '.join(bits)}{RST}")
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def cmd_code(args) -> None:
|
|
382
|
+
con = connect_or_die()
|
|
383
|
+
rows = query.code_search(con, args.query, args.limit)
|
|
384
|
+
if args.json:
|
|
385
|
+
out_json(rows)
|
|
386
|
+
return
|
|
387
|
+
for r in rows:
|
|
388
|
+
print(f"{BOLD}{r['path']}{RST} {DIM}{r['exploit_id'] or ''}{RST}")
|
|
389
|
+
print(f" {r['snippet']}")
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def cmd_bin(args) -> None:
|
|
393
|
+
con = connect_or_die()
|
|
394
|
+
rows = query.toolkit_search(con, args.query, args.source, args.limit)
|
|
395
|
+
if args.json:
|
|
396
|
+
out_json(rows)
|
|
397
|
+
return
|
|
398
|
+
for r in rows:
|
|
399
|
+
print(f"{BOLD}{r['name']}{RST} {DIM}{r['source']}{RST} "
|
|
400
|
+
f"{textwrap.shorten(r['description'] or '', args.width - 30, placeholder='…')}")
|
|
401
|
+
print(f" {DIM}{r['path']} {r['tags']}{RST}")
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def cmd_debian(args) -> None:
|
|
405
|
+
con = connect_or_die()
|
|
406
|
+
rows = query.debian_lookup(con, args.package, args.cve)
|
|
407
|
+
if args.json:
|
|
408
|
+
out_json(rows)
|
|
409
|
+
return
|
|
410
|
+
for r in rows:
|
|
411
|
+
print(f"{BOLD}{r['cve']}{RST} {r['release']:<14} {r['status']:<16} "
|
|
412
|
+
f"{DIM}fixed {r['fixed'] or '-'}{RST} {r['urgency'] or ''}")
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def cmd_poc(args) -> None:
|
|
416
|
+
"""Public GitHub PoC repositories (metadata only, never executed)."""
|
|
417
|
+
con = connect_or_die()
|
|
418
|
+
if args.cve:
|
|
419
|
+
rows = query.poc_search(con, args.query or "", cve=args.cve,
|
|
420
|
+
language=args.language, min_stars=args.min_stars,
|
|
421
|
+
limit=args.limit)
|
|
422
|
+
elif args.top:
|
|
423
|
+
rows = query.poc_cves(con, args.limit)
|
|
424
|
+
if args.json:
|
|
425
|
+
out_json(rows)
|
|
426
|
+
return
|
|
427
|
+
for r in rows:
|
|
428
|
+
print(f"{BOLD}{r['cve_id']}{RST} {DIM}×{r['poc_count']} repos{RST} "
|
|
429
|
+
f"{r['severity'] or '-'} {r['cvss'] if r['cvss'] is not None else '-'} "
|
|
430
|
+
f"{r['description'] or ''}")
|
|
431
|
+
return
|
|
432
|
+
else:
|
|
433
|
+
rows = query.poc_search(con, args.query or "", language=args.language,
|
|
434
|
+
min_stars=args.min_stars, limit=args.limit)
|
|
435
|
+
if args.json:
|
|
436
|
+
out_json(rows)
|
|
437
|
+
return
|
|
438
|
+
for r in rows:
|
|
439
|
+
print(f"{BOLD}{r['cve_id']}{RST} {DIM}★{r['stars']}{RST} {r['repo']}")
|
|
440
|
+
print(f" {DIM}{r['url']}{RST} {r['language'] or ''} {r['description'] or ''}")
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def cmd_sources(args) -> None:
|
|
444
|
+
"""Provenance: which source filled which table, with license and hash."""
|
|
445
|
+
con = connect_or_die()
|
|
446
|
+
rows = [dict(r) for r in con.execute(
|
|
447
|
+
"SELECT source, url, path, license, sha256, fetched_at, loaded_at, "
|
|
448
|
+
"record_count, status, note FROM source_state ORDER BY source")]
|
|
449
|
+
if args.json:
|
|
450
|
+
out_json(rows)
|
|
451
|
+
return
|
|
452
|
+
if not rows:
|
|
453
|
+
print("no source_state rows (run: crowbar update)")
|
|
454
|
+
return
|
|
455
|
+
for r in rows:
|
|
456
|
+
print(f"{BOLD}{r['source']}{RST} {r['status']} {r['record_count']:,} records "
|
|
457
|
+
f"{DIM}license={r['license'] or '?'}{RST}")
|
|
458
|
+
print(f" {DIM}{r['url']}{RST}")
|
|
459
|
+
print(f" {DIM}sha256 {r['sha256'] or '-'}{RST}")
|
|
460
|
+
if r["note"]:
|
|
461
|
+
print(f" {DIM}{r['note']}{RST}")
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def cmd_map(args) -> None:
|
|
465
|
+
if args.tables:
|
|
466
|
+
for t, desc in catalog.TABLES.items():
|
|
467
|
+
print(f"{BOLD}{t:<20}{RST} {desc}")
|
|
468
|
+
return
|
|
469
|
+
manifest = datasets.load_manifest().get("datasets", {})
|
|
470
|
+
items = datasets.registry(with_extra=True, with_bulk=True)
|
|
471
|
+
rows = []
|
|
472
|
+
for d in items:
|
|
473
|
+
present, size = catalog.disk_state(d.dest)
|
|
474
|
+
g = catalog.describe(d.name)
|
|
475
|
+
rows.append({
|
|
476
|
+
"name": d.name, "group": d.group, "tier": d.tier,
|
|
477
|
+
"path": d.dest, "present": present, "bytes": size,
|
|
478
|
+
"fetched_at": (manifest.get(d.name) or {}).get("fetched_at"),
|
|
479
|
+
"tables": g["tables"], "commands": g["commands"], "note": d.note,
|
|
480
|
+
})
|
|
481
|
+
if args.json:
|
|
482
|
+
out_json(rows)
|
|
483
|
+
return
|
|
484
|
+
table(["dataset", "tier", "status", "size", "indexes", "command"],
|
|
485
|
+
[[r["name"], r["tier"], "ok" if r["present"] else "-",
|
|
486
|
+
human_bytes(r["bytes"]), ",".join(r["tables"]) or "-",
|
|
487
|
+
(r["commands"] or ["-"])[0]] for r in rows])
|
|
488
|
+
print(f"\n{DIM}raw path root: {paths.DATA}{RST}")
|
|
489
|
+
print(f"{DIM}table meanings: crowbar map --tables{RST}")
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def human_bytes(n: float) -> str:
|
|
493
|
+
for unit in ("B", "K", "M", "G"):
|
|
494
|
+
if n < 1024 or unit == "G":
|
|
495
|
+
return f"{n:.0f}{unit}" if unit == "B" else f"{n:.1f}{unit}"
|
|
496
|
+
n /= 1024
|
|
497
|
+
return f"{n:.1f}G"
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def cmd_sql(args) -> None:
|
|
501
|
+
con = connect_or_die()
|
|
502
|
+
try:
|
|
503
|
+
rows = query.sql_readonly(con, args.query, args.limit)
|
|
504
|
+
except (ValueError, Exception) as exc: # noqa: BLE001
|
|
505
|
+
die(f"sql error: {exc}")
|
|
506
|
+
if args.json:
|
|
507
|
+
out_json(rows)
|
|
508
|
+
return
|
|
509
|
+
if rows:
|
|
510
|
+
table(list(rows[0]), [list(r.values()) for r in rows])
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def cmd_stats(args) -> None:
|
|
514
|
+
con = connect_or_die()
|
|
515
|
+
s = query.stats(con)
|
|
516
|
+
if args.json:
|
|
517
|
+
out_json(s)
|
|
518
|
+
return
|
|
519
|
+
meta = s.pop("meta", {})
|
|
520
|
+
fresh = s.pop("freshness", {})
|
|
521
|
+
print(f"{BOLD}corpus{RST}")
|
|
522
|
+
for k in ("cve", "cve_kev", "cve_with_epss", "cve_with_exploit", "cve_with_nuclei",
|
|
523
|
+
"cve_with_msf", "cpe_match", "kev", "exploit", "shellcode", "nuclei", "msf",
|
|
524
|
+
"attack", "cwe", "capec", "model", "ssvc"):
|
|
525
|
+
if k in s:
|
|
526
|
+
print(f" {k:<18} {s[k]:>10,}")
|
|
527
|
+
print(f" {'db':<18} {s['db_bytes'] / 1e6:>8.0f} MB {DIM}{paths.DB}{RST}")
|
|
528
|
+
print(f"\n{BOLD}state{RST}")
|
|
529
|
+
for k in ("schema", "built_at", "updated_at"):
|
|
530
|
+
print(f" {k:<12} {meta.get(k, ' - ')}")
|
|
531
|
+
if fresh:
|
|
532
|
+
newest = max((v["fetched_at"] or "" for v in fresh.values()), default="")
|
|
533
|
+
print(f" {'datasets':<12} {len(fresh)} files, newest {newest}")
|
|
534
|
+
print(f"\n{BOLD}coverage{RST}")
|
|
535
|
+
if s.get("cve"):
|
|
536
|
+
for k in ("cve_kev", "cve_with_epss", "cve_with_exploit", "cve_with_nuclei", "cve_with_msf"):
|
|
537
|
+
print(f" {k:<18} {100 * s.get(k, 0) / s['cve']:>5.1f}%")
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def cmd_where(args) -> None:
|
|
541
|
+
info = {
|
|
542
|
+
"home": str(paths.ROOT),
|
|
543
|
+
"home_link": str(paths.HOME_LINK) + (" (symlink)" if paths.HOME_LINK.is_symlink() else ""),
|
|
544
|
+
"db": str(paths.DB),
|
|
545
|
+
"data": str(paths.DATA),
|
|
546
|
+
"manifest": str(paths.MANIFEST),
|
|
547
|
+
"drive": paths.is_drive_root(),
|
|
548
|
+
"datasets": {k: v["path"] for k, v in datasets.freshness().items()},
|
|
549
|
+
}
|
|
550
|
+
if args.json:
|
|
551
|
+
out_json(info)
|
|
552
|
+
return
|
|
553
|
+
print(f"{BOLD}root{RST} {info['home']}")
|
|
554
|
+
print(f"{BOLD}home link{RST} {info['home_link']}")
|
|
555
|
+
print(f"{BOLD}index{RST} {info['db']}")
|
|
556
|
+
print(f"{BOLD}data{RST} {info['data']}")
|
|
557
|
+
print(f"{BOLD}drive{RST} {'yes' if info['drive'] else 'no'}")
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
# ---------------------------------------------------------------- data ops
|
|
561
|
+
def _report(r: dict) -> None:
|
|
562
|
+
if r["status"] == "FAIL":
|
|
563
|
+
print(f" FAIL {r['name']:<22} {r.get('error', '')}", file=sys.stderr)
|
|
564
|
+
elif r["status"] in ("same", "exists"):
|
|
565
|
+
print(f" same {r['name']:<22}")
|
|
566
|
+
else:
|
|
567
|
+
print(f" ok {r['name']:<22} {datasets.human(r.get('bytes', 0)):>9s}")
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _record_tier(tier: str) -> None:
|
|
571
|
+
m = datasets.load_manifest()
|
|
572
|
+
m["tier"] = tier
|
|
573
|
+
datasets.save_manifest(m)
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _writer_lock() -> None:
|
|
577
|
+
if not datasets.acquire_update_lock():
|
|
578
|
+
die("another crowbar sync/build/update is running - wait for it to finish")
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def cmd_sync(args) -> None:
|
|
582
|
+
_writer_lock()
|
|
583
|
+
years = None
|
|
584
|
+
if args.years:
|
|
585
|
+
years = []
|
|
586
|
+
for part in args.years.split(","):
|
|
587
|
+
if "-" in part:
|
|
588
|
+
a, b = part.split("-", 1)
|
|
589
|
+
years += list(range(int(a), int(b) + 1))
|
|
590
|
+
else:
|
|
591
|
+
years.append(int(part))
|
|
592
|
+
if getattr(args, "tier", None):
|
|
593
|
+
items = datasets.tier_plan(args.tier)
|
|
594
|
+
_record_tier(args.tier)
|
|
595
|
+
else:
|
|
596
|
+
items = datasets.plan(
|
|
597
|
+
only=[s for s in (args.only or "").split(",") if s] or None,
|
|
598
|
+
years=years, with_extra=args.extra, with_bulk=args.bulk)
|
|
599
|
+
if not items:
|
|
600
|
+
die("nothing selected")
|
|
601
|
+
print(f"syncing {len(items)} datasets -> {paths.DATA}"
|
|
602
|
+
+ (f" (tier {args.tier})" if getattr(args, "tier", None) else ""))
|
|
603
|
+
results = datasets.sync(items, force=args.force, jobs=args.jobs,
|
|
604
|
+
pull=args.pull, progress=_report)
|
|
605
|
+
bad = [r for r in results if r["status"] == "FAIL"]
|
|
606
|
+
print(f"{len(results) - len(bad)}/{len(results)} ready")
|
|
607
|
+
if bad:
|
|
608
|
+
die(", ".join(r["name"] for r in bad))
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def cmd_build(args) -> None:
|
|
612
|
+
from . import indexer
|
|
613
|
+
if not paths.DATA.exists() or not any(paths.DATA.iterdir()):
|
|
614
|
+
die("no data - run: crowbar sync")
|
|
615
|
+
_writer_lock()
|
|
616
|
+
if getattr(args, "tier", None):
|
|
617
|
+
_record_tier(args.tier)
|
|
618
|
+
missing = datasets.tier_missing(args.tier)
|
|
619
|
+
if missing and not getattr(args, "allow_incomplete", False):
|
|
620
|
+
die(f"tier {args.tier} is missing {', '.join(missing)} - nothing to build yet. "
|
|
621
|
+
f"Run: crowbar sync --tier {args.tier} --force (or build --allow-incomplete)")
|
|
622
|
+
if missing:
|
|
623
|
+
print(f"{YEL}WARNING: building an incomplete {args.tier} index "
|
|
624
|
+
f"(missing {', '.join(missing)}){RST}", file=sys.stderr)
|
|
625
|
+
_prog = (lambda msg: print(msg, file=sys.stderr)) if args.json else print
|
|
626
|
+
stats = indexer.build(quick=args.quick, with_extra=args.extra,
|
|
627
|
+
with_bulk=args.bulk, progress=_prog)
|
|
628
|
+
if args.json:
|
|
629
|
+
out_json(stats)
|
|
630
|
+
return
|
|
631
|
+
print(f"\nbuilt {stats['cve']:,} CVEs · {stats.get('cpe_match', 0):,} CPE ranges · "
|
|
632
|
+
f"{datasets.human(stats['db_bytes'])} in {stats['build_seconds']}s")
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def cmd_update(args) -> None:
|
|
636
|
+
from . import indexer
|
|
637
|
+
if args.if_mounted and not _mounted():
|
|
638
|
+
return
|
|
639
|
+
_writer_lock()
|
|
640
|
+
status = datasets.index_status()
|
|
641
|
+
tier = status["tier"]
|
|
642
|
+
if status["stale"]:
|
|
643
|
+
if tier:
|
|
644
|
+
# the modified feed only reaches back 8 days: re-sync the tier and rebuild
|
|
645
|
+
ns = argparse.Namespace(tier=tier, years=None, force=False, jobs=args.jobs, pull=False,
|
|
646
|
+
allow_incomplete=True, quick=False, extra=False, bulk=False,
|
|
647
|
+
json=args.json)
|
|
648
|
+
cmd_sync(ns)
|
|
649
|
+
cmd_build(ns)
|
|
650
|
+
return
|
|
651
|
+
print(f"{YEL}index is {status['age_days']} days old, past the 8-day NVD modified feed - "
|
|
652
|
+
f"CVEs changed in the gap stay stale until: crowbar sync && crowbar build{RST}",
|
|
653
|
+
file=sys.stderr)
|
|
654
|
+
groups = ["nvd", "kev", "epss", "exploitdb", "models", "cveorg", "poc"]
|
|
655
|
+
if args.deep:
|
|
656
|
+
groups += ["nuclei", "metasploit", "attack", "cwe", "capec"]
|
|
657
|
+
if tier and tier != "extreme":
|
|
658
|
+
groups = [g for g in groups if g in datasets.tier_groups(tier)]
|
|
659
|
+
items = [d for d in datasets.plan(with_extra=args.extra)
|
|
660
|
+
if d.group in groups and d.name != "nvd-2002"]
|
|
661
|
+
# only the incremental NVD feeds here; year snapshots are for `crowbar build`
|
|
662
|
+
items = [d for d in items if not (d.group == "nvd" and d.name not in ("nvd-recent", "nvd-modified"))]
|
|
663
|
+
if not args.no_sync:
|
|
664
|
+
if not args.quiet:
|
|
665
|
+
print(f"syncing {len(items)} datasets")
|
|
666
|
+
datasets.sync(items, jobs=args.jobs, pull=args.deep, progress=None if args.quiet else _report)
|
|
667
|
+
stats = indexer.update(deep=args.deep, progress=None if args.quiet else print)
|
|
668
|
+
if args.json:
|
|
669
|
+
out_json(stats)
|
|
670
|
+
elif not args.quiet:
|
|
671
|
+
print(f"updated {stats['rows_touched']} CVE rows in {stats['seconds']}s "
|
|
672
|
+
f"({stats['cve']:,} total)")
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def cmd_doctor(args) -> None:
|
|
676
|
+
ok = True
|
|
677
|
+
print(f"{BOLD}crowbar doctor{RST}")
|
|
678
|
+
print(f" python {sys.version.split()[0]}")
|
|
679
|
+
print(f" root {paths.ROOT}")
|
|
680
|
+
print(f" drive {'yes' if paths.is_drive_root() else 'no'}")
|
|
681
|
+
if paths.HOME_LINK.is_symlink():
|
|
682
|
+
print(f" home link {paths.HOME_LINK} -> {os.readlink(paths.HOME_LINK)}")
|
|
683
|
+
if not paths.DB.exists():
|
|
684
|
+
print(" index MISSING - run: crowbar sync && crowbar build")
|
|
685
|
+
sys.exit(1)
|
|
686
|
+
con = db.connect()
|
|
687
|
+
ver = db.schema_version(con)
|
|
688
|
+
print(f" schema v{ver} {'ok' if ver == paths.SCHEMA_VERSION else 'MISMATCH - rebuild'}")
|
|
689
|
+
ok = ok and ver == paths.SCHEMA_VERSION
|
|
690
|
+
_integ = con.execute("PRAGMA quick_check").fetchone()[0]
|
|
691
|
+
print(f" integrity {_integ}")
|
|
692
|
+
ok = ok and _integ == "ok"
|
|
693
|
+
s = query.stats(con)
|
|
694
|
+
print(f" cves {s.get('cve', 0):,} · kev {s.get('cve_kev', 0):,} · "
|
|
695
|
+
f"exploits {s.get('cve_with_exploit', 0):,} · cpe ranges {s.get('cpe_match', 0):,}")
|
|
696
|
+
m = datasets.load_manifest()
|
|
697
|
+
missing = [v["path"] for v in m.get("datasets", {}).values() if not (paths.DATA / v["path"]).exists()]
|
|
698
|
+
print(f" datasets {len(m.get('datasets', {}))} tracked, {len(missing)} missing")
|
|
699
|
+
if m.get("tier"):
|
|
700
|
+
print(f" tier {m['tier']} ({datasets.TIER_SIZES.get(m['tier'], '?')})")
|
|
701
|
+
incomplete = datasets.tier_missing(m["tier"])
|
|
702
|
+
if incomplete:
|
|
703
|
+
ok = False
|
|
704
|
+
shown = ", ".join(incomplete[:6]) + (" …" if len(incomplete) > 6 else "")
|
|
705
|
+
print(f" {YEL}incomplete - tier {m['tier']} is missing {len(incomplete)} "
|
|
706
|
+
f"dataset(s): {shown}{RST}")
|
|
707
|
+
print(f" {YEL}re-run: crowbar sync --tier {m['tier']} && crowbar build --tier {m['tier']}{RST}")
|
|
708
|
+
for p in missing[:5]:
|
|
709
|
+
print(f" {DIM}missing {p}{RST}")
|
|
710
|
+
if paths.HOME_LINK.is_symlink() and paths.HOME_LINK.exists():
|
|
711
|
+
try:
|
|
712
|
+
(paths.HOME_LINK / "index/state.json").read_text()
|
|
713
|
+
except OSError:
|
|
714
|
+
print(f" {YEL}drive unreadable by this process - for the launchd agent,"
|
|
715
|
+
f" grant Full Disk Access to /bin/sh{RST}")
|
|
716
|
+
fresh = datasets.index_status()
|
|
717
|
+
if fresh["age_days"] is not None:
|
|
718
|
+
print(f" fresh {fresh['age_days']} days since update"
|
|
719
|
+
f"{'' if fresh['auto_update'] else ' (auto-update off)'}")
|
|
720
|
+
if fresh["stale"]:
|
|
721
|
+
print(f" {YEL}stale - run: crowbar update{RST}")
|
|
722
|
+
free = shutil.disk_usage(paths.ROOT).free
|
|
723
|
+
print(f" disk {datasets.human(free)} free")
|
|
724
|
+
if free < 3 * 1024 ** 3:
|
|
725
|
+
print(f" {YEL}low disk - index rebuild needs ~2x current db size{RST}")
|
|
726
|
+
print(f"\n{'ready' if ok else 'needs attention'}")
|
|
727
|
+
sys.exit(0 if ok else 1)
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
# ---------------------------------------------------------------- project links
|
|
731
|
+
def _find_home() -> pathlib.Path:
|
|
732
|
+
return paths.HOME_LINK if paths.HOME_LINK.exists() else paths.ROOT
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
def cmd_link(args) -> None:
|
|
736
|
+
target = pathlib.Path(args.dir).expanduser().resolve()
|
|
737
|
+
if not target.is_dir():
|
|
738
|
+
die(f"{target} is not a directory")
|
|
739
|
+
if target == paths.ROOT or paths.ROOT in target.parents:
|
|
740
|
+
die("refusing to link the harness into itself")
|
|
741
|
+
src = _find_home()
|
|
742
|
+
link = target / ".crowbar"
|
|
743
|
+
if link.is_symlink() or link.exists():
|
|
744
|
+
if not args.force:
|
|
745
|
+
die(f"{link} already exists (use --force)")
|
|
746
|
+
link.unlink() if link.is_symlink() else shutil.rmtree(link)
|
|
747
|
+
link.symlink_to(src, target_is_directory=True)
|
|
748
|
+
|
|
749
|
+
skill_dst = target / ".opencode" / "skills" / "crowbar"
|
|
750
|
+
if not skill_dst.exists() and paths.SKILL.exists():
|
|
751
|
+
skill_dst.parent.mkdir(parents=True, exist_ok=True)
|
|
752
|
+
skill_dst.symlink_to(paths.SKILL, target_is_directory=True)
|
|
753
|
+
|
|
754
|
+
print(f"linked {link} -> {src}")
|
|
755
|
+
print(" cli .crowbar/bin/crowbar search \"...\"")
|
|
756
|
+
print(" docs .crowbar/AGENTS.md")
|
|
757
|
+
print(f" remove crowbar unlink {target}")
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def cmd_unlink(args) -> None:
|
|
761
|
+
target = pathlib.Path(args.dir).expanduser().resolve()
|
|
762
|
+
removed = []
|
|
763
|
+
link = target / ".crowbar"
|
|
764
|
+
if link.is_symlink():
|
|
765
|
+
link.unlink()
|
|
766
|
+
removed.append(str(link))
|
|
767
|
+
skill = target / ".opencode" / "skills" / "crowbar"
|
|
768
|
+
if skill.is_symlink() and str(skill.resolve()).startswith(str(paths.ROOT)):
|
|
769
|
+
skill.unlink()
|
|
770
|
+
removed.append(str(skill))
|
|
771
|
+
if not removed:
|
|
772
|
+
die("nothing to unlink")
|
|
773
|
+
print("removed " + ", ".join(removed))
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
# ---------------------------------------------------------------- drive ops
|
|
777
|
+
def _mounted() -> bool:
|
|
778
|
+
if paths.HOME_LINK.is_symlink() and not paths.HOME_LINK.exists():
|
|
779
|
+
return False
|
|
780
|
+
return paths.ROOT.exists()
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
CODE = ["bin", "crowbar", "mcp", "scripts", "docs", ".opencode",
|
|
784
|
+
"AGENTS.md", "README.md", "Makefile", ".gitignore"]
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def _copy_db(src: pathlib.Path, dst: pathlib.Path) -> None:
|
|
788
|
+
"""Consistent snapshot even if a writer is active (sqlite backup API)."""
|
|
789
|
+
if dst.exists():
|
|
790
|
+
dst.unlink()
|
|
791
|
+
s = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
|
|
792
|
+
d = sqlite3.connect(dst)
|
|
793
|
+
try:
|
|
794
|
+
with d:
|
|
795
|
+
s.backup(d)
|
|
796
|
+
finally:
|
|
797
|
+
d.close()
|
|
798
|
+
s.close()
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def _db_ok(path: pathlib.Path) -> bool:
|
|
802
|
+
journal = path.with_name(path.name + "-journal")
|
|
803
|
+
if journal.exists() and journal.stat().st_size > 0:
|
|
804
|
+
# unclean shutdown: open read-write once so SQLite rolls the hot journal back
|
|
805
|
+
try:
|
|
806
|
+
w = sqlite3.connect(path, timeout=30)
|
|
807
|
+
w.execute("PRAGMA quick_check").fetchone()
|
|
808
|
+
w.close()
|
|
809
|
+
except sqlite3.DatabaseError:
|
|
810
|
+
return False
|
|
811
|
+
try:
|
|
812
|
+
con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
813
|
+
res = con.execute("PRAGMA integrity_check").fetchone()[0]
|
|
814
|
+
con.close()
|
|
815
|
+
return res == "ok"
|
|
816
|
+
except sqlite3.DatabaseError:
|
|
817
|
+
return False
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _verify(vault: pathlib.Path) -> list[tuple[str, str]]:
|
|
821
|
+
bad: list[tuple[str, str]] = []
|
|
822
|
+
db = vault / "index/crowbar.db"
|
|
823
|
+
if not db.exists():
|
|
824
|
+
bad.append(("crowbar.db", "missing"))
|
|
825
|
+
elif db.stat().st_size == 0:
|
|
826
|
+
bad.append(("crowbar.db", "empty (0 bytes) - rebuild"))
|
|
827
|
+
elif not _db_ok(db):
|
|
828
|
+
bad.append(("crowbar.db", "integrity check failed"))
|
|
829
|
+
else:
|
|
830
|
+
try:
|
|
831
|
+
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
|
832
|
+
tabs = {r[0] for r in con.execute(
|
|
833
|
+
"SELECT name FROM sqlite_master WHERE type='table'")}
|
|
834
|
+
if "cve" not in tabs:
|
|
835
|
+
bad.append(("crowbar.db", "no schema - empty or wrong DB; rebuild"))
|
|
836
|
+
elif con.execute("SELECT count(*) FROM cve").fetchone()[0] == 0:
|
|
837
|
+
bad.append(("crowbar.db", "0 CVEs - build incomplete; rebuild"))
|
|
838
|
+
con.close()
|
|
839
|
+
except sqlite3.DatabaseError as exc:
|
|
840
|
+
bad.append(("crowbar.db", f"unreadable: {exc}"))
|
|
841
|
+
import json as _json
|
|
842
|
+
man = vault / "index/manifest.json"
|
|
843
|
+
if man.exists():
|
|
844
|
+
for name, rec in _json.loads(man.read_text()).get("datasets", {}).items():
|
|
845
|
+
p = vault / "data" / rec["path"]
|
|
846
|
+
if not p.exists():
|
|
847
|
+
bad.append((name, "missing"))
|
|
848
|
+
elif rec.get("sha256") and datasets.sha256(p) != rec["sha256"]:
|
|
849
|
+
bad.append((name, "hash mismatch"))
|
|
850
|
+
return bad
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def cmd_verify(args) -> None:
|
|
854
|
+
problems = _verify(paths.ROOT)
|
|
855
|
+
if args.json:
|
|
856
|
+
out_json({"root": str(paths.ROOT), "ok": not problems,
|
|
857
|
+
"problems": [{"dataset": n, "issue": i} for n, i in problems]})
|
|
858
|
+
return
|
|
859
|
+
if not problems:
|
|
860
|
+
print(f"{GRN}ok{RST} - index and all datasets match the manifest ({paths.ROOT})")
|
|
861
|
+
return
|
|
862
|
+
for n, i in problems:
|
|
863
|
+
print(f"{RED}FAIL{RST} {n:<22} {i}")
|
|
864
|
+
print(f"\n{len(problems)} problems - try: crowbar sync --only <dataset> --force && crowbar build")
|
|
865
|
+
sys.exit(1)
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def cmd_selftest(args) -> None:
|
|
869
|
+
"""Offline gate: static checks + every test suite + index/dataset verify.
|
|
870
|
+
Never touches the network - run `crowbar sync`/`crowbar update` for that."""
|
|
871
|
+
root = paths.ROOT
|
|
872
|
+
steps = [
|
|
873
|
+
("static checks", [sys.executable, "scripts/check.py"]),
|
|
874
|
+
("unit tests", [sys.executable, "-m", "tests.test_unit"]),
|
|
875
|
+
("schema-v3 tests", [sys.executable, "-m", "tests.test_v3"]),
|
|
876
|
+
("corpus tests", [sys.executable, "-m", "tests.test_corpus"]),
|
|
877
|
+
]
|
|
878
|
+
failed = []
|
|
879
|
+
for label, cmd in steps:
|
|
880
|
+
print(f"{CYN}▶ {label}{RST}")
|
|
881
|
+
if subprocess.run(cmd, cwd=str(root)).returncode != 0:
|
|
882
|
+
failed.append(label)
|
|
883
|
+
print(f"{CYN}▶ index + dataset verify{RST}")
|
|
884
|
+
if paths.DB.exists():
|
|
885
|
+
problems = _verify(root)
|
|
886
|
+
if problems:
|
|
887
|
+
for n, i in problems:
|
|
888
|
+
print(f"{RED}FAIL{RST} {n:<22} {i}")
|
|
889
|
+
failed.append("verify")
|
|
890
|
+
else:
|
|
891
|
+
print(f"{GRN}ok{RST} - index and all datasets match the manifest")
|
|
892
|
+
else:
|
|
893
|
+
print(f"{YEL}skip{RST} - no index/crowbar.db (run: crowbar sync && crowbar build)")
|
|
894
|
+
if failed:
|
|
895
|
+
print(f"\n{RED}selftest FAILED{RST}: {', '.join(failed)}")
|
|
896
|
+
sys.exit(1)
|
|
897
|
+
print(f"\n{GRN}selftest passed{RST} - static checks, tests, and verify all green")
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
START_HERE = """# HACKVAULT - portable vulnerability index
|
|
901
|
+
|
|
902
|
+
This drive holds a complete, self-contained copy of the crowbar corpus:
|
|
903
|
+
every CVE (2002-today) with CPE version ranges, CISA KEV, EPSS, Exploit-DB,
|
|
904
|
+
Nuclei templates, Metasploit modules, MITRE ATT&CK (enterprise/mobile/ICS),
|
|
905
|
+
CWE, CAPEC, and the LLM model catalog.
|
|
906
|
+
|
|
907
|
+
Built {date}. Size: ~{size}.
|
|
908
|
+
|
|
909
|
+
## Use it
|
|
910
|
+
|
|
911
|
+
```bash
|
|
912
|
+
# from this folder
|
|
913
|
+
./bin/crowbar doctor
|
|
914
|
+
./bin/crowbar search "nginx rce" --kev
|
|
915
|
+
./bin/crowbar product openssh --version 9.7
|
|
916
|
+
./bin/crowbar cve CVE-2021-44228
|
|
917
|
+
./bin/crowbar browse
|
|
918
|
+
```
|
|
919
|
+
|
|
920
|
+
`~/.crowbar` points here, so projects can link to it:
|
|
921
|
+
|
|
922
|
+
```bash
|
|
923
|
+
cd /path/to/project
|
|
924
|
+
~/.crowbar/bin/crowbar link .
|
|
925
|
+
.crowbar/bin/crowbar product <name> --version <ver>
|
|
926
|
+
~/.crowbar/bin/crowbar unlink .
|
|
927
|
+
```
|
|
928
|
+
|
|
929
|
+
Agents get the MCP tools from the opencode config on the host
|
|
930
|
+
(`crowbar connect` registers them); the skill is at `.opencode/skills/crowbar/`.
|
|
931
|
+
|
|
932
|
+
## Update it
|
|
933
|
+
|
|
934
|
+
On the host (drive plugged in):
|
|
935
|
+
|
|
936
|
+
```bash
|
|
937
|
+
~/.crowbar/bin/crowbar update # ~10-60 s: modified CVEs + KEV + EPSS + Exploit-DB
|
|
938
|
+
~/.crowbar/bin/crowbar update --deep # also Nuclei / Metasploit / ATT&CK
|
|
939
|
+
~/.crowbar/bin/crowbar verify # index + dataset hashes
|
|
940
|
+
```
|
|
941
|
+
|
|
942
|
+
Scheduling is declared in nix (`launchd.agents.crowbar`, see `docs/NIX.md`),
|
|
943
|
+
not installed imperatively. Until it is wired up, run `crowbar update` by hand after
|
|
944
|
+
plugging the drive in. For a full rebuild: `crowbar sync && crowbar build`.
|
|
945
|
+
|
|
946
|
+
## Verify before trusting it
|
|
947
|
+
|
|
948
|
+
```bash
|
|
949
|
+
./bin/crowbar doctor # schema, integrity, freshness, disk
|
|
950
|
+
./scripts/selftest.py 2>/dev/null || python3 scripts/selftest.py
|
|
951
|
+
```
|
|
952
|
+
|
|
953
|
+
## Care
|
|
954
|
+
|
|
955
|
+
- Eject cleanly (`diskutil eject /Volumes/VAULT`) so the SQLite index is not
|
|
956
|
+
torn mid-write.
|
|
957
|
+
- `data/` is upstream third-party content; see `docs/DATASETS.md` for licenses.
|
|
958
|
+
- Reference data is not authorization: only test systems you may test.
|
|
959
|
+
"""
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
def cmd_setup_drive(args) -> None:
|
|
963
|
+
vault = pathlib.Path(args.path).expanduser().resolve()
|
|
964
|
+
vault.mkdir(parents=True, exist_ok=True)
|
|
965
|
+
if not os.access(vault, os.W_OK):
|
|
966
|
+
die(f"{vault} is not writable")
|
|
967
|
+
if any(vault.iterdir()) and not args.force:
|
|
968
|
+
die(f"{vault} is not empty - pass --force to merge into it")
|
|
969
|
+
print(f"installing vault -> {vault}")
|
|
970
|
+
for name in CODE:
|
|
971
|
+
src = paths.ROOT / name
|
|
972
|
+
if not src.exists():
|
|
973
|
+
continue
|
|
974
|
+
dst = vault / name
|
|
975
|
+
if src.is_dir():
|
|
976
|
+
shutil.copytree(src, dst, dirs_exist_ok=True) # refresh code in place
|
|
977
|
+
else:
|
|
978
|
+
shutil.copy2(src, dst)
|
|
979
|
+
(vault / paths.DRIVE_MARKER).write_text(
|
|
980
|
+
f"crowbar drive\ncreated {time.strftime('%Y-%m-%d')}\n")
|
|
981
|
+
|
|
982
|
+
if not args.no_data:
|
|
983
|
+
data_src, data_dst = paths.DATA, vault / "data"
|
|
984
|
+
if not data_dst.exists():
|
|
985
|
+
print(f"copying data ({datasets.human(_du(data_src))})…")
|
|
986
|
+
shutil.copytree(data_src, data_dst)
|
|
987
|
+
elif shutil.which("rsync"):
|
|
988
|
+
# refresh an existing drive incrementally; keep drive-only files
|
|
989
|
+
print("refreshing data (rsync)…")
|
|
990
|
+
subprocess.run(["rsync", "-a", f"{data_src}/", f"{data_dst}/"], check=True)
|
|
991
|
+
else:
|
|
992
|
+
print("refreshing data (copy)…")
|
|
993
|
+
shutil.copytree(data_src, data_dst, dirs_exist_ok=True)
|
|
994
|
+
idx_dst = vault / "index"
|
|
995
|
+
idx_dst.mkdir(exist_ok=True)
|
|
996
|
+
for f in ("manifest.json", "state.json", "stats.json"):
|
|
997
|
+
if (paths.INDEX / f).exists():
|
|
998
|
+
shutil.copy2(paths.INDEX / f, idx_dst / f)
|
|
999
|
+
print(f"copying index ({datasets.human(paths.DB.stat().st_size)})…")
|
|
1000
|
+
_copy_db(paths.DB, idx_dst / "crowbar.db")
|
|
1001
|
+
|
|
1002
|
+
# verify and repair any file that was re-written while copying
|
|
1003
|
+
import json as _json
|
|
1004
|
+
man = _json.loads((idx_dst / "manifest.json").read_text()).get("datasets", {}) \
|
|
1005
|
+
if (idx_dst / "manifest.json").exists() else {}
|
|
1006
|
+
for name, rec in man.items():
|
|
1007
|
+
src_file, dst_file = paths.DATA / rec["path"], data_dst / rec["path"]
|
|
1008
|
+
if rec.get("sha256") and src_file.exists() and dst_file.exists() \
|
|
1009
|
+
and datasets.sha256(dst_file) != rec["sha256"]:
|
|
1010
|
+
print(f"repairing {name}")
|
|
1011
|
+
shutil.copy2(src_file, dst_file)
|
|
1012
|
+
|
|
1013
|
+
(vault / "START-HERE.md").write_text(
|
|
1014
|
+
START_HERE.format(date=time.strftime("%Y-%m-%d"),
|
|
1015
|
+
size=datasets.human(_du(vault))))
|
|
1016
|
+
|
|
1017
|
+
link = paths.HOME_LINK
|
|
1018
|
+
if link.is_symlink() or link.exists():
|
|
1019
|
+
if not args.force:
|
|
1020
|
+
print(f"{link} exists - not replacing (use --force)")
|
|
1021
|
+
link = None
|
|
1022
|
+
else:
|
|
1023
|
+
link.unlink() if link.is_symlink() else shutil.rmtree(link)
|
|
1024
|
+
if link is not None:
|
|
1025
|
+
link.symlink_to(vault, target_is_directory=True)
|
|
1026
|
+
print(f"linked ~/.crowbar -> {vault}")
|
|
1027
|
+
print("\nnext:")
|
|
1028
|
+
print(f" {vault}/bin/crowbar doctor")
|
|
1029
|
+
print(f" {vault}/bin/crowbar update # when the drive is mounted")
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def _du(p: pathlib.Path) -> int:
|
|
1033
|
+
return sum(f.stat().st_size for f in p.rglob("*") if f.is_file())
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
PLIST = """<?xml version="1.0" encoding="UTF-8"?>
|
|
1037
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
1038
|
+
<plist version="1.0"><dict>
|
|
1039
|
+
<key>Label</key><string>sh.crowbar.update</string>
|
|
1040
|
+
<key>ProgramArguments</key><array><string>{wrapper}</string></array>
|
|
1041
|
+
<key>StartInterval</key><integer>{interval}</integer>
|
|
1042
|
+
<key>RunAtLoad</key><true/>
|
|
1043
|
+
<key>WatchPaths</key><array><string>/Volumes</string></array>
|
|
1044
|
+
<key>StandardOutPath</key><string>{log}</string>
|
|
1045
|
+
<key>StandardErrorPath</key><string>{log}</string>
|
|
1046
|
+
</dict></plist>
|
|
1047
|
+
"""
|
|
1048
|
+
|
|
1049
|
+
WRAPPER = """#!/bin/sh
|
|
1050
|
+
# crowbar auto-update.
|
|
1051
|
+
# Prefer the drive copy; fall back to the local copy when the drive is absent
|
|
1052
|
+
# or unreadable (macOS TCC blocks launchd jobs from /Volumes until "Full Disk
|
|
1053
|
+
# Access" is granted to /bin/sh).
|
|
1054
|
+
LOG="$HOME/Library/Logs/crowbar.log"
|
|
1055
|
+
DRIVE="$HOME/.crowbar/bin/crowbar"
|
|
1056
|
+
LOCAL="{root}/bin/crowbar"
|
|
1057
|
+
PY="{python}"
|
|
1058
|
+
run() {{ "$PY" "$1" update --quiet --if-mounted; }}
|
|
1059
|
+
|
|
1060
|
+
if [ -x "$DRIVE" ]; then
|
|
1061
|
+
run "$DRIVE" && exit 0
|
|
1062
|
+
echo "$(date '+%F %T') drive update failed (grant Full Disk Access to /bin/sh for /Volumes access); using local copy" >> "$LOG"
|
|
1063
|
+
fi
|
|
1064
|
+
if [ -x "$LOCAL" ]; then run "$LOCAL"
|
|
1065
|
+
else PYTHONPATH="{pkg}" "$PY" -m crowbar.cli update --quiet; fi
|
|
1066
|
+
exit 0
|
|
1067
|
+
"""
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
def _write_wrapper() -> pathlib.Path:
|
|
1071
|
+
wrapper = pathlib.Path.home() / ".local/bin/crowbar-autoupdate"
|
|
1072
|
+
wrapper.parent.mkdir(parents=True, exist_ok=True)
|
|
1073
|
+
wrapper.write_text(WRAPPER.format(root=paths.ROOT, python=sys.executable,
|
|
1074
|
+
pkg=pathlib.Path(__file__).resolve().parents[1]))
|
|
1075
|
+
wrapper.chmod(0o755)
|
|
1076
|
+
return wrapper
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
def cmd_install_agent(args) -> None:
|
|
1080
|
+
if pathlib.Path("/run/current-system").exists():
|
|
1081
|
+
print(f"{YEL}note:{RST} this machine is managed by nix. "
|
|
1082
|
+
f"Declaring the agent in home-manager is cleaner - see docs/NIX.md.", file=sys.stderr)
|
|
1083
|
+
wrapper = _write_wrapper()
|
|
1084
|
+
if sys.platform == "darwin":
|
|
1085
|
+
plist = pathlib.Path.home() / "Library/LaunchAgents/sh.crowbar.update.plist"
|
|
1086
|
+
plist.parent.mkdir(parents=True, exist_ok=True)
|
|
1087
|
+
log = pathlib.Path.home() / "Library/Logs/crowbar.log"
|
|
1088
|
+
plist.write_text(PLIST.format(wrapper=wrapper, interval=args.interval, log=log))
|
|
1089
|
+
subprocess.run(["launchctl", "bootout", f"gui/{os.getuid()}", str(plist)],
|
|
1090
|
+
capture_output=True)
|
|
1091
|
+
res = subprocess.run(["launchctl", "bootstrap", f"gui/{os.getuid()}", str(plist)],
|
|
1092
|
+
capture_output=True, text=True)
|
|
1093
|
+
if res.returncode != 0:
|
|
1094
|
+
die(f"launchctl: {res.stderr.strip()}")
|
|
1095
|
+
print(f"launchd agent installed: {plist}")
|
|
1096
|
+
print(f" wrapper: {wrapper}")
|
|
1097
|
+
print(f" every {args.interval}s and whenever /Volumes changes")
|
|
1098
|
+
print(f" log: {log}")
|
|
1099
|
+
else:
|
|
1100
|
+
print("add to crontab -e:\n"
|
|
1101
|
+
f" */{max(1, args.interval // 60)} * * * * {wrapper}")
|
|
1102
|
+
|
|
1103
|
+
|
|
1104
|
+
def cmd_uninstall_agent(args) -> None:
|
|
1105
|
+
if sys.platform == "darwin":
|
|
1106
|
+
plist = pathlib.Path.home() / "Library/LaunchAgents/sh.crowbar.update.plist"
|
|
1107
|
+
subprocess.run(["launchctl", "bootout", f"gui/{os.getuid()}", str(plist)],
|
|
1108
|
+
capture_output=True)
|
|
1109
|
+
if plist.exists():
|
|
1110
|
+
plist.unlink()
|
|
1111
|
+
wrapper = pathlib.Path.home() / ".local/bin/crowbar-autoupdate"
|
|
1112
|
+
if wrapper.exists():
|
|
1113
|
+
wrapper.unlink()
|
|
1114
|
+
print("removed auto-update agent")
|
|
1115
|
+
|
|
1116
|
+
|
|
1117
|
+
def cmd_watch(args) -> None:
|
|
1118
|
+
print(f"watching {paths.HOME_LINK} every {args.interval}s (ctrl-c to stop)")
|
|
1119
|
+
present = _mounted()
|
|
1120
|
+
while True:
|
|
1121
|
+
time.sleep(args.interval)
|
|
1122
|
+
now = _mounted()
|
|
1123
|
+
if now and not present:
|
|
1124
|
+
print(f"[{time.strftime('%H:%M:%S')}] drive appeared - updating")
|
|
1125
|
+
subprocess.run([sys.executable, str(paths.ROOT / "bin/crowbar"), "update", "--quiet"])
|
|
1126
|
+
elif not now and present:
|
|
1127
|
+
print(f"[{time.strftime('%H:%M:%S')}] drive removed")
|
|
1128
|
+
present = now
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
# ---------------------------------------------------------------- opencode
|
|
1132
|
+
def _mcp_command(local: bool = False) -> list[str]:
|
|
1133
|
+
"""Portable launch command for the MCP server. Defaults to the published
|
|
1134
|
+
package via uvx; --local uses the source checkout's mcp/server.py."""
|
|
1135
|
+
if local and (paths.ROOT / "mcp/server.py").exists():
|
|
1136
|
+
return [sys.executable, str(paths.ROOT / "mcp/server.py")]
|
|
1137
|
+
return ["uvx", "--from", "crowbar-cli", "crowbar-mcp"]
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
def cmd_connect(args) -> None:
|
|
1141
|
+
host = getattr(args, "host", "opencode")
|
|
1142
|
+
cmd = _mcp_command(local=getattr(args, "local", False))
|
|
1143
|
+
if getattr(args, "print", False):
|
|
1144
|
+
host = "print"
|
|
1145
|
+
|
|
1146
|
+
if host in ("print", "claude", "codex", "cursor", "gemini"):
|
|
1147
|
+
block = {"mcpServers": {"crowbar": {"command": cmd[0], "args": cmd[1:]}}}
|
|
1148
|
+
if host == "claude":
|
|
1149
|
+
print("# Claude Code - run this, or paste the block into .mcp.json:")
|
|
1150
|
+
print(f"claude mcp add crowbar -- {' '.join(cmd)}\n")
|
|
1151
|
+
elif host != "print":
|
|
1152
|
+
print(f"# {host}: add this to the host's MCP server config:")
|
|
1153
|
+
out_json(block)
|
|
1154
|
+
if host in ("claude", "codex") and paths.SKILL.exists():
|
|
1155
|
+
print(f"\n# skill: {paths.SKILL / 'SKILL.md'} (install into the host's skills dir)")
|
|
1156
|
+
return
|
|
1157
|
+
|
|
1158
|
+
# opencode: write the config file and install the skill
|
|
1159
|
+
block = {"type": "local", "command": cmd, "enabled": True, "timeout": 15000}
|
|
1160
|
+
target = (pathlib.Path.home() / ".config/opencode/opencode.json"
|
|
1161
|
+
if args.global_ else pathlib.Path(args.target).resolve())
|
|
1162
|
+
cfg = {}
|
|
1163
|
+
if target.exists():
|
|
1164
|
+
try:
|
|
1165
|
+
cfg = json.loads(target.read_text() or "{}")
|
|
1166
|
+
except json.JSONDecodeError:
|
|
1167
|
+
die(f"{target} is not valid JSON - fix it first")
|
|
1168
|
+
cfg.setdefault("$schema", "https://opencode.ai/config.json")
|
|
1169
|
+
cfg.setdefault("mcp", {})["crowbar"] = block
|
|
1170
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1171
|
+
target.write_text(json.dumps(cfg, indent=2) + "\n")
|
|
1172
|
+
print(f"mcp server registered -> {target}")
|
|
1173
|
+
if not args.no_skill and paths.SKILL.exists():
|
|
1174
|
+
dst = target.parent / ".opencode/skills/crowbar"
|
|
1175
|
+
dst.mkdir(parents=True, exist_ok=True)
|
|
1176
|
+
shutil.copy2(paths.SKILL / "SKILL.md", dst / "SKILL.md")
|
|
1177
|
+
print(f"skill installed -> {dst / 'SKILL.md'}")
|
|
1178
|
+
print("restart opencode; the crowbar tools will appear automatically")
|
|
1179
|
+
|
|
1180
|
+
|
|
1181
|
+
def cmd_mcp(args) -> None:
|
|
1182
|
+
from . import mcp
|
|
1183
|
+
mcp.serve()
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
# ---------------------------------------------------------------- parser
|
|
1187
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
1188
|
+
from crowbar import __version__
|
|
1189
|
+
p = argparse.ArgumentParser(prog="crowbar", description="crowbar - local vulnerability index (terminal only)")
|
|
1190
|
+
p.add_argument("--version", action="version", version=f"crowbar {__version__}")
|
|
1191
|
+
p.add_argument("--width", type=int, default=110)
|
|
1192
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
1193
|
+
|
|
1194
|
+
def add(name, fn, help_):
|
|
1195
|
+
s = sub.add_parser(name, help=help_)
|
|
1196
|
+
s.set_defaults(fn=fn)
|
|
1197
|
+
return s
|
|
1198
|
+
|
|
1199
|
+
s = add("search", cmd_search, "full-text CVE search")
|
|
1200
|
+
s.add_argument("query", nargs="?")
|
|
1201
|
+
s.add_argument("--limit", type=int, default=20)
|
|
1202
|
+
s.add_argument("--offset", type=int, default=0)
|
|
1203
|
+
s.add_argument("--sort", choices=["relevance", "newest", "oldest", "cvss", "epss"], default="relevance")
|
|
1204
|
+
s.add_argument("--severity")
|
|
1205
|
+
s.add_argument("--year", type=int)
|
|
1206
|
+
s.add_argument("--since")
|
|
1207
|
+
s.add_argument("--kev", action="store_true")
|
|
1208
|
+
s.add_argument("--exploit", action="store_true")
|
|
1209
|
+
s.add_argument("--epss", type=float)
|
|
1210
|
+
s.add_argument("--cvss", type=float)
|
|
1211
|
+
s.add_argument("--cwe")
|
|
1212
|
+
s.add_argument("--product")
|
|
1213
|
+
s.add_argument("--vendor")
|
|
1214
|
+
s.add_argument("--ssvc", help="CISA SSVC exploitation: none|poc|active")
|
|
1215
|
+
s.add_argument("--poc", action="store_true", help="only CVEs with a public PoC")
|
|
1216
|
+
s.add_argument("--json", action="store_true")
|
|
1217
|
+
|
|
1218
|
+
s = add("browse", cmd_browse, "interactive fzf browser")
|
|
1219
|
+
s.add_argument("query", nargs="?")
|
|
1220
|
+
s.add_argument("--limit", type=int, default=500)
|
|
1221
|
+
s.add_argument("--kev", action="store_true")
|
|
1222
|
+
s.add_argument("--exploit", action="store_true")
|
|
1223
|
+
|
|
1224
|
+
s = add("cve", cmd_cve, "one CVE in full")
|
|
1225
|
+
s.add_argument("id")
|
|
1226
|
+
s.add_argument("--json", action="store_true")
|
|
1227
|
+
|
|
1228
|
+
s = add("product", cmd_product, "CVEs affecting a product, optionally a version")
|
|
1229
|
+
s.add_argument("name")
|
|
1230
|
+
s.add_argument("--version", help="exact version to test against CPE ranges, e.g. 1.18.0")
|
|
1231
|
+
s.add_argument("--limit", type=int, default=25)
|
|
1232
|
+
s.add_argument("--json", action="store_true")
|
|
1233
|
+
|
|
1234
|
+
s = add("kev", cmd_kev, "CISA Known Exploited Vulnerabilities")
|
|
1235
|
+
s.add_argument("query", nargs="?")
|
|
1236
|
+
s.add_argument("--limit", type=int, default=25)
|
|
1237
|
+
s.add_argument("--json", action="store_true")
|
|
1238
|
+
|
|
1239
|
+
s = add("exploit", cmd_exploit, "search Exploit-DB, or fetch one by id")
|
|
1240
|
+
s.add_argument("query")
|
|
1241
|
+
s.add_argument("--limit", type=int, default=20)
|
|
1242
|
+
s.add_argument("--json", action="store_true")
|
|
1243
|
+
|
|
1244
|
+
s = add("nuclei", cmd_nuclei, "search Nuclei templates")
|
|
1245
|
+
s.add_argument("query")
|
|
1246
|
+
s.add_argument("--domain", help="http, network, dns, file, headless, cloud, javascript, ssl, code")
|
|
1247
|
+
s.add_argument("--limit", type=int, default=20)
|
|
1248
|
+
s.add_argument("--json", action="store_true")
|
|
1249
|
+
|
|
1250
|
+
s = add("msf", cmd_msf, "search Metasploit modules")
|
|
1251
|
+
s.add_argument("query")
|
|
1252
|
+
s.add_argument("--limit", type=int, default=30)
|
|
1253
|
+
s.add_argument("--json", action="store_true")
|
|
1254
|
+
|
|
1255
|
+
s = add("models", cmd_models, "LLM catalog: context, price, capabilities")
|
|
1256
|
+
s.add_argument("query", nargs="?")
|
|
1257
|
+
s.add_argument("--provider")
|
|
1258
|
+
s.add_argument("--limit", type=int, default=40)
|
|
1259
|
+
s.add_argument("--json", action="store_true")
|
|
1260
|
+
|
|
1261
|
+
s = add("attack", cmd_attack, "MITRE ATT&CK techniques")
|
|
1262
|
+
s.add_argument("query")
|
|
1263
|
+
s.add_argument("--domain", choices=["enterprise", "mobile", "ics"])
|
|
1264
|
+
s.add_argument("--limit", type=int, default=10)
|
|
1265
|
+
s.add_argument("--json", action="store_true")
|
|
1266
|
+
|
|
1267
|
+
s = add("cwe", cmd_cwe, "CWE weakness lookup")
|
|
1268
|
+
s.add_argument("query", nargs="?")
|
|
1269
|
+
s.add_argument("--limit", type=int, default=30)
|
|
1270
|
+
s.add_argument("--json", action="store_true")
|
|
1271
|
+
|
|
1272
|
+
s = add("capec", cmd_capec, "CAPEC attack patterns")
|
|
1273
|
+
s.add_argument("query")
|
|
1274
|
+
s.add_argument("--limit", type=int, default=15)
|
|
1275
|
+
s.add_argument("--json", action="store_true")
|
|
1276
|
+
|
|
1277
|
+
s = add("os", cmd_os, "Windows / iOS / macOS vendor advisories")
|
|
1278
|
+
s.add_argument("query", nargs="?", default=None)
|
|
1279
|
+
s.add_argument("--platform", choices=["windows", "ios", "macos", "apple"])
|
|
1280
|
+
s.add_argument("--impact", help="RCE, EoP, DoS, InfoDisclosure, Spoofing, Bypass")
|
|
1281
|
+
s.add_argument("--exploited", action="store_true")
|
|
1282
|
+
s.add_argument("--since")
|
|
1283
|
+
s.add_argument("--limit", type=int, default=40)
|
|
1284
|
+
s.add_argument("--json", action="store_true")
|
|
1285
|
+
|
|
1286
|
+
s = add("advisory", cmd_advisory, "OSV/GHSA advisories by package or ecosystem")
|
|
1287
|
+
s.add_argument("query", nargs="?", default=None)
|
|
1288
|
+
s.add_argument("--ecosystem")
|
|
1289
|
+
s.add_argument("--package")
|
|
1290
|
+
s.add_argument("--limit", type=int, default=30)
|
|
1291
|
+
s.add_argument("--json", action="store_true")
|
|
1292
|
+
|
|
1293
|
+
s = add("code", cmd_code, "full-text search inside Exploit-DB sources")
|
|
1294
|
+
s.add_argument("query")
|
|
1295
|
+
s.add_argument("--limit", type=int, default=20)
|
|
1296
|
+
s.add_argument("--json", action="store_true")
|
|
1297
|
+
|
|
1298
|
+
s = add("bin", cmd_bin, "GTFOBins / LOLBAS / PayloadsAllTheThings")
|
|
1299
|
+
s.add_argument("query")
|
|
1300
|
+
s.add_argument("--source", choices=["gtfobins", "lolbas", "payloads"])
|
|
1301
|
+
s.add_argument("--limit", type=int, default=25)
|
|
1302
|
+
s.add_argument("--json", action="store_true")
|
|
1303
|
+
|
|
1304
|
+
s = add("debian", cmd_debian, "Debian tracker: fixed versions per release")
|
|
1305
|
+
s.add_argument("package")
|
|
1306
|
+
s.add_argument("--cve")
|
|
1307
|
+
s.add_argument("--json", action="store_true")
|
|
1308
|
+
|
|
1309
|
+
s = add("poc", cmd_poc, "public GitHub PoC repositories (metadata only)")
|
|
1310
|
+
s.add_argument("query", nargs="?", default="")
|
|
1311
|
+
s.add_argument("--cve")
|
|
1312
|
+
s.add_argument("--language")
|
|
1313
|
+
s.add_argument("--min-stars", type=int, default=0)
|
|
1314
|
+
s.add_argument("--top", action="store_true", help="CVEs with the most PoCs")
|
|
1315
|
+
s.add_argument("--limit", type=int, default=25)
|
|
1316
|
+
s.add_argument("--json", action="store_true")
|
|
1317
|
+
|
|
1318
|
+
s = add("sources", cmd_sources, "source provenance: url, license, sha256, row counts")
|
|
1319
|
+
s.add_argument("--json", action="store_true")
|
|
1320
|
+
|
|
1321
|
+
s = add("map", cmd_map, "dataset -> path -> index table map")
|
|
1322
|
+
s.add_argument("--tables", action="store_true", help="explain every table")
|
|
1323
|
+
s.add_argument("--json", action="store_true")
|
|
1324
|
+
|
|
1325
|
+
s = add("sql", cmd_sql, "read-only SQL escape hatch")
|
|
1326
|
+
s.add_argument("query")
|
|
1327
|
+
s.add_argument("--limit", type=int, default=200)
|
|
1328
|
+
s.add_argument("--json", action="store_true")
|
|
1329
|
+
|
|
1330
|
+
s = add("stats", cmd_stats, "corpus statistics and freshness")
|
|
1331
|
+
s.add_argument("--json", action="store_true")
|
|
1332
|
+
|
|
1333
|
+
add("where", cmd_where, "print paths").add_argument("--json", action="store_true")
|
|
1334
|
+
add("doctor", cmd_doctor, "health check")
|
|
1335
|
+
|
|
1336
|
+
s = add("sync", cmd_sync, "download datasets")
|
|
1337
|
+
s.add_argument("--tier", choices=datasets.TIER_NAMES,
|
|
1338
|
+
help="size tier: lite ~150MB, medium ~500MB, heavy ~2GB, extreme full (each a superset)")
|
|
1339
|
+
s.add_argument("--only")
|
|
1340
|
+
s.add_argument("--years")
|
|
1341
|
+
s.add_argument("--extra", action="store_true", help="ATT&CK mobile/ics + CISA Vulnrichment")
|
|
1342
|
+
s.add_argument("--bulk", action="store_true", help="+ OSV ecosystems, Debian tracker, full Exploit-DB")
|
|
1343
|
+
s.add_argument("--force", action="store_true")
|
|
1344
|
+
s.add_argument("--pull", action="store_true", help="git pull existing clones")
|
|
1345
|
+
s.add_argument("--jobs", type=int, default=4)
|
|
1346
|
+
|
|
1347
|
+
s = add("build", cmd_build, "build the index from downloaded data")
|
|
1348
|
+
s.add_argument("--tier", choices=datasets.TIER_NAMES, help="record the size tier this index was built for")
|
|
1349
|
+
s.add_argument("--allow-incomplete", action="store_true", help="build even if some tier datasets are missing")
|
|
1350
|
+
s.add_argument("--quick", action="store_true", help="CVEs + exploits only")
|
|
1351
|
+
s.add_argument("--extra", action="store_true")
|
|
1352
|
+
s.add_argument("--bulk", action="store_true")
|
|
1353
|
+
s.add_argument("--json", action="store_true")
|
|
1354
|
+
|
|
1355
|
+
s = add("update", cmd_update, "incremental refresh (modified CVEs + intel)")
|
|
1356
|
+
s.add_argument("--deep", action="store_true", help="also reindex Nuclei/MSF/ATT&CK")
|
|
1357
|
+
s.add_argument("--if-mounted", action="store_true", help="no-op when the drive is absent")
|
|
1358
|
+
s.add_argument("--no-sync", action="store_true")
|
|
1359
|
+
s.add_argument("--extra", action="store_true")
|
|
1360
|
+
s.add_argument("--jobs", type=int, default=4)
|
|
1361
|
+
s.add_argument("--quiet", action="store_true")
|
|
1362
|
+
s.add_argument("--json", action="store_true")
|
|
1363
|
+
|
|
1364
|
+
s = add("link", cmd_link, "link the corpus into a project as .crowbar")
|
|
1365
|
+
s.add_argument("dir", nargs="?", default=".")
|
|
1366
|
+
s.add_argument("--force", action="store_true")
|
|
1367
|
+
|
|
1368
|
+
s = add("unlink", cmd_unlink, "remove the project link")
|
|
1369
|
+
s.add_argument("dir", nargs="?", default=".")
|
|
1370
|
+
|
|
1371
|
+
s = add("verify", cmd_verify, "check index + dataset hashes")
|
|
1372
|
+
s.add_argument("--json", action="store_true")
|
|
1373
|
+
|
|
1374
|
+
add("selftest", cmd_selftest, "offline gate: static checks + tests + verify (no sync)")
|
|
1375
|
+
|
|
1376
|
+
s = add("setup-drive", cmd_setup_drive, "install the harness on a USB drive")
|
|
1377
|
+
s.add_argument("path")
|
|
1378
|
+
s.add_argument("--no-data", action="store_true", help="code only, no data/index copy")
|
|
1379
|
+
s.add_argument("--force", action="store_true")
|
|
1380
|
+
|
|
1381
|
+
s = add("install-agent", cmd_install_agent, "refresh automatically when the drive appears")
|
|
1382
|
+
s.add_argument("--interval", type=int, default=600)
|
|
1383
|
+
|
|
1384
|
+
add("uninstall-agent", cmd_uninstall_agent, "remove the launchd agent")
|
|
1385
|
+
|
|
1386
|
+
s = add("watch", cmd_watch, "foreground mount watcher")
|
|
1387
|
+
s.add_argument("--interval", type=int, default=60)
|
|
1388
|
+
|
|
1389
|
+
s = add("connect", cmd_connect, "register the MCP server with an agent host")
|
|
1390
|
+
s.add_argument("--host", choices=["opencode", "claude", "codex", "cursor", "gemini", "print"],
|
|
1391
|
+
default="opencode", help="which agent host to wire up (default: opencode)")
|
|
1392
|
+
s.add_argument("--local", action="store_true", help="launch from this source checkout instead of uvx")
|
|
1393
|
+
s.add_argument("--target", default=str(paths.ROOT.parent / "opencode.json"))
|
|
1394
|
+
s.add_argument("--global", dest="global_", action="store_true")
|
|
1395
|
+
s.add_argument("--no-skill", action="store_true")
|
|
1396
|
+
s.add_argument("--print", action="store_true")
|
|
1397
|
+
|
|
1398
|
+
add("mcp", cmd_mcp, "run the MCP server on stdio")
|
|
1399
|
+
return p
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
def main() -> None:
|
|
1403
|
+
args = build_parser().parse_args()
|
|
1404
|
+
try:
|
|
1405
|
+
args.fn(args)
|
|
1406
|
+
except BrokenPipeError:
|
|
1407
|
+
pass
|
|
1408
|
+
except KeyboardInterrupt:
|
|
1409
|
+
sys.exit(130)
|
|
1410
|
+
except db.NotIndexed as exc:
|
|
1411
|
+
die(str(exc))
|
|
1412
|
+
except ValueError as exc:
|
|
1413
|
+
die(str(exc))
|
|
1414
|
+
|
|
1415
|
+
|
|
1416
|
+
if __name__ == "__main__":
|
|
1417
|
+
main()
|