crowbar-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
crowbar/cli.py ADDED
@@ -0,0 +1,1417 @@
1
+ """Command line interface. Terminal only - no web UI, no TUI dependency."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import os
7
+ import pathlib
8
+ import shutil
9
+ import sqlite3
10
+ import subprocess
11
+ import sys
12
+ import textwrap
13
+ import time
14
+ from typing import NoReturn
15
+
16
+ from . import catalog, datasets, db, paths, query
17
+
18
+ TTY = sys.stdout.isatty() and os.environ.get("NO_COLOR") is None
19
+ DIM, BOLD, RED, YEL, GRN, CYN, MAG, RST = (
20
+ ("\033[2m", "\033[1m", "\033[31m", "\033[33m", "\033[32m", "\033[36m", "\033[35m", "\033[0m")
21
+ if TTY else ("",) * 8)
22
+ SEV_COLOR = {"CRITICAL": RED, "HIGH": YEL, "MEDIUM": YEL, "LOW": GRN, "NONE": DIM}
23
+
24
+
25
+ # ---------------------------------------------------------------- formatting
26
+ def out_json(obj) -> None:
27
+ print(json.dumps(obj, indent=2, ensure_ascii=False, default=str))
28
+
29
+
30
+ def die(msg: str, code: int = 1) -> NoReturn:
31
+ print(msg, file=sys.stderr)
32
+ sys.exit(code)
33
+
34
+
35
+ def money(v) -> str:
36
+ if v is None:
37
+ return "-"
38
+ if v == 0:
39
+ return "free"
40
+ return f"{v:.2f}" if v >= 0.01 else f"{v:.4f}"
41
+
42
+
43
+ def table(headers: list[str], rows: list[list], wrap_last: int = 0) -> None:
44
+ if not rows:
45
+ print(f"{DIM}no results{RST}")
46
+ return
47
+ cells = [[("" if c is None else str(c)) for c in r] for r in rows]
48
+ if wrap_last:
49
+ cells = [r[:-1] + [textwrap.shorten(r[-1], wrap_last, placeholder="…")] for r in cells]
50
+ widths = [max(len(h), *(len(r[i]) for r in cells)) for i, h in enumerate(headers)]
51
+ print(" ".join(f"{BOLD}{h:<{w}}{RST}" for h, w in zip(headers, widths)))
52
+ print(f"{DIM}{' '.join('─' * w for w in widths)}{RST}")
53
+ for r in cells:
54
+ print(" ".join(f"{c:<{w}}" for c, w in zip(r, widths)))
55
+
56
+
57
+ def sev_badge(sev: str | None, cvss: float | None) -> str:
58
+ s = (sev or "").upper()
59
+ label = {"CRITICAL": "CRIT", "HIGH": "HIGH", "MEDIUM": "MED", "LOW": "LOW"}.get(s, s[:4] or "?")
60
+ return f"{SEV_COLOR.get(s, DIM)}{label:<4}{RST} {('%.1f' % cvss) if cvss is not None else ' - '}"
61
+
62
+
63
+ def risk_bits(r) -> str:
64
+ bits = []
65
+ if r["kev"]:
66
+ bits.append(f"{RED}KEV{RST}")
67
+ if r["epss"] is not None:
68
+ bits.append(f"epss {r['epss']:.2f}")
69
+ return " ".join(bits)
70
+
71
+
72
+ def connect_or_die():
73
+ try:
74
+ con = db.connect()
75
+ except db.NotIndexed as exc:
76
+ die(f"{exc}")
77
+ if db.schema_version(con) != paths.SCHEMA_VERSION:
78
+ die(f"index schema v{db.schema_version(con)} != v{paths.SCHEMA_VERSION} - run: crowbar build")
79
+ if datasets.refresh_in_background():
80
+ print(f"{DIM}index is over a day old - refreshing in the background{RST}", file=sys.stderr)
81
+ return con
82
+
83
+
84
+ # ---------------------------------------------------------------- read commands
85
+ def cmd_search(args) -> None:
86
+ con = connect_or_die()
87
+ rows = query.search(
88
+ con, args.query or "", limit=args.limit, offset=args.offset, sort=args.sort,
89
+ severity=args.severity, year=args.year, since=args.since, kev=args.kev,
90
+ exploit=args.exploit, epss=args.epss, cvss=args.cvss, cwe=args.cwe,
91
+ product=args.product, vendor=args.vendor, ssvc=args.ssvc, poc=args.poc)
92
+ if args.json:
93
+ out_json(rows)
94
+ return
95
+ for r in rows:
96
+ tag = f" {CYN}exp×{r['exploits']}{RST}" if r["exploits"] else ""
97
+ tag += f" {DIM}nuc×{r['nuclei_count']}{RST}" if r["nuclei_count"] else ""
98
+ tag += f" {MAG}poc×{r['poc_count']}{RST}" if r.get("poc_count") else ""
99
+ print(f"{BOLD}{r['id']}{RST} {sev_badge(r['severity'], r['cvss'])} {risk_bits(r)}{tag}".rstrip())
100
+ print(f" {DIM}{(r['published'] or '')[:10]}{RST} "
101
+ f"{textwrap.shorten(r['description'] or '', args.width - 4, placeholder='…')}")
102
+ print(f"\n{DIM}{len(rows)} shown · next page: --offset {args.offset + args.limit}{RST}")
103
+
104
+
105
+ def cmd_browse(args) -> None:
106
+ """Interactive fzf browser: type to filter, preview the CVE, enter to open."""
107
+ con = connect_or_die()
108
+ fzf = shutil.which("fzf")
109
+ rows = query.search(con, args.query or "", limit=args.limit, kev=args.kev,
110
+ exploit=args.exploit, sort="relevance" if args.query else "newest")
111
+ if not fzf or not (sys.stdin.isatty() and sys.stdout.isatty()):
112
+ for r in rows:
113
+ print(f"{r['id']} {sev_badge(r['severity'], r['cvss'])} {risk_bits(r)} "
114
+ f"{(r['description'] or '')[:80]}")
115
+ if not fzf:
116
+ print("\n(fzf not installed - showing plain list)", file=sys.stderr)
117
+ return
118
+ lines = "\n".join(
119
+ f"{r['id']}\t{r['severity'] or '-'}\t{r['cvss'] if r['cvss'] is not None else '-'}\t"
120
+ f"{'KEV' if r['kev'] else ''}\t{(r['description'] or '')[:110]}"
121
+ for r in rows)
122
+ hv = shutil.which("crowbar") or str(paths.ROOT / "bin" / "crowbar")
123
+ pick = subprocess.run(
124
+ [fzf, "--ansi", "--delimiter=\t", "--with-nth=1,2,3,4,5",
125
+ "--header", f"{len(rows)} CVEs - enter to open, esc to quit",
126
+ "--preview", f"{hv} cve {{1}} --width 100",
127
+ "--preview-window", "right:62%:wrap"],
128
+ input=lines, text=True, capture_output=True)
129
+ cid = (pick.stdout or "").split("\t")[0].strip()
130
+ if cid:
131
+ subprocess.run([hv, "cve", cid])
132
+
133
+
134
+ def cmd_cve(args) -> None:
135
+ con = connect_or_die()
136
+ try:
137
+ d = query.get_cve(con, args.id)
138
+ except KeyError as exc:
139
+ msg = exc.args[0]
140
+ die(msg if " " in msg else f"{msg} not in index")
141
+ if args.json:
142
+ out_json(d)
143
+ return
144
+ print(f"{BOLD}{d['id']}{RST} {sev_badge(d['severity'], d['cvss'])} {risk_bits(d)}")
145
+ print(f"{BOLD}{textwrap.shorten(d['description'] or '', args.width, placeholder='…')}{RST}\n")
146
+ print(f" {DIM}published{RST} {d['published']} {DIM}modified{RST} {d['modified']}"
147
+ f" {DIM}status{RST} {d['status']}")
148
+ if d["cvss_vector"]:
149
+ print(f" {DIM}vector{RST} {d['cvss_vector']}")
150
+ if d["epss"] is not None:
151
+ print(f" {DIM}epss{RST} {d['epss']:.4f} (percentile {d['epss_pct']:.2f})")
152
+ if d["cwe"]:
153
+ print(f" {DIM}weakness{RST} {', '.join(d['cwe'])}")
154
+ if d["affects"]:
155
+ labels = [a["label"] for a in d["affects"]]
156
+ print(f" {DIM}affects{RST} {textwrap.shorten(', '.join(labels[:10]), args.width - 14, placeholder=f'… (+{len(labels) - 10} ranges)')}")
157
+ if d["ssvc"]:
158
+ s = d["ssvc"]
159
+ print(f" {DIM}ssvc{RST} exploitation={s['exploitation']} automatable={s['automatable']}"
160
+ f" impact={s['technical_impact']}")
161
+ print()
162
+ if d["kev_detail"]:
163
+ k = d["kev_detail"]
164
+ print(f"{RED}▌ CISA KEV{RST} added {k['date_added']} due {k['due']} ransomware={k['ransomware']}")
165
+ print(f" {k['vendor']} {k['product']} - {k['name']}")
166
+ if k["action"]:
167
+ print(f" {DIM}action:{RST} {textwrap.shorten(k['action'], args.width - 10, placeholder='…')}")
168
+ print()
169
+ if d["exploits"]:
170
+ print(f"{MAG}▌ Exploit-DB ({len(d['exploits'])}){RST}")
171
+ for e in d["exploits"][:8]:
172
+ print(f" {e['id']:>6} {(e['platform'] or '-'):<10} {(e['type'] or '-'):<8} "
173
+ f"{textwrap.shorten(e['description'] or '', args.width - 34, placeholder='…')}")
174
+ print()
175
+ if d["nuclei"]:
176
+ print(f"{CYN}▌ Nuclei ({len(d['nuclei'])}){RST}")
177
+ for n in d["nuclei"][:8]:
178
+ print(f" {(n['severity'] or '?'):<8} {n['id']} {DIM}{n['path']}{RST}")
179
+ print()
180
+ if d["metasploit"]:
181
+ print(f"{CYN}▌ Metasploit ({len(d['metasploit'])}){RST}")
182
+ for m in d["metasploit"][:8]:
183
+ print(f" {m['type']:<9} {m['name']}")
184
+ print()
185
+ if d.get("pocs"):
186
+ print(f"{MAG}▌ Public PoCs ({d['poc_count']}){RST}")
187
+ for p in d["pocs"][:10]:
188
+ print(f" ★{p['stars']:<6} {p['repo']} {DIM}{p['url']}{RST}")
189
+ print()
190
+ if d.get("cna") and d["cna"].get("description") and d["cna"]["description"] != d["description"]:
191
+ print(f"{DIM}CNA record ({d['cna']['assigner'] or 'unknown'}):{RST} "
192
+ f"{textwrap.shorten(d['cna']['description'], args.width - 4, placeholder='…')}\n")
193
+ if d.get("aliases"):
194
+ print(f"{DIM}aliases{RST} {', '.join(d['aliases'][:20])}")
195
+ if d["references"]:
196
+ print(f"{DIM}references ({len(d['references'])}){RST}")
197
+ for ref in d["references"][:10]:
198
+ print(f" {DIM}·{RST} {ref['url']} {DIM}{ref['tags']}{RST}")
199
+
200
+
201
+ def cmd_product(args) -> None:
202
+ con = connect_or_die()
203
+ rows = query.products(con, args.name, version=args.version, limit=args.limit)
204
+ if args.json:
205
+ out_json(rows)
206
+ return
207
+ head = f"CVEs affecting {args.name!r}"
208
+ if args.version:
209
+ head += f" version {args.version}"
210
+ print(f"{DIM}{head}{RST}\n")
211
+ for r in rows:
212
+ print(f"{BOLD}{r['id']}{RST} {sev_badge(r['severity'], r['cvss'])} {risk_bits(r)} "
213
+ f"{DIM}{(r['published'] or '')[:10]}{RST}")
214
+ if r["matched"]:
215
+ print(f" {DIM}matched{RST} {', '.join(r['matched'])}")
216
+ print(f" {textwrap.shorten(r['description'] or '', args.width - 2, placeholder='…')}")
217
+
218
+
219
+ def cmd_kev(args) -> None:
220
+ con = connect_or_die()
221
+ rows = query.kev_list(con, args.query, args.limit)
222
+ if args.json:
223
+ out_json(rows)
224
+ return
225
+ for r in rows:
226
+ rw = f"{RED}ransom{RST}" if (r["ransomware"] or "").lower() == "known" else ""
227
+ print(f"{BOLD}{r['cve']}{RST} {DIM}{r['date_added']}{RST} {rw}")
228
+ print(f" {r['vendor']} {r['product']} - {r['name']}")
229
+ print(f"\n{DIM}{len(rows)} entries{RST}")
230
+
231
+
232
+ def cmd_exploit(args) -> None:
233
+ con = connect_or_die()
234
+ if args.query.isdigit():
235
+ try:
236
+ e = query.exploit_fetch(con, args.query)
237
+ except KeyError:
238
+ die(f"exploit {args.query} not found")
239
+ if args.json:
240
+ out_json(e)
241
+ else:
242
+ print(f"{BOLD}{e['id']}{RST} {e['platform']} {e['type']} {e['description']}")
243
+ print(f"{DIM}local:{RST} {e['local_path']}")
244
+ return
245
+ rows = query.exploit_search(con, args.query, args.limit)
246
+ if args.json:
247
+ out_json([{k: v for k, v in r.items() if k != "rank"} for r in rows])
248
+ return
249
+ for r in rows:
250
+ v = f"{GRN}✓{RST}" if r["verified"] else " "
251
+ print(f"{v} {BOLD}{r['id']:>6}{RST} {DIM}{(r['date'] or '')[:10]:<10}{RST} "
252
+ f"{(r['platform'] or '-'):<12} "
253
+ f"{textwrap.shorten(r['description'] or '', args.width - 40, placeholder='…')}")
254
+ if r["codes"]:
255
+ print(f" {DIM}{r['codes']}{RST}")
256
+
257
+
258
+ def cmd_nuclei(args) -> None:
259
+ con = connect_or_die()
260
+ rows = query.nuclei_search(con, args.query, args.limit, args.domain)
261
+ if args.json:
262
+ out_json(rows)
263
+ return
264
+ for r in rows:
265
+ print(f"{BOLD}{r['id']}{RST} [{r['severity'] or '?'}] {r['domain']:<9} "
266
+ f"{textwrap.shorten(r['name'] or '', args.width - 40, placeholder='…')}")
267
+ print(f" {DIM}{r['path']} {r['cve']}{RST}")
268
+
269
+
270
+ def cmd_msf(args) -> None:
271
+ con = connect_or_die()
272
+ rows = query.msf_search(con, args.query, args.limit)
273
+ if args.json:
274
+ out_json(rows)
275
+ return
276
+ for r in rows:
277
+ print(f"{r['type']:<10} {BOLD}{r['name']}{RST}")
278
+
279
+
280
+ def cmd_models(args) -> None:
281
+ con = connect_or_die()
282
+ rows = query.models_list(con, args.query, args.provider, args.limit)
283
+ if args.json:
284
+ out_json(rows)
285
+ return
286
+ table(["provider", "model", "ctx", "out", "$in/M", "$out/M", "cap"],
287
+ [[r["provider"], r["name"], f"{(r['context'] or 0) // 1000}k",
288
+ f"{(r['output_limit'] or 0) // 1000}k", money(r["input_cost"]),
289
+ money(r["output_cost"]),
290
+ "".join(x for x, on in (("R", r["reasoning"]), ("T", r["tool_call"]), ("V", r["vision"])) if on)]
291
+ for r in rows])
292
+
293
+
294
+ def cmd_attack(args) -> None:
295
+ con = connect_or_die()
296
+ rows = query.attack_lookup(con, args.query, args.limit, args.domain)
297
+ if args.json:
298
+ out_json(rows)
299
+ return
300
+ for r in rows:
301
+ print(f"{BOLD}{r['id']}{RST} {r['name']} {DIM}{r['domain']} · {r['tactics']}{RST}")
302
+ print(f" {textwrap.shorten(r['description'] or '', args.width - 2, placeholder='…')}")
303
+
304
+
305
+ def cmd_cwe(args) -> None:
306
+ con = connect_or_die()
307
+ try:
308
+ res = query.cwe_get(con, args.query, args.limit)
309
+ except KeyError as exc:
310
+ die(f"{exc.args[0]} not found")
311
+ if args.json:
312
+ out_json(res)
313
+ return
314
+ if isinstance(res, dict):
315
+ print(f"{BOLD}{res['id']}{RST} {res['name']} {DIM}{res['abstraction']} {res['status']}{RST}\n")
316
+ print(textwrap.fill(res["description"] or "", args.width))
317
+ return
318
+ for r in res:
319
+ print(f"{r['id']:<9} {r['name']}")
320
+
321
+
322
+ def cmd_capec(args) -> None:
323
+ con = connect_or_die()
324
+ rows = query.capec_search(con, args.query, args.limit)
325
+ if args.json:
326
+ out_json(rows)
327
+ return
328
+ for r in rows:
329
+ print(f"{r['id']:<12} {r['name']} {DIM}{r['severity'] or ''} {r['cwe']}{RST}")
330
+
331
+
332
+ def cmd_os(args) -> None:
333
+ con = connect_or_die()
334
+ rows = query.os_search(con, args.query, args.platform, args.impact,
335
+ args.exploited, args.since, args.limit)
336
+ if args.json:
337
+ out_json(rows)
338
+ return
339
+ head = "OS advisories"
340
+ if args.platform:
341
+ head += f" · {args.platform}"
342
+ if args.impact:
343
+ head += f" · {args.impact}"
344
+ if args.exploited:
345
+ head += " · exploited"
346
+ print(f"{DIM}{head}{RST}\n")
347
+ for r in rows:
348
+ cve = r["cve"] or " - "
349
+ print(f"{BOLD}{cve:<16}{RST} {r['platform']:<6} {r['severity'] or '-':<9} "
350
+ f"{(r['impact'] or '-'):<14} {DIM}{(r['released'] or '')[:10]}{RST} "
351
+ f"{textwrap.shorten(r['product'] or '', args.width - 70, placeholder='…')}")
352
+ if r["version"] and r["version"] not in (r["product"] or ""):
353
+ print(f" {DIM}{r['version']}{RST}")
354
+ print(f"\n{DIM}{len(rows)} rows{RST}")
355
+
356
+
357
+ def cmd_advisory(args) -> None:
358
+ con = connect_or_die()
359
+ rows = query.advisory_search(con, args.query, args.ecosystem, args.package, args.limit)
360
+ if args.json:
361
+ out_json(rows)
362
+ return
363
+ for r in rows:
364
+ cve = f" {DIM}{r['cve']}{RST}" if r["cve"] else ""
365
+ print(f"{BOLD}{r['id']}{RST} [{r['ecosystem'] or '?'}] {r['package']}{cve}")
366
+ if r["summary"]:
367
+ print(f" {textwrap.shorten(r['summary'], args.width - 2, placeholder='…')}")
368
+ for x in r.get("ranges", [])[:4]:
369
+ bits = []
370
+ if x["introduced"]:
371
+ bits.append(f">={x['introduced']}")
372
+ if x["fixed"]:
373
+ bits.append(f"fixed {x['fixed']}")
374
+ if x["last_affected"]:
375
+ bits.append(f"<={x['last_affected']}")
376
+ if x["versions"]:
377
+ bits.append(x["versions"][:80])
378
+ print(f" {DIM}{x['package']} {' '.join(bits)}{RST}")
379
+
380
+
381
+ def cmd_code(args) -> None:
382
+ con = connect_or_die()
383
+ rows = query.code_search(con, args.query, args.limit)
384
+ if args.json:
385
+ out_json(rows)
386
+ return
387
+ for r in rows:
388
+ print(f"{BOLD}{r['path']}{RST} {DIM}{r['exploit_id'] or ''}{RST}")
389
+ print(f" {r['snippet']}")
390
+
391
+
392
+ def cmd_bin(args) -> None:
393
+ con = connect_or_die()
394
+ rows = query.toolkit_search(con, args.query, args.source, args.limit)
395
+ if args.json:
396
+ out_json(rows)
397
+ return
398
+ for r in rows:
399
+ print(f"{BOLD}{r['name']}{RST} {DIM}{r['source']}{RST} "
400
+ f"{textwrap.shorten(r['description'] or '', args.width - 30, placeholder='…')}")
401
+ print(f" {DIM}{r['path']} {r['tags']}{RST}")
402
+
403
+
404
+ def cmd_debian(args) -> None:
405
+ con = connect_or_die()
406
+ rows = query.debian_lookup(con, args.package, args.cve)
407
+ if args.json:
408
+ out_json(rows)
409
+ return
410
+ for r in rows:
411
+ print(f"{BOLD}{r['cve']}{RST} {r['release']:<14} {r['status']:<16} "
412
+ f"{DIM}fixed {r['fixed'] or '-'}{RST} {r['urgency'] or ''}")
413
+
414
+
415
+ def cmd_poc(args) -> None:
416
+ """Public GitHub PoC repositories (metadata only, never executed)."""
417
+ con = connect_or_die()
418
+ if args.cve:
419
+ rows = query.poc_search(con, args.query or "", cve=args.cve,
420
+ language=args.language, min_stars=args.min_stars,
421
+ limit=args.limit)
422
+ elif args.top:
423
+ rows = query.poc_cves(con, args.limit)
424
+ if args.json:
425
+ out_json(rows)
426
+ return
427
+ for r in rows:
428
+ print(f"{BOLD}{r['cve_id']}{RST} {DIM}×{r['poc_count']} repos{RST} "
429
+ f"{r['severity'] or '-'} {r['cvss'] if r['cvss'] is not None else '-'} "
430
+ f"{r['description'] or ''}")
431
+ return
432
+ else:
433
+ rows = query.poc_search(con, args.query or "", language=args.language,
434
+ min_stars=args.min_stars, limit=args.limit)
435
+ if args.json:
436
+ out_json(rows)
437
+ return
438
+ for r in rows:
439
+ print(f"{BOLD}{r['cve_id']}{RST} {DIM}★{r['stars']}{RST} {r['repo']}")
440
+ print(f" {DIM}{r['url']}{RST} {r['language'] or ''} {r['description'] or ''}")
441
+
442
+
443
+ def cmd_sources(args) -> None:
444
+ """Provenance: which source filled which table, with license and hash."""
445
+ con = connect_or_die()
446
+ rows = [dict(r) for r in con.execute(
447
+ "SELECT source, url, path, license, sha256, fetched_at, loaded_at, "
448
+ "record_count, status, note FROM source_state ORDER BY source")]
449
+ if args.json:
450
+ out_json(rows)
451
+ return
452
+ if not rows:
453
+ print("no source_state rows (run: crowbar update)")
454
+ return
455
+ for r in rows:
456
+ print(f"{BOLD}{r['source']}{RST} {r['status']} {r['record_count']:,} records "
457
+ f"{DIM}license={r['license'] or '?'}{RST}")
458
+ print(f" {DIM}{r['url']}{RST}")
459
+ print(f" {DIM}sha256 {r['sha256'] or '-'}{RST}")
460
+ if r["note"]:
461
+ print(f" {DIM}{r['note']}{RST}")
462
+
463
+
464
+ def cmd_map(args) -> None:
465
+ if args.tables:
466
+ for t, desc in catalog.TABLES.items():
467
+ print(f"{BOLD}{t:<20}{RST} {desc}")
468
+ return
469
+ manifest = datasets.load_manifest().get("datasets", {})
470
+ items = datasets.registry(with_extra=True, with_bulk=True)
471
+ rows = []
472
+ for d in items:
473
+ present, size = catalog.disk_state(d.dest)
474
+ g = catalog.describe(d.name)
475
+ rows.append({
476
+ "name": d.name, "group": d.group, "tier": d.tier,
477
+ "path": d.dest, "present": present, "bytes": size,
478
+ "fetched_at": (manifest.get(d.name) or {}).get("fetched_at"),
479
+ "tables": g["tables"], "commands": g["commands"], "note": d.note,
480
+ })
481
+ if args.json:
482
+ out_json(rows)
483
+ return
484
+ table(["dataset", "tier", "status", "size", "indexes", "command"],
485
+ [[r["name"], r["tier"], "ok" if r["present"] else "-",
486
+ human_bytes(r["bytes"]), ",".join(r["tables"]) or "-",
487
+ (r["commands"] or ["-"])[0]] for r in rows])
488
+ print(f"\n{DIM}raw path root: {paths.DATA}{RST}")
489
+ print(f"{DIM}table meanings: crowbar map --tables{RST}")
490
+
491
+
492
+ def human_bytes(n: float) -> str:
493
+ for unit in ("B", "K", "M", "G"):
494
+ if n < 1024 or unit == "G":
495
+ return f"{n:.0f}{unit}" if unit == "B" else f"{n:.1f}{unit}"
496
+ n /= 1024
497
+ return f"{n:.1f}G"
498
+
499
+
500
+ def cmd_sql(args) -> None:
501
+ con = connect_or_die()
502
+ try:
503
+ rows = query.sql_readonly(con, args.query, args.limit)
504
+ except (ValueError, Exception) as exc: # noqa: BLE001
505
+ die(f"sql error: {exc}")
506
+ if args.json:
507
+ out_json(rows)
508
+ return
509
+ if rows:
510
+ table(list(rows[0]), [list(r.values()) for r in rows])
511
+
512
+
513
+ def cmd_stats(args) -> None:
514
+ con = connect_or_die()
515
+ s = query.stats(con)
516
+ if args.json:
517
+ out_json(s)
518
+ return
519
+ meta = s.pop("meta", {})
520
+ fresh = s.pop("freshness", {})
521
+ print(f"{BOLD}corpus{RST}")
522
+ for k in ("cve", "cve_kev", "cve_with_epss", "cve_with_exploit", "cve_with_nuclei",
523
+ "cve_with_msf", "cpe_match", "kev", "exploit", "shellcode", "nuclei", "msf",
524
+ "attack", "cwe", "capec", "model", "ssvc"):
525
+ if k in s:
526
+ print(f" {k:<18} {s[k]:>10,}")
527
+ print(f" {'db':<18} {s['db_bytes'] / 1e6:>8.0f} MB {DIM}{paths.DB}{RST}")
528
+ print(f"\n{BOLD}state{RST}")
529
+ for k in ("schema", "built_at", "updated_at"):
530
+ print(f" {k:<12} {meta.get(k, ' - ')}")
531
+ if fresh:
532
+ newest = max((v["fetched_at"] or "" for v in fresh.values()), default="")
533
+ print(f" {'datasets':<12} {len(fresh)} files, newest {newest}")
534
+ print(f"\n{BOLD}coverage{RST}")
535
+ if s.get("cve"):
536
+ for k in ("cve_kev", "cve_with_epss", "cve_with_exploit", "cve_with_nuclei", "cve_with_msf"):
537
+ print(f" {k:<18} {100 * s.get(k, 0) / s['cve']:>5.1f}%")
538
+
539
+
540
+ def cmd_where(args) -> None:
541
+ info = {
542
+ "home": str(paths.ROOT),
543
+ "home_link": str(paths.HOME_LINK) + (" (symlink)" if paths.HOME_LINK.is_symlink() else ""),
544
+ "db": str(paths.DB),
545
+ "data": str(paths.DATA),
546
+ "manifest": str(paths.MANIFEST),
547
+ "drive": paths.is_drive_root(),
548
+ "datasets": {k: v["path"] for k, v in datasets.freshness().items()},
549
+ }
550
+ if args.json:
551
+ out_json(info)
552
+ return
553
+ print(f"{BOLD}root{RST} {info['home']}")
554
+ print(f"{BOLD}home link{RST} {info['home_link']}")
555
+ print(f"{BOLD}index{RST} {info['db']}")
556
+ print(f"{BOLD}data{RST} {info['data']}")
557
+ print(f"{BOLD}drive{RST} {'yes' if info['drive'] else 'no'}")
558
+
559
+
560
+ # ---------------------------------------------------------------- data ops
561
+ def _report(r: dict) -> None:
562
+ if r["status"] == "FAIL":
563
+ print(f" FAIL {r['name']:<22} {r.get('error', '')}", file=sys.stderr)
564
+ elif r["status"] in ("same", "exists"):
565
+ print(f" same {r['name']:<22}")
566
+ else:
567
+ print(f" ok {r['name']:<22} {datasets.human(r.get('bytes', 0)):>9s}")
568
+
569
+
570
+ def _record_tier(tier: str) -> None:
571
+ m = datasets.load_manifest()
572
+ m["tier"] = tier
573
+ datasets.save_manifest(m)
574
+
575
+
576
+ def _writer_lock() -> None:
577
+ if not datasets.acquire_update_lock():
578
+ die("another crowbar sync/build/update is running - wait for it to finish")
579
+
580
+
581
+ def cmd_sync(args) -> None:
582
+ _writer_lock()
583
+ years = None
584
+ if args.years:
585
+ years = []
586
+ for part in args.years.split(","):
587
+ if "-" in part:
588
+ a, b = part.split("-", 1)
589
+ years += list(range(int(a), int(b) + 1))
590
+ else:
591
+ years.append(int(part))
592
+ if getattr(args, "tier", None):
593
+ items = datasets.tier_plan(args.tier)
594
+ _record_tier(args.tier)
595
+ else:
596
+ items = datasets.plan(
597
+ only=[s for s in (args.only or "").split(",") if s] or None,
598
+ years=years, with_extra=args.extra, with_bulk=args.bulk)
599
+ if not items:
600
+ die("nothing selected")
601
+ print(f"syncing {len(items)} datasets -> {paths.DATA}"
602
+ + (f" (tier {args.tier})" if getattr(args, "tier", None) else ""))
603
+ results = datasets.sync(items, force=args.force, jobs=args.jobs,
604
+ pull=args.pull, progress=_report)
605
+ bad = [r for r in results if r["status"] == "FAIL"]
606
+ print(f"{len(results) - len(bad)}/{len(results)} ready")
607
+ if bad:
608
+ die(", ".join(r["name"] for r in bad))
609
+
610
+
611
+ def cmd_build(args) -> None:
612
+ from . import indexer
613
+ if not paths.DATA.exists() or not any(paths.DATA.iterdir()):
614
+ die("no data - run: crowbar sync")
615
+ _writer_lock()
616
+ if getattr(args, "tier", None):
617
+ _record_tier(args.tier)
618
+ missing = datasets.tier_missing(args.tier)
619
+ if missing and not getattr(args, "allow_incomplete", False):
620
+ die(f"tier {args.tier} is missing {', '.join(missing)} - nothing to build yet. "
621
+ f"Run: crowbar sync --tier {args.tier} --force (or build --allow-incomplete)")
622
+ if missing:
623
+ print(f"{YEL}WARNING: building an incomplete {args.tier} index "
624
+ f"(missing {', '.join(missing)}){RST}", file=sys.stderr)
625
+ _prog = (lambda msg: print(msg, file=sys.stderr)) if args.json else print
626
+ stats = indexer.build(quick=args.quick, with_extra=args.extra,
627
+ with_bulk=args.bulk, progress=_prog)
628
+ if args.json:
629
+ out_json(stats)
630
+ return
631
+ print(f"\nbuilt {stats['cve']:,} CVEs · {stats.get('cpe_match', 0):,} CPE ranges · "
632
+ f"{datasets.human(stats['db_bytes'])} in {stats['build_seconds']}s")
633
+
634
+
635
+ def cmd_update(args) -> None:
636
+ from . import indexer
637
+ if args.if_mounted and not _mounted():
638
+ return
639
+ _writer_lock()
640
+ status = datasets.index_status()
641
+ tier = status["tier"]
642
+ if status["stale"]:
643
+ if tier:
644
+ # the modified feed only reaches back 8 days: re-sync the tier and rebuild
645
+ ns = argparse.Namespace(tier=tier, years=None, force=False, jobs=args.jobs, pull=False,
646
+ allow_incomplete=True, quick=False, extra=False, bulk=False,
647
+ json=args.json)
648
+ cmd_sync(ns)
649
+ cmd_build(ns)
650
+ return
651
+ print(f"{YEL}index is {status['age_days']} days old, past the 8-day NVD modified feed - "
652
+ f"CVEs changed in the gap stay stale until: crowbar sync && crowbar build{RST}",
653
+ file=sys.stderr)
654
+ groups = ["nvd", "kev", "epss", "exploitdb", "models", "cveorg", "poc"]
655
+ if args.deep:
656
+ groups += ["nuclei", "metasploit", "attack", "cwe", "capec"]
657
+ if tier and tier != "extreme":
658
+ groups = [g for g in groups if g in datasets.tier_groups(tier)]
659
+ items = [d for d in datasets.plan(with_extra=args.extra)
660
+ if d.group in groups and d.name != "nvd-2002"]
661
+ # only the incremental NVD feeds here; year snapshots are for `crowbar build`
662
+ items = [d for d in items if not (d.group == "nvd" and d.name not in ("nvd-recent", "nvd-modified"))]
663
+ if not args.no_sync:
664
+ if not args.quiet:
665
+ print(f"syncing {len(items)} datasets")
666
+ datasets.sync(items, jobs=args.jobs, pull=args.deep, progress=None if args.quiet else _report)
667
+ stats = indexer.update(deep=args.deep, progress=None if args.quiet else print)
668
+ if args.json:
669
+ out_json(stats)
670
+ elif not args.quiet:
671
+ print(f"updated {stats['rows_touched']} CVE rows in {stats['seconds']}s "
672
+ f"({stats['cve']:,} total)")
673
+
674
+
675
+ def cmd_doctor(args) -> None:
676
+ ok = True
677
+ print(f"{BOLD}crowbar doctor{RST}")
678
+ print(f" python {sys.version.split()[0]}")
679
+ print(f" root {paths.ROOT}")
680
+ print(f" drive {'yes' if paths.is_drive_root() else 'no'}")
681
+ if paths.HOME_LINK.is_symlink():
682
+ print(f" home link {paths.HOME_LINK} -> {os.readlink(paths.HOME_LINK)}")
683
+ if not paths.DB.exists():
684
+ print(" index MISSING - run: crowbar sync && crowbar build")
685
+ sys.exit(1)
686
+ con = db.connect()
687
+ ver = db.schema_version(con)
688
+ print(f" schema v{ver} {'ok' if ver == paths.SCHEMA_VERSION else 'MISMATCH - rebuild'}")
689
+ ok = ok and ver == paths.SCHEMA_VERSION
690
+ _integ = con.execute("PRAGMA quick_check").fetchone()[0]
691
+ print(f" integrity {_integ}")
692
+ ok = ok and _integ == "ok"
693
+ s = query.stats(con)
694
+ print(f" cves {s.get('cve', 0):,} · kev {s.get('cve_kev', 0):,} · "
695
+ f"exploits {s.get('cve_with_exploit', 0):,} · cpe ranges {s.get('cpe_match', 0):,}")
696
+ m = datasets.load_manifest()
697
+ missing = [v["path"] for v in m.get("datasets", {}).values() if not (paths.DATA / v["path"]).exists()]
698
+ print(f" datasets {len(m.get('datasets', {}))} tracked, {len(missing)} missing")
699
+ if m.get("tier"):
700
+ print(f" tier {m['tier']} ({datasets.TIER_SIZES.get(m['tier'], '?')})")
701
+ incomplete = datasets.tier_missing(m["tier"])
702
+ if incomplete:
703
+ ok = False
704
+ shown = ", ".join(incomplete[:6]) + (" …" if len(incomplete) > 6 else "")
705
+ print(f" {YEL}incomplete - tier {m['tier']} is missing {len(incomplete)} "
706
+ f"dataset(s): {shown}{RST}")
707
+ print(f" {YEL}re-run: crowbar sync --tier {m['tier']} && crowbar build --tier {m['tier']}{RST}")
708
+ for p in missing[:5]:
709
+ print(f" {DIM}missing {p}{RST}")
710
+ if paths.HOME_LINK.is_symlink() and paths.HOME_LINK.exists():
711
+ try:
712
+ (paths.HOME_LINK / "index/state.json").read_text()
713
+ except OSError:
714
+ print(f" {YEL}drive unreadable by this process - for the launchd agent,"
715
+ f" grant Full Disk Access to /bin/sh{RST}")
716
+ fresh = datasets.index_status()
717
+ if fresh["age_days"] is not None:
718
+ print(f" fresh {fresh['age_days']} days since update"
719
+ f"{'' if fresh['auto_update'] else ' (auto-update off)'}")
720
+ if fresh["stale"]:
721
+ print(f" {YEL}stale - run: crowbar update{RST}")
722
+ free = shutil.disk_usage(paths.ROOT).free
723
+ print(f" disk {datasets.human(free)} free")
724
+ if free < 3 * 1024 ** 3:
725
+ print(f" {YEL}low disk - index rebuild needs ~2x current db size{RST}")
726
+ print(f"\n{'ready' if ok else 'needs attention'}")
727
+ sys.exit(0 if ok else 1)
728
+
729
+
730
+ # ---------------------------------------------------------------- project links
731
+ def _find_home() -> pathlib.Path:
732
+ return paths.HOME_LINK if paths.HOME_LINK.exists() else paths.ROOT
733
+
734
+
735
+ def cmd_link(args) -> None:
736
+ target = pathlib.Path(args.dir).expanduser().resolve()
737
+ if not target.is_dir():
738
+ die(f"{target} is not a directory")
739
+ if target == paths.ROOT or paths.ROOT in target.parents:
740
+ die("refusing to link the harness into itself")
741
+ src = _find_home()
742
+ link = target / ".crowbar"
743
+ if link.is_symlink() or link.exists():
744
+ if not args.force:
745
+ die(f"{link} already exists (use --force)")
746
+ link.unlink() if link.is_symlink() else shutil.rmtree(link)
747
+ link.symlink_to(src, target_is_directory=True)
748
+
749
+ skill_dst = target / ".opencode" / "skills" / "crowbar"
750
+ if not skill_dst.exists() and paths.SKILL.exists():
751
+ skill_dst.parent.mkdir(parents=True, exist_ok=True)
752
+ skill_dst.symlink_to(paths.SKILL, target_is_directory=True)
753
+
754
+ print(f"linked {link} -> {src}")
755
+ print(" cli .crowbar/bin/crowbar search \"...\"")
756
+ print(" docs .crowbar/AGENTS.md")
757
+ print(f" remove crowbar unlink {target}")
758
+
759
+
760
+ def cmd_unlink(args) -> None:
761
+ target = pathlib.Path(args.dir).expanduser().resolve()
762
+ removed = []
763
+ link = target / ".crowbar"
764
+ if link.is_symlink():
765
+ link.unlink()
766
+ removed.append(str(link))
767
+ skill = target / ".opencode" / "skills" / "crowbar"
768
+ if skill.is_symlink() and str(skill.resolve()).startswith(str(paths.ROOT)):
769
+ skill.unlink()
770
+ removed.append(str(skill))
771
+ if not removed:
772
+ die("nothing to unlink")
773
+ print("removed " + ", ".join(removed))
774
+
775
+
776
+ # ---------------------------------------------------------------- drive ops
777
+ def _mounted() -> bool:
778
+ if paths.HOME_LINK.is_symlink() and not paths.HOME_LINK.exists():
779
+ return False
780
+ return paths.ROOT.exists()
781
+
782
+
783
+ CODE = ["bin", "crowbar", "mcp", "scripts", "docs", ".opencode",
784
+ "AGENTS.md", "README.md", "Makefile", ".gitignore"]
785
+
786
+
787
+ def _copy_db(src: pathlib.Path, dst: pathlib.Path) -> None:
788
+ """Consistent snapshot even if a writer is active (sqlite backup API)."""
789
+ if dst.exists():
790
+ dst.unlink()
791
+ s = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
792
+ d = sqlite3.connect(dst)
793
+ try:
794
+ with d:
795
+ s.backup(d)
796
+ finally:
797
+ d.close()
798
+ s.close()
799
+
800
+
801
+ def _db_ok(path: pathlib.Path) -> bool:
802
+ journal = path.with_name(path.name + "-journal")
803
+ if journal.exists() and journal.stat().st_size > 0:
804
+ # unclean shutdown: open read-write once so SQLite rolls the hot journal back
805
+ try:
806
+ w = sqlite3.connect(path, timeout=30)
807
+ w.execute("PRAGMA quick_check").fetchone()
808
+ w.close()
809
+ except sqlite3.DatabaseError:
810
+ return False
811
+ try:
812
+ con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
813
+ res = con.execute("PRAGMA integrity_check").fetchone()[0]
814
+ con.close()
815
+ return res == "ok"
816
+ except sqlite3.DatabaseError:
817
+ return False
818
+
819
+
820
+ def _verify(vault: pathlib.Path) -> list[tuple[str, str]]:
821
+ bad: list[tuple[str, str]] = []
822
+ db = vault / "index/crowbar.db"
823
+ if not db.exists():
824
+ bad.append(("crowbar.db", "missing"))
825
+ elif db.stat().st_size == 0:
826
+ bad.append(("crowbar.db", "empty (0 bytes) - rebuild"))
827
+ elif not _db_ok(db):
828
+ bad.append(("crowbar.db", "integrity check failed"))
829
+ else:
830
+ try:
831
+ con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
832
+ tabs = {r[0] for r in con.execute(
833
+ "SELECT name FROM sqlite_master WHERE type='table'")}
834
+ if "cve" not in tabs:
835
+ bad.append(("crowbar.db", "no schema - empty or wrong DB; rebuild"))
836
+ elif con.execute("SELECT count(*) FROM cve").fetchone()[0] == 0:
837
+ bad.append(("crowbar.db", "0 CVEs - build incomplete; rebuild"))
838
+ con.close()
839
+ except sqlite3.DatabaseError as exc:
840
+ bad.append(("crowbar.db", f"unreadable: {exc}"))
841
+ import json as _json
842
+ man = vault / "index/manifest.json"
843
+ if man.exists():
844
+ for name, rec in _json.loads(man.read_text()).get("datasets", {}).items():
845
+ p = vault / "data" / rec["path"]
846
+ if not p.exists():
847
+ bad.append((name, "missing"))
848
+ elif rec.get("sha256") and datasets.sha256(p) != rec["sha256"]:
849
+ bad.append((name, "hash mismatch"))
850
+ return bad
851
+
852
+
853
+ def cmd_verify(args) -> None:
854
+ problems = _verify(paths.ROOT)
855
+ if args.json:
856
+ out_json({"root": str(paths.ROOT), "ok": not problems,
857
+ "problems": [{"dataset": n, "issue": i} for n, i in problems]})
858
+ return
859
+ if not problems:
860
+ print(f"{GRN}ok{RST} - index and all datasets match the manifest ({paths.ROOT})")
861
+ return
862
+ for n, i in problems:
863
+ print(f"{RED}FAIL{RST} {n:<22} {i}")
864
+ print(f"\n{len(problems)} problems - try: crowbar sync --only <dataset> --force && crowbar build")
865
+ sys.exit(1)
866
+
867
+
868
+ def cmd_selftest(args) -> None:
869
+ """Offline gate: static checks + every test suite + index/dataset verify.
870
+ Never touches the network - run `crowbar sync`/`crowbar update` for that."""
871
+ root = paths.ROOT
872
+ steps = [
873
+ ("static checks", [sys.executable, "scripts/check.py"]),
874
+ ("unit tests", [sys.executable, "-m", "tests.test_unit"]),
875
+ ("schema-v3 tests", [sys.executable, "-m", "tests.test_v3"]),
876
+ ("corpus tests", [sys.executable, "-m", "tests.test_corpus"]),
877
+ ]
878
+ failed = []
879
+ for label, cmd in steps:
880
+ print(f"{CYN}▶ {label}{RST}")
881
+ if subprocess.run(cmd, cwd=str(root)).returncode != 0:
882
+ failed.append(label)
883
+ print(f"{CYN}▶ index + dataset verify{RST}")
884
+ if paths.DB.exists():
885
+ problems = _verify(root)
886
+ if problems:
887
+ for n, i in problems:
888
+ print(f"{RED}FAIL{RST} {n:<22} {i}")
889
+ failed.append("verify")
890
+ else:
891
+ print(f"{GRN}ok{RST} - index and all datasets match the manifest")
892
+ else:
893
+ print(f"{YEL}skip{RST} - no index/crowbar.db (run: crowbar sync && crowbar build)")
894
+ if failed:
895
+ print(f"\n{RED}selftest FAILED{RST}: {', '.join(failed)}")
896
+ sys.exit(1)
897
+ print(f"\n{GRN}selftest passed{RST} - static checks, tests, and verify all green")
898
+
899
+
900
+ START_HERE = """# HACKVAULT - portable vulnerability index
901
+
902
+ This drive holds a complete, self-contained copy of the crowbar corpus:
903
+ every CVE (2002-today) with CPE version ranges, CISA KEV, EPSS, Exploit-DB,
904
+ Nuclei templates, Metasploit modules, MITRE ATT&CK (enterprise/mobile/ICS),
905
+ CWE, CAPEC, and the LLM model catalog.
906
+
907
+ Built {date}. Size: ~{size}.
908
+
909
+ ## Use it
910
+
911
+ ```bash
912
+ # from this folder
913
+ ./bin/crowbar doctor
914
+ ./bin/crowbar search "nginx rce" --kev
915
+ ./bin/crowbar product openssh --version 9.7
916
+ ./bin/crowbar cve CVE-2021-44228
917
+ ./bin/crowbar browse
918
+ ```
919
+
920
+ `~/.crowbar` points here, so projects can link to it:
921
+
922
+ ```bash
923
+ cd /path/to/project
924
+ ~/.crowbar/bin/crowbar link .
925
+ .crowbar/bin/crowbar product <name> --version <ver>
926
+ ~/.crowbar/bin/crowbar unlink .
927
+ ```
928
+
929
+ Agents get the MCP tools from the opencode config on the host
930
+ (`crowbar connect` registers them); the skill is at `.opencode/skills/crowbar/`.
931
+
932
+ ## Update it
933
+
934
+ On the host (drive plugged in):
935
+
936
+ ```bash
937
+ ~/.crowbar/bin/crowbar update # ~10-60 s: modified CVEs + KEV + EPSS + Exploit-DB
938
+ ~/.crowbar/bin/crowbar update --deep # also Nuclei / Metasploit / ATT&CK
939
+ ~/.crowbar/bin/crowbar verify # index + dataset hashes
940
+ ```
941
+
942
+ Scheduling is declared in nix (`launchd.agents.crowbar`, see `docs/NIX.md`),
943
+ not installed imperatively. Until it is wired up, run `crowbar update` by hand after
944
+ plugging the drive in. For a full rebuild: `crowbar sync && crowbar build`.
945
+
946
+ ## Verify before trusting it
947
+
948
+ ```bash
949
+ ./bin/crowbar doctor # schema, integrity, freshness, disk
950
+ ./scripts/selftest.py 2>/dev/null || python3 scripts/selftest.py
951
+ ```
952
+
953
+ ## Care
954
+
955
+ - Eject cleanly (`diskutil eject /Volumes/VAULT`) so the SQLite index is not
956
+ torn mid-write.
957
+ - `data/` is upstream third-party content; see `docs/DATASETS.md` for licenses.
958
+ - Reference data is not authorization: only test systems you may test.
959
+ """
960
+
961
+
962
+ def cmd_setup_drive(args) -> None:
963
+ vault = pathlib.Path(args.path).expanduser().resolve()
964
+ vault.mkdir(parents=True, exist_ok=True)
965
+ if not os.access(vault, os.W_OK):
966
+ die(f"{vault} is not writable")
967
+ if any(vault.iterdir()) and not args.force:
968
+ die(f"{vault} is not empty - pass --force to merge into it")
969
+ print(f"installing vault -> {vault}")
970
+ for name in CODE:
971
+ src = paths.ROOT / name
972
+ if not src.exists():
973
+ continue
974
+ dst = vault / name
975
+ if src.is_dir():
976
+ shutil.copytree(src, dst, dirs_exist_ok=True) # refresh code in place
977
+ else:
978
+ shutil.copy2(src, dst)
979
+ (vault / paths.DRIVE_MARKER).write_text(
980
+ f"crowbar drive\ncreated {time.strftime('%Y-%m-%d')}\n")
981
+
982
+ if not args.no_data:
983
+ data_src, data_dst = paths.DATA, vault / "data"
984
+ if not data_dst.exists():
985
+ print(f"copying data ({datasets.human(_du(data_src))})…")
986
+ shutil.copytree(data_src, data_dst)
987
+ elif shutil.which("rsync"):
988
+ # refresh an existing drive incrementally; keep drive-only files
989
+ print("refreshing data (rsync)…")
990
+ subprocess.run(["rsync", "-a", f"{data_src}/", f"{data_dst}/"], check=True)
991
+ else:
992
+ print("refreshing data (copy)…")
993
+ shutil.copytree(data_src, data_dst, dirs_exist_ok=True)
994
+ idx_dst = vault / "index"
995
+ idx_dst.mkdir(exist_ok=True)
996
+ for f in ("manifest.json", "state.json", "stats.json"):
997
+ if (paths.INDEX / f).exists():
998
+ shutil.copy2(paths.INDEX / f, idx_dst / f)
999
+ print(f"copying index ({datasets.human(paths.DB.stat().st_size)})…")
1000
+ _copy_db(paths.DB, idx_dst / "crowbar.db")
1001
+
1002
+ # verify and repair any file that was re-written while copying
1003
+ import json as _json
1004
+ man = _json.loads((idx_dst / "manifest.json").read_text()).get("datasets", {}) \
1005
+ if (idx_dst / "manifest.json").exists() else {}
1006
+ for name, rec in man.items():
1007
+ src_file, dst_file = paths.DATA / rec["path"], data_dst / rec["path"]
1008
+ if rec.get("sha256") and src_file.exists() and dst_file.exists() \
1009
+ and datasets.sha256(dst_file) != rec["sha256"]:
1010
+ print(f"repairing {name}")
1011
+ shutil.copy2(src_file, dst_file)
1012
+
1013
+ (vault / "START-HERE.md").write_text(
1014
+ START_HERE.format(date=time.strftime("%Y-%m-%d"),
1015
+ size=datasets.human(_du(vault))))
1016
+
1017
+ link = paths.HOME_LINK
1018
+ if link.is_symlink() or link.exists():
1019
+ if not args.force:
1020
+ print(f"{link} exists - not replacing (use --force)")
1021
+ link = None
1022
+ else:
1023
+ link.unlink() if link.is_symlink() else shutil.rmtree(link)
1024
+ if link is not None:
1025
+ link.symlink_to(vault, target_is_directory=True)
1026
+ print(f"linked ~/.crowbar -> {vault}")
1027
+ print("\nnext:")
1028
+ print(f" {vault}/bin/crowbar doctor")
1029
+ print(f" {vault}/bin/crowbar update # when the drive is mounted")
1030
+
1031
+
1032
+ def _du(p: pathlib.Path) -> int:
1033
+ return sum(f.stat().st_size for f in p.rglob("*") if f.is_file())
1034
+
1035
+
1036
+ PLIST = """<?xml version="1.0" encoding="UTF-8"?>
1037
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
1038
+ <plist version="1.0"><dict>
1039
+ <key>Label</key><string>sh.crowbar.update</string>
1040
+ <key>ProgramArguments</key><array><string>{wrapper}</string></array>
1041
+ <key>StartInterval</key><integer>{interval}</integer>
1042
+ <key>RunAtLoad</key><true/>
1043
+ <key>WatchPaths</key><array><string>/Volumes</string></array>
1044
+ <key>StandardOutPath</key><string>{log}</string>
1045
+ <key>StandardErrorPath</key><string>{log}</string>
1046
+ </dict></plist>
1047
+ """
1048
+
1049
+ WRAPPER = """#!/bin/sh
1050
+ # crowbar auto-update.
1051
+ # Prefer the drive copy; fall back to the local copy when the drive is absent
1052
+ # or unreadable (macOS TCC blocks launchd jobs from /Volumes until "Full Disk
1053
+ # Access" is granted to /bin/sh).
1054
+ LOG="$HOME/Library/Logs/crowbar.log"
1055
+ DRIVE="$HOME/.crowbar/bin/crowbar"
1056
+ LOCAL="{root}/bin/crowbar"
1057
+ PY="{python}"
1058
+ run() {{ "$PY" "$1" update --quiet --if-mounted; }}
1059
+
1060
+ if [ -x "$DRIVE" ]; then
1061
+ run "$DRIVE" && exit 0
1062
+ echo "$(date '+%F %T') drive update failed (grant Full Disk Access to /bin/sh for /Volumes access); using local copy" >> "$LOG"
1063
+ fi
1064
+ if [ -x "$LOCAL" ]; then run "$LOCAL"
1065
+ else PYTHONPATH="{pkg}" "$PY" -m crowbar.cli update --quiet; fi
1066
+ exit 0
1067
+ """
1068
+
1069
+
1070
+ def _write_wrapper() -> pathlib.Path:
1071
+ wrapper = pathlib.Path.home() / ".local/bin/crowbar-autoupdate"
1072
+ wrapper.parent.mkdir(parents=True, exist_ok=True)
1073
+ wrapper.write_text(WRAPPER.format(root=paths.ROOT, python=sys.executable,
1074
+ pkg=pathlib.Path(__file__).resolve().parents[1]))
1075
+ wrapper.chmod(0o755)
1076
+ return wrapper
1077
+
1078
+
1079
+ def cmd_install_agent(args) -> None:
1080
+ if pathlib.Path("/run/current-system").exists():
1081
+ print(f"{YEL}note:{RST} this machine is managed by nix. "
1082
+ f"Declaring the agent in home-manager is cleaner - see docs/NIX.md.", file=sys.stderr)
1083
+ wrapper = _write_wrapper()
1084
+ if sys.platform == "darwin":
1085
+ plist = pathlib.Path.home() / "Library/LaunchAgents/sh.crowbar.update.plist"
1086
+ plist.parent.mkdir(parents=True, exist_ok=True)
1087
+ log = pathlib.Path.home() / "Library/Logs/crowbar.log"
1088
+ plist.write_text(PLIST.format(wrapper=wrapper, interval=args.interval, log=log))
1089
+ subprocess.run(["launchctl", "bootout", f"gui/{os.getuid()}", str(plist)],
1090
+ capture_output=True)
1091
+ res = subprocess.run(["launchctl", "bootstrap", f"gui/{os.getuid()}", str(plist)],
1092
+ capture_output=True, text=True)
1093
+ if res.returncode != 0:
1094
+ die(f"launchctl: {res.stderr.strip()}")
1095
+ print(f"launchd agent installed: {plist}")
1096
+ print(f" wrapper: {wrapper}")
1097
+ print(f" every {args.interval}s and whenever /Volumes changes")
1098
+ print(f" log: {log}")
1099
+ else:
1100
+ print("add to crontab -e:\n"
1101
+ f" */{max(1, args.interval // 60)} * * * * {wrapper}")
1102
+
1103
+
1104
+ def cmd_uninstall_agent(args) -> None:
1105
+ if sys.platform == "darwin":
1106
+ plist = pathlib.Path.home() / "Library/LaunchAgents/sh.crowbar.update.plist"
1107
+ subprocess.run(["launchctl", "bootout", f"gui/{os.getuid()}", str(plist)],
1108
+ capture_output=True)
1109
+ if plist.exists():
1110
+ plist.unlink()
1111
+ wrapper = pathlib.Path.home() / ".local/bin/crowbar-autoupdate"
1112
+ if wrapper.exists():
1113
+ wrapper.unlink()
1114
+ print("removed auto-update agent")
1115
+
1116
+
1117
+ def cmd_watch(args) -> None:
1118
+ print(f"watching {paths.HOME_LINK} every {args.interval}s (ctrl-c to stop)")
1119
+ present = _mounted()
1120
+ while True:
1121
+ time.sleep(args.interval)
1122
+ now = _mounted()
1123
+ if now and not present:
1124
+ print(f"[{time.strftime('%H:%M:%S')}] drive appeared - updating")
1125
+ subprocess.run([sys.executable, str(paths.ROOT / "bin/crowbar"), "update", "--quiet"])
1126
+ elif not now and present:
1127
+ print(f"[{time.strftime('%H:%M:%S')}] drive removed")
1128
+ present = now
1129
+
1130
+
1131
+ # ---------------------------------------------------------------- opencode
1132
+ def _mcp_command(local: bool = False) -> list[str]:
1133
+ """Portable launch command for the MCP server. Defaults to the published
1134
+ package via uvx; --local uses the source checkout's mcp/server.py."""
1135
+ if local and (paths.ROOT / "mcp/server.py").exists():
1136
+ return [sys.executable, str(paths.ROOT / "mcp/server.py")]
1137
+ return ["uvx", "--from", "crowbar-cli", "crowbar-mcp"]
1138
+
1139
+
1140
+ def cmd_connect(args) -> None:
1141
+ host = getattr(args, "host", "opencode")
1142
+ cmd = _mcp_command(local=getattr(args, "local", False))
1143
+ if getattr(args, "print", False):
1144
+ host = "print"
1145
+
1146
+ if host in ("print", "claude", "codex", "cursor", "gemini"):
1147
+ block = {"mcpServers": {"crowbar": {"command": cmd[0], "args": cmd[1:]}}}
1148
+ if host == "claude":
1149
+ print("# Claude Code - run this, or paste the block into .mcp.json:")
1150
+ print(f"claude mcp add crowbar -- {' '.join(cmd)}\n")
1151
+ elif host != "print":
1152
+ print(f"# {host}: add this to the host's MCP server config:")
1153
+ out_json(block)
1154
+ if host in ("claude", "codex") and paths.SKILL.exists():
1155
+ print(f"\n# skill: {paths.SKILL / 'SKILL.md'} (install into the host's skills dir)")
1156
+ return
1157
+
1158
+ # opencode: write the config file and install the skill
1159
+ block = {"type": "local", "command": cmd, "enabled": True, "timeout": 15000}
1160
+ target = (pathlib.Path.home() / ".config/opencode/opencode.json"
1161
+ if args.global_ else pathlib.Path(args.target).resolve())
1162
+ cfg = {}
1163
+ if target.exists():
1164
+ try:
1165
+ cfg = json.loads(target.read_text() or "{}")
1166
+ except json.JSONDecodeError:
1167
+ die(f"{target} is not valid JSON - fix it first")
1168
+ cfg.setdefault("$schema", "https://opencode.ai/config.json")
1169
+ cfg.setdefault("mcp", {})["crowbar"] = block
1170
+ target.parent.mkdir(parents=True, exist_ok=True)
1171
+ target.write_text(json.dumps(cfg, indent=2) + "\n")
1172
+ print(f"mcp server registered -> {target}")
1173
+ if not args.no_skill and paths.SKILL.exists():
1174
+ dst = target.parent / ".opencode/skills/crowbar"
1175
+ dst.mkdir(parents=True, exist_ok=True)
1176
+ shutil.copy2(paths.SKILL / "SKILL.md", dst / "SKILL.md")
1177
+ print(f"skill installed -> {dst / 'SKILL.md'}")
1178
+ print("restart opencode; the crowbar tools will appear automatically")
1179
+
1180
+
1181
+ def cmd_mcp(args) -> None:
1182
+ from . import mcp
1183
+ mcp.serve()
1184
+
1185
+
1186
+ # ---------------------------------------------------------------- parser
1187
+ def build_parser() -> argparse.ArgumentParser:
1188
+ from crowbar import __version__
1189
+ p = argparse.ArgumentParser(prog="crowbar", description="crowbar - local vulnerability index (terminal only)")
1190
+ p.add_argument("--version", action="version", version=f"crowbar {__version__}")
1191
+ p.add_argument("--width", type=int, default=110)
1192
+ sub = p.add_subparsers(dest="cmd", required=True)
1193
+
1194
+ def add(name, fn, help_):
1195
+ s = sub.add_parser(name, help=help_)
1196
+ s.set_defaults(fn=fn)
1197
+ return s
1198
+
1199
+ s = add("search", cmd_search, "full-text CVE search")
1200
+ s.add_argument("query", nargs="?")
1201
+ s.add_argument("--limit", type=int, default=20)
1202
+ s.add_argument("--offset", type=int, default=0)
1203
+ s.add_argument("--sort", choices=["relevance", "newest", "oldest", "cvss", "epss"], default="relevance")
1204
+ s.add_argument("--severity")
1205
+ s.add_argument("--year", type=int)
1206
+ s.add_argument("--since")
1207
+ s.add_argument("--kev", action="store_true")
1208
+ s.add_argument("--exploit", action="store_true")
1209
+ s.add_argument("--epss", type=float)
1210
+ s.add_argument("--cvss", type=float)
1211
+ s.add_argument("--cwe")
1212
+ s.add_argument("--product")
1213
+ s.add_argument("--vendor")
1214
+ s.add_argument("--ssvc", help="CISA SSVC exploitation: none|poc|active")
1215
+ s.add_argument("--poc", action="store_true", help="only CVEs with a public PoC")
1216
+ s.add_argument("--json", action="store_true")
1217
+
1218
+ s = add("browse", cmd_browse, "interactive fzf browser")
1219
+ s.add_argument("query", nargs="?")
1220
+ s.add_argument("--limit", type=int, default=500)
1221
+ s.add_argument("--kev", action="store_true")
1222
+ s.add_argument("--exploit", action="store_true")
1223
+
1224
+ s = add("cve", cmd_cve, "one CVE in full")
1225
+ s.add_argument("id")
1226
+ s.add_argument("--json", action="store_true")
1227
+
1228
+ s = add("product", cmd_product, "CVEs affecting a product, optionally a version")
1229
+ s.add_argument("name")
1230
+ s.add_argument("--version", help="exact version to test against CPE ranges, e.g. 1.18.0")
1231
+ s.add_argument("--limit", type=int, default=25)
1232
+ s.add_argument("--json", action="store_true")
1233
+
1234
+ s = add("kev", cmd_kev, "CISA Known Exploited Vulnerabilities")
1235
+ s.add_argument("query", nargs="?")
1236
+ s.add_argument("--limit", type=int, default=25)
1237
+ s.add_argument("--json", action="store_true")
1238
+
1239
+ s = add("exploit", cmd_exploit, "search Exploit-DB, or fetch one by id")
1240
+ s.add_argument("query")
1241
+ s.add_argument("--limit", type=int, default=20)
1242
+ s.add_argument("--json", action="store_true")
1243
+
1244
+ s = add("nuclei", cmd_nuclei, "search Nuclei templates")
1245
+ s.add_argument("query")
1246
+ s.add_argument("--domain", help="http, network, dns, file, headless, cloud, javascript, ssl, code")
1247
+ s.add_argument("--limit", type=int, default=20)
1248
+ s.add_argument("--json", action="store_true")
1249
+
1250
+ s = add("msf", cmd_msf, "search Metasploit modules")
1251
+ s.add_argument("query")
1252
+ s.add_argument("--limit", type=int, default=30)
1253
+ s.add_argument("--json", action="store_true")
1254
+
1255
+ s = add("models", cmd_models, "LLM catalog: context, price, capabilities")
1256
+ s.add_argument("query", nargs="?")
1257
+ s.add_argument("--provider")
1258
+ s.add_argument("--limit", type=int, default=40)
1259
+ s.add_argument("--json", action="store_true")
1260
+
1261
+ s = add("attack", cmd_attack, "MITRE ATT&CK techniques")
1262
+ s.add_argument("query")
1263
+ s.add_argument("--domain", choices=["enterprise", "mobile", "ics"])
1264
+ s.add_argument("--limit", type=int, default=10)
1265
+ s.add_argument("--json", action="store_true")
1266
+
1267
+ s = add("cwe", cmd_cwe, "CWE weakness lookup")
1268
+ s.add_argument("query", nargs="?")
1269
+ s.add_argument("--limit", type=int, default=30)
1270
+ s.add_argument("--json", action="store_true")
1271
+
1272
+ s = add("capec", cmd_capec, "CAPEC attack patterns")
1273
+ s.add_argument("query")
1274
+ s.add_argument("--limit", type=int, default=15)
1275
+ s.add_argument("--json", action="store_true")
1276
+
1277
+ s = add("os", cmd_os, "Windows / iOS / macOS vendor advisories")
1278
+ s.add_argument("query", nargs="?", default=None)
1279
+ s.add_argument("--platform", choices=["windows", "ios", "macos", "apple"])
1280
+ s.add_argument("--impact", help="RCE, EoP, DoS, InfoDisclosure, Spoofing, Bypass")
1281
+ s.add_argument("--exploited", action="store_true")
1282
+ s.add_argument("--since")
1283
+ s.add_argument("--limit", type=int, default=40)
1284
+ s.add_argument("--json", action="store_true")
1285
+
1286
+ s = add("advisory", cmd_advisory, "OSV/GHSA advisories by package or ecosystem")
1287
+ s.add_argument("query", nargs="?", default=None)
1288
+ s.add_argument("--ecosystem")
1289
+ s.add_argument("--package")
1290
+ s.add_argument("--limit", type=int, default=30)
1291
+ s.add_argument("--json", action="store_true")
1292
+
1293
+ s = add("code", cmd_code, "full-text search inside Exploit-DB sources")
1294
+ s.add_argument("query")
1295
+ s.add_argument("--limit", type=int, default=20)
1296
+ s.add_argument("--json", action="store_true")
1297
+
1298
+ s = add("bin", cmd_bin, "GTFOBins / LOLBAS / PayloadsAllTheThings")
1299
+ s.add_argument("query")
1300
+ s.add_argument("--source", choices=["gtfobins", "lolbas", "payloads"])
1301
+ s.add_argument("--limit", type=int, default=25)
1302
+ s.add_argument("--json", action="store_true")
1303
+
1304
+ s = add("debian", cmd_debian, "Debian tracker: fixed versions per release")
1305
+ s.add_argument("package")
1306
+ s.add_argument("--cve")
1307
+ s.add_argument("--json", action="store_true")
1308
+
1309
+ s = add("poc", cmd_poc, "public GitHub PoC repositories (metadata only)")
1310
+ s.add_argument("query", nargs="?", default="")
1311
+ s.add_argument("--cve")
1312
+ s.add_argument("--language")
1313
+ s.add_argument("--min-stars", type=int, default=0)
1314
+ s.add_argument("--top", action="store_true", help="CVEs with the most PoCs")
1315
+ s.add_argument("--limit", type=int, default=25)
1316
+ s.add_argument("--json", action="store_true")
1317
+
1318
+ s = add("sources", cmd_sources, "source provenance: url, license, sha256, row counts")
1319
+ s.add_argument("--json", action="store_true")
1320
+
1321
+ s = add("map", cmd_map, "dataset -> path -> index table map")
1322
+ s.add_argument("--tables", action="store_true", help="explain every table")
1323
+ s.add_argument("--json", action="store_true")
1324
+
1325
+ s = add("sql", cmd_sql, "read-only SQL escape hatch")
1326
+ s.add_argument("query")
1327
+ s.add_argument("--limit", type=int, default=200)
1328
+ s.add_argument("--json", action="store_true")
1329
+
1330
+ s = add("stats", cmd_stats, "corpus statistics and freshness")
1331
+ s.add_argument("--json", action="store_true")
1332
+
1333
+ add("where", cmd_where, "print paths").add_argument("--json", action="store_true")
1334
+ add("doctor", cmd_doctor, "health check")
1335
+
1336
+ s = add("sync", cmd_sync, "download datasets")
1337
+ s.add_argument("--tier", choices=datasets.TIER_NAMES,
1338
+ help="size tier: lite ~150MB, medium ~500MB, heavy ~2GB, extreme full (each a superset)")
1339
+ s.add_argument("--only")
1340
+ s.add_argument("--years")
1341
+ s.add_argument("--extra", action="store_true", help="ATT&CK mobile/ics + CISA Vulnrichment")
1342
+ s.add_argument("--bulk", action="store_true", help="+ OSV ecosystems, Debian tracker, full Exploit-DB")
1343
+ s.add_argument("--force", action="store_true")
1344
+ s.add_argument("--pull", action="store_true", help="git pull existing clones")
1345
+ s.add_argument("--jobs", type=int, default=4)
1346
+
1347
+ s = add("build", cmd_build, "build the index from downloaded data")
1348
+ s.add_argument("--tier", choices=datasets.TIER_NAMES, help="record the size tier this index was built for")
1349
+ s.add_argument("--allow-incomplete", action="store_true", help="build even if some tier datasets are missing")
1350
+ s.add_argument("--quick", action="store_true", help="CVEs + exploits only")
1351
+ s.add_argument("--extra", action="store_true")
1352
+ s.add_argument("--bulk", action="store_true")
1353
+ s.add_argument("--json", action="store_true")
1354
+
1355
+ s = add("update", cmd_update, "incremental refresh (modified CVEs + intel)")
1356
+ s.add_argument("--deep", action="store_true", help="also reindex Nuclei/MSF/ATT&CK")
1357
+ s.add_argument("--if-mounted", action="store_true", help="no-op when the drive is absent")
1358
+ s.add_argument("--no-sync", action="store_true")
1359
+ s.add_argument("--extra", action="store_true")
1360
+ s.add_argument("--jobs", type=int, default=4)
1361
+ s.add_argument("--quiet", action="store_true")
1362
+ s.add_argument("--json", action="store_true")
1363
+
1364
+ s = add("link", cmd_link, "link the corpus into a project as .crowbar")
1365
+ s.add_argument("dir", nargs="?", default=".")
1366
+ s.add_argument("--force", action="store_true")
1367
+
1368
+ s = add("unlink", cmd_unlink, "remove the project link")
1369
+ s.add_argument("dir", nargs="?", default=".")
1370
+
1371
+ s = add("verify", cmd_verify, "check index + dataset hashes")
1372
+ s.add_argument("--json", action="store_true")
1373
+
1374
+ add("selftest", cmd_selftest, "offline gate: static checks + tests + verify (no sync)")
1375
+
1376
+ s = add("setup-drive", cmd_setup_drive, "install the harness on a USB drive")
1377
+ s.add_argument("path")
1378
+ s.add_argument("--no-data", action="store_true", help="code only, no data/index copy")
1379
+ s.add_argument("--force", action="store_true")
1380
+
1381
+ s = add("install-agent", cmd_install_agent, "refresh automatically when the drive appears")
1382
+ s.add_argument("--interval", type=int, default=600)
1383
+
1384
+ add("uninstall-agent", cmd_uninstall_agent, "remove the launchd agent")
1385
+
1386
+ s = add("watch", cmd_watch, "foreground mount watcher")
1387
+ s.add_argument("--interval", type=int, default=60)
1388
+
1389
+ s = add("connect", cmd_connect, "register the MCP server with an agent host")
1390
+ s.add_argument("--host", choices=["opencode", "claude", "codex", "cursor", "gemini", "print"],
1391
+ default="opencode", help="which agent host to wire up (default: opencode)")
1392
+ s.add_argument("--local", action="store_true", help="launch from this source checkout instead of uvx")
1393
+ s.add_argument("--target", default=str(paths.ROOT.parent / "opencode.json"))
1394
+ s.add_argument("--global", dest="global_", action="store_true")
1395
+ s.add_argument("--no-skill", action="store_true")
1396
+ s.add_argument("--print", action="store_true")
1397
+
1398
+ add("mcp", cmd_mcp, "run the MCP server on stdio")
1399
+ return p
1400
+
1401
+
1402
+ def main() -> None:
1403
+ args = build_parser().parse_args()
1404
+ try:
1405
+ args.fn(args)
1406
+ except BrokenPipeError:
1407
+ pass
1408
+ except KeyboardInterrupt:
1409
+ sys.exit(130)
1410
+ except db.NotIndexed as exc:
1411
+ die(str(exc))
1412
+ except ValueError as exc:
1413
+ die(str(exc))
1414
+
1415
+
1416
+ if __name__ == "__main__":
1417
+ main()