openfix-cli 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
openfix/ingest.py ADDED
@@ -0,0 +1,512 @@
1
+ """Turn scanner output into Findings.
2
+
3
+ Supported inputs (auto-detected):
4
+ * SARIF 2.1.0 - Semgrep, CodeQL, Snyk Code, Trivy, Checkov, Bandit, gitleaks, ...
5
+ * Semgrep JSON (``semgrep --json``)
6
+ * Snyk Open Source JSON (``snyk test --json``)
7
+ * Trivy JSON (``trivy fs --format json``)
8
+ * OSV-Scanner JSON (``osv-scanner --format json``)
9
+ * Gitleaks JSON report
10
+ * GitHub code-scanning alerts and Dependabot alerts (REST API export)
11
+ * OpenFix findings JSON (``{"openfix": "findings/v1", "findings": [...]}``)
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import re
18
+ from collections.abc import Callable, Iterable
19
+ from pathlib import Path
20
+ from typing import Any
21
+
22
+ from .findings import Category, Finding, Location, Package, Severity, severity_from
23
+ from .redact import redact, redact_text
24
+
25
+ MAX_INPUT_BYTES = 64 * 1024 * 1024
26
+ CWE_RE = re.compile(r"CWE-\d+", re.IGNORECASE)
27
+ ADVISORY_RE = re.compile(r"\b(?:CVE-\d{4}-\d{4,}|GHSA(?:-[23456789cfghjmpqrvwx]{4}){3})\b", re.IGNORECASE)
28
+
29
+ ECOSYSTEM_MANIFESTS = {
30
+ "npm": "package.json",
31
+ "pypi": "requirements.txt",
32
+ "pip": "requirements.txt",
33
+ "go": "go.mod",
34
+ "maven": "pom.xml",
35
+ "rubygems": "Gemfile",
36
+ "cargo": "Cargo.toml",
37
+ "crates.io": "Cargo.toml",
38
+ "nuget": "packages.config",
39
+ "composer": "composer.json",
40
+ }
41
+
42
+
43
+ class UnsupportedFormat(ValueError):
44
+ pass
45
+
46
+
47
+ def _cwes(*values: Any) -> list[str]:
48
+ found: list[str] = []
49
+ for v in values:
50
+ for item in v if isinstance(v, list) else [v]:
51
+ if item is None:
52
+ continue
53
+ for m in CWE_RE.findall(str(item)):
54
+ cwe = m.upper()
55
+ if cwe not in found:
56
+ found.append(cwe)
57
+ return found
58
+
59
+
60
+ def _advisories(*values: Any) -> list[str]:
61
+ found: list[str] = []
62
+ for v in values:
63
+ for item in v if isinstance(v, list) else [v]:
64
+ if item is None:
65
+ continue
66
+ for m in ADVISORY_RE.findall(str(item)):
67
+ adv = m.upper() if m.upper().startswith("CVE") else m
68
+ if adv not in found:
69
+ found.append(adv)
70
+ return found
71
+
72
+
73
+ def _rel(path: str | None) -> str | None:
74
+ if not path:
75
+ return None
76
+ path = re.sub(r"^file://", "", str(path))
77
+ path = path.replace("\\", "/")
78
+ while path.startswith("./"):
79
+ path = path[2:]
80
+ return path
81
+
82
+
83
+ def _category_for(text: str, default: Category = Category.sast) -> Category:
84
+ t = text.lower()
85
+ if re.search(
86
+ r"secret|hard-?coded[-_ ]?(password|credential|key|token)|api[-_ ]?key|private[-_ ]key|gitleaks"
87
+ r"|leaked[-_ ]credential",
88
+ t,
89
+ ):
90
+ return Category.secrets
91
+ if re.search(r"terraform|cloudformation|kubernetes|k8s|helm|dockerfile|iac|checkov|tfsec|kics", t):
92
+ return Category.iac
93
+ if re.search(r"\bcve-|\bghsa-|dependency|vulnerable package|sca\b", t):
94
+ return Category.sca
95
+ return default
96
+
97
+
98
+ # --------------------------------------------------------------------------- SARIF
99
+
100
+
101
+ def parse_sarif(doc: dict[str, Any]) -> list[Finding]:
102
+ out: list[Finding] = []
103
+ for run in doc.get("runs") or []:
104
+ driver = (run.get("tool") or {}).get("driver") or {}
105
+ tool = driver.get("name") or "sarif"
106
+ rules = {r.get("id"): r for r in driver.get("rules") or [] if isinstance(r, dict)}
107
+ rules_by_index = driver.get("rules") or []
108
+ for result in run.get("results") or []:
109
+ rule_id = result.get("ruleId") or (result.get("rule") or {}).get("id") or "unknown"
110
+ rule = rules.get(rule_id)
111
+ if rule is None and isinstance(result.get("ruleIndex"), int) and result["ruleIndex"] < len(rules_by_index):
112
+ rule = rules_by_index[result["ruleIndex"]]
113
+ rule = rule or {}
114
+ props = {**(rule.get("properties") or {}), **(result.get("properties") or {})}
115
+ tags = props.get("tags") or []
116
+ loc = ((result.get("locations") or [{}])[0] or {}).get("physicalLocation") or {}
117
+ region = loc.get("region") or {}
118
+ path = _rel((loc.get("artifactLocation") or {}).get("uri"))
119
+ snippet = (region.get("snippet") or {}).get("text")
120
+ message = (result.get("message") or {}).get("text") or ""
121
+ title = (rule.get("shortDescription") or {}).get("text") or rule.get("name") or message[:120] or rule_id
122
+ level = result.get("level") or (rule.get("defaultConfiguration") or {}).get("level")
123
+ score = props.get("security-severity") or props.get("security_severity")
124
+ text = " ".join([tool, rule_id, title, " ".join(map(str, tags)), str(props.get("category", ""))])
125
+ category = _category_for(text)
126
+ explicit = str(props.get("category", "")).lower()
127
+ if explicit in Category.__members__:
128
+ category = Category(explicit)
129
+ pkg = None
130
+ if category is Category.sca and props.get("packageName"):
131
+ pkg = Package(name=props["packageName"], version=props.get("packageVersion"))
132
+ out.append(
133
+ Finding(
134
+ tool=tool,
135
+ rule_id=rule_id,
136
+ title=title,
137
+ message=message,
138
+ severity=severity_from(props.get("severity") or level, score),
139
+ category=category,
140
+ cwe=_cwes(props.get("cwe"), tags, rule.get("relationships")),
141
+ advisories=_advisories(rule_id, message, tags),
142
+ location=Location(
143
+ path=path or "",
144
+ start_line=region.get("startLine"),
145
+ end_line=region.get("endLine") or region.get("startLine"),
146
+ snippet=redact_text(snippet) if snippet else None,
147
+ )
148
+ if path
149
+ else None,
150
+ package=pkg,
151
+ help_uri=rule.get("helpUri"),
152
+ source_format="sarif",
153
+ raw=redact(result),
154
+ )
155
+ )
156
+ return out
157
+
158
+
159
+ # --------------------------------------------------------------------------- Semgrep JSON
160
+
161
+
162
+ def parse_semgrep(doc: dict[str, Any]) -> list[Finding]:
163
+ out = []
164
+ for r in doc.get("results") or []:
165
+ extra = r.get("extra") or {}
166
+ meta = extra.get("metadata") or {}
167
+ rule_id = r.get("check_id") or "semgrep"
168
+ msg = extra.get("message") or ""
169
+ out.append(
170
+ Finding(
171
+ tool="semgrep",
172
+ rule_id=rule_id,
173
+ title=msg.split("\n")[0][:160] or rule_id,
174
+ message=msg,
175
+ severity=severity_from(meta.get("impact") or extra.get("severity")),
176
+ category=_category_for(" ".join([rule_id, str(meta.get("category", ""))])),
177
+ cwe=_cwes(meta.get("cwe")),
178
+ advisories=_advisories(meta.get("references")),
179
+ location=Location(
180
+ path=_rel(r.get("path")) or "",
181
+ start_line=(r.get("start") or {}).get("line"),
182
+ end_line=(r.get("end") or {}).get("line"),
183
+ snippet=redact_text(extra["lines"]) if extra.get("lines") else None,
184
+ ),
185
+ help_uri=meta.get("source"),
186
+ source_format="semgrep-json",
187
+ raw=redact(r),
188
+ )
189
+ )
190
+ return out
191
+
192
+
193
+ # --------------------------------------------------------------------------- Snyk Open Source
194
+
195
+
196
+ def parse_snyk(doc: Any) -> list[Finding]:
197
+ projects = doc if isinstance(doc, list) else [doc]
198
+ out = []
199
+ for project in projects:
200
+ manifest = project.get("displayTargetFile") or project.get("targetFile")
201
+ eco = project.get("packageManager")
202
+ for v in project.get("vulnerabilities") or []:
203
+ ids = v.get("identifiers") or {}
204
+ fixed = [str(x) for x in v.get("fixedIn") or []]
205
+ out.append(
206
+ Finding(
207
+ tool="snyk",
208
+ rule_id=v.get("id") or "snyk",
209
+ title=v.get("title") or v.get("id") or "Vulnerable dependency",
210
+ message=(v.get("description") or "")[:2000],
211
+ severity=severity_from(v.get("severity"), v.get("cvssScore")),
212
+ category=Category.sca,
213
+ cwe=_cwes(ids.get("CWE")),
214
+ advisories=_advisories(ids.get("CVE"), ids.get("GHSA")) or [v.get("id")],
215
+ location=Location(path=_rel(manifest) or manifest) if manifest else None,
216
+ package=Package(
217
+ name=v.get("packageName") or v.get("moduleName") or v.get("name"),
218
+ version=v.get("version"),
219
+ ecosystem=eco,
220
+ fixed_versions=fixed,
221
+ manifest=_rel(manifest),
222
+ ),
223
+ source_format="snyk-json",
224
+ raw=redact({k: v.get(k) for k in ("id", "from", "upgradePath", "isUpgradable", "isPatchable")}),
225
+ )
226
+ )
227
+ return out
228
+
229
+
230
+ # --------------------------------------------------------------------------- Trivy
231
+
232
+
233
+ def parse_trivy(doc: dict[str, Any]) -> list[Finding]:
234
+ out = []
235
+ for res in doc.get("Results") or []:
236
+ target = _rel(res.get("Target"))
237
+ for v in res.get("Vulnerabilities") or []:
238
+ fixed = [x.strip() for x in str(v.get("FixedVersion") or "").split(",") if x.strip()]
239
+ out.append(
240
+ Finding(
241
+ tool="trivy",
242
+ rule_id=v.get("VulnerabilityID") or "trivy-vuln",
243
+ title=v.get("Title") or v.get("VulnerabilityID") or "Vulnerable package",
244
+ message=(v.get("Description") or "")[:2000],
245
+ severity=severity_from(v.get("Severity")),
246
+ category=Category.container if res.get("Class") == "os-pkgs" else Category.sca,
247
+ cwe=_cwes(v.get("CweIDs")),
248
+ advisories=_advisories(v.get("VulnerabilityID")) or [v.get("VulnerabilityID")],
249
+ location=Location(path=target) if target else None,
250
+ package=Package(
251
+ name=v.get("PkgName"),
252
+ version=v.get("InstalledVersion"),
253
+ ecosystem=res.get("Type"),
254
+ fixed_versions=fixed,
255
+ manifest=target,
256
+ ),
257
+ help_uri=v.get("PrimaryURL"),
258
+ source_format="trivy-json",
259
+ raw=redact(
260
+ {k: v.get(k) for k in ("VulnerabilityID", "PkgName", "InstalledVersion", "FixedVersion")}
261
+ ),
262
+ )
263
+ )
264
+ for m in res.get("Misconfigurations") or []:
265
+ cause = m.get("CauseMetadata") or {}
266
+ out.append(
267
+ Finding(
268
+ tool="trivy",
269
+ rule_id=m.get("ID") or m.get("AVDID") or "trivy-misconfig",
270
+ title=m.get("Title") or "Misconfiguration",
271
+ message=m.get("Message") or m.get("Description") or "",
272
+ severity=severity_from(m.get("Severity")),
273
+ category=Category.iac,
274
+ location=Location(
275
+ path=target or "", start_line=cause.get("StartLine"), end_line=cause.get("EndLine")
276
+ ),
277
+ help_uri=m.get("PrimaryURL"),
278
+ source_format="trivy-json",
279
+ raw=redact(m),
280
+ )
281
+ )
282
+ for s in res.get("Secrets") or []:
283
+ out.append(
284
+ Finding(
285
+ tool="trivy",
286
+ rule_id=s.get("RuleID") or "trivy-secret",
287
+ title=s.get("Title") or "Exposed secret",
288
+ severity=severity_from(s.get("Severity")),
289
+ category=Category.secrets,
290
+ location=Location(path=target or "", start_line=s.get("StartLine"), end_line=s.get("EndLine")),
291
+ source_format="trivy-json",
292
+ raw={"RuleID": s.get("RuleID")}, # never keep the secret itself
293
+ )
294
+ )
295
+ return out
296
+
297
+
298
+ # --------------------------------------------------------------------------- OSV-Scanner
299
+
300
+
301
+ def _osv_fixed(vuln: dict[str, Any], name: str) -> list[str]:
302
+ fixed: list[str] = []
303
+ for aff in vuln.get("affected") or []:
304
+ if (aff.get("package") or {}).get("name") not in (None, name):
305
+ continue
306
+ for rng in aff.get("ranges") or []:
307
+ for ev in rng.get("events") or []:
308
+ if ev.get("fixed") and ev["fixed"] not in fixed:
309
+ fixed.append(ev["fixed"])
310
+ return fixed
311
+
312
+
313
+ def parse_osv(doc: dict[str, Any]) -> list[Finding]:
314
+ out = []
315
+ for res in doc.get("results") or []:
316
+ manifest = _rel((res.get("source") or {}).get("path"))
317
+ for p in res.get("packages") or []:
318
+ pkg = p.get("package") or {}
319
+ groups = p.get("groups") or []
320
+ for v in p.get("vulnerabilities") or []:
321
+ score = None
322
+ for g in groups:
323
+ if v.get("id") in (g.get("ids") or []):
324
+ score = g.get("max_severity")
325
+ sev_word = (v.get("database_specific") or {}).get("severity")
326
+ out.append(
327
+ Finding(
328
+ tool="osv-scanner",
329
+ rule_id=v.get("id") or "osv",
330
+ title=v.get("summary") or v.get("id") or "Vulnerable package",
331
+ message=(v.get("details") or "")[:2000],
332
+ severity=severity_from(sev_word, score),
333
+ category=Category.sca,
334
+ cwe=_cwes((v.get("database_specific") or {}).get("cwe_ids")),
335
+ advisories=_advisories(v.get("id"), v.get("aliases")) or [v.get("id")],
336
+ location=Location(path=manifest) if manifest else None,
337
+ package=Package(
338
+ name=pkg.get("name") or "unknown",
339
+ version=pkg.get("version"),
340
+ ecosystem=(pkg.get("ecosystem") or "").lower() or None,
341
+ fixed_versions=_osv_fixed(v, pkg.get("name") or ""),
342
+ manifest=manifest,
343
+ ),
344
+ source_format="osv-json",
345
+ raw={"id": v.get("id"), "aliases": v.get("aliases")},
346
+ )
347
+ )
348
+ return out
349
+
350
+
351
+ # --------------------------------------------------------------------------- Gitleaks
352
+
353
+
354
+ def parse_gitleaks(doc: list[dict[str, Any]]) -> list[Finding]:
355
+ return [
356
+ Finding(
357
+ tool="gitleaks",
358
+ rule_id=g.get("RuleID") or "gitleaks",
359
+ title=g.get("Description") or "Hardcoded secret",
360
+ severity=Severity.high,
361
+ category=Category.secrets,
362
+ location=Location(path=_rel(g.get("File")) or "", start_line=g.get("StartLine"), end_line=g.get("EndLine")),
363
+ source_format="gitleaks-json",
364
+ raw={"RuleID": g.get("RuleID"), "Fingerprint": g.get("Fingerprint"), "Commit": g.get("Commit")},
365
+ )
366
+ for g in doc
367
+ ]
368
+
369
+
370
+ # --------------------------------------------------------------------------- GitHub
371
+
372
+
373
+ def parse_github_code_scanning(doc: list[dict[str, Any]]) -> list[Finding]:
374
+ out = []
375
+ for a in doc:
376
+ rule = a.get("rule") or {}
377
+ inst = a.get("most_recent_instance") or {}
378
+ loc = inst.get("location") or {}
379
+ tool = (a.get("tool") or {}).get("name") or "github-code-scanning"
380
+ out.append(
381
+ Finding(
382
+ tool=tool,
383
+ rule_id=rule.get("id") or "unknown",
384
+ title=rule.get("description") or (inst.get("message") or {}).get("text") or "Code scanning alert",
385
+ message=(inst.get("message") or {}).get("text") or "",
386
+ severity=severity_from(rule.get("security_severity_level") or rule.get("severity")),
387
+ category=_category_for(" ".join([tool, rule.get("id") or "", " ".join(rule.get("tags") or [])])),
388
+ cwe=_cwes(rule.get("tags")),
389
+ location=Location(
390
+ path=_rel(loc.get("path")) or "", start_line=loc.get("start_line"), end_line=loc.get("end_line")
391
+ ),
392
+ help_uri=a.get("html_url"),
393
+ source_format="github-code-scanning",
394
+ raw=redact({"number": a.get("number"), "state": a.get("state"), "commit_sha": inst.get("commit_sha")}),
395
+ )
396
+ )
397
+ return out
398
+
399
+
400
+ def parse_dependabot(doc: list[dict[str, Any]]) -> list[Finding]:
401
+ out = []
402
+ for a in doc:
403
+ adv = a.get("security_advisory") or {}
404
+ vuln = a.get("security_vulnerability") or {}
405
+ pkg = vuln.get("package") or {}
406
+ dep = a.get("dependency") or {}
407
+ patched = (vuln.get("first_patched_version") or {}).get("identifier")
408
+ manifest = _rel(dep.get("manifest_path"))
409
+ out.append(
410
+ Finding(
411
+ tool="dependabot",
412
+ rule_id=adv.get("ghsa_id") or "dependabot",
413
+ title=adv.get("summary") or "Vulnerable dependency",
414
+ message=(adv.get("description") or "")[:2000],
415
+ severity=severity_from(adv.get("severity"), (adv.get("cvss") or {}).get("score")),
416
+ category=Category.sca,
417
+ cwe=_cwes([c.get("cwe_id") for c in adv.get("cwes") or []]),
418
+ advisories=_advisories(adv.get("ghsa_id"), adv.get("cve_id")),
419
+ location=Location(path=manifest) if manifest else None,
420
+ package=Package(
421
+ name=pkg.get("name") or "unknown",
422
+ ecosystem=pkg.get("ecosystem"),
423
+ version=None,
424
+ fixed_versions=[patched] if patched else [],
425
+ manifest=manifest,
426
+ ),
427
+ help_uri=a.get("html_url"),
428
+ source_format="github-dependabot",
429
+ raw={"number": a.get("number"), "vulnerable_version_range": vuln.get("vulnerable_version_range")},
430
+ )
431
+ )
432
+ return out
433
+
434
+
435
+ # --------------------------------------------------------------------------- OpenFix native
436
+
437
+
438
+ def parse_openfix(doc: dict[str, Any]) -> list[Finding]:
439
+ return [Finding.model_validate({**f, "source_format": "openfix"}) for f in doc.get("findings") or []]
440
+
441
+
442
+ # --------------------------------------------------------------------------- dispatch
443
+
444
+
445
+ def detect(doc: Any) -> tuple[str, Callable[[Any], list[Finding]]]:
446
+ if isinstance(doc, dict):
447
+ if str(doc.get("openfix", "")).startswith("findings/"):
448
+ return "openfix", parse_openfix
449
+ if isinstance(doc.get("runs"), list) and ("version" in doc or "$schema" in doc):
450
+ return "sarif", parse_sarif
451
+ if (
452
+ isinstance(doc.get("results"), list)
453
+ and (not doc["results"] or "check_id" in (doc["results"][0] or {}))
454
+ and "errors" in doc
455
+ ):
456
+ return "semgrep-json", parse_semgrep
457
+ if isinstance(doc.get("results"), list) and doc["results"] and "packages" in doc["results"][0]:
458
+ return "osv-json", parse_osv
459
+ if isinstance(doc.get("Results"), list) or doc.get("ArtifactName"):
460
+ return "trivy-json", parse_trivy
461
+ if isinstance(doc.get("vulnerabilities"), list):
462
+ return "snyk-json", parse_snyk
463
+ if isinstance(doc, list):
464
+ if not doc:
465
+ return "empty", lambda _d: []
466
+ first = doc[0] if isinstance(doc[0], dict) else {}
467
+ if "vulnerabilities" in first and ("packageManager" in first or "projectName" in first):
468
+ return "snyk-json", parse_snyk
469
+ if "RuleID" in first and ("File" in first or "Secret" in first):
470
+ return "gitleaks-json", parse_gitleaks
471
+ if "most_recent_instance" in first:
472
+ return "github-code-scanning", parse_github_code_scanning
473
+ if "security_advisory" in first and "security_vulnerability" in first:
474
+ return "github-dependabot", parse_dependabot
475
+ raise UnsupportedFormat(
476
+ "Unrecognised findings format. Supported: SARIF 2.1, Semgrep JSON, Snyk JSON, Trivy JSON, "
477
+ "OSV-Scanner JSON, Gitleaks JSON, GitHub code-scanning/Dependabot alerts, OpenFix findings JSON."
478
+ )
479
+
480
+
481
+ SECRET_CWES = {"CWE-798", "CWE-259", "CWE-321", "CWE-522"}
482
+
483
+
484
+ def load_document(doc: Any) -> list[Finding]:
485
+ _, parser = detect(doc)
486
+ findings = parser(doc)
487
+ for f in findings:
488
+ if f.category is Category.sast and SECRET_CWES & set(f.cwe):
489
+ f.category = Category.secrets
490
+ return findings
491
+
492
+
493
+ def load_file(path: str | Path) -> list[Finding]:
494
+ p = Path(path)
495
+ size = p.stat().st_size
496
+ if size > MAX_INPUT_BYTES:
497
+ raise ValueError(f"{p} is {size} bytes; the limit is {MAX_INPUT_BYTES}")
498
+ try:
499
+ doc = json.loads(p.read_text(encoding="utf-8"))
500
+ except json.JSONDecodeError as exc:
501
+ raise ValueError(f"{p} is not valid JSON ({exc.msg} at line {exc.lineno})") from None
502
+ return load_document(doc)
503
+
504
+
505
+ def load_files(paths: Iterable[str | Path]) -> list[Finding]:
506
+ """Load and de-duplicate findings from several reports (same fingerprint -> one finding)."""
507
+ seen: dict[str, Finding] = {}
508
+ for path in paths:
509
+ for f in load_file(path):
510
+ if f.id not in seen or f.severity.rank > seen[f.id].severity.rank:
511
+ seen[f.id] = f
512
+ return list(seen.values())