sast-eval 0.1.0__py3-none-any.whl

sast_eval/__init__.py

@@ -0,0 +1,5 @@
+"""sast-eval — unified SAST evaluation harness across 5 vulnerability benchmarks.
+
+Public entry point: ``sast_eval.cli:main`` (the ``sast-eval`` console script).
+"""
+__version__ = "0.1.0"

sast_eval/adapters/common.py

@@ -0,0 +1,66 @@
+"""Shared helpers for adapters and importers.
+
+CWE normalization (§2.1) and bounty-value parsing (reusing the rules of
+corpus/bountytasks/calculate_bounties.py).
+"""
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+# Matches the first integer in a CWE-ish string, with or without the "CWE-" prefix.
+_CWE_INT = re.compile(r"(\d+)")
+
+
+def normalize_cwe(raw: Optional[str]) -> str:
+    """Normalize a raw CWE string to canonical ``CWE-<int>`` form.
+
+    Per §2.1: extract the integer -> ``CWE-<int>`` (zero-padded to 3 for
+    display). Empty/unparseable -> ``CWE-UNKNOWN``.
+
+    Examples (from §2.1):
+        "CWE-22: Path Traversal"        -> "CWE-022"
+        "400: Denial of Service"        -> "CWE-400"
+        "CWE-29: Path Traversal: '\\..\\filename'" -> "CWE-029"
+        ""                             -> "CWE-UNKNOWN"
+        "22"                           -> "CWE-022"
+        None                           -> "CWE-UNKNOWN"
+    """
+    if raw is None:
+        return "CWE-UNKNOWN"
+    s = str(raw).strip()
+    if not s:
+        return "CWE-UNKNOWN"
+    m = _CWE_INT.search(s)
+    if not m:
+        return "CWE-UNKNOWN"
+    return f"CWE-{int(m.group(1)):03d}"
+
+
+def parse_bounty_value(value) -> float:
+    """Parse a bounty value from various formats.
+
+    Reuses the rules of ``corpus/bountytasks/calculate_bounties.py``:
+    - None / "" -> 0.0
+    - already a number -> float
+    - leading "$" stripped
+    - "X to Y" range -> average of the two ends
+    - otherwise best-effort float(), 0.0 on failure
+    """
+    if value is None or value == "":
+        return 0.0
+    if isinstance(value, (int, float)):
+        return float(value)
+    if isinstance(value, str) and value.startswith("$"):
+        value = value[1:]
+    if isinstance(value, str) and " to " in value:
+        parts = value.split(" to ")
+        if len(parts) == 2:
+            try:
+                return (float(parts[0]) + float(parts[1])) / 2
+            except ValueError:
+                pass
+    try:
+        return float(value)
+    except (ValueError, TypeError):
+        return 0.0

sast_eval/adapters/owasp_adapter.py

@@ -0,0 +1,129 @@
+"""OWASP BenchmarkJava adapter — the only end-to-end adapter in this repo (§3.2).
+
+Parses ``expectedresults-1.2.csv`` and emits one unified task record (§2) per row
+to ``tasks/owasp.jsonl``.
+
+Per §3.2 / Step 3b:
+- One task per CSV row. Columns: test name, category, real vulnerability, cwe.
+- ``source_root`` = the benchmark repo root (tools scan the whole repo; matching
+  is per test-case file).
+- ``vulnerable_files`` = ``src/main/java/org/owasp/benchmark/testcode/<TestName>.java``
+  (directory is ``testcode``, not ``testcases``).
+- ``real=false`` -> ``fp_trap: true`` (~1,000 of 2,740 rows are FP traps).
+- Category code -> CWE via the fixed OWASP mapping; the CSV's own ``cwe``
+  column is preferred when present (it is authoritative per-row).
+- No weights (all tasks equal).
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import os
+from pathlib import Path
+
+from sast_eval.adapters.common import normalize_cwe
+
+# Fixed OWASP category -> CWE mapping (§3.2). Used as a fallback when the CSV's
+# per-row ``cwe`` column is absent/empty. Values are zero-padded to 3 for display.
+OWASP_CATEGORY_TO_CWE = {
+    "sqli": "CWE-089",
+    "xss": "CWE-079",
+    "cmdi": "CWE-078",
+    "pathtraver": "CWE-022",
+    "ldapi": "CWE-090",
+    "hash": "CWE-328",
+    "weakrand": "CWE-330",
+    "trustbound": "CWE-501",
+    "securecookie": "CWE-614",
+    "crypto": "CWE-327",
+    "xpathi": "CWE-643",
+}
+
+TESTCODE_REL = "src/main/java/org/owasp/benchmark/testcode"
+
+
+def _category_to_cwe(category: str) -> str:
+    return OWASP_CATEGORY_TO_CWE.get(category, "CWE-UNKNOWN")
+
+
+def build_tasks(root: str) -> list[dict]:
+    root_path = Path(root).resolve()
+    csv_path = root_path / "expectedresults-1.2.csv"
+    if not csv_path.exists():
+        raise FileNotFoundError(f"expectedresults-1.2.csv not found at {csv_path}")
+
+    tasks: list[dict] = []
+    with open(csv_path, newline="", encoding="utf-8") as f:
+        reader = csv.reader(f)
+        header = next(reader, None)  # the '#' comment line
+        if not header or not header[0].lstrip().startswith("#"):
+            # Defensive: if the comment line is missing, the first row is data.
+            # Re-open and treat all rows as data. (The known file always has it.)
+            f.seek(0)
+            reader = csv.reader(f)
+
+        for row in reader:
+            if not row or all(not c.strip() for c in row):
+                continue
+            # Columns: # test name, category, real vulnerability, cwe
+            test_name = row[0].strip()
+            category = row[1].strip() if len(row) > 1 else ""
+            real = row[2].strip().lower() if len(row) > 2 else "true"
+            cwe_raw = row[3].strip() if len(row) > 3 else ""
+
+            # Prefer the CSV's per-row cwe column (authoritative); fall back to
+            # the fixed category mapping.
+            cwe = normalize_cwe(cwe_raw) if cwe_raw else _category_to_cwe(category)
+
+            fp_trap = (real == "false")
+            vuln_file = f"{TESTCODE_REL}/{test_name}.java"
+
+            task = {
+                "task_id": f"owasp/{test_name}",
+                "benchmark": "owasp",
+                "language": "java",
+                "source_root": str(root_path),
+                "vcs": {
+                    "type": "git",
+                    "commit": "",  # filled by PROVENANCE from corpus hash
+                    "checked_out": True,
+                },
+                "ground_truth": {
+                    "cwe": cwe,
+                    "cwe_raw": cwe_raw or category,
+                    "cve": None,
+                    "vulnerable_files": [vuln_file],
+                    "vulnerable_methods": [],
+                    "fp_trap": fp_trap,
+                    "notes": "OWASP BenchmarkJava v1.2; category=%s" % category,
+                },
+                "weights": {},
+                "meta": {
+                    "category": category,
+                    "real_vulnerability": not fp_trap,
+                },
+            }
+            tasks.append(task)
+    return tasks
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Build OWASP BenchmarkJava task records")
+    parser.add_argument("--root", required=True, help="Path to BenchmarkJava repo root")
+    parser.add_argument("--out", required=True, help="Output JSONL path (tasks/owasp.jsonl)")
+    args = parser.parse_args(argv)
+
+    tasks = build_tasks(args.root)
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w", encoding="utf-8") as f:
+        for t in tasks:
+            f.write(json.dumps(t, ensure_ascii=False, sort_keys=True) + "\n")
+
+    # Idempotency note: deterministic output (sorted keys, CSV order preserved).
+    print(f"Wrote {len(tasks)} OWASP task records to {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sast_eval/adapters/sastbench_adapter.py

@@ -0,0 +1,231 @@
+"""SASTbench adapter — end-to-end (this repo owns matching/scoring).
+
+SASTbench evaluates SAST tools on agentic codebases. Its cases are
+self-contained JSON files (``cases/<track>/<caseType>/<id>/case.json``) with
+region-level ground truth: each case annotates one or more regions in the
+source, labelled ``vulnerable`` (scanners must detect) or ``capability_safe``
+(scanners must NOT flag — guarded dangerous code is an FP trap).
+
+Per the SASTbench case schema (``schema/case.schema.json``):
+- ``canonicalKind`` is one of 6 SASTbench kinds (command_injection,
+  path_traversal, ssrf, auth_bypass, authz_bypass, sql_injection). We map each
+  to its primary CWE via ``taxonomy/canonical_kinds.json`` (``cweMappings[0]``).
+- ``regions`` carry ``path``, ``startLine``/``endLine``, ``label``,
+  ``acceptedKinds`` (for vulnerable regions) and ``requiredGuards``/``capability``
+  (for capability_safe regions).
+- ``expectedOutcome.mustDetectRegionIds`` / ``mustNotFlagRegionIds`` state the
+  ground truth explicitly.
+- ``files.root`` is the scannable project root (``project/`` for core,
+  ``../../../../.repos/<snapshot>/`` for full track).
+
+Mapping to the unified task schema (§2):
+- ``task_id`` = ``sastbench/<case-id>`` (e.g. ``sastbench/SB-PY-SV-001``).
+- ``source_root`` = absolute path to ``<case_dir>/<files.root>``.
+- ``ground_truth.cwe`` = primary CWE for ``canonicalKind``.
+- ``ground_truth.vulnerable_files`` = paths of ``vulnerable`` regions.
+- ``ground_truth.vulnerable_regions`` = full region list (NEW field; the
+  matcher uses this for region-level overlap matching — see
+  ``matching/matcher.py``).
+- ``ground_truth.fp_trap`` = ``True`` for ``capability_safe`` cases (no
+  vulnerable regions; any finding is a FP). For ``mixed_intent`` cases this is
+  ``False`` (they have vulnerable regions), but their ``capability_safe``
+  regions are still FP traps at the region level — handled by the matcher.
+- ``meta`` carries ``track``, ``case_type``, ``canonical_kind``, ``agentic``,
+  ``profile``, and (for real-world cases) ``repo``, ``cve``, ``ghsa``,
+  disclosure dates for knowledge-cutoff gating.
+- ``vcs.checked_out`` = ``False`` + ``meta.source_missing`` = ``True`` when
+  ``source_root`` doesn't exist on disk (Full Track snapshots live under
+  ``.repos/`` and must be fetched via ``scripts/setup_repos.py`` — see
+  ``tools/fetch_sources.py``).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+from sast_eval.adapters.common import normalize_cwe
+
+# SASTbench canonical kind -> primary CWE (first entry in
+# taxonomy/canonical_kinds.json ``cweMappings``). Used for the unified
+# ``ground_truth.cwe`` field. Region-level ``acceptedKinds`` are matched
+# against the finding's CWE via the same map in the matcher.
+KIND_TO_PRIMARY_CWE: dict[str, str] = {
+    "command_injection": "CWE-078",
+    "path_traversal": "CWE-022",
+    "ssrf": "CWE-918",
+    "auth_bypass": "CWE-287",
+    "authz_bypass": "CWE-862",
+    "sql_injection": "CWE-089",
+}
+
+# Reverse: CWE -> SASTbench kind, for matching a finding's CWE against a
+# region's acceptedKinds. A finding counts as matching a region if its CWE
+# maps to any of the region's acceptedKinds. Built from the full
+# cweMappings in taxonomy/canonical_kinds.json at load time (see below).
+_CWE_TO_KINDS: dict[str, list[str]] = {}
+
+
+def _load_kind_cwe_map(taxonomy_dir: Path) -> None:
+    """Populate ``_CWE_TO_KINDS`` from taxonomy/canonical_kinds.json."""
+    global _CWE_TO_KINDS
+    if _CWE_TO_KINDS:
+        return
+    kinds_file = taxonomy_dir / "canonical_kinds.json"
+    if not kinds_file.is_file():
+        return
+    data = json.loads(kinds_file.read_text(encoding="utf-8"))
+    for kind in data.get("canonicalKinds", []):
+        kid = kind.get("id", "")
+        for cwe in kind.get("cweMappings", []):
+            _CWE_TO_KINDS.setdefault(normalize_cwe(cwe), []).append(kid)
+
+
+def cwe_to_kinds(cwe: str) -> list[str]:
+    """Return the SASTbench kinds that a CWE maps to (for region matching)."""
+    return _CWE_TO_KINDS.get(cwe, [])
+
+
+def _language_map(raw: str) -> str:
+    """Map SASTbench language labels to our language field."""
+    m = {
+        "python": "python",
+        "typescript": "typescript",
+        "rust": "rust",
+        "swift": "swift",
+        "go": "go",
+        "java": "java",
+        "clojure": "clojure",
+    }
+    return m.get((raw or "").strip().lower(), (raw or "").strip().lower() or "unknown")
+
+
+def _profile_for(case: dict) -> str:
+    """Return the SASTbench profile: 'agentic', 'generic', or 'all'."""
+    if case.get("caseType") == "real_world_generic":
+        return "generic"
+    return "agentic" if case.get("agentic", True) else "generic"
+
+
+def _build_region(r: dict) -> dict:
+    """Normalize one SASTbench region into our ground-truth region record."""
+    return {
+        "id": r.get("id", ""),
+        "path": r.get("path", ""),
+        "start": r.get("startLine", 0),
+        "end": r.get("endLine", 0),
+        "label": r.get("label", ""),  # "vulnerable" | "capability_safe"
+        "accepted_kinds": r.get("acceptedKinds", []),
+        "capability": r.get("capability", ""),
+        "required_guards": r.get("requiredGuards", []),
+    }
+
+
+def build_tasks(root: str) -> list[dict]:
+    """Build unified task records from SASTbench case definitions.
+
+    ``root`` is the sast-bench repo root (containing ``cases/`` and
+    ``taxonomy/``).
+    """
+    root_path = Path(root).resolve()
+    cases_dir = root_path / "cases"
+    if not cases_dir.is_dir():
+        raise FileNotFoundError(f"cases/ not found at {cases_dir}")
+
+    _load_kind_cwe_map(root_path / "taxonomy")
+
+    tasks: list[dict] = []
+    for case_json in sorted(cases_dir.rglob("case.json")):
+        case = json.loads(case_json.read_text(encoding="utf-8"))
+        case_dir = case_json.parent
+        case_id = case.get("id", "")
+        if not case_id:
+            continue
+
+        kind = case.get("canonicalKind", "")
+        cwe = KIND_TO_PRIMARY_CWE.get(kind, "CWE-UNKNOWN")
+
+        regions = [_build_region(r) for r in case.get("regions", [])]
+        vuln_regions = [r for r in regions if r["label"] == "vulnerable"]
+        cap_safe_regions = [r for r in regions if r["label"] == "capability_safe"]
+        vuln_files = sorted({r["path"] for r in vuln_regions if r["path"]})
+
+        # capability_safe cases have no vulnerable regions — any finding is a FP.
+        # mixed_intent cases have both; they are NOT fp_trap at the task level
+        # (they have real vulns to find), but their capability_safe regions are
+        # FP traps at the region level (handled by the matcher).
+        fp_trap = (case.get("caseType") == "capability_safe")
+
+        files_root = case.get("files", {}).get("root", "")
+        source_root = (case_dir / files_root).resolve() if files_root else case_dir
+        source_missing = not source_root.is_dir()
+
+        real_world = case.get("realWorld") or {}
+        disclosure = real_world.get("disclosure") or {}
+
+        meta = {
+            "track": case.get("track", ""),
+            "case_type": case.get("caseType", ""),
+            "canonical_kind": kind,
+            "agentic": case.get("agentic", True),
+            "profile": _profile_for(case),
+            "title": case.get("title", ""),
+            "source_missing": source_missing,
+        }
+        if real_world:
+            meta["repo"] = real_world.get("repo", "")
+            meta["cve"] = real_world.get("cve")
+            meta["ghsa"] = real_world.get("ghsa")
+            meta["vulnerable_commit"] = real_world.get("vulnerableCommit", "")
+            meta["fix_commit"] = real_world.get("fixCommit", "")
+            meta["disclosure_ghsa_published"] = disclosure.get("ghsaPublished")
+            meta["disclosure_fix_commit_date"] = disclosure.get("fixCommitDate")
+            meta["disclosure_cve_published"] = disclosure.get("cvePublished")
+
+        task = {
+            "task_id": f"sastbench/{case_id}",
+            "benchmark": "sastbench",
+            "language": _language_map(case.get("language", "")),
+            "source_root": str(source_root),
+            "vcs": {
+                "type": "git",
+                "commit": real_world.get("vulnerableCommit", ""),
+                "checked_out": not source_missing,
+            },
+            "ground_truth": {
+                "cwe": cwe,
+                "cwe_raw": kind,
+                "cve": real_world.get("cve"),
+                "vulnerable_files": vuln_files,
+                "vulnerable_methods": [],
+                "vulnerable_regions": regions,
+                "fp_trap": fp_trap,
+                "notes": case.get("description", ""),
+            },
+            "weights": {},
+            "meta": meta,
+        }
+        tasks.append(task)
+    return tasks
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Build SASTbench task records from case definitions")
+    parser.add_argument("--root", required=True, help="Path to sast-bench repo root (contains cases/ and taxonomy/)")
+    parser.add_argument("--out", required=True, help="Output JSONL path (tasks/sastbench.jsonl)")
+    args = parser.parse_args(argv)
+
+    tasks = build_tasks(args.root)
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w", encoding="utf-8") as f:
+        for t in tasks:
+            f.write(json.dumps(t, ensure_ascii=False, sort_keys=True) + "\n")
+
+    # Idempotency note: deterministic output (sorted keys, rglob case order).
+    print(f"Wrote {len(tasks)} SASTbench task records to {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sast_eval/cli.py

@@ -0,0 +1,190 @@
+"""Unified CLI for the sast-eval harness.
+
+Subcommands map to the existing module mains (each accepts ``argv: list[str]``):
+
+    sast-eval build       build all 5 benchmarks' tasks (tasks/*.jsonl)
+    sast-eval fetch       fetch source for bountytasks/CWE-Bench/CyberGym/SASTbench
+    sast-eval package     build per-task .tar.gz codebases for SAST analysis
+    sast-eval match       match SARIF results against ground truth
+    sast-eval exploit     run exploit-validation oracles on matched results
+    sast-eval score       render the scorecard from matched + exploit results
+    sast-eval all         build + fetch + package + match + exploit + score
+
+Run ``sast-eval <subcommand> --help`` for per-subcommand options.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+# Default layout (matches the repo's Makefile defaults).
+CORPUS = "corpus"
+TASKS = "tasks"
+IMPORTED = "imported"
+RESULTS = "results"
+REPORTS = "reports"
+CODEBASES = "codebases"
+
+
+def _run(module: str, func: str, argv: list[str]) -> int:
+    """Import module.func and call it with argv."""
+    import importlib
+
+    mod = importlib.import_module(module)
+    fn = getattr(mod, func)
+    return fn(argv)
+
+
+def _add_corpus_args(p: argparse.ArgumentParser) -> None:
+    """Add --corpus/--tasks/--imported/--results/--reports/--codebases overrides."""
+    p.add_argument("--corpus", default=CORPUS, help="Corpus repos root (default: corpus)")
+    p.add_argument("--tasks", default=TASKS, help="Tasks dir (default: tasks)")
+    p.add_argument("--imported", default=IMPORTED, help="Imported results dir (default: imported)")
+    p.add_argument("--results", default=RESULTS, help="Results root (default: results)")
+    p.add_argument("--reports", default=REPORTS, help="Reports dir (default: reports)")
+    p.add_argument("--codebases", default=CODEBASES, help="Codebases dir (default: codebases)")
+
+
+def cmd_build(args: argparse.Namespace) -> int:
+    rc = 0
+    Path(args.tasks).mkdir(parents=True, exist_ok=True)
+    Path(args.imported).mkdir(parents=True, exist_ok=True)
+    rc |= _run("sast_eval.adapters.owasp_adapter", "main",
+                ["--root", str(Path(args.corpus) / "BenchmarkJava"), "--out", f"{args.tasks}/owasp.jsonl"])
+    rc |= _run("sast_eval.importers.bountytasks_importer", "main",
+                ["--metadata-root", str(Path(args.corpus) / "bountytasks"),
+                 "--tasks-out", f"{args.tasks}/bountytasks.jsonl",
+                 "--imported-out", f"{args.imported}/bountytasks.jsonl"])
+    rc |= _run("sast_eval.importers.cwebench_importer", "main",
+                ["--root", str(Path(args.corpus) / "cwe-bench-java"),
+                 "--tasks-out", f"{args.tasks}/cwebench.jsonl",
+                 "--imported-out", f"{args.imported}/cwebench.jsonl"])
+    rc |= _run("sast_eval.importers.cybergym_importer", "main",
+                ["--root", str(Path(args.corpus) / "cybergym"),
+                 "--tasks-out", f"{args.tasks}/cybergym.jsonl",
+                 "--imported-out", f"{args.imported}/cybergym.jsonl"])
+    rc |= _run("sast_eval.adapters.sastbench_adapter", "main",
+                ["--root", str(Path(args.corpus) / "sast-bench"), "--out", f"{args.tasks}/sastbench.jsonl"])
+    return rc
+
+
+def cmd_fetch(args: argparse.Namespace) -> int:
+    argv = [
+        "--bountytasks", str(Path(args.corpus) / "bountytasks"),
+        "--cwebench", str(Path(args.corpus) / "cwe-bench-java"),
+        "--cybergym", str(Path(args.corpus) / "cybergym"),
+        "--sastbench", str(Path(args.corpus) / "sast-bench"),
+    ]
+    if args.cybergym_limit is not None:
+        argv += ["--cybergym-limit", str(args.cybergym_limit)]
+    return _run("sast_eval.tools.fetch_sources", "main", argv)
+
+
+def cmd_package(args: argparse.Namespace) -> int:
+    Path(args.codebases).mkdir(parents=True, exist_ok=True)
+    return _run("sast_eval.tools.package_codebases", "main",
+                ["--tasks", args.tasks, "--out", args.codebases])
+
+
+def cmd_match(args: argparse.Namespace) -> int:
+    out = f"{args.results}/matched/{args.tool}"
+    Path(out).mkdir(parents=True, exist_ok=True)
+    argv = ["--tasks", args.tasks, "--results", f"{args.results}/raw/{args.tool}", "--out", out, "--tool", args.tool]
+    rules_path = Path(f"tools/{args.tool}/rules.json")
+    if rules_path.exists():
+        argv += ["--rules", str(rules_path)]
+    return _run("sast_eval.matching.matcher", "main", argv)
+
+
+def cmd_exploit(args: argparse.Namespace) -> int:
+    out = f"{args.results}/exploits/{args.tool}"
+    Path(out).mkdir(parents=True, exist_ok=True)
+    return _run("sast_eval.exploit.oracle", "main",
+                ["--matched", f"{args.results}/matched/{args.tool}",
+                 "--tasks", args.tasks, "--out", out, "--codebases", args.codebases])
+
+
+def cmd_score(args: argparse.Namespace) -> int:
+    Path(args.reports).mkdir(parents=True, exist_ok=True)
+    out = f"{args.reports}/scorecard.md"
+    argv = ["--matched", f"{args.results}/matched/{args.tool}",
+            "--imported", args.imported, "--tasks", args.tasks, "--out", out]
+    exploits_dir = Path(f"{args.results}/exploits")
+    if exploits_dir.is_dir():
+        argv += ["--exploits", str(exploits_dir)]
+    return _run("sast_eval.scoring.metrics", "main", argv)
+
+
+def cmd_all(args: argparse.Namespace) -> int:
+    rc = cmd_build(args)
+    rc |= cmd_fetch(args)
+    rc |= cmd_package(args)
+    Path(f"{args.results}/raw/{args.tool}").mkdir(parents=True, exist_ok=True)
+    rc |= cmd_match(args)
+    rc |= cmd_exploit(args)
+    rc |= cmd_score(args)
+    return rc
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="sast-eval",
+        description="Unified SAST evaluation harness across 5 vulnerability benchmarks.",
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    # build
+    p = sub.add_parser("build", help="Build all 5 benchmarks' tasks (tasks/*.jsonl)")
+    _add_corpus_args(p)
+    p.set_defaults(func=cmd_build)
+
+    # fetch
+    p = sub.add_parser("fetch", help="Fetch source for bountytasks/CWE-Bench/CyberGym/SASTbench")
+    _add_corpus_args(p)
+    p.add_argument("--cybergym-limit", type=int, default=None,
+                   help="Cap CyberGym tasks fetched from HuggingFace (empty = all)")
+    p.set_defaults(func=cmd_fetch)
+
+    # package
+    p = sub.add_parser("package", help="Build per-task .tar.gz codebases for SAST analysis")
+    _add_corpus_args(p)
+    p.set_defaults(func=cmd_package)
+
+    # match
+    p = sub.add_parser("match", help="Match SARIF results against ground truth")
+    _add_corpus_args(p)
+    p.add_argument("--tool", required=True, help="Tool name (results/raw/<tool>/)")
+    p.set_defaults(func=cmd_match)
+
+    # exploit
+    p = sub.add_parser("exploit", help="Run exploit-validation oracles on matched results")
+    _add_corpus_args(p)
+    p.add_argument("--tool", required=True, help="Tool name (results/matched/<tool>/)")
+    p.set_defaults(func=cmd_exploit)
+
+    # score
+    p = sub.add_parser("score", help="Render the scorecard from matched + exploit results")
+    _add_corpus_args(p)
+    p.add_argument("--tool", required=True, help="Tool name (results/matched/<tool>/)")
+    p.set_defaults(func=cmd_score)
+
+    # all
+    p = sub.add_parser("all", help="build + fetch + package + match + exploit + score")
+    _add_corpus_args(p)
+    p.add_argument("--tool", required=True, help="Tool name")
+    p.add_argument("--cybergym-limit", type=int, default=None,
+                   help="Cap CyberGym tasks fetched from HuggingFace (empty = all)")
+    p.set_defaults(func=cmd_all)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

sast_eval/exploit/__init__.py

@@ -0,0 +1,5 @@
+"""Exploit-validation oracles (§10) — 4-tier oracle engine.
+
+Importing this package registers the generic Tier 1 reachability oracle and
+makes the bespoke oracle subpackage importable.
+"""