jscope 0.4.2__py3-none-any.whl

jscope/assessment_engine.py

@@ -0,0 +1,285 @@
+"""Deterministic, scope-aware assessment orchestration primitives.
+
+This module intentionally ships conservative baseline/passive stages and module
+contracts. Active checks are opt-in modules and must use the supplied guarded
+transport; discovery never grants authorization.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable
+from urllib.parse import urljoin, urlsplit
+
+from jscope.security_transport import PolicyError, TransportPolicy, GuardedSession, normalize_host
+
+
+class AssessmentError(RuntimeError):
+    pass
+
+
+def _stable_id(prefix: str, value: Any) -> str:
+    blob = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode()
+    return f"{prefix}-{hashlib.sha256(blob).hexdigest()[:12]}"
+
+
+@dataclass(frozen=True)
+class ModuleMetadata:
+    name: str
+    version: str = "1.0"
+    required_permissions: tuple[str, ...] = ()
+    traffic: str = "passive"
+    authentication: str = "optional"
+    impact: str = "low"
+    input_types: tuple[str, ...] = ("endpoint",)
+    evidence_required: bool = True
+    request_budget: int = 10
+    time_budget_seconds: float = 30.0
+
+
+@dataclass
+class Endpoint:
+    url: str
+    method: str = "GET"
+    source: str = "observed"
+    parameters: list[str] = field(default_factory=list)
+    auth_context: str | None = None
+    status: int | None = None
+    content_type: str | None = None
+    endpoint_id: str = ""
+
+    def __post_init__(self):
+        if not self.endpoint_id:
+            self.endpoint_id = _stable_id("EP", [self.method.upper(), self.url])
+
+
+@dataclass
+class Relationship:
+    source_id: str
+    target_id: str
+    relation: str
+    provenance: dict[str, Any]
+    confidence: str = "medium"
+    observed: bool = False
+    relationship_id: str = ""
+
+    def __post_init__(self):
+        if not self.relationship_id:
+            self.relationship_id = _stable_id("REL", [self.source_id, self.target_id, self.relation, self.provenance])
+
+
+@dataclass
+class FindingRecord:
+    title: str
+    category: str
+    severity: str
+    confidence: str
+    endpoint: str
+    method: str
+    summary: str
+    expected: str = ""
+    observed: str = ""
+    remediation: str = ""
+    evidence: list[dict[str, Any]] = field(default_factory=list)
+    related_endpoints: list[str] = field(default_factory=list)
+    status: str = "hypothesis"
+    finding_id: str = ""
+    validated_at: str | None = None
+
+    def __post_init__(self):
+        if not self.finding_id:
+            self.finding_id = _stable_id("JSF", [self.category, self.method.upper(), self.endpoint, self.title])
+
+
+class EvidenceStore:
+    """Stores sanitized, content-addressed evidence records."""
+    SENSITIVE = re.compile(r"(?i)(authorization|cookie|set-cookie|token|api[_-]?key|password|secret)(\s*[:=]\s*)([^&\s,;]+)")
+
+    def __init__(self):
+        self.records: dict[str, dict[str, Any]] = {}
+
+    @classmethod
+    def sanitize(cls, value: str, limit: int = 4000) -> str:
+        value = re.sub(r"(?im)(authorization|proxy-authorization|cookie|set-cookie)(\s*[:=]\s*)[^\r\n]+", r"\1\2[REDACTED]", value)
+        value = cls.SENSITIVE.sub(r"\1\2[REDACTED]", value)
+        value = re.sub(r"(?i)([?&](?:token|key|password|secret|access_token)=)[^&#\s]+", r"\1[REDACTED]", value)
+        return value[:limit]
+
+    def add(self, kind: str, content: str, metadata: dict[str, Any] | None = None) -> str:
+        clean = self.sanitize(content)
+        digest = hashlib.sha256(clean.encode()).hexdigest()
+        evidence_id = "E-" + digest[:16]
+        self.records.setdefault(evidence_id, {
+            "evidence_id": evidence_id, "kind": kind, "sha256": digest,
+            "content": clean, "metadata": metadata or {},
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        })
+        return evidence_id
+
+
+class ModuleRegistry:
+    def __init__(self):
+        self._modules: dict[str, tuple[ModuleMetadata, Callable]] = {}
+
+    def register(self, metadata: ModuleMetadata, runner: Callable) -> None:
+        if metadata.name in self._modules:
+            raise AssessmentError(f"duplicate module: {metadata.name}")
+        if metadata.traffic not in {"passive", "active"} or metadata.impact not in {"low", "moderate", "high"}:
+            raise AssessmentError("invalid module capability metadata")
+        if metadata.request_budget < 0 or metadata.time_budget_seconds <= 0:
+            raise AssessmentError("invalid module budgets")
+        self._modules[metadata.name] = (metadata, runner)
+
+    def get(self, name: str):
+        return self._modules[name]
+
+    def all(self):
+        return [metadata for metadata, _ in self._modules.values()]
+
+
+class CorrelationEngine:
+    """Creates traceable relationships; references are hypotheses, not authorization."""
+    def correlate(self, resources: list[dict[str, Any]]) -> list[Relationship]:
+        by_url = {}
+        by_path = {}
+        for item in resources:
+            if item.get("url"):
+                by_url[item["url"]] = item
+                try:
+                    by_path[urlsplit(item["url"]).path] = item
+                except ValueError:
+                    pass
+        links = []
+        for item in resources:
+            source_id = item.get("id") or _stable_id("R", item)
+            text = item.get("content", "")
+            base = item.get("url", "")
+            pattern = r"""(?:fetch|axios\.(?:get|post|put|delete))\s*\(\s*['"]([^'"]+)['"]|(?:src|href)\s*=\s*['"]([^'"]+)['"]"""
+            for match in re.finditer(pattern, text, re.I):
+                raw = match.group(1) or match.group(2)
+                absolute = urljoin(base, raw) if base else raw
+                target = by_url.get(absolute) or by_path.get(urlsplit(absolute).path)
+                if target:
+                    target_id = target.get("id") or _stable_id("R", target)
+                    links.append(Relationship(source_id, target_id, "references", {
+                        "source_url": base, "reference": raw, "resolved_url": absolute,
+                        "method": "static-string-reference",
+                    }, confidence="medium", observed=False))
+        return links
+
+
+class AssessmentOrchestrator:
+    STAGES = ("authorization", "inventory", "passive_discovery", "baseline", "low_impact",
+              "correlation", "additional_approval", "active_testing", "validation", "reporting")
+
+    def __init__(self, assessment_id: str, manifest_path: str | Path,
+                 policy: TransportPolicy, registry: ModuleRegistry | None = None):
+        self.assessment_id = assessment_id
+        self.manifest_path = Path(manifest_path)
+        self.policy = policy
+        self.registry = registry or ModuleRegistry()
+        self.evidence = EvidenceStore()
+        self.endpoints: dict[str, Endpoint] = {}
+        self.relationships: list[Relationship] = []
+        self.findings: dict[str, FindingRecord] = {}
+        self.events: list[dict[str, Any]] = []
+        self.cancelled = False
+        if policy.assessment_id != assessment_id:
+            raise AssessmentError("transport assessment ID mismatch")
+        self._validate_authorization()
+
+    def _validate_authorization(self):
+        try:
+            data = json.loads(self.manifest_path.read_text(encoding="utf-8"))
+            stat = self.manifest_path.stat()
+        except (OSError, ValueError) as exc:
+            raise AssessmentError("authorization manifest unavailable or corrupt") from exc
+        if data.get("schema") != "jscope-assessment/1" or data.get("assessment_id") != self.assessment_id:
+            raise AssessmentError("authorization manifest identity/schema mismatch")
+        auth = data.get("authorization")
+        if not isinstance(auth, dict) or auth.get("fresh_for_this_run") is not True:
+            raise AssessmentError("assessment lacks fresh authorization")
+        approved = auth.get("approved_hosts")
+        if not isinstance(approved, list) or not approved:
+            raise AssessmentError("assessment has no approved hosts")
+        normalized = {normalize_host(h) for h in approved}
+        if normalized != {normalize_host(h) for h in self.policy.hosts}:
+            raise AssessmentError("transport scope does not match approved manifest hosts")
+        if data.get("status") != "approved":
+            raise AssessmentError("assessment is not in approved state")
+        self.manifest_digest = hashlib.sha256(self.manifest_path.read_bytes()).hexdigest()
+        self._manifest_mtime_ns = stat.st_mtime_ns
+        self._manifest_size = stat.st_size
+        self.manifest = data
+
+    def revalidate(self):
+        try:
+            stat = self.manifest_path.stat()
+            digest = hashlib.sha256(self.manifest_path.read_bytes()).hexdigest()
+        except OSError as exc:
+            raise AssessmentError("authorization manifest unavailable") from exc
+        if (stat.st_mtime_ns != self._manifest_mtime_ns or stat.st_size != self._manifest_size
+                or digest != self.manifest_digest):
+            raise AssessmentError("authorization manifest changed; restart approval review")
+        self._validate_authorization()
+
+    def add_endpoint(self, endpoint: Endpoint):
+        self.revalidate()
+        host = normalize_host(urlsplit(endpoint.url).hostname or "")
+        if host not in {normalize_host(h) for h in self.policy.hosts}:
+            raise AssessmentError("endpoint host is not explicitly approved")
+        self.endpoints.setdefault(endpoint.endpoint_id, endpoint)
+
+    def add_finding(self, finding: FindingRecord):
+        # Dedupe key includes category, method and endpoint; title alone is insufficient.
+        self.findings.setdefault(finding.finding_id, finding)
+
+    def cancel(self):
+        self.cancelled = True
+        self.policy.event("assessment-cancelled", outcome="info", reason="operator-request")
+
+    def run_modules(self, names: list[str], context: dict[str, Any] | None = None):
+        results = []
+        for name in names:
+            if self.cancelled:
+                break
+            self.revalidate()
+            metadata, runner = self.registry.get(name)
+            if metadata.traffic == "active":
+                if "active_testing" not in self.manifest.get("authorization", {}).get("permissions", ["active_testing"]):
+                    raise AssessmentError(f"active permission missing for module {name}")
+                if self.manifest.get("status") != "approved":
+                    raise AssessmentError("active testing requires approved assessment")
+            start = time.monotonic()
+            before = self.policy._count
+            global_budget = self.policy.total_budget
+            self.policy.total_budget = min(global_budget, before + metadata.request_budget)
+            try:
+                result = runner(context or {}, self)
+                if time.monotonic() - start > metadata.time_budget_seconds:
+                    raise AssessmentError(f"module time budget exceeded: {name}")
+                results.append({"module": name, "status": "completed", "result": result})
+            except Exception as exc:
+                results.append({"module": name, "status": "failed", "error": type(exc).__name__})
+                self.policy.event("module-failed", outcome="error", reason=type(exc).__name__)
+            finally:
+                self.policy.total_budget = global_budget
+        return results
+
+    def to_dict(self):
+        return {
+            "schema_version": "jscope-assessment-result/1",
+            "assessment_id": self.assessment_id,
+            "manifest_sha256": self.manifest_digest,
+            "status": "cancelled" if self.cancelled else "completed",
+            "endpoints": [asdict(x) for x in self.endpoints.values()],
+            "relationships": [asdict(x) for x in self.relationships],
+            "findings": [asdict(x) for x in self.findings.values()],
+            "evidence": list(self.evidence.records.values()),
+            "coverage": {"known_endpoints": len(self.endpoints), "finding_count": len(self.findings)},
+        }

jscope/assessment_reporting.py

@@ -0,0 +1,71 @@
+"""Versioned JSON and Markdown exports for assessment-engine results."""
+from __future__ import annotations
+import html
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def assessment_payload(result: dict[str, Any], tool_version: str | None = None) -> dict[str, Any]:
+    from jscope import __version__
+    out = dict(result)
+    out["schema_version"] = out.get("schema_version", "jscope-assessment-result/1")
+    out["tool_version"] = tool_version or __version__
+    out["generated_at"] = datetime.now(timezone.utc).isoformat()
+    findings = out.get("findings", [])
+    counts = {}
+    for finding in findings:
+        sev = str(finding.get("severity", "unknown")).lower()
+        counts[sev] = counts.get(sev, 0) + 1
+    out["summary"] = {"finding_count": len(findings), "by_severity": counts,
+                      "endpoint_count": len(out.get("endpoints", []))}
+    return out
+
+
+def render_json(result: dict[str, Any]) -> str:
+    return json.dumps(assessment_payload(result), indent=2, sort_keys=True)
+
+
+def render_markdown(result: dict[str, Any]) -> str:
+    p = assessment_payload(result)
+    lines = ["# JScope Assessment Report", "",
+             f"- Assessment: `{p.get('assessment_id', 'unknown')}`",
+             f"- Status: **{p.get('status', 'unknown')}**",
+             f"- Generated: {p['generated_at']}",
+             f"- Endpoints inventoried: {p['summary']['endpoint_count']}",
+             f"- Findings: {p['summary']['finding_count']}", "",
+             "## Coverage and limitations", "",
+             "Static references and inferred relationships are not proof of authorization or vulnerability.",
+             "Untested components must not be interpreted as secure.", "", "## Findings", ""]
+    findings = p.get("findings", [])
+    if not findings:
+        lines.append("No findings were recorded. This is not a statement that the application is secure.")
+    for f in findings:
+        lines += [f"### {f.get('title', 'Untitled finding')}",
+                  f"- ID: `{f.get('finding_id', 'unknown')}`",
+                  f"- Severity / confidence: **{f.get('severity', 'unknown')}** / {f.get('confidence', 'unknown')}",
+                  f"- Status: {f.get('status', 'hypothesis')}",
+                  f"- Endpoint: `{f.get('method', 'GET')} {f.get('endpoint', 'unknown')}`",
+                  "", f.get("summary", ""), ""]
+        if f.get("expected"): lines += [f"**Expected:** {f['expected']}", ""]
+        if f.get("observed"): lines += [f"**Observed:** {f['observed']}", ""]
+        if f.get("remediation"): lines += [f"**Remediation:** {f['remediation']}", ""]
+        if f.get("evidence"):
+            lines.append("Evidence references: " + ", ".join(f"`{e}`" for e in f["evidence"]))
+            lines.append("")
+    lines += ["## Endpoints", ""]
+    for ep in p.get("endpoints", []):
+        lines.append(f"- `{ep.get('method', 'GET')} {ep.get('url', '')}` ({ep.get('source', 'unknown')})")
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def write_report(result: dict[str, Any], path: str | Path, fmt: str = "json") -> Path:
+    fmt = fmt.lower()
+    if fmt not in {"json", "md", "markdown"}:
+        raise ValueError("format must be json or markdown")
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    content = render_json(result) if fmt == "json" else render_markdown(result)
+    out.write_text(content, encoding="utf-8")
+    return out

jscope/benchmark/__init__.py

@@ -0,0 +1 @@
+"""Real-world accuracy benchmark (FR-06)."""

jscope/benchmark/fixtures.py

@@ -0,0 +1,105 @@
+"""Benchmark fixtures (FR-06): small, hand-verified applications with documented ground truth.
+
+Each fixture is a tiny project plus a `BenchmarkEntry` that states what JScope SHOULD conclude and why. Fixtures come in
+matched vulnerable/patched/absent triples wherever practical, because a detector that cannot tell them apart is not
+measuring anything. `difficult` fixtures are known-hard cases (documented limitations, not failures) that are reported
+separately so they never silently drag down the headline numbers.
+
+Real downloadable third-party applications are deliberately NOT vendored here (licensing + reproducibility): the
+benchmark is built from synthetic, framework-representative fixtures modelled on common real-world patterns, which is
+what FR-06 asks for when "licensing and test reproducibility permit" -- synthetic fixtures always do.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class BenchmarkEntry:
+    name: str
+    vuln_class: str                       # dom_xss | open_redirect | sql_injection | command_injection | idor | none
+    files: dict[str, str]
+    expect_flag: bool                     # should JScope (at default min-confidence) report this vuln_class present?
+    ground_truth_basis: str               # why we know this (hand-verified code review of the fixture)
+    test_conditions: str = "default analyzer config, min_confidence=LOW, dataflow_analyzer enabled"
+    difficult: bool = False               # a known-hard case; reported separately, not folded into headline precision/recall
+    note: str = ""
+    framework: str = "vanilla"
+    minified: bool = False
+    tags: tuple[str, ...] = field(default_factory=tuple)
+
+
+def _entries() -> list[BenchmarkEntry]:
+    e: list[BenchmarkEntry] = []
+
+    e.append(BenchmarkEntry("dom_xss_vulnerable", "dom_xss", {
+        "app.js": "const q = new URLSearchParams(location.search).get('q');\ndocument.getElementById('out').innerHTML = q;\n"
+    }, True, "innerHTML assigned directly from an unsanitised URL query parameter"))
+    e.append(BenchmarkEntry("dom_xss_patched_sanitized", "dom_xss", {
+        "app.js": "import DOMPurify from 'dompurify';\nconst q = new URLSearchParams(location.search).get('q');\n"
+                 "document.getElementById('out').innerHTML = DOMPurify.sanitize(q);\n"
+    }, False, "value passes through DOMPurify.sanitize() before reaching innerHTML"))
+    e.append(BenchmarkEntry("dom_xss_patched_textcontent", "dom_xss", {
+        "app.js": "const q = new URLSearchParams(location.search).get('q');\ndocument.getElementById('out').textContent = q;\n"
+    }, False, "textContent is not an HTML sink; value is rendered as text, not markup"))
+    e.append(BenchmarkEntry("dom_xss_vulnerable_react", "dom_xss", {
+        "Page.jsx": "import React from 'react';\nexport function Page() {\n  const h = new URLSearchParams(window.location.search).get('h') ?? '';\n"
+                   "  return <div dangerouslySetInnerHTML={{ __html: h }} />;\n}\n"
+    }, True, "dangerouslySetInnerHTML fed directly from the URL", framework="react"))
+    e.append(BenchmarkEntry("dom_xss_safe_react_text_prop", "dom_xss", {
+        "Page.jsx": "import React from 'react';\nexport function Page({ title }) { const h = new URLSearchParams(window.location.search).get('h');"
+                   "\n  return <div title={h}>{title}</div>;\n}\n"
+    }, False, "URL value flows only into a text prop, never into dangerouslySetInnerHTML", framework="react"))
+
+    e.append(BenchmarkEntry("open_redirect_vulnerable", "open_redirect", {
+        "nav.js": "const next = new URLSearchParams(location.search).get('next');\nlocation.href = next;\n"
+    }, True, "location.href assigned directly from an unsanitised query parameter"))
+    e.append(BenchmarkEntry("open_redirect_patched_allowlist", "open_redirect", {
+        "nav.js": "const next = new URLSearchParams(location.search).get('next');\nconst ALLOWED = ['/home', '/profile'];\n"
+                 "if (ALLOWED.includes(next)) { location.href = next; }\n"
+    }, False, "destination is checked against a fixed allow-list of literal paths before navigation"))
+
+    e.append(BenchmarkEntry("sql_injection_vulnerable", "sql_injection", {
+        "db.js": "const mysql = require('mysql');\nfunction getUser(id) { return conn.query('SELECT * FROM users WHERE id = ' + id); }\n"
+                "getUser(new URLSearchParams(location.search).get('id'));\n"
+    }, True, "query string concatenated directly into SQL text"))
+    e.append(BenchmarkEntry("sql_injection_patched_parameterized", "sql_injection", {
+        "db.js": "const mysql = require('mysql');\nfunction getUser(id) { return conn.query('SELECT * FROM users WHERE id = ?', [id]); }\n"
+                "getUser(new URLSearchParams(location.search).get('id'));\n"
+    }, False, "value passed as a bound parameter, never concatenated into the query text"))
+
+    e.append(BenchmarkEntry("command_injection_vulnerable", "command_injection", {
+        "run.js": "const cp = require('child_process');\nconst name = new URLSearchParams(location.search).get('name');\n"
+                 "cp.exec('echo ' + name);\n"
+    }, True, "shell command string built via concatenation with untrusted input"))
+    e.append(BenchmarkEntry("command_injection_safe_no_shell_package", "command_injection", {
+        "run.js": "const name = new URLSearchParams(location.search).get('name');\nthing.exec('echo ' + name);\n"
+    }, False, "no child_process/execa/etc. package is used; `.exec()` here is an unrelated API"))
+
+    e.append(BenchmarkEntry("clean_app_no_findings", "none", {
+        "app.js": "function add(a, b) { return a + b; }\nconsole.log(add(1, 2));\nconst el = document.getElementById('x');\n"
+                 "el.textContent = 'static label';\n"
+    }, False, "no untrusted source reaches any sink at all"))
+
+    # -- difficult / documented-limitation cases: reported separately, never silently lowering headline numbers --
+    e.append(BenchmarkEntry("difficult_minified_bundle", "dom_xss", {
+        "bundle.min.js": "function a(b){return document.getElementById(\"x\").innerHTML=b}a(new URLSearchParams(location.search).get(\"q\"))"
+    }, True, "same vulnerable pattern, minified: tests that minification alone does not defeat detection",
+        difficult=True, minified=True, note="minified identifiers; verifies the engine isn't name-dependent"))
+    e.append(BenchmarkEntry("difficult_dynamic_alias_unresolved", "dom_xss", {
+        "app.js": "import { render } from '@app/render';\nrender(location.hash);\n"       # no tsconfig/alias config provided on purpose
+    }, False, "alias has no configuration in this fixture, so the import is legitimately UNRESOLVED; JScope must not "
+             "guess a vulnerability out of an unresolved import -- expected outcome is 'not flagged', not a false negative",
+        difficult=True, note="documents the resolver's honest behaviour on an unconfigured bundler alias"))
+    return e
+
+
+ENTRIES: list[BenchmarkEntry] = _entries()
+
+
+def by_class() -> dict[str, list[BenchmarkEntry]]:
+    out: dict[str, list[BenchmarkEntry]] = {}
+    for e in ENTRIES:
+        out.setdefault(e.vuln_class, []).append(e)
+    return out

jscope/benchmark/runner.py

@@ -0,0 +1,155 @@
+"""Benchmark runner (FR-06): executes fixtures, scores against ground truth, measures reproducibility and performance.
+
+`vuln_class -> sink_class` lets a single lookup table decide whether a fixture's expected vulnerability class was (or
+was not) reported, from the dataflow payload already on AnalysisResult -- no string-matching on finding text.
+"""
+
+from __future__ import annotations
+
+import statistics
+import time
+import tracemalloc
+from dataclasses import dataclass, field
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
+
+from jscope.benchmark.fixtures import ENTRIES, BenchmarkEntry
+from jscope.config import Config
+from jscope.pipeline import run_analysis
+
+CLASS_TO_SINK = {"dom_xss": "html", "open_redirect": "navigation", "sql_injection": "sql", "command_injection": "command",
+                 "code_injection": "code"}
+
+
+@dataclass
+class EntryResult:
+    name: str
+    vuln_class: str
+    difficult: bool
+    expected: bool
+    actual: bool
+    outcome: str                        # true_positive | true_negative | false_positive | false_negative
+    duration_s: float
+    findings: int
+    note: str = ""
+
+
+@dataclass
+class ClassMetrics:
+    vuln_class: str
+    tp: int = 0
+    fp: int = 0
+    fn: int = 0
+    tn: int = 0
+
+    @property
+    def precision(self) -> float | None:
+        return self.tp / (self.tp + self.fp) if (self.tp + self.fp) else None
+
+    @property
+    def recall(self) -> float | None:
+        return self.tp / (self.tp + self.fn) if (self.tp + self.fn) else None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"vuln_class": self.vuln_class, "true_positives": self.tp, "false_positives": self.fp,
+                "false_negatives": self.fn, "true_negatives": self.tn, "precision": self.precision, "recall": self.recall}
+
+
+@dataclass
+class BenchmarkReport:
+    entries: list[EntryResult] = field(default_factory=list)
+    by_class: dict[str, ClassMetrics] = field(default_factory=dict)
+    difficult: list[EntryResult] = field(default_factory=list)
+    duration_s: float = 0.0
+    peak_memory_mb: float = 0.0
+    reproducible: bool | None = None
+    reproducibility_detail: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        agg_tp = sum(m.tp for m in self.by_class.values())
+        agg_fp = sum(m.fp for m in self.by_class.values())
+        agg_fn = sum(m.fn for m in self.by_class.values())
+        return {
+            "schema": "jscope-benchmark/1", "entries": len(self.entries), "difficult_entries": len(self.difficult),
+            "duration_s": round(self.duration_s, 3), "peak_memory_mb": round(self.peak_memory_mb, 1),
+            "reproducible": self.reproducible, "reproducibility_detail": self.reproducibility_detail,
+            "overall": {"true_positives": agg_tp, "false_positives": agg_fp, "false_negatives": agg_fn,
+                        "precision": (agg_tp / (agg_tp + agg_fp)) if (agg_tp + agg_fp) else None,
+                        "recall": (agg_tp / (agg_tp + agg_fn)) if (agg_tp + agg_fn) else None},
+            "by_class": {k: v.to_dict() for k, v in sorted(self.by_class.items())},
+            "results": [e.__dict__ for e in self.entries],
+            "difficult_results": [e.__dict__ for e in self.difficult],
+        }
+
+    def regressions_against(self, baseline: dict) -> list[dict]:
+        """Entries that passed in `baseline` but fail now, by name -- each a line item CI can block a release on."""
+        base_by_name = {r["name"]: r["outcome"] for r in baseline.get("results", [])}
+        out = []
+        for e in self.entries:
+            prev = base_by_name.get(e.name)
+            if prev in ("true_positive", "true_negative") and e.outcome not in ("true_positive", "true_negative"):
+                out.append({"name": e.name, "was": prev, "now": e.outcome})
+        return out
+
+
+def _run_one(entry: BenchmarkEntry) -> tuple[bool, int, float]:
+    with TemporaryDirectory(prefix="jscope-bench-") as d:
+        root = Path(d)
+        for rel, src in entry.files.items():
+            p = root / rel
+            p.parent.mkdir(parents=True, exist_ok=True)
+            p.write_text(src)
+        cfg = Config()
+        cfg.data["analyzers"]["enabled"] = ["framework_detector", "dataflow_analyzer"]
+        cfg.data["min_confidence"] = "LOW"
+        t0 = time.monotonic()
+        res = run_analysis([str(root)], cfg)
+        dt = time.monotonic() - t0
+    wanted_sink = CLASS_TO_SINK.get(entry.vuln_class)
+    flagged = wanted_sink is not None and any(
+        f.category == "dataflow" and f.metadata.get("sink_class") == wanted_sink for f in res.findings)
+    return flagged, len(res.findings), dt
+
+
+def run_benchmark(entries: list[BenchmarkEntry] | None = None, check_reproducibility: bool = True) -> BenchmarkReport:
+    entries = entries if entries is not None else ENTRIES
+    report = BenchmarkReport()
+    tracemalloc.start()
+    t0 = time.monotonic()
+    for entry in entries:
+        actual, n, dt = _run_one(entry)
+        if entry.expect_flag and actual:
+            outcome = "true_positive"
+        elif entry.expect_flag and not actual:
+            outcome = "false_negative"
+        elif not entry.expect_flag and actual:
+            outcome = "false_positive"
+        else:
+            outcome = "true_negative"
+        er = EntryResult(entry.name, entry.vuln_class, entry.difficult, entry.expect_flag, actual, outcome, dt, n, entry.note)
+        if entry.difficult:
+            report.difficult.append(er)
+            continue
+        report.entries.append(er)
+        m = report.by_class.setdefault(entry.vuln_class, ClassMetrics(entry.vuln_class))
+        setattr(m, {"true_positive": "tp", "false_positive": "fp", "false_negative": "fn", "true_negative": "tn"}[outcome],
+               getattr(m, {"true_positive": "tp", "false_positive": "fp", "false_negative": "fn", "true_negative": "tn"}[outcome]) + 1)
+    report.duration_s = time.monotonic() - t0
+    report.peak_memory_mb = tracemalloc.get_traced_memory()[1] / (1024 * 1024)
+    tracemalloc.stop()
+
+    if check_reproducibility:
+        sample = [e for e in entries if not e.difficult][:4] or entries[:4]
+        runs = []
+        for entry in sample:
+            flagged, n, _ = _run_one(entry)
+            runs.append((flagged, n))
+        runs2 = []
+        for entry in sample:
+            flagged, n, _ = _run_one(entry)
+            runs2.append((flagged, n))
+        report.reproducible = runs == runs2
+        report.reproducibility_detail = "repeated runs of the same fixtures produced identical flagged/finding-count results" \
+            if report.reproducible else f"non-deterministic output detected: {runs} != {runs2}"
+    return report