switch-trust-cli 1.7.0__py3-none-any.whl

switch_trust/cli/runner.py

@@ -0,0 +1,274 @@
+"""
+CLI evaluation runner.
+
+Runs evaluations without a database, using the JsonRepository for
+configuration and printing progress to the console.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+
+from dataclasses_json import dataclass_json
+from rich.panel import Panel
+from rich.progress import Progress
+from rich.table import Table
+
+from switch_trust.cli.console import CLI_WIDTH, console, score_style, status_style
+from switch_trust.cli.output_formatters import (
+    OutputFormat,
+    get_eval_output_formatter,
+)
+from switch_trust.cli.rich_observer import RichObserver, pad_description
+from switch_trust.eval.core.eval.evaluation import (
+    EvaluationResult,
+    EvaluationStatus,
+    EvaluationSummary,
+)
+from switch_trust.eval.db.base.eval.model_eval_types import DbModelEvaluation
+from switch_trust.eval.db.json.repository_json import JsonRepository
+
+logger = logging.getLogger(__name__)
+
+MAX_CONSECUTIVE_RUN_FAILURES = 3
+
+
+@dataclass_json
+@dataclass
+class CliRunResult:
+    model_evaluation_id: str
+    model_evaluation_name: str
+    model: dict = field(default_factory=dict)
+    evaluation: dict = field(default_factory=dict)
+    summary: EvaluationSummary | None = None
+    results: list[EvaluationResult] = field(
+        default_factory=list,
+    )
+
+
+async def run_cli_evaluation(
+    model_evaluation: DbModelEvaluation,
+    store: JsonRepository,
+    concurrency: int,
+    progress: Progress,
+    desc_width: int,
+) -> CliRunResult:
+    db_eval = store.evaluations.get(
+        model_evaluation.evaluation_id,
+    )
+    db_model = store.models.get(
+        model_evaluation.model_id,
+    )
+
+    logger.info("Running evaluation: %s", db_eval.name)
+
+    desc = pad_description(db_eval.name, desc_width)
+    task_id = progress.add_task(desc, total=None)
+
+    evaluation = store.evaluations.get_evaluation(
+        model_evaluation.evaluation_id,
+        message_collection_repo=store.message_collections,
+        detector_repo=store.detectors,
+    )
+    model = store.models.get_model(model_evaluation.model_id)
+
+    observer = RichObserver(progress, task_id)
+
+    try:
+        await evaluation.init()
+        total = evaluation.get_summary().total_evaluations
+        progress.update(task_id, total=total)
+        evaluation.add_observer(observer)
+        await evaluation.run(model, concurrency)
+    except Exception as e:
+        logger.error(
+            "Evaluation failed: %s (%s: %s)",
+            db_eval.name,
+            type(e).__name__,
+            e,
+        )
+        progress.console.print(
+            f"[red]Error ({db_eval.name}): {e}[/red]",
+        )
+    finally:
+        evaluation.remove_observer(observer)
+        observer.finish(evaluation)
+
+    summary = evaluation.get_summary()
+    results = evaluation.get_results()
+
+    logger.info(
+        "Evaluation finished: %s (score=%s)",
+        db_eval.name,
+        summary.score if summary else "N/A",
+    )
+
+    return CliRunResult(
+        model_evaluation_id=model_evaluation.id,
+        model_evaluation_name=model_evaluation.name,
+        model={
+            "id": db_model.id,
+            "name": db_model.name,
+            "type": db_model.type.value,
+        },
+        evaluation={
+            "id": db_eval.id,
+            "name": db_eval.name,
+            "type": db_eval.type.value,
+        },
+        summary=summary,
+        results=results,
+    )
+
+
+def _aggregate_summary(
+    results: list[CliRunResult],
+) -> EvaluationSummary:
+    total = 0
+    finished = 0
+    errors = 0
+    max_score = 0.0
+    achieved = 0.0
+    error_msgs: list[str] = []
+    has_error = False
+    has_non_error = False
+
+    for r in results:
+        s = r.summary
+        if s is None:
+            continue
+        total += s.total_evaluations
+        finished += s.finished_evaluations
+        errors += s.error_evaluations
+        error_msgs.extend(s.error_messages)
+
+        if s.status == EvaluationStatus.ERROR:
+            has_error = True
+        else:
+            has_non_error = True
+            max_score += s.max_score
+            achieved += s.achieved_score
+
+    if has_error and not has_non_error:
+        status = EvaluationStatus.ERROR
+    elif total > 0 and finished + errors >= total:
+        status = EvaluationStatus.FINISHED
+    else:
+        status = EvaluationStatus.RUNNING
+
+    return EvaluationSummary(
+        status=status,
+        total_evaluations=total,
+        finished_evaluations=finished,
+        error_evaluations=errors,
+        max_score=max_score,
+        achieved_score=achieved,
+        error_messages=error_msgs,
+    )
+
+
+def _make_summary_panel(
+    summary: EvaluationSummary,
+    title: str,
+    width: int | None = None,
+) -> Panel:
+    grid = Table.grid(padding=(0, 1))
+    grid.add_column(style="key", justify="right")
+    grid.add_column()
+
+    st = summary.status.value.upper()
+    grid.add_row("Status", f"[{status_style(summary.status.value)}]{st}[/]")
+
+    if summary.score is not None:
+        pct = summary.score * 100
+        style = score_style(summary.score)
+        grid.add_row("Score", f"[{style}]{summary.score:.2f} ({pct:.0f}%)[/]")
+
+    completed = summary.finished_evaluations
+    total = summary.total_evaluations
+    errors = summary.error_evaluations
+    grid.add_row(
+        "Completed",
+        f"{completed} [dim]/ {total}[/dim]",
+    )
+    if errors:
+        grid.add_row("Errors", f"[red]{errors}[/red]")
+
+    return Panel(grid, title=f"[bold]{title}[/bold]", width=width)
+
+
+def log_run_summary(results: list[CliRunResult]) -> None:
+    for r in results:
+        s = r.summary
+        name = r.evaluation.get("name", "?")
+        if s is None:
+            logger.info("Evaluation: %s | status=no_result", name)
+            continue
+
+        score_str = f"{s.score:.2f}" if s.score is not None else "N/A"
+        error_str = f" | error={s.error_messages[0]}" if s.error_messages else ""
+        logger.info(
+            "Evaluation: %s | status=%s | score=%s | completed=%d/%d%s",
+            name,
+            s.status.value,
+            score_str,
+            s.finished_evaluations,
+            s.total_evaluations,
+            error_str,
+        )
+
+    overall = _aggregate_summary(results)
+    overall_score = f"{overall.score:.2f}" if overall.score is not None else "N/A"
+    logger.info(
+        "Overall: status=%s | score=%s | completed=%d/%d | errors=%d",
+        overall.status.value,
+        overall_score,
+        overall.finished_evaluations,
+        overall.total_evaluations,
+        overall.error_evaluations,
+    )
+
+
+def print_results(results: list[CliRunResult]) -> None:
+    with_summary = [r for r in results if r.summary is not None]
+    console.print()
+
+    if with_summary:
+        cols_per_row = min(len(with_summary), 2)
+        col_width = CLI_WIDTH // cols_per_row
+
+        renderables = [
+            _make_summary_panel(
+                r.summary,
+                r.evaluation.get("name", "?"),
+                width=col_width,
+            )
+            for r in with_summary
+        ]
+
+        for row_start in range(0, len(renderables), cols_per_row):
+            row = renderables[row_start : row_start + cols_per_row]
+            grid = Table.grid(padding=0)
+            for _ in row:
+                grid.add_column()
+            grid.add_row(*row)
+            console.print(grid)
+
+    overall = _aggregate_summary(results)
+    console.print(
+        _make_summary_panel(overall, "Overall", width=CLI_WIDTH),
+    )
+
+
+def write_output(
+    runs: list[CliRunResult],
+    config_path: str,
+    output_path: str,
+    fmt: str = "json",
+) -> None:
+
+    formatter = get_eval_output_formatter(OutputFormat(fmt))
+    content = formatter.format(runs, config_path)
+    with open(output_path, "w") as f:
+        f.write(content)

switch_trust/cli/scan_cli.py

@@ -0,0 +1,314 @@
+"""
+Subcommands for `switch-trust scan`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import datetime
+import logging
+import os
+from typing import TYPE_CHECKING
+
+from rich.panel import Panel
+from rich.table import Table
+
+from switch_trust.cli.console import CLI_WIDTH, console, severity_style
+from switch_trust.cli.file_filter import FileType, RelevantFile, find_relevant_files
+from switch_trust.cli.output_formatters import (
+    OutputFormat,
+    get_scan_output_formatter,
+)
+
+if TYPE_CHECKING:
+    from switch_trust.scan.schema import ScanReport
+
+logger = logging.getLogger(__name__)
+
+
+def register(subparsers: argparse._SubParsersAction) -> None:
+    scan_parser = subparsers.add_parser(
+        "scan",
+        help="Agent security scanning commands",
+    )
+    scan_parser.add_argument(
+        "path",
+        help="Path to a file or folder to scan",
+    )
+    scan_parser.add_argument(
+        "--output",
+        "-o",
+        help="Output file path (default: scan_<timestamp>.<format>)",
+    )
+    scan_parser.add_argument(
+        "--format",
+        "-f",
+        choices=["json", "sarif"],
+        default="json",
+        help="Output format (default: json)",
+    )
+    scan_parser.add_argument(
+        "--log",
+        help="Log file path (default: switch_trust_<timestamp>.log)",
+    )
+
+
+def write_output(report: ScanReport, output_path: str, fmt: str = "json"):
+
+    formatter = get_scan_output_formatter(OutputFormat(fmt))
+    content = formatter.format(report)
+    with open(output_path, "w") as f:
+        f.write(content)
+
+
+def print_report(report: ScanReport) -> None:
+    console.print()
+
+    # ── Summary panel ────────────────────────────────────────────
+    grid = Table.grid(padding=(0, 2))
+    grid.add_column(style="key", justify="right")
+    grid.add_column()
+
+    grid.add_row("Framework", report.framework_detected or "unknown")
+
+    meta = report.scan_metadata or {}
+    py_files = meta.get("python_files", 0)
+    total_files = meta.get("total_files_scanned", 0)
+    grid.add_row("Files", f"{py_files} Python, {total_files} total scanned")
+
+    total = len(report.findings)
+    pre_triage = (
+        len(report.pre_triage_findings) if report.pre_triage_findings else total
+    )
+    if pre_triage != total:
+        grid.add_row("Findings", f"{total} [dim]({pre_triage} pre-triage)[/dim]")
+    else:
+        grid.add_row("Findings", str(total))
+
+    tools = meta.get("tools_used", [])
+    if tools:
+        grid.add_row("Tools", ", ".join(tools))
+
+    skipped = meta.get("tools_skipped") or []
+    if skipped:
+        grid.add_row(
+            "Skipped",
+            "[bold yellow]"
+            + ", ".join(s.get("tool", "unknown") for s in skipped)
+            + "[/bold yellow]",
+        )
+
+    if meta.get("ai_summary"):
+        grid.add_row("AI summary", meta["ai_summary"])
+
+    console.print(Panel(grid, title="[bold]Scan Summary[/bold]", width=CLI_WIDTH))
+
+    # ── Partial-scan warning ─────────────────────────────────────
+    # A skipped tool means whole rule families never ran, so the findings
+    # below are incomplete. Say so loudly — the log file is not enough.
+    if skipped:
+        warn = Table.grid(padding=(0, 1))
+        warn.add_column()
+        plural = "s were" if len(skipped) > 1 else " was"
+        warn.add_row(
+            f"[bold yellow]This scan is incomplete — "
+            f"{len(skipped)} analysis tool{plural} skipped.[/bold yellow]"
+        )
+        for s in skipped:
+            warn.add_row("")
+            warn.add_row(
+                f"[bold]{s.get('tool', 'unknown')}[/bold] — {s.get('reason', '')}"
+            )
+        console.print(
+            Panel(
+                warn,
+                title="[bold yellow]Warning[/bold yellow]",
+                border_style="yellow",
+                width=CLI_WIDTH,
+            )
+        )
+
+    # ── Findings table ───────────────────────────────────────────
+    if not report.findings:
+        console.print("[dim]No findings.[/dim]")
+        console.print()
+        return
+
+    table = Table(
+        title="Findings",
+        show_lines=True,
+        width=CLI_WIDTH,
+    )
+    table.add_column("Sev", max_width=10, ratio=1)
+    table.add_column("CVSS", max_width=6, ratio=1)
+    table.add_column("Title", style="bold", ratio=5)
+    table.add_column("File", ratio=3)
+    table.add_column("Source", style="dim", ratio=1)
+
+    for f in report.findings:
+        sev = (
+            f.ai_spm_severity
+            if isinstance(f, dataclasses.Field) or hasattr(f, "ai_spm_severity")
+            else f.get("ai_spm_severity", "")
+        )
+        cvss = ""
+        title = ""
+        source = ""
+        file_name = ""
+
+        if isinstance(f, dict):
+            sev = f.get("ai_spm_severity", "")
+            cvss_scores = f.get("cvss_scores", {})
+            cvss = str(cvss_scores.get("base_score", "")) if cvss_scores else ""
+            title = f.get("title", "") or f.get("description", "")[:80]
+            source = f.get("source", "")
+            components = f.get("affected_components", [])
+            if components:
+                file_name = os.path.basename(components[0].get("name", ""))
+        else:
+            sev = f.ai_spm_severity
+            cvss = str(f.cvss_scores.base_score) if f.cvss_scores else ""
+            title = f.title or f.description[:80]
+            source = f.source
+            if f.affected_components:
+                file_name = os.path.basename(f.affected_components[0].name)
+
+        sev_display = f"[{severity_style(sev)}]{sev}[/]"
+        source_label = (
+            source.replace("_", " ")
+            .replace("ai reasoning", "AI")
+            .replace("static ", "")
+        )
+
+        table.add_row(sev_display, cvss, title, file_name, source_label)
+
+    console.print(table)
+
+    # ── Category summary ─────────────────────────────────────────
+    non_zero = {
+        k: v
+        for k, v in (report.category_summary or {}).items()
+        if (v.get("count", 0) if isinstance(v, dict) else getattr(v, "count", 0)) > 0
+    }
+
+    if non_zero:
+        cat_table = Table(
+            title="Category Summary",
+            show_lines=False,
+            width=CLI_WIDTH,
+        )
+        cat_table.add_column("Category", style="bold", ratio=5)
+        cat_table.add_column("Total", justify="right", ratio=1)
+        cat_table.add_column("Crit", justify="right", ratio=1)
+        cat_table.add_column("High", justify="right", ratio=1)
+        cat_table.add_column("Med", justify="right", ratio=1)
+        cat_table.add_column("Low", justify="right", ratio=1)
+
+        for v in non_zero.values():
+            if isinstance(v, dict):
+                asi_title = v.get("asi_title", "")
+                count = v.get("count", 0)
+                crit = v.get("critical", 0)
+                high = v.get("high", 0)
+                med = v.get("medium", 0)
+                low = v.get("low", 0)
+            else:
+                asi_title = getattr(v, "asi_title", "")
+                count = getattr(v, "count", 0)
+                crit = getattr(v, "critical", 0)
+                high = getattr(v, "high", 0)
+                med = getattr(v, "medium", 0)
+                low = getattr(v, "low", 0)
+
+            cat_table.add_row(
+                asi_title,
+                str(count),
+                f"[{severity_style('critical')}]{crit}[/]" if crit else str(crit),
+                f"[{severity_style('high')}]{high}[/]" if high else str(high),
+                f"[{severity_style('medium')}]{med}[/]" if med else str(med),
+                str(low),
+            )
+
+        console.print(cat_table)
+
+    console.print()
+
+
+def handle_scan(args: argparse.Namespace) -> str:
+    # Imported lazily: pulling in the agent scanner loads the Google ADK stack,
+    # which is expensive and only needed when a scan actually runs.
+    from switch_trust.scan.agent_scanner import (  # noqa: PLC0415 - deferred import cost
+        run_core,
+    )
+    from switch_trust.schema import RepoFile
+
+    path: str = os.path.abspath(os.path.expanduser(args.path))
+
+    if not os.path.exists(path):
+        # Raise instead of sys.exit so the error flows through main's unified
+        # exception handler (record_error -> telemetry, error panel, logging).
+        # sys.exit raises SystemExit, a BaseException that bypasses that path.
+        raise FileNotFoundError(f"Path does not exist: {path}")
+
+    files: list[RelevantFile] = find_relevant_files(path)
+
+    if not files:
+        # Raise (rather than sys.exit) so the error reaches main's unified
+        # handler and is recorded to telemetry. See the path check above.
+        raise ValueError("No scannable targets found")
+
+    model = os.environ.get("GENERATOR_MODEL")
+
+    logger.info(
+        "Starting agent scan on %s files with model '%s'", len(files), model or "none"
+    )
+
+    python_files: list[RepoFile] = []
+    requirements_files: list[RepoFile] = []
+    for rf in files:
+        with open(rf.path, encoding="utf-8", errors="replace") as fh:
+            content = fh.read()
+        repo_file = RepoFile(path=rf.path, content=content, size=len(content))
+        if rf.type == FileType.REQUIREMENTS:
+            requirements_files.append(repo_file)
+        elif rf.type == FileType.PYTHON:
+            python_files.append(repo_file)
+
+    logger.info(
+        "Files categorized: %d Python, %d requirements",
+        len(python_files),
+        len(requirements_files),
+    )
+
+    fw_counts: dict[str, int] = {}
+    for rf in files:
+        if rf.framework:
+            fw_counts[rf.framework] = fw_counts.get(rf.framework, 0) + 1
+    frameworks = list(fw_counts)
+    primary = max(fw_counts, key=fw_counts.get) if fw_counts else "unknown"
+
+    logger.info("Frameworks detected: %s (primary: %s)", ", ".join(frameworks), primary)
+
+    # Let a scan failure propagate to main's unified exception handler so it is
+    # recorded to telemetry; previously it was swallowed into sys.exit(1).
+    report = run_core(
+        python_files,
+        requirements_files,
+        skip_triage=False,
+        agentic=True,
+        model_string=model,
+        repo_name=os.path.basename(path),
+        primary_framework=primary,
+        frameworks_detected=frameworks,
+        total_files_scanned=len(files),
+    )
+
+    print_report(report)
+    fmt = getattr(args, "format", "json") or "json"
+    ext = fmt if fmt != "json" else "json"
+    output_path = args.output or (
+        f"scan_{datetime.datetime.now().strftime('%Y%m%dT%H%M%S')}.{ext}"
+    )
+    write_output(report, output_path, fmt=fmt)
+    return output_path