maskflow-cli 0.9.0__tar.gz → 0.10.0__tar.gz

{maskflow_cli-0.9.0 → maskflow_cli-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: maskflow-cli
-Version: 0.9.0
+Version: 0.10.0
 Summary: Command-line interface for MaskFlow: .maskflowrc config validation and inspection
 License: MIT
 Requires-Python: >=3.10
@@ -13,6 +13,8 @@ Requires-Dist: tomli-w>=1.0
 Requires-Dist: typer>=0.12
 Provides-Extra: agent
 Requires-Dist: maskflow-agent<0.2,>=0.1.0; extra == 'agent'
+Provides-Extra: audit
+Requires-Dist: maskflow-audit<0.2,>=0.1.0; extra == 'audit'
 Provides-Extra: bench
 Requires-Dist: maskflow-bench<0.2,>=0.1.0; extra == 'bench'
 Provides-Extra: dev

{maskflow_cli-0.9.0 → maskflow_cli-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "maskflow-cli"
-version = "0.9.0"
+version = "0.10.0"
 description = "Command-line interface for MaskFlow: .maskflowrc config validation and inspection"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -86,6 +86,15 @@ postgres = [
 bench = [
     "maskflow-bench>=0.1.0,<0.2",
 ]
+# `maskflow audit` -- static discovery of LLM egress paths plus (from M2) the
+# canary harness that proves which of them leak. Same policy as [bench] and
+# [evidence] above -- opt-in so a bare `pip install maskflow-cli` (and the
+# standalone PyInstaller binary) doesn't carry an AST analyser, a PyYAML
+# dependency, and eventually an interceptor; audit_cmd.py imports it lazily
+# and every other command runs without it.
+audit = [
+    "maskflow-audit>=0.1.0,<0.2",
+]
 # `maskflow agent install|uninstall|doctor` -- passthrough to maskflow-agent's
 # own console script (Claude Code PII guard). Lazy-imported by agent_cmd.py.
 agent = [
@@ -94,6 +103,7 @@ agent = [
 
 [tool.uv.sources]
 maskflow-agent = { workspace = true }
+maskflow-audit = { workspace = true }
 maskflow-core = { workspace = true }
 maskflow-pack-intl = { workspace = true }
 maskflow-pack-india = { workspace = true }

{maskflow_cli-0.9.0 → maskflow_cli-0.10.0}/src/maskflow_cli/app.py

@@ -12,6 +12,7 @@ import maskflow_pack_india  # noqa: F401 -- import side effect registers pack-in
 import typer
 
 from .commands.agent_cmd import agent
+from .commands.audit_cmd import audit
 from .commands.bench_cmd import bench
 from .commands.config_cmd import app as config_app
 from .commands.doctor_cmd import doctor
@@ -24,6 +25,7 @@ app.command("doctor")(doctor)
 app.command("explain")(explain)
 app.command("scan")(scan)
 app.command("bench")(bench)
+app.command("audit")(audit)
 app.command(
     "agent",
     context_settings={"allow_extra_args": True, "ignore_unknown_options": True},

maskflow_cli-0.10.0/src/maskflow_cli/audit_render.py

@@ -0,0 +1,208 @@
+"""Rich rendering for `maskflow audit`. Kept separate from
+commands/audit_cmd.py so the command body stays thin, same split as
+doctor.py/doctor_render.py and bench_render.py.
+
+Only type-checked against maskflow-audit (TYPE_CHECKING guard below,
+`from __future__ import annotations` keeps every annotation an unevaluated
+string at runtime) -- this module never constructs an AuditSummary itself,
+only duck-types over one handed in by audit_cmd.py, so it never needs the
+optional [audit] extra installed just to be imported.
+
+Nothing here reads a value: an AuditSummary carries identifiers and counts
+only (see maskflow_audit.report.summary's docstring), so this renderer
+cannot print PII even by accident."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+
+if TYPE_CHECKING:
+    from maskflow_audit.baseline.diff import CiVerdict
+    from maskflow_audit.fix.loop import FixReport
+    from maskflow_audit.report.summary import AuditSummary
+
+# Same rationale as doctor_render.py's: a fixed width so output doesn't
+# wrap unpredictably under a non-tty (CI logs, CliRunner in tests).
+CONSOLE_WIDTH = 100
+
+#: Longest source list shown per flow before it is summarised. The full
+#: list is always in audit.json; the table is a triage view, and a cell
+#: that folds character-by-character (which is what happens when five
+#: column names meet a long fixture path at a fixed console width) is
+#: worse than no cell at all.
+MAX_SOURCES_SHOWN = 2
+
+
+#: How a status reads in the terminal, and the colour it reads in.
+_STATUS_STYLE: dict[str, tuple[str, str]] = {
+    "unmasked": ("UNMASKED", "bold red"),
+    "unknown": ("unknown", "yellow"),
+    "masked_via_mask_call": ("masked: mask()", "green"),
+    "masked_via_gateway_base_url": ("masked: gateway", "green"),
+    "masked_via_integration": ("masked: integration", "green"),
+}
+
+#: A proven leak overrides the static status entirely -- it is the one
+#: line in the output that is evidence rather than inference.
+_LEAK_STYLE = ("LEAK PROVEN", "bold white on red")
+
+#: What a flow's canary verdict adds to its detail line. `not_exercised`
+#: is spelled out because it is the one a reader is most likely to
+#: misread as safe.
+_CANARY_NOTE: dict[str, str] = {
+    "escaped": "canary escaped",
+    "contained": "canary contained",
+    "not_exercised": "not exercised by the run",
+    "not_run": "",
+}
+
+
+def _sources(flow: object) -> str:
+    """A compact source summary: at most MAX_SOURCES_SHOWN, then a count."""
+    sources = flow.sources  # type: ignore[attr-defined]
+    shown = [f"{s.kind}:{s.detail}" if s.detail else s.kind for s in sources]
+    if not shown:
+        return ""
+    if len(shown) <= MAX_SOURCES_SHOWN:
+        return ", ".join(shown)
+    return ", ".join(shown[:MAX_SOURCES_SHOWN]) + f" (+{len(shown) - MAX_SOURCES_SHOWN})"
+
+
+def make_console(*, stderr: bool = True) -> Console:
+    """Human output goes to stderr so `--out -` stays pipeable."""
+    return Console(width=CONSOLE_WIDTH, stderr=stderr, highlight=False, soft_wrap=False)
+
+
+def render_summary(console: Console, summary: AuditSummary) -> None:
+    """The headline, the flow table, and the honest caveat."""
+    coverage = summary.coverage
+    console.print()
+    console.print(f"[bold]{summary.headline}[/bold]")
+
+    if coverage.total_flows == 0:
+        console.print(
+            "[yellow]No LLM egress paths found.[/yellow] That means none of the sinks in "
+            "the registry appear in this tree -- not that none exist. See the limits note "
+            "in the report."
+        )
+        return
+
+    parts = []
+    if coverage.confirmed_leaks:
+        # Listed first and loudest: it is the only number here that is
+        # evidence rather than inference.
+        parts.append(f"[bold white on red] {coverage.confirmed_leaks} PROVEN LEAK(S) [/]")
+    if coverage.unmasked:
+        parts.append(f"[bold red]{coverage.unmasked} unmasked[/bold red]")
+    if coverage.unknown:
+        parts.append(f"[yellow]{coverage.unknown} unknown[/yellow]")
+    if coverage.masked:
+        parts.append(f"[green]{coverage.masked} masked[/green]")
+    console.print(" · ".join(parts))
+    console.print()
+
+    # Deliberately not a rich Table. At a fixed console width a table
+    # redistributes space between columns, so one long repo-relative path
+    # starves its neighbours until a cell folds one character per line --
+    # which is what a real audit of a real tree produces. A two-line block
+    # per flow stays readable at any width and never truncates a path.
+    for flow in summary.flows:
+        if flow.canary_status == "escaped":
+            label, style = _LEAK_STYLE
+        else:
+            label, style = _STATUS_STYLE.get(flow.masking_status, (flow.masking_status, ""))
+        badge = f"[{style}]{label:<20}[/{style}]" if style else f"{label:<20}"
+        console.print(
+            f"{badge} {flow.file}:{flow.line}  [dim]{flow.sink_id} · {flow.flow_id}[/dim]"
+        )
+        sources = _sources(flow)
+        detail = f"from {sources}" if sources else "source not traced"
+        note = _CANARY_NOTE.get(flow.canary_status, "")
+        suffix = f" · {note}" if note else ""
+        console.print(f"{'':<20} [dim]{detail} · conf: {flow.confidence}{suffix}[/dim]")
+        for escape in flow.canary_escapes[:1]:
+            entities = ", ".join(sorted({e.entity_type for e in flow.canary_escapes}))
+            console.print(
+                f"{'':<20} [red]canary {entities} reached {escape.host}{escape.path}"
+                f" ({escape.form})[/red]"
+            )
+            for frame in escape.app_stack[-3:]:
+                console.print(f"{'':<22} [dim]{frame}[/dim]")
+
+    run = summary.canary_run
+    if not summary.scope.canaries_run:
+        console.print(
+            "[dim]Canaries were not run, so every finding above is predicted, not "
+            "proven. Re-run with --prove to confirm which paths actually leak.[/dim]"
+        )
+    elif run is not None:
+        console.print(
+            f"[dim]Canary run: seed {run.seed} · {run.requests_intercepted} request(s) "
+            f"intercepted (none sent) · {run.escapes_observed} escape(s) observed.[/dim]"
+        )
+        if run.unattributed_escapes:
+            # Discovery missed a sink. This is the most useful bug report
+            # the tool can produce about itself, so it is asked for.
+            console.print(
+                f"[yellow]{run.unattributed_escapes} escape(s) matched no discovered "
+                "flow -- a canary got out through a path static analysis did not find. "
+                "Please report it.[/yellow]"
+            )
+        if run.note:
+            console.print(f"[yellow]{run.note}[/yellow]")
+        for warning in run.warnings:
+            console.print(f"[yellow]--allow-env override: {warning}[/yellow]")
+    if summary.skipped_files:
+        console.print(
+            f"[dim]{len(summary.skipped_files)} file(s) skipped (unparseable or "
+            f"unreadable); see skipped_files in the report.[/dim]"
+        )
+
+
+def render_ci(console: Console, verdict: CiVerdict, summary: AuditSummary) -> None:
+    """The `--ci` gate's output: short, and PR-friendly.
+
+    GitHub Actions workflow commands (`::error::`, `::notice::`) so a
+    failure annotates the run rather than hiding in a log, matching the
+    scan action's behaviour.
+    """
+    if verdict.skipped:
+        console.print(f"[yellow]{verdict.summary_line()}[/yellow]")
+        console.print(f"::notice::{verdict.summary_line()}")
+        return
+
+    if verdict.ok:
+        console.print(f"[green]{verdict.summary_line()}[/green]")
+        console.print(f"::notice::{summary.headline}")
+        return
+
+    console.print(f"[bold red]{verdict.summary_line()}[/bold red]\n")
+    for problem in verdict.problems:
+        console.print(f"  [red]•[/red] {problem}")
+    console.print(f"\n::error::{verdict.summary_line()}")
+    console.print(
+        "[dim]If these paths are intentional, re-run `maskflow audit --baseline` "
+        "and commit the updated baseline.[/dim]"
+    )
+
+
+def render_fix(console: Console, report: FixReport) -> None:
+    """What the fix loop did, or why it did nothing."""
+    if not report.model_available:
+        console.print(f"\n[yellow]{report.note}[/yellow]")
+        return
+
+    console.print()
+    for result in report.fixed:
+        gates = result.attempts[-1].gates
+        console.print(f"[green]FIXED[/green] {result.file}:{result.line} ({result.flow_id})")
+        if gates is not None:
+            console.print(f"{'':<6}[dim]{gates.summary()}[/dim]")
+    for result in report.unfixed:
+        console.print(f"[yellow]NOT FIXED[/yellow] {result.file}:{result.line}")
+        console.print(f"{'':<10}[dim]{result.reason}[/dim]")
+    if report.fixed:
+        console.print("\n[dim]Suggested PR body:[/dim]")
+        console.print(report.pr_body())

maskflow_cli-0.10.0/src/maskflow_cli/commands/audit_cmd.py

@@ -0,0 +1,233 @@
+"""`maskflow audit <path>` -- find every place a codebase sends text to an
+LLM, prove which of those paths leak, and fix them.
+
+`maskflow scan` asks what PII already reached a provider, from logs. This
+asks the forward question, from source: where can text leave, and is it
+guarded? With `--prove` it stops guessing -- it seeds synthetic identities,
+runs the project, intercepts every provider-bound request before it leaves
+the process, and reports which canary reached which host from which line.
+
+`maskflow-audit` is an **optional** dependency (`pip install
+'maskflow-cli[audit]'`) -- imported lazily here, same policy as
+`maskflow_cli.evidence` and `bench_cmd.py`, so a bare CLI install (and the
+standalone PyInstaller binary) doesn't require it and every other command
+runs without it.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import typer
+
+from ..audit_render import make_console, render_ci, render_fix, render_summary
+
+_MISSING = (
+    "`maskflow audit` needs the audit extra -- install it with: pip install 'maskflow-cli[audit]'"
+)
+
+_EVIDENCE_MISSING = (
+    "--evidence needs the evidence extra too -- install it with: "
+    "pip install 'maskflow-cli[audit,evidence]'"
+)
+
+# Ruff B008: ruff allows a bare typer.Argument()/Option() in a default, but
+# not a nested call inside one (the Path(".") below), so the argument and
+# the list/Path-defaulted options are module-level singletons -- same
+# pattern as scan/cmd.py's and explain_cmd.py's.
+_PATH_ARGUMENT = typer.Argument(Path("."), help="Directory or file to audit.")
+_OUT_OPTION = typer.Option(
+    None, "--out", "-o", help="Write the report here ('-' for stdout). Default: audit.<format>."
+)
+_FORMAT_OPTION = typer.Option("json", "--format", help="json | html.")
+_EXCLUDE_OPTION = typer.Option(
+    [], "--exclude", help="Directory name to skip, repeatable. Adds to the defaults."
+)
+_PROVE_OPTION = typer.Option(
+    False,
+    "--prove",
+    help="Seed canaries, run the project, and confirm which paths actually leak.",
+)
+_FLOWS_OPTION = typer.Option(
+    None,
+    "--flows",
+    help="Command --prove should run (default: python -m pytest -q).",
+)
+_SEED_OPTION = typer.Option(
+    None, "--seed", help="Canary seed, for reproducing a run. Default: fresh each time."
+)
+_IDENTITIES_OPTION = typer.Option(
+    1, "--identities", min=1, help="How many synthetic people to seed."
+)
+_ALLOW_ENV_OPTION = typer.Option(
+    False,
+    "--allow-env",
+    help="Seed even though the environment looks like a real deployment.",
+)
+_BASELINE_OPTION = typer.Option(
+    False, "--baseline", help="Write maskflow-audit.baseline.json and exit."
+)
+_CI_OPTION = typer.Option(
+    False, "--ci", help="Fail the build on a new unmasked path or a new confirmed leak."
+)
+_BASELINE_FILE_OPTION = typer.Option(
+    None, "--baseline-file", help="Baseline path (default: maskflow-audit.baseline.json)."
+)
+_FIX_OPTION = typer.Option(False, "--fix", help="Agentic fix loop. Needs your own model key.")
+_HANDOFF_OPTION = typer.Option(
+    False,
+    "--handoff",
+    help="Write maskflow-audit-tasks.md for your own coding agent. No key needed.",
+)
+_EVIDENCE_OPTION = typer.Option(
+    False, "--evidence", help="Emit metadata-only evidence events for this run."
+)
+_QUIET_OPTION = typer.Option(False, "--quiet", help="Suppress the terminal summary.")
+
+_FORMATS = ("json", "html")
+
+
+def _die(message: str, code: int = 1) -> None:
+    typer.echo(f"error: {message}", err=True)
+    raise typer.Exit(code=code)
+
+
+def audit(  # noqa: PLR0913 -- a CLI surface, each option is deliberate
+    path: Path = _PATH_ARGUMENT,
+    out: Path | None = _OUT_OPTION,
+    fmt: str = _FORMAT_OPTION,
+    exclude: list[str] = _EXCLUDE_OPTION,
+    prove: bool = _PROVE_OPTION,
+    flows: str | None = _FLOWS_OPTION,
+    seed: int | None = _SEED_OPTION,
+    identities: int = _IDENTITIES_OPTION,
+    allow_env: bool = _ALLOW_ENV_OPTION,
+    baseline: bool = _BASELINE_OPTION,
+    ci: bool = _CI_OPTION,
+    baseline_file: Path | None = _BASELINE_FILE_OPTION,
+    fix: bool = _FIX_OPTION,
+    handoff: bool = _HANDOFF_OPTION,
+    evidence: bool = _EVIDENCE_OPTION,
+    quiet: bool = _QUIET_OPTION,
+) -> None:
+    """Audits a source tree for unmasked LLM egress paths.
+
+    Writes a machine-readable report (audit.json) and prints a summary.
+    Your code is parsed, never executed; with --prove the project's own
+    outbound requests are intercepted before they reach a provider, so
+    nothing is ever transmitted. See docs/audit.md for the full reference."""
+    try:
+        from maskflow_audit.api import run_audit
+        from maskflow_audit.baseline import BASELINE_FILENAME, BaselineError, diff_against
+        from maskflow_audit.baseline import write_baseline as write_baseline_file
+        from maskflow_audit.canary.manifest import SelfCheckFailure
+        from maskflow_audit.discovery.walk import DEFAULT_EXCLUDES, discover
+        from maskflow_audit.fix import run_fix_loop, write_handoff
+        from maskflow_audit.fix.handoff import HANDOFF_FILENAME
+        from maskflow_audit.prove import parse_command, run_canary_proof
+        from maskflow_audit.report import render
+        from maskflow_audit.seeding.safety import ProductionLooksLikely
+        from maskflow_audit.sinks.loader import RegistryError, default_registry
+    except ModuleNotFoundError as exc:
+        typer.echo(_MISSING, err=True)
+        raise typer.Exit(code=1) from exc
+
+    if fmt not in _FORMATS:
+        _die(f"--format must be one of {', '.join(_FORMATS)} (got {fmt!r})")
+    if not path.exists():
+        _die(f"{path}: no such file or directory", code=2)
+    if fix and handoff:
+        _die("--fix and --handoff are alternatives; pass one or the other")
+
+    excludes = DEFAULT_EXCLUDES | frozenset(exclude)
+    console = make_console()
+
+    try:
+        registry = default_registry()
+    except RegistryError as exc:
+        # A malformed shipped registry is a packaging bug, not user error.
+        _die(str(exc), code=1)
+        return
+
+    proof = None
+    if prove:
+        discovered, _, _ = discover(path, registry, exclude=excludes)
+        try:
+            proof = run_canary_proof(
+                path,
+                discovered,
+                command=parse_command(flows),
+                seed=seed,
+                identities=identities,
+                allow_env=allow_env,
+            )
+        except ProductionLooksLikely as exc:
+            _die(str(exc), code=2)
+            return
+        except SelfCheckFailure as exc:
+            # A canary MaskFlow would not detect proves nothing, so this is
+            # fatal rather than a warning.
+            _die(str(exc), code=1)
+            return
+
+    summary = run_audit(path, sinks=registry, exclude=excludes, prove=proof)
+
+    baseline_path = baseline_file or (Path.cwd() / BASELINE_FILENAME)
+
+    if baseline:
+        write_baseline_file(baseline_path, summary)
+        if not quiet:
+            console.print(
+                f"wrote {baseline_path} -- commit it, then gate your builds with "
+                "`maskflow audit --ci`"
+            )
+        return
+
+    if ci:
+        try:
+            verdict = diff_against(summary, baseline_path)
+        except BaselineError as exc:
+            _die(str(exc), code=1)
+            return
+        render_ci(console, verdict, summary)
+        raise typer.Exit(code=verdict.exit_code)
+
+    rendered = render(summary, fmt)
+    destination = out if out is not None else Path(f"audit.{fmt}")
+    if str(destination) == "-":
+        sys.stdout.write(rendered)
+    else:
+        destination.write_text(rendered, encoding="utf-8")
+
+    if handoff:
+        handoff_path = Path.cwd() / HANDOFF_FILENAME
+        tasks = write_handoff(handoff_path, summary)
+        if not quiet:
+            console.print(f"wrote {handoff_path} ({tasks} task(s))")
+
+    if fix:
+        # No model is wired in by default: the audit never carries a
+        # provider dependency or a key. `model=None` reports that plainly
+        # and points at --handoff, which needs neither.
+        report = run_fix_loop(path, summary, model=None)
+        render_fix(console, report)
+
+    if evidence:
+        try:
+            from maskflow_audit.evidence import EvidenceExtraMissing, emit_audit_evidence
+
+            count = emit_audit_evidence(summary, enabled=True)
+        except ModuleNotFoundError as exc:
+            typer.echo(_EVIDENCE_MISSING, err=True)
+            raise typer.Exit(code=1) from exc
+        except EvidenceExtraMissing as exc:
+            typer.echo(_EVIDENCE_MISSING, err=True)
+            raise typer.Exit(code=1) from exc
+        if not quiet:
+            console.print(f"emitted {count} metadata-only evidence event(s)")
+
+    if not quiet:
+        render_summary(console, summary)
+        if str(destination) != "-":
+            console.print(f"\nwrote {destination}")

maskflow_cli-0.10.0/tests/audit/test_audit_extra_is_optional.py

@@ -0,0 +1,69 @@
+"""`maskflow-audit` must stay an optional dependency.
+
+A bare `pip install maskflow-cli`, and the standalone PyInstaller binary,
+must keep working with every other command intact -- the binary's spec
+collects every `maskflow_cli` submodule, so a module-level
+`import maskflow_audit` anywhere in the CLI would either bundle the audit
+package into the binary or break its build.
+
+Checked by AST inspection rather than by uninstalling the package, so the
+guarantee holds in a dev environment where the extra IS installed (which
+is every environment this suite runs in).
+"""
+
+from __future__ import annotations
+
+import ast
+import pathlib
+
+import maskflow_cli.audit_render as audit_render_mod
+import maskflow_cli.commands.audit_cmd as audit_cmd_mod
+from maskflow_cli.app import app
+from typer.testing import CliRunner
+
+runner = CliRunner()
+
+_MODULES = (audit_cmd_mod, audit_render_mod)
+
+
+def _toplevel_imports(module: object) -> set[str]:
+    """Root package names imported at module level (not inside a function).
+
+    A TYPE_CHECKING block counts as module level textually but never runs,
+    so it is skipped -- that is exactly how audit_render types itself
+    against maskflow-audit without importing it.
+    """
+    source = pathlib.Path(module.__file__).read_text(encoding="utf-8")  # type: ignore[attr-defined]
+    tree = ast.parse(source)
+    roots: set[str] = set()
+    for node in tree.body:
+        if isinstance(node, ast.Import):
+            roots.update(alias.name.split(".")[0] for alias in node.names)
+        elif isinstance(node, ast.ImportFrom) and node.module and not node.level:
+            roots.add(node.module.split(".")[0])
+    return roots
+
+
+def test_no_cli_module_imports_maskflow_audit_at_module_level() -> None:
+    for module in _MODULES:
+        assert "maskflow_audit" not in _toplevel_imports(module), (
+            f"{module.__name__} imports maskflow_audit eagerly; it must be "
+            "lazy-imported inside the command body"
+        )
+
+
+def test_the_guard_can_actually_see_imports() -> None:
+    """Non-vacuity: the helper must find the imports that ARE there."""
+    assert "typer" in _toplevel_imports(audit_cmd_mod)
+    assert "rich" in _toplevel_imports(audit_render_mod)
+
+
+def test_the_missing_extra_message_names_the_install_command() -> None:
+    # A user who hits this needs the exact command, not a concept.
+    assert "pip install 'maskflow-cli[audit]'" in audit_cmd_mod._MISSING
+
+
+def test_every_other_command_still_works() -> None:
+    # The binary ships without the extra; `doctor` must not care.
+    result = runner.invoke(app, ["doctor"], env={"HOME": "/nonexistent-home"})
+    assert result.exit_code == 0, result.output

maskflow_cli-0.10.0/tests/audit/test_cli_audit.py

@@ -0,0 +1,342 @@
+"""`maskflow audit` through the real root app."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from maskflow_cli.app import app
+from typer.testing import CliRunner
+
+runner = CliRunner()
+
+_APP = (
+    "from openai import OpenAI\n"
+    "client = OpenAI()\n"
+    "def summarize(cur):\n"
+    "    row = cur.fetchone()\n"
+    "    return client.chat.completions.create(messages=row['aadhaar'])\n"
+)
+
+
+@pytest.fixture
+def project(tmp_path: Path) -> Path:
+    (tmp_path / "app.py").write_text(_APP, encoding="utf-8")
+    return tmp_path
+
+
+def test_writes_a_report_and_exits_zero(project: Path, tmp_path: Path) -> None:
+    out = tmp_path / "audit.json"
+    result = runner.invoke(app, ["audit", str(project), "--out", str(out)])
+    assert result.exit_code == 0, result.output
+    data = json.loads(out.read_text(encoding="utf-8"))
+    assert data["coverage"]["total_flows"] == 1
+    assert data["flows"][0]["masking_status"] == "unmasked"
+
+
+def _flat(output: str) -> str:
+    """Rich wraps at a fixed width, so collapse whitespace before matching
+    a phrase -- otherwise the assertion pins the wrap points instead."""
+    return " ".join(output.split())
+
+
+def test_prints_the_headline_and_the_unproven_caveat(project: Path, tmp_path: Path) -> None:
+    result = runner.invoke(app, ["audit", str(project), "--out", str(tmp_path / "a.json")])
+    assert result.exit_code == 0, result.output
+    assert "0 of 1 LLM egress paths masked" in result.output
+    # The report must never let a reader think a static finding was proven.
+    assert "predicted, not proven" in _flat(result.output)
+
+
+def test_quiet_suppresses_the_summary(project: Path, tmp_path: Path) -> None:
+    out = tmp_path / "a.json"
+    result = runner.invoke(app, ["audit", str(project), "--out", str(out), "--quiet"])
+    assert result.exit_code == 0, result.output
+    assert "LLM egress paths" not in result.output
+    assert out.is_file()
+
+
+def test_out_dash_writes_to_stdout(project: Path) -> None:
+    result = runner.invoke(app, ["audit", str(project), "--out", "-", "--quiet"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.stdout)["schema_version"] == "1"
+
+
+def test_default_output_path_is_audit_json(
+    project: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    workdir = tmp_path / "cwd"
+    workdir.mkdir()
+    monkeypatch.chdir(workdir)
+    result = runner.invoke(app, ["audit", str(project), "--quiet"])
+    assert result.exit_code == 0, result.output
+    assert (workdir / "audit.json").is_file()
+
+
+def test_a_missing_path_exits_two(tmp_path: Path) -> None:
+    result = runner.invoke(app, ["audit", str(tmp_path / "nope")])
+    assert result.exit_code == 2, result.output
+    assert "no such file or directory" in result.output
+
+
+def test_an_unknown_format_exits_one(project: Path) -> None:
+    result = runner.invoke(app, ["audit", str(project), "--format", "pdf"])
+    assert result.exit_code == 1, result.output
+    assert "--format must be one of" in result.output
+
+
+def test_exclude_is_repeatable(tmp_path: Path) -> None:
+    (tmp_path / "app.py").write_text(_APP, encoding="utf-8")
+    for name in ("vendor", "examples"):
+        (tmp_path / name).mkdir()
+        (tmp_path / name / "x.py").write_text(_APP, encoding="utf-8")
+
+    out = tmp_path / "a.json"
+    result = runner.invoke(
+        app,
+        [
+            "audit",
+            str(tmp_path),
+            "--exclude",
+            "vendor",
+            "--exclude",
+            "examples",
+            "--out",
+            str(out),
+            "--quiet",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    files = [f["file"] for f in json.loads(out.read_text(encoding="utf-8"))["flows"]]
+    assert files == ["app.py"]
+
+
+def test_a_clean_tree_reports_zero_without_failing(tmp_path: Path) -> None:
+    (tmp_path / "plain.py").write_text("def f():\n    return 1\n", encoding="utf-8")
+    result = runner.invoke(app, ["audit", str(tmp_path), "--out", str(tmp_path / "a.json")])
+    assert result.exit_code == 0, result.output
+    # A clean report must not read as a clean bill of health.
+    assert "not that none exist" in _flat(result.output)
+
+
+def test_audit_is_registered_on_the_root_app() -> None:
+    result = runner.invoke(app, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "audit" in result.output
+
+
+class TestNewSurfaces:
+    """--prove / --baseline / --ci / --handoff / --format html."""
+
+    _LEAKY = (
+        "import os, sqlite3, httpx\n"
+        "DB = os.path.join(os.path.dirname(__file__), 'app.db')\n"
+        "client = httpx.Client(base_url='https://api.openai.com')\n"
+        "def summarize():\n"
+        "    row = sqlite3.connect(DB).execute("
+        "'SELECT customer_name, aadhaar FROM tickets').fetchone()\n"
+        "    return client.post('/v1/chat/completions',\n"
+        "        json={'messages': [{'role': 'user', 'content': f'{row[0]} {row[1]}'}]})\n"
+    )
+
+    @pytest.fixture
+    def leaky_project(self, tmp_path: Path) -> Path:
+        import sqlite3
+
+        (tmp_path / "app.py").write_text(self._LEAKY, encoding="utf-8")
+        (tmp_path / "run_flows.py").write_text("import app\napp.summarize()\n", encoding="utf-8")
+        connection = sqlite3.connect(tmp_path / "app.db")
+        connection.execute(
+            "CREATE TABLE tickets (id INTEGER PRIMARY KEY, customer_name TEXT, aadhaar TEXT)"
+        )
+        connection.commit()
+        connection.close()
+        return tmp_path
+
+    def test_html_is_self_contained(self, project: Path, tmp_path: Path) -> None:
+        out = tmp_path / "r.html"
+        result = runner.invoke(
+            app, ["audit", str(project), "--format", "html", "--out", str(out), "--quiet"]
+        )
+        assert result.exit_code == 0, result.output
+        html = out.read_text(encoding="utf-8")
+        assert html.startswith("<!doctype html>")
+        for needle in ("http://", "https://fonts", "cdn.", "<script src", "<link "):
+            assert needle not in html, needle
+        assert "DPDP Rule 6" in html
+        assert "<!-- DPDP_RULE6_APPENDIX -->" in html
+        assert "No raw PII value" in html
+
+    def test_prove_confirms_the_leak_with_a_call_stack(
+        self, leaky_project: Path, tmp_path: Path
+    ) -> None:
+        out = tmp_path / "a.json"
+        result = runner.invoke(
+            app,
+            [
+                "audit",
+                str(leaky_project),
+                "--prove",
+                "--flows",
+                "python run_flows.py",
+                "--seed",
+                "7",
+                "--out",
+                str(out),
+            ],
+        )
+        assert result.exit_code == 0, result.output
+        data = json.loads(out.read_text(encoding="utf-8"))
+        assert data["coverage"]["confirmed_leaks"] == 1
+        flow = data["flows"][0]
+        assert flow["canary_status"] == "escaped"
+        assert flow["confidence"] == "confirmed"
+        escapes = flow["canary_escapes"]
+        assert escapes and escapes[0]["host"] == "api.openai.com"
+        assert escapes[0]["app_stack"], "the call stack is the evidence; it must be present"
+        assert "LEAK PROVEN" in result.output
+
+    def test_prove_writes_no_canary_value_into_the_report(
+        self, leaky_project: Path, tmp_path: Path
+    ) -> None:
+        """The report is a document a DPO retains; it must carry no value."""
+        out = tmp_path / "a.json"
+        result = runner.invoke(
+            app,
+            [
+                "audit",
+                str(leaky_project),
+                "--prove",
+                "--flows",
+                "python run_flows.py",
+                "--seed",
+                "7",
+                "--quiet",
+                "--out",
+                str(out),
+            ],
+        )
+        assert result.exit_code == 0, result.output
+        from maskflow_audit.canary.manifest import build_manifest
+
+        rendered = out.read_text(encoding="utf-8")
+        for value in build_manifest(seed=7).values():
+            assert value not in rendered
+
+    def test_prove_restores_the_tree(self, leaky_project: Path, tmp_path: Path) -> None:
+        """An audit that left canary rows behind would be a bug."""
+        import sqlite3
+
+        result = runner.invoke(
+            app,
+            [
+                "audit",
+                str(leaky_project),
+                "--prove",
+                "--flows",
+                "python run_flows.py",
+                "--quiet",
+                "--out",
+                str(tmp_path / "a.json"),
+            ],
+        )
+        assert result.exit_code == 0, result.output
+        connection = sqlite3.connect(leaky_project / "app.db")
+        assert connection.execute("SELECT COUNT(*) FROM tickets").fetchone()[0] == 0
+
+    def test_baseline_then_ci_passes_unchanged(self, project: Path, tmp_path: Path) -> None:
+        base = tmp_path / "base.json"
+        written = runner.invoke(
+            app, ["audit", str(project), "--baseline", "--baseline-file", str(base), "--quiet"]
+        )
+        assert written.exit_code == 0, written.output
+        assert base.is_file()
+
+        gate = runner.invoke(app, ["audit", str(project), "--ci", "--baseline-file", str(base)])
+        assert gate.exit_code == 0, gate.output
+        assert "no new unmasked" in gate.output
+
+    def test_ci_fails_on_a_newly_added_unmasked_call(self, project: Path, tmp_path: Path) -> None:
+        base = tmp_path / "base.json"
+        runner.invoke(
+            app, ["audit", str(project), "--baseline", "--baseline-file", str(base), "--quiet"]
+        )
+        (project / "app.py").write_text(
+            _APP + "def added(t):\n    client.chat.completions.create(messages=t)\n",
+            encoding="utf-8",
+        )
+        gate = runner.invoke(app, ["audit", str(project), "--ci", "--baseline-file", str(base)])
+        assert gate.exit_code == 1, gate.output
+        assert "NEW unguarded egress path" in gate.output
+        assert "::error::" in gate.output
+
+    def test_ci_skips_without_a_baseline(self, project: Path, tmp_path: Path) -> None:
+        gate = runner.invoke(
+            app, ["audit", str(project), "--ci", "--baseline-file", str(tmp_path / "none.json")]
+        )
+        assert gate.exit_code == 0, gate.output
+        assert "skipped" in gate.output
+
+    def test_handoff_writes_a_task_file(
+        self, project: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        workdir = tmp_path / "cwd"
+        workdir.mkdir()
+        monkeypatch.chdir(workdir)
+        result = runner.invoke(
+            app, ["audit", str(project), "--handoff", "--quiet", "--out", str(tmp_path / "a.json")]
+        )
+        assert result.exit_code == 0, result.output
+        tasks = (workdir / "maskflow-audit-tasks.md").read_text(encoding="utf-8")
+        assert "## Task 1" in tasks
+        assert "Never edit a test" in tasks
+
+    def test_fix_without_a_model_points_at_handoff(self, project: Path, tmp_path: Path) -> None:
+        result = runner.invoke(
+            app, ["audit", str(project), "--fix", "--out", str(tmp_path / "a.json")]
+        )
+        assert result.exit_code == 0, result.output
+        assert "--handoff" in _flat(result.output)
+
+    def test_fix_and_handoff_are_mutually_exclusive(self, project: Path) -> None:
+        result = runner.invoke(app, ["audit", str(project), "--fix", "--handoff"])
+        assert result.exit_code == 1, result.output
+        assert "alternatives" in result.output
+
+    def test_allow_env_is_required_in_a_production_looking_env(
+        self, leaky_project: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """Seeding production would be a data incident caused by the tool
+        that exists to prevent data incidents."""
+        monkeypatch.setenv("APP_ENV", "production")
+        refused = runner.invoke(
+            app,
+            [
+                "audit",
+                str(leaky_project),
+                "--prove",
+                "--flows",
+                "python run_flows.py",
+                "--out",
+                str(tmp_path / "a.json"),
+            ],
+        )
+        assert refused.exit_code == 2, refused.output
+        assert "refusing to seed" in refused.output
+
+        allowed = runner.invoke(
+            app,
+            [
+                "audit",
+                str(leaky_project),
+                "--prove",
+                "--flows",
+                "python run_flows.py",
+                "--allow-env",
+                "--quiet",
+                "--out",
+                str(tmp_path / "b.json"),
+            ],
+        )
+        assert allowed.exit_code == 0, allowed.output

{maskflow_cli-0.9.0 → maskflow_cli-0.10.0}/tests/test_cli_agent.py

@@ -10,6 +10,6 @@ def test_agent_passthrough_reaches_maskflow_agent() -> None:
 
 def test_agent_install_writes_hooks(tmp_path, monkeypatch) -> None:
     monkeypatch.chdir(tmp_path)
-    result = CliRunner().invoke(app, ["agent", "install", "--scope", "local"])
+    result = CliRunner().invoke(app, ["agent", "install", "--scope", "local", "--no-warm-up"])
     assert result.exit_code == 0, result.output
     assert (tmp_path / ".claude" / "settings.local.json").is_file()