vesk-cli 0.2.0__tar.gz → 0.2.1__tar.gz

{vesk_cli-0.2.0 → vesk_cli-0.2.1}/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.2.1 - 2026-11-02
+
+### Added
+- `vesk scan` prints live per-request progress to stderr (prompt excerpt, verdict, latency); `-q/--quiet` turns it off. Model responses are never printed; `--redact` withholds the prompt excerpt.
+
 ## 0.2.0 - 2026-11-02
 
 See docs/schema-v2.md for the upgrade note.

{vesk_cli-0.2.0 → vesk_cli-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vesk-cli
-Version: 0.2.0
+Version: 0.2.1
 Summary: Vesk Verify: test LLM endpoints and produce compliance-mapped reports.
 Author: Vesk
 License-Expression: Apache-2.0

{vesk_cli-0.2.0 → vesk_cli-0.2.1}/src/vesk/__init__.py

@@ -1,3 +1,3 @@
 """Vesk Verify: LLM endpoint testing with compliance-mapped reports."""
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"

{vesk_cli-0.2.0 → vesk_cli-0.2.1}/src/vesk/cli.py

@@ -81,6 +81,9 @@ def scan(
     judge: bool = typer.Option(
         False, "--judge", help="Also ask the judge from the config (sends responses to it)."
     ),
+    quiet: bool = typer.Option(
+        False, "-q", "--quiet", help="No progress output; print only the final summary."
+    ),
     env_file: Path | None = typer.Option(
         None,
         "--env-file",
@@ -144,7 +147,7 @@ def scan(
                 cfg,
                 probes,
                 client,
-                progress=lambda m: typer.echo(m, err=True),
+                progress=None if quiet else (lambda m: typer.echo(m, err=True)),
                 options=RunOptions(
                     system_prompt_source=source,
                     include_system_prompt=include_system_prompt,

{vesk_cli-0.2.0 → vesk_cli-0.2.1}/src/vesk/runner.py

@@ -206,6 +206,23 @@ def _store_text(probe: Probe, case, text: str, opts: RunOptions) -> str:
     return REDACTED.format(n=len(text)) if opts.redact and text else text
 
 
+PREVIEW_CHARS = 100
+
+
+def _preview(probe: Probe, case, opts: RunOptions) -> str:
+    """One-line excerpt of the prompt sent, for the console. Never the response.
+
+    Uses the probe's own redaction (planted PII is masked) and strips secrets. With
+    --redact the text is withheld entirely.
+    """
+    if opts.redact:
+        return "[prompt withheld: --redact]"
+    text = " ".join(_scrub(probe.redact(case, case.prompt), opts.secrets).split())
+    if len(text) > PREVIEW_CHARS:
+        text = text[: PREVIEW_CHARS - 1] + "…"
+    return f'"{text}"'
+
+
 async def _run_probe(
     probe: Probe,
     cfg: Config,
@@ -222,26 +239,29 @@ async def _run_probe(
         for t in range(trials)
     ]
     total = sum(len(c) for c in by_trial)
-    done = 0
-    step = max(1, total // 4)
+    started = 0
+
+    def say(msg: str) -> None:
+        if progress:
+            progress(msg)
 
     async def one(t: int, case) -> tuple[str, TrialResult]:
-        nonlocal done
+        nonlocal started
         base = dict(index=t, request_messages=_store_messages(probe, case, opts))
         async with sem:
+            started += 1
+            tag = f"  [{started}/{total}]"
+            say(f"{tag} sending {case.id} (trial {t + 1}/{trials}): {_preview(probe, case, opts)}")
             try:
                 comp = await client.complete_detailed(case.messages)
             except EndpointError as e:
+                say(f"{tag} ERROR endpoint_error")
                 return case.id, TrialResult(
                     **base,
                     status="error",
                     status_reason="endpoint_error",
                     error=_scrub(str(e), opts.secrets),
                 )
-            finally:
-                done += 1
-                if progress and (done == total or done % step == 0):
-                    progress(f"  {probe.name}: {done}/{total} requests")
         verdict = probe.evaluate(case, comp.content, comp.reasoning)
         invalid, reason = classify_completion(comp.content, comp.finish_reason)
         if verdict.failed:
@@ -252,6 +272,11 @@ async def _run_probe(
             status, reason = "inconclusive", verdict.reason
         else:
             status, reason = "pass", ""
+        say(
+            f"{tag} {status.upper()}"
+            + (f" ({reason})" if reason else "")
+            + f" {comp.latency_s:.1f}s"
+        )
         judge = None
         if judge_client is not None and comp.content.strip():
             judge = await judge_trial(judge_client, probe.description, case.prompt, comp.content)
@@ -360,7 +385,8 @@ async def run_scan(
     options: RunOptions | None = None,
     judge_client: ChatClient | None = None,
 ) -> ScanResults:
-    """`progress` receives short status lines: probe names and counts only, never content."""
+    """`progress` receives status lines: probe names, and one line per request with a short
+    prompt excerpt (masked, secrets removed) and its verdict. Model responses are never sent."""
     opts = options or RunOptions()
     started = _now()
     sem = asyncio.Semaphore(cfg.scan.max_concurrency)

{vesk_cli-0.2.0 → vesk_cli-0.2.1}/tests/integration/test_scan_cli.py

@@ -113,10 +113,47 @@ def test_progress_shows_status_without_sensitive_data(cfg, tmp_path):
     r, out = scan(cfg, tmp_path)
     assert "Scanning llm.test (model m) with 3 probe(s)" in r.output
     assert "[1/3]" in r.output and "[3/3]" in r.output
-    assert "8/8 requests" in r.output
     assert "secret-key" not in r.output
     assert "/v1" not in r.output.split("Scanning", 1)[1].splitlines()[0]
-    assert not MARKER.search(r.output)
+
+
+@respx.mock
+def test_each_request_is_announced_with_prompt_and_verdict(cfg, tmp_path):
+    respx.post(URL).mock(side_effect=vulnerable)
+    r, _ = scan(cfg, tmp_path, "--probe", "prompt_injection")
+    lines = r.output.splitlines()
+    sending = [x for x in lines if "] sending " in x]
+    verdicts = [x for x in lines if re.search(r"\] (PASS|FAIL|ERROR|INCONCLUSIVE)", x)]
+    assert len(sending) == len(verdicts) == 8
+    assert "[1/8] sending" in r.output and "[8/8]" in r.output
+    assert all('"' in x for x in sending)
+    assert all(len(x) < 220 for x in sending)
+
+
+@respx.mock
+def test_console_never_shows_responses(cfg, tmp_path):
+    respx.post(URL).mock(side_effect=vulnerable)
+    r, out = scan(cfg, tmp_path, "--probe", "pii_leakage")
+    data = json.loads(out.read_text())
+    for c in data["probes"][0]["cases"]:
+        for t in c["trials"]:
+            assert t["response"] not in r.output
+    assert "sending" in r.output
+
+
+@respx.mock
+def test_redact_withholds_prompt_preview(cfg, tmp_path):
+    respx.post(URL).mock(side_effect=hardened)
+    r, _ = scan(cfg, tmp_path, "--redact", "--probe", "prompt_injection")
+    assert "prompt withheld" in r.output
+
+
+@respx.mock
+def test_quiet_suppresses_progress(cfg, tmp_path):
+    respx.post(URL).mock(side_effect=hardened)
+    r, _ = scan(cfg, tmp_path, "-q")
+    assert "sending" not in r.output and "Scanning" not in r.output
+    assert "prompt_injection: pass" in r.output
 
 
 @respx.mock