vesk-cli 0.1.3__tar.gz → 0.1.5__tar.gz

{vesk_cli-0.1.3 → vesk_cli-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vesk-cli
-Version: 0.1.3
+Version: 0.1.5
 Summary: Vesk Verify: test LLM endpoints and produce compliance-mapped reports.
 Author: Vesk
 License-Expression: Apache-2.0
@@ -53,7 +53,7 @@ vesk scan                      # run probes, write results.json
 vesk report results.json -o report.md
 ```
 
-`api_key_env` holds the *name* of an environment variable (e.g. `VESK_API_KEY`), not a file path or the key. vesk does not read `.env` files; export the variable in your shell (or load your `.env` first, e.g. `set -a; . ./.env; set +a`). Omit `api_key_env` if the endpoint needs no key.
+`api_key_env` holds the *name* of an environment variable (e.g. `VESK_API_KEY`), not a file path or the key. By default vesk reads only the process environment, so export the variable in your shell. To use a `.env` file instead, pass it explicitly: `vesk scan --env-file .env`. Variables already set in the environment take precedence over the file. Omit `api_key_env` if the endpoint needs no key.
 
 ## Commands
 
@@ -71,6 +71,9 @@ Runs the configured probes against the target and writes a results JSON file.
 - `-c, --config PATH`: config file (default `vesk.yaml`).
 - `--probe NAME`: run only this probe; repeatable.
 - `--out PATH`: results file (default `results.json`).
+- `--env-file PATH`: optional dotenv-style file (`KEY=VALUE` lines) to read the API key from. Not read unless given; real environment variables win over the file.
+
+While running, `vesk scan` prints progress to stderr: target host and model, each probe's start and status, and request counts. It never prints prompts, responses, API keys, URL paths or credentials; those stay in the results file only (with PII masked).
 
 Exit codes:
 

{vesk_cli-0.1.3 → vesk_cli-0.1.5}/README.md

@@ -22,7 +22,7 @@ vesk scan                      # run probes, write results.json
 vesk report results.json -o report.md
 ```
 
-`api_key_env` holds the *name* of an environment variable (e.g. `VESK_API_KEY`), not a file path or the key. vesk does not read `.env` files; export the variable in your shell (or load your `.env` first, e.g. `set -a; . ./.env; set +a`). Omit `api_key_env` if the endpoint needs no key.
+`api_key_env` holds the *name* of an environment variable (e.g. `VESK_API_KEY`), not a file path or the key. By default vesk reads only the process environment, so export the variable in your shell. To use a `.env` file instead, pass it explicitly: `vesk scan --env-file .env`. Variables already set in the environment take precedence over the file. Omit `api_key_env` if the endpoint needs no key.
 
 ## Commands
 
@@ -40,6 +40,9 @@ Runs the configured probes against the target and writes a results JSON file.
 - `-c, --config PATH`: config file (default `vesk.yaml`).
 - `--probe NAME`: run only this probe; repeatable.
 - `--out PATH`: results file (default `results.json`).
+- `--env-file PATH`: optional dotenv-style file (`KEY=VALUE` lines) to read the API key from. Not read unless given; real environment variables win over the file.
+
+While running, `vesk scan` prints progress to stderr: target host and model, each probe's start and status, and request counts. It never prints prompts, responses, API keys, URL paths or credentials; those stay in the results file only (with PII masked).
 
 Exit codes:
 

{vesk_cli-0.1.3 → vesk_cli-0.1.5}/src/vesk/__init__.py

@@ -1,3 +1,3 @@
 """Vesk Verify: LLM endpoint testing with compliance-mapped reports."""
 
-__version__ = "0.1.3"
+__version__ = "0.1.5"

{vesk_cli-0.1.3 → vesk_cli-0.1.5}/src/vesk/cli.py

@@ -54,6 +54,12 @@ def scan(
     config: Path = typer.Option(Path("vesk.yaml"), "-c", "--config", help="Config file."),
     probe: list[str] | None = typer.Option(None, "--probe", help="Probe to run (repeatable)."),
     out: Path = typer.Option(Path("results.json"), "--out", help="Results JSON path."),
+    env_file: Path | None = typer.Option(
+        None,
+        "--env-file",
+        help="Optional .env file to read the API key from. "
+        "Variables already set in the environment win. Without it, only the environment is used.",
+    ),
 ) -> None:
     """Run probes against the target and write a results JSON file."""
     import asyncio
@@ -65,7 +71,7 @@ def scan(
     try:
         cfg = load_config(config)
         probes = select_probes(cfg, probe)
-        api_key = cfg.api_key()
+        api_key = cfg.api_key(env_file)
     except ConfigError as e:
         typer.echo(f"vesk scan: {e}", err=True)
         raise typer.Exit(2) from e
@@ -79,7 +85,7 @@ def scan(
             max_retries=cfg.target.max_retries,
         )
         try:
-            return await run_scan(cfg, probes, client)
+            return await run_scan(cfg, probes, client, progress=lambda m: typer.echo(m, err=True))
         finally:
             await client.aclose()
 

{vesk_cli-0.1.3 → vesk_cli-0.1.5}/src/vesk/config.py

@@ -51,8 +51,12 @@ class Config(_Strict):
     scan: ScanConfig = Field(default_factory=lambda: ScanConfig())
     report: ReportConfig = Field(default_factory=lambda: ReportConfig())
 
-    def api_key(self) -> str | None:
-        """Read the API key from the environment variable named in the config."""
+    def api_key(self, env_file: Path | None = None) -> str | None:
+        """Read the API key from the environment variable named in the config.
+
+        If env_file is given, its values fill in variables missing from the process
+        environment; variables already set in the environment take precedence.
+        """
         name = self.target.api_key_env
         if not name:
             return None
@@ -63,17 +67,47 @@ class Config(_Strict):
                 "the key itself, then export that variable: export VESK_API_KEY=<your key>"
             )
         key = os.environ.get(name)
+        if not key and env_file is not None:
+            key = read_env_file(env_file).get(name)
         if not key:
             raise ConfigError(
                 f"environment variable {name} is not set or is empty "
                 f"(named by target.api_key_env in your config). "
                 f"Export it in this shell first: export {name}=<your key>. "
-                "vesk does not read .env files. If the endpoint needs no key, "
-                "remove api_key_env from the config."
+                "Or put it in a .env file and pass --env-file .env "
+                "(vesk only reads .env files when you pass --env-file). "
+                "If the endpoint needs no key, remove api_key_env from the config."
             )
         return key
 
 
+def read_env_file(path: Path) -> dict[str, str]:
+    """Parse a dotenv-style file: KEY=VALUE lines, # comments, optional `export `, quotes."""
+    try:
+        text = path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        raise ConfigError(f"env file not found: {path}") from None
+    except (OSError, UnicodeDecodeError) as e:
+        raise ConfigError(f"cannot read env file {path}: {e}") from e
+    values: dict[str, str] = {}
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("export "):
+            line = line[len("export ") :].lstrip()
+        name, sep, value = line.partition("=")
+        name, value = name.strip(), value.strip()
+        if not sep or not name.isidentifier():
+            continue
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        elif " #" in value:
+            value = value.split(" #", 1)[0].rstrip()
+        values[name] = value
+    return values
+
+
 def load_config(path: Path) -> Config:
     try:
         raw = yaml.safe_load(path.read_text(encoding="utf-8"))

{vesk_cli-0.1.3 → vesk_cli-0.1.5}/src/vesk/runner.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 import asyncio
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import Callable
+from urllib.parse import urlsplit
 
 from pydantic import BaseModel, Field
 
@@ -78,17 +80,35 @@ def _excerpt(text: str) -> str:
     return text if len(text) <= RESPONSE_LIMIT else text[:RESPONSE_LIMIT] + "…"
 
 
-async def _run_probe(probe: Probe, cfg: Config, client: ChatClient, sem: asyncio.Semaphore):
+def display_host(base_url: str) -> str:
+    """Host[:port] only: drops credentials, path and query so nothing sensitive is echoed."""
+    return urlsplit(base_url).netloc.rpartition("@")[2] or "target"
+
+
+async def _run_probe(
+    probe: Probe,
+    cfg: Config,
+    client: ChatClient,
+    sem: asyncio.Semaphore,
+    progress: Progress | None = None,
+):
     cases = probe.cases(cfg.target.system_prompt, cfg.scan.samples_per_probe, cfg.scan.seed)
     errors: list[str] = []
+    done = 0
+    step = max(1, len(cases) // 4)
 
     async def one(case) -> CaseResult | None:
+        nonlocal done
         async with sem:
             try:
                 response = await client.complete(case.messages)
             except EndpointError as e:
                 errors.append(str(e))
                 return None
+            finally:
+                done += 1
+                if progress and (done == len(cases) or done % step == 0):
+                    progress(f"  {probe.name}: {done}/{len(cases)} requests")
         verdict = probe.evaluate(case, response)
         return CaseResult(
             id=case.id,
@@ -118,12 +138,28 @@ async def _run_probe(probe: Probe, cfg: Config, client: ChatClient, sem: asyncio
     )
 
 
+Progress = Callable[[str], None]
+
+
 async def run_scan(
-    cfg: Config, probes: list[Probe], client: ChatClient
+    cfg: Config, probes: list[Probe], client: ChatClient, progress: Progress | None = None
 ) -> ScanResults:
+    """`progress` receives short status lines: probe names and counts only, never content."""
     started = _now()
     sem = asyncio.Semaphore(cfg.scan.max_concurrency)
-    runs = [await _run_probe(p, cfg, client, sem) for p in probes]
+    if progress:
+        progress(
+            f"Scanning {display_host(cfg.target.base_url)} (model {cfg.target.model}) "
+            f"with {len(probes)} probe(s)"
+        )
+    runs = []
+    for i, p in enumerate(probes, 1):
+        if progress:
+            progress(f"[{i}/{len(probes)}] {p.name}: running")
+        run = await _run_probe(p, cfg, client, sem, progress)
+        if progress:
+            progress(f"[{i}/{len(probes)}] {p.name}: {run.status}")
+        runs.append(run)
     return ScanResults(
         target=cfg.target.base_url,
         model=cfg.target.model,

{vesk_cli-0.1.3 → vesk_cli-0.1.5}/tests/integration/test_scan_cli.py

@@ -120,6 +120,18 @@ def test_endpoint_down_exits_3_and_still_writes_results(cfg, tmp_path):
     assert all(p["status"] == "error" and p["error"] for p in data["probes"])
 
 
+@respx.mock
+def test_progress_shows_status_without_sensitive_data(cfg, tmp_path):
+    respx.post(URL).mock(side_effect=vulnerable)
+    r, out = scan(cfg, tmp_path)
+    assert "Scanning llm.test (model m) with 3 probe(s)" in r.output
+    assert "[1/3]" in r.output and "[3/3]" in r.output
+    assert "4/4 requests" in r.output
+    assert "secret-key" not in r.output
+    assert "/v1" not in r.output.split("Scanning", 1)[1].splitlines()[0]
+    assert not MARKER.search(r.output)
+
+
 @respx.mock
 def test_sends_bearer_key(cfg, tmp_path):
     route = respx.post(URL).mock(side_effect=hardened)
@@ -145,3 +157,20 @@ def test_missing_api_key_env_exits_2(cfg, tmp_path, monkeypatch):
     r, _ = scan(cfg, tmp_path)
     assert r.exit_code == 2
     assert "VESK_TEST_KEY" in r.output
+
+
+@respx.mock
+def test_env_file_supplies_key_when_env_unset(cfg, tmp_path, monkeypatch):
+    monkeypatch.delenv("VESK_TEST_KEY")
+    envf = tmp_path / ".env"
+    envf.write_text("VESK_TEST_KEY=file-key\n")
+    respx.post(URL).mock(side_effect=hardened)
+    r, _ = scan(cfg, tmp_path, "--env-file", str(envf))
+    assert r.exit_code == 0, r.output
+
+
+def test_env_file_missing_exits_2(cfg, tmp_path, monkeypatch):
+    monkeypatch.delenv("VESK_TEST_KEY")
+    r, _ = scan(cfg, tmp_path, "--env-file", str(tmp_path / "nope.env"))
+    assert r.exit_code == 2
+    assert "env file not found" in r.output

{vesk_cli-0.1.3 → vesk_cli-0.1.5}/tests/unit/test_scan.py

@@ -221,3 +221,22 @@ async def test_client_retries_timeouts():
 
     with pytest.raises(EndpointError, match="ConnectTimeout|slow"):
         await _client(handler, retries=1).complete([])
+
+
+def test_api_key_from_env_file(tmp_path, monkeypatch):
+    cfg = load_config(write(tmp_path, MINIMAL + "  api_key_env: MY_KEY\n"))
+    monkeypatch.delenv("MY_KEY", raising=False)
+    env = tmp_path / ".env"
+    env.write_text("# c\nOTHER=1\nexport MY_KEY='from-file'\n")
+    assert cfg.api_key(env) == "from-file"
+    with pytest.raises(ConfigError, match="--env-file"):
+        cfg.api_key()  # file is never read unless passed
+    monkeypatch.setenv("MY_KEY", "from-env")
+    assert cfg.api_key(env) == "from-env"  # environment wins
+
+
+def test_env_file_missing(tmp_path, monkeypatch):
+    cfg = load_config(write(tmp_path, MINIMAL + "  api_key_env: MY_KEY\n"))
+    monkeypatch.delenv("MY_KEY", raising=False)
+    with pytest.raises(ConfigError, match="env file not found"):
+        cfg.api_key(tmp_path / "nope.env")