vesk-cli 0.1.2__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/PKG-INFO +4 -1
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/README.md +3 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/__init__.py +1 -1
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/cli.py +7 -1
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/config.py +49 -3
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/tests/integration/test_scan_cli.py +17 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/tests/unit/test_scan.py +25 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/.gitignore +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/LICENSE +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/NOTICE +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/docs/.gitkeep +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/examples/.gitkeep +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/pyproject.toml +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/client.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/data/.gitkeep +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/__init__.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/base.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/data/.gitkeep +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/data/jailbreak.yaml +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/data/pii_leakage.yaml +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/data/prompt_injection.yaml +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/jailbreak.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/pii_leakage.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/probes/prompt_injection.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/report/__init__.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/report/model.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/report/render.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/report/templates/report.html.j2 +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/report/templates/report.md.j2 +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/runner.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/src/vesk/scaffold.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/tests/unit/test_cli.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/tests/unit/test_init.py +0 -0
- {vesk_cli-0.1.2 → vesk_cli-0.1.4}/tests/unit/test_report.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: vesk-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Vesk Verify: test LLM endpoints and produce compliance-mapped reports.
|
|
5
5
|
Author: Vesk
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -53,6 +53,8 @@ vesk scan # run probes, write results.json
|
|
|
53
53
|
vesk report results.json -o report.md
|
|
54
54
|
```
|
|
55
55
|
|
|
56
|
+
`api_key_env` holds the *name* of an environment variable (e.g. `VESK_API_KEY`), not a file path or the key. By default vesk reads only the process environment, so export the variable in your shell. To use a `.env` file instead, pass it explicitly: `vesk scan --env-file .env`. Variables already set in the environment take precedence over the file. Omit `api_key_env` if the endpoint needs no key.
|
|
57
|
+
|
|
56
58
|
## Commands
|
|
57
59
|
|
|
58
60
|
### `vesk init`
|
|
@@ -69,6 +71,7 @@ Runs the configured probes against the target and writes a results JSON file.
|
|
|
69
71
|
- `-c, --config PATH`: config file (default `vesk.yaml`).
|
|
70
72
|
- `--probe NAME`: run only this probe; repeatable.
|
|
71
73
|
- `--out PATH`: results file (default `results.json`).
|
|
74
|
+
- `--env-file PATH`: optional dotenv-style file (`KEY=VALUE` lines) to read the API key from. Not read unless given; real environment variables win over the file.
|
|
72
75
|
|
|
73
76
|
Exit codes:
|
|
74
77
|
|
|
@@ -22,6 +22,8 @@ vesk scan # run probes, write results.json
|
|
|
22
22
|
vesk report results.json -o report.md
|
|
23
23
|
```
|
|
24
24
|
|
|
25
|
+
`api_key_env` holds the *name* of an environment variable (e.g. `VESK_API_KEY`), not a file path or the key. By default vesk reads only the process environment, so export the variable in your shell. To use a `.env` file instead, pass it explicitly: `vesk scan --env-file .env`. Variables already set in the environment take precedence over the file. Omit `api_key_env` if the endpoint needs no key.
|
|
26
|
+
|
|
25
27
|
## Commands
|
|
26
28
|
|
|
27
29
|
### `vesk init`
|
|
@@ -38,6 +40,7 @@ Runs the configured probes against the target and writes a results JSON file.
|
|
|
38
40
|
- `-c, --config PATH`: config file (default `vesk.yaml`).
|
|
39
41
|
- `--probe NAME`: run only this probe; repeatable.
|
|
40
42
|
- `--out PATH`: results file (default `results.json`).
|
|
43
|
+
- `--env-file PATH`: optional dotenv-style file (`KEY=VALUE` lines) to read the API key from. Not read unless given; real environment variables win over the file.
|
|
41
44
|
|
|
42
45
|
Exit codes:
|
|
43
46
|
|
|
@@ -54,6 +54,12 @@ def scan(
|
|
|
54
54
|
config: Path = typer.Option(Path("vesk.yaml"), "-c", "--config", help="Config file."),
|
|
55
55
|
probe: list[str] | None = typer.Option(None, "--probe", help="Probe to run (repeatable)."),
|
|
56
56
|
out: Path = typer.Option(Path("results.json"), "--out", help="Results JSON path."),
|
|
57
|
+
env_file: Path | None = typer.Option(
|
|
58
|
+
None,
|
|
59
|
+
"--env-file",
|
|
60
|
+
help="Optional .env file to read the API key from. "
|
|
61
|
+
"Variables already set in the environment win. Without it, only the environment is used.",
|
|
62
|
+
),
|
|
57
63
|
) -> None:
|
|
58
64
|
"""Run probes against the target and write a results JSON file."""
|
|
59
65
|
import asyncio
|
|
@@ -65,7 +71,7 @@ def scan(
|
|
|
65
71
|
try:
|
|
66
72
|
cfg = load_config(config)
|
|
67
73
|
probes = select_probes(cfg, probe)
|
|
68
|
-
api_key = cfg.api_key()
|
|
74
|
+
api_key = cfg.api_key(env_file)
|
|
69
75
|
except ConfigError as e:
|
|
70
76
|
typer.echo(f"vesk scan: {e}", err=True)
|
|
71
77
|
raise typer.Exit(2) from e
|
|
@@ -51,17 +51,63 @@ class Config(_Strict):
|
|
|
51
51
|
scan: ScanConfig = Field(default_factory=lambda: ScanConfig())
|
|
52
52
|
report: ReportConfig = Field(default_factory=lambda: ReportConfig())
|
|
53
53
|
|
|
54
|
-
def api_key(self) -> str | None:
|
|
55
|
-
"""Read the API key from the environment variable named in the config.
|
|
54
|
+
def api_key(self, env_file: Path | None = None) -> str | None:
|
|
55
|
+
"""Read the API key from the environment variable named in the config.
|
|
56
|
+
|
|
57
|
+
If env_file is given, its values fill in variables missing from the process
|
|
58
|
+
environment; variables already set in the environment take precedence.
|
|
59
|
+
"""
|
|
56
60
|
name = self.target.api_key_env
|
|
57
61
|
if not name:
|
|
58
62
|
return None
|
|
63
|
+
if not name.isidentifier():
|
|
64
|
+
raise ConfigError(
|
|
65
|
+
f"target.api_key_env is {name!r}, which is not an environment variable name. "
|
|
66
|
+
"Set it to the NAME of a variable (for example VESK_API_KEY), not a file or "
|
|
67
|
+
"the key itself, then export that variable: export VESK_API_KEY=<your key>"
|
|
68
|
+
)
|
|
59
69
|
key = os.environ.get(name)
|
|
70
|
+
if not key and env_file is not None:
|
|
71
|
+
key = read_env_file(env_file).get(name)
|
|
60
72
|
if not key:
|
|
61
|
-
raise ConfigError(
|
|
73
|
+
raise ConfigError(
|
|
74
|
+
f"environment variable {name} is not set or is empty "
|
|
75
|
+
f"(named by target.api_key_env in your config). "
|
|
76
|
+
f"Export it in this shell first: export {name}=<your key>. "
|
|
77
|
+
"Or put it in a .env file and pass --env-file .env "
|
|
78
|
+
"(vesk only reads .env files when you pass --env-file). "
|
|
79
|
+
"If the endpoint needs no key, remove api_key_env from the config."
|
|
80
|
+
)
|
|
62
81
|
return key
|
|
63
82
|
|
|
64
83
|
|
|
84
|
+
def read_env_file(path: Path) -> dict[str, str]:
|
|
85
|
+
"""Parse a dotenv-style file: KEY=VALUE lines, # comments, optional `export `, quotes."""
|
|
86
|
+
try:
|
|
87
|
+
text = path.read_text(encoding="utf-8")
|
|
88
|
+
except FileNotFoundError:
|
|
89
|
+
raise ConfigError(f"env file not found: {path}") from None
|
|
90
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
91
|
+
raise ConfigError(f"cannot read env file {path}: {e}") from e
|
|
92
|
+
values: dict[str, str] = {}
|
|
93
|
+
for line in text.splitlines():
|
|
94
|
+
line = line.strip()
|
|
95
|
+
if not line or line.startswith("#"):
|
|
96
|
+
continue
|
|
97
|
+
if line.startswith("export "):
|
|
98
|
+
line = line[len("export ") :].lstrip()
|
|
99
|
+
name, sep, value = line.partition("=")
|
|
100
|
+
name, value = name.strip(), value.strip()
|
|
101
|
+
if not sep or not name.isidentifier():
|
|
102
|
+
continue
|
|
103
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
|
104
|
+
value = value[1:-1]
|
|
105
|
+
elif " #" in value:
|
|
106
|
+
value = value.split(" #", 1)[0].rstrip()
|
|
107
|
+
values[name] = value
|
|
108
|
+
return values
|
|
109
|
+
|
|
110
|
+
|
|
65
111
|
def load_config(path: Path) -> Config:
|
|
66
112
|
try:
|
|
67
113
|
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
@@ -145,3 +145,20 @@ def test_missing_api_key_env_exits_2(cfg, tmp_path, monkeypatch):
|
|
|
145
145
|
r, _ = scan(cfg, tmp_path)
|
|
146
146
|
assert r.exit_code == 2
|
|
147
147
|
assert "VESK_TEST_KEY" in r.output
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@respx.mock
|
|
151
|
+
def test_env_file_supplies_key_when_env_unset(cfg, tmp_path, monkeypatch):
|
|
152
|
+
monkeypatch.delenv("VESK_TEST_KEY")
|
|
153
|
+
envf = tmp_path / ".env"
|
|
154
|
+
envf.write_text("VESK_TEST_KEY=file-key\n")
|
|
155
|
+
respx.post(URL).mock(side_effect=hardened)
|
|
156
|
+
r, _ = scan(cfg, tmp_path, "--env-file", str(envf))
|
|
157
|
+
assert r.exit_code == 0, r.output
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def test_env_file_missing_exits_2(cfg, tmp_path, monkeypatch):
|
|
161
|
+
monkeypatch.delenv("VESK_TEST_KEY")
|
|
162
|
+
r, _ = scan(cfg, tmp_path, "--env-file", str(tmp_path / "nope.env"))
|
|
163
|
+
assert r.exit_code == 2
|
|
164
|
+
assert "env file not found" in r.output
|
|
@@ -49,6 +49,12 @@ def test_api_key_comes_from_env(tmp_path, monkeypatch):
|
|
|
49
49
|
assert cfg.api_key() == "abc"
|
|
50
50
|
|
|
51
51
|
|
|
52
|
+
def test_api_key_env_must_be_a_variable_name(tmp_path):
|
|
53
|
+
cfg = load_config(write(tmp_path, MINIMAL + " api_key_env: .env\n"))
|
|
54
|
+
with pytest.raises(ConfigError, match="not an environment variable name"):
|
|
55
|
+
cfg.api_key()
|
|
56
|
+
|
|
57
|
+
|
|
52
58
|
def test_select_probes_unknown(tmp_path):
|
|
53
59
|
cfg = load_config(write(tmp_path, MINIMAL))
|
|
54
60
|
with pytest.raises(ConfigError, match="unknown probe"):
|
|
@@ -215,3 +221,22 @@ async def test_client_retries_timeouts():
|
|
|
215
221
|
|
|
216
222
|
with pytest.raises(EndpointError, match="ConnectTimeout|slow"):
|
|
217
223
|
await _client(handler, retries=1).complete([])
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def test_api_key_from_env_file(tmp_path, monkeypatch):
|
|
227
|
+
cfg = load_config(write(tmp_path, MINIMAL + " api_key_env: MY_KEY\n"))
|
|
228
|
+
monkeypatch.delenv("MY_KEY", raising=False)
|
|
229
|
+
env = tmp_path / ".env"
|
|
230
|
+
env.write_text("# c\nOTHER=1\nexport MY_KEY='from-file'\n")
|
|
231
|
+
assert cfg.api_key(env) == "from-file"
|
|
232
|
+
with pytest.raises(ConfigError, match="--env-file"):
|
|
233
|
+
cfg.api_key() # file is never read unless passed
|
|
234
|
+
monkeypatch.setenv("MY_KEY", "from-env")
|
|
235
|
+
assert cfg.api_key(env) == "from-env" # environment wins
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def test_env_file_missing(tmp_path, monkeypatch):
|
|
239
|
+
cfg = load_config(write(tmp_path, MINIMAL + " api_key_env: MY_KEY\n"))
|
|
240
|
+
monkeypatch.delenv("MY_KEY", raising=False)
|
|
241
|
+
with pytest.raises(ConfigError, match="env file not found"):
|
|
242
|
+
cfg.api_key(tmp_path / "nope.env")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|