vesk-cli 0.1.2__py3-none-any.whl

vesk/report/model.py

@@ -0,0 +1,43 @@
+"""Schema of the results JSON consumed by `vesk report`."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+Status = Literal["pass", "fail", "error"]
+
+
+class Case(BaseModel):
+    id: str
+    passed: bool
+    prompt: str = ""
+    response: str = ""
+    evidence: str = ""
+
+
+class ProbeResult(BaseModel):
+    id: str
+    name: str = ""
+    category: str = ""
+    status: Status
+    description: str = ""
+    error: str = ""
+    cases: list[Case] = Field(default_factory=list)
+
+    @property
+    def failed_cases(self) -> list[Case]:
+        return [c for c in self.cases if not c.passed]
+
+
+class Results(BaseModel):
+    vesk_version: str = ""
+    target: str = ""
+    model: str = ""
+    started_at: str = ""
+    finished_at: str = ""
+    probes: list[ProbeResult] = Field(default_factory=list)
+
+    def count(self, status: Status) -> int:
+        return sum(1 for p in self.probes if p.status == status)

vesk/report/render.py

@@ -0,0 +1,52 @@
+"""Render a Results object to Markdown or HTML."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+from pydantic import ValidationError
+
+from vesk.report.model import Results
+
+DISCLAIMER = (
+    "This report is generated automatically from automated test results. It is not legal "
+    "or compliance advice, and it does not certify compliance with any law, regulation or "
+    "standard. Consult qualified professionals before relying on it."
+)
+
+FORMATS = ("md", "html")
+
+
+class ReportError(Exception):
+    """Results file could not be read or rendered."""
+
+
+def load_results(path: Path) -> Results:
+    try:
+        raw = path.read_text(encoding="utf-8")
+    except OSError as e:
+        raise ReportError(f"cannot read {path}: {e.strerror or e}") from e
+    try:
+        return Results.model_validate(json.loads(raw))
+    except json.JSONDecodeError as e:
+        raise ReportError(f"{path} is not valid JSON: {e}") from e
+    except ValidationError as e:
+        raise ReportError(f"{path} is not a valid results file:\n{e}") from e
+
+
+def _env(fmt: str) -> Environment:
+    return Environment(
+        loader=FileSystemLoader(Path(__file__).parent / "templates"),
+        autoescape=fmt == "html",
+        trim_blocks=True,
+        lstrip_blocks=True,
+        keep_trailing_newline=True,
+    )
+
+
+def render(results: Results, fmt: str = "md") -> str:
+    if fmt not in FORMATS:
+        raise ReportError(f"unknown format {fmt!r}; choose one of: {', '.join(FORMATS)}")
+    return _env(fmt).get_template(f"report.{fmt}.j2").render(r=results, disclaimer=DISCLAIMER)

vesk/report/templates/report.html.j2

@@ -0,0 +1,48 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Vesk Verify report</title>
+<style>
+body{font-family:sans-serif;max-width:60rem;margin:2rem auto;padding:0 1rem}
+table{border-collapse:collapse;width:100%}td,th{border:1px solid #ccc;padding:.3rem .5rem;text-align:left}
+.pass{color:#0a6b2d}.fail,.error{color:#b00020}
+.disclaimer{border:1px solid #c90;background:#fff8e1;padding:.6rem}
+pre{white-space:pre-wrap}
+</style>
+</head>
+<body>
+<h1>Vesk Verify report</h1>
+<p class="disclaimer"><strong>Disclaimer:</strong> {{ disclaimer }}</p>
+<ul>
+<li>Target: {{ r.target or "n/a" }}</li>
+<li>Model: {{ r.model or "n/a" }}</li>
+<li>Run: {{ r.started_at or "n/a" }} to {{ r.finished_at or "n/a" }}</li>
+<li>Vesk version: {{ r.vesk_version or "n/a" }}</li>
+</ul>
+<h2>Summary</h2>
+<p>{{ r.count("pass") }} passed, {{ r.count("fail") }} failed, {{ r.count("error") }} errored ({{ r.probes|length }} probes).</p>
+<table>
+<tr><th>Probe</th><th>Category</th><th>Status</th><th>Failed cases</th></tr>
+{% for p in r.probes %}
+<tr><td>{{ p.name or p.id }}</td><td>{{ p.category }}</td><td class="{{ p.status }}">{{ p.status|upper }}</td><td>{{ p.failed_cases|length }}/{{ p.cases|length }}</td></tr>
+{% endfor %}
+</table>
+<h2>Details</h2>
+{% for p in r.probes if p.status != "pass" %}
+<h3>{{ p.name or p.id }} (<span class="{{ p.status }}">{{ p.status|upper }}</span>)</h3>
+{% if p.description %}<p>{{ p.description }}</p>{% endif %}
+{% if p.error %}<p>Error: {{ p.error }}</p>{% endif %}
+{% for c in p.failed_cases %}
+<h4>Case {{ c.id }}</h4>
+<p>Prompt:</p><pre>{{ c.prompt }}</pre>
+<p>Response:</p><pre>{{ c.response }}</pre>
+{% if c.evidence %}<p>Evidence: {{ c.evidence }}</p>{% endif %}
+{% endfor %}
+{% else %}
+<p>No failures or errors.</p>
+{% endfor %}
+<hr>
+<p class="disclaimer">{{ disclaimer }}</p>
+</body>
+</html>

vesk/report/templates/report.md.j2

@@ -0,0 +1,48 @@
+# Vesk Verify report
+
+> **Disclaimer:** {{ disclaimer }}
+
+- Target: {{ r.target or "n/a" }}
+- Model: {{ r.model or "n/a" }}
+- Run: {{ r.started_at or "n/a" }} to {{ r.finished_at or "n/a" }}
+- Vesk version: {{ r.vesk_version or "n/a" }}
+
+## Summary
+
+{{ r.count("pass") }} passed, {{ r.count("fail") }} failed, {{ r.count("error") }} errored ({{ r.probes|length }} probes).
+
+| Probe | Category | Status | Failed cases |
+|---|---|---|---|
+{% for p in r.probes %}
+| {{ (p.name or p.id)|replace("|", "\\|") }} | {{ p.category|replace("|", "\\|") }} | {{ p.status|upper }} | {{ p.failed_cases|length }}/{{ p.cases|length }} |
+{% endfor %}
+
+## Details
+{% for p in r.probes if p.status != "pass" %}
+
+### {{ p.name or p.id }} ({{ p.status|upper }})
+{% if p.description %}
+
+{{ p.description }}
+{% endif %}
+{% if p.error %}
+
+Error: {{ p.error }}
+{% endif %}
+{% for c in p.failed_cases %}
+
+- Case `{{ c.id }}`
+  - Prompt: `{{ c.prompt|replace("`", "'")|replace("\n", " ") }}`
+  - Response: `{{ c.response|replace("`", "'")|replace("\n", " ") }}`
+{% if c.evidence %}
+  - Evidence: {{ c.evidence|replace("\n", " ") }}
+{% endif %}
+{% endfor %}
+{% else %}
+
+No failures or errors.
+{% endfor %}
+
+---
+
+{{ disclaimer }}

vesk/runner.py

@@ -0,0 +1,134 @@
+"""Runs probes against a chat client and builds the results consumed by `vesk report`."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timezone
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from vesk import __version__
+from vesk.client import ChatClient, EndpointError
+from vesk.config import SEVERITY_RANK, Config, ConfigError
+from vesk.probes import REGISTRY, Probe
+
+RESPONSE_LIMIT = 500
+
+
+class CaseResult(BaseModel):
+    id: str
+    passed: bool
+    prompt: str = ""
+    response: str = ""
+    evidence: str = ""
+
+
+class ProbeRun(BaseModel):
+    id: str
+    name: str
+    category: str
+    severity: str
+    status: str  # pass | fail | error
+    description: str = ""
+    error: str = ""
+    cases: list[CaseResult] = Field(default_factory=list)
+
+
+class ScanResults(BaseModel):
+    schema_version: int = 1
+    vesk_version: str = __version__
+    target: str
+    model: str
+    system_name: str = ""
+    started_at: str
+    finished_at: str
+    probes: list[ProbeRun]
+
+    def exit_code(self, fail_on: str) -> int:
+        """3 = incomplete scan (endpoint errors), 1 = findings at/above fail_on, else 0."""
+        if any(p.error for p in self.probes):
+            return 3
+        if fail_on == "none":
+            return 0
+        floor = SEVERITY_RANK[fail_on]
+        failing = (p for p in self.probes if p.status == "fail")
+        return 1 if any(SEVERITY_RANK[p.severity] >= floor for p in failing) else 0
+
+    def write(self, path: Path) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(self.model_dump_json(indent=2) + "\n", encoding="utf-8")
+
+
+def select_probes(cfg: Config, override: list[str] | None) -> list[Probe]:
+    names = override or cfg.scan.probes or list(REGISTRY)
+    unknown = [n for n in names if n not in REGISTRY]
+    if unknown:
+        raise ConfigError(
+            f"unknown probe(s): {', '.join(unknown)}. Available: {', '.join(REGISTRY)}"
+        )
+    return [REGISTRY[n]() for n in dict.fromkeys(names)]
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _excerpt(text: str) -> str:
+    return text if len(text) <= RESPONSE_LIMIT else text[:RESPONSE_LIMIT] + "…"
+
+
+async def _run_probe(probe: Probe, cfg: Config, client: ChatClient, sem: asyncio.Semaphore):
+    cases = probe.cases(cfg.target.system_prompt, cfg.scan.samples_per_probe, cfg.scan.seed)
+    errors: list[str] = []
+
+    async def one(case) -> CaseResult | None:
+        async with sem:
+            try:
+                response = await client.complete(case.messages)
+            except EndpointError as e:
+                errors.append(str(e))
+                return None
+        verdict = probe.evaluate(case, response)
+        return CaseResult(
+            id=case.id,
+            passed=not verdict.failed,
+            prompt=probe.redact(case, case.prompt),
+            response=_excerpt(probe.redact(case, response)),
+            evidence=verdict.evidence,
+        )
+
+    results = [r for r in await asyncio.gather(*(one(c) for c in cases)) if r is not None]
+    if not results:
+        status = "error"
+    else:
+        status = "fail" if any(not r.passed for r in results) else "pass"
+    error = ""
+    if errors:
+        error = f"{len(errors)} of {len(cases)} requests failed: {errors[0]}"
+    return ProbeRun(
+        id=probe.name,
+        name=probe.name.replace("_", " ").title(),
+        category=probe.category,
+        severity=probe.severity,
+        status=status,
+        description=probe.description,
+        error=error,
+        cases=results,
+    )
+
+
+async def run_scan(
+    cfg: Config, probes: list[Probe], client: ChatClient
+) -> ScanResults:
+    started = _now()
+    sem = asyncio.Semaphore(cfg.scan.max_concurrency)
+    runs = [await _run_probe(p, cfg, client, sem) for p in probes]
+    return ScanResults(
+        target=cfg.target.base_url,
+        model=cfg.target.model,
+        system_name=cfg.report.system_name,
+        started_at=started,
+        finished_at=_now(),
+        probes=runs,
+    )

vesk/scaffold.py

@@ -0,0 +1,48 @@
+"""Starter config written by `vesk init`."""
+
+from pathlib import Path
+
+STARTER_CONFIG = """\
+# Vesk Verify configuration. Docs: https://github.com/vesk-ai/vesk-cli
+version: 1
+
+target:
+  # Any OpenAI-compatible chat completions endpoint.
+  base_url: https://api.example.com/v1
+  model: my-model
+  # Name of the environment variable holding the API key.
+  # The key itself must never be written in this file.
+  api_key_env: VESK_API_KEY
+  # Optional system prompt your application uses (helps injection/leak probes).
+  # system_prompt: "You are a support bot."
+  timeout_s: 30
+  max_retries: 2
+
+scan:
+  # Probes to run. Available: prompt_injection, jailbreak, pii_leakage.
+  probes: [prompt_injection, jailbreak, pii_leakage]
+  samples_per_probe: 20
+  max_concurrency: 4
+  seed: 0
+  # Exit with code 1 if any finding is at or above this severity: none|low|medium|high
+  fail_on: high
+
+report:
+  frameworks: [eu_ai_act, dpdp, nist_ai_rmf]
+  system_name: "My AI system"
+"""
+
+
+class ConfigExistsError(FileExistsError):
+    """Raised when the target file exists and overwrite was not requested."""
+
+
+def write_starter_config(path: Path, force: bool = False) -> Path:
+    """Write the starter config to `path`, creating parent directories."""
+    if path.exists() and not force:
+        raise ConfigExistsError(f"{path} already exists (use --force to overwrite)")
+    if path.is_dir():
+        raise IsADirectoryError(f"{path} is a directory")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(STARTER_CONFIG, encoding="utf-8")
+    return path

vesk_cli-0.1.2.dist-info/METADATA

@@ -0,0 +1,148 @@
+Metadata-Version: 2.5
+Name: vesk-cli
+Version: 0.1.2
+Summary: Vesk Verify: test LLM endpoints and produce compliance-mapped reports.
+Author: Vesk
+License-Expression: Apache-2.0
+License-File: LICENSE
+License-File: NOTICE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: jinja2>=3.1
+Requires-Dist: pydantic>=2
+Requires-Dist: pyyaml>=6
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Requires-Dist: respx; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: types-pyyaml; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# vesk
+
+**Vesk Verify** tests an OpenAI-compatible LLM endpoint with prompt-injection, jailbreak and
+PII-leakage probes, then renders a report mapped to EU AI Act, DPDP and NIST AI RMF.
+
+> Status: alpha (0.1.x). Interfaces may change between minor versions.
+
+## Install
+
+```
+pip install vesk-cli
+```
+
+Requires Python 3.10 or newer.
+
+## Quick start
+
+```
+vesk init                      # write a commented vesk.yaml
+export VESK_API_KEY=...        # the env var named by target.api_key_env
+vesk scan                      # run probes, write results.json
+vesk report results.json -o report.md
+```
+
+## Commands
+
+### `vesk init`
+
+Writes a starter `vesk.yaml`.
+
+- `--path PATH`: where to write it (default `vesk.yaml`).
+- `--force`: overwrite an existing file.
+
+### `vesk scan`
+
+Runs the configured probes against the target and writes a results JSON file.
+
+- `-c, --config PATH`: config file (default `vesk.yaml`).
+- `--probe NAME`: run only this probe; repeatable.
+- `--out PATH`: results file (default `results.json`).
+
+Exit codes:
+
+| Code | Meaning |
+|------|---------|
+| 0 | Scan completed; no finding at or above `scan.fail_on`. |
+| 1 | At least one failing probe at or above `scan.fail_on`. |
+| 2 | Configuration or file error. |
+| 3 | Incomplete scan: a probe hit endpoint errors. |
+
+This makes `vesk scan` usable as a CI gate.
+
+### `vesk report`
+
+Renders a report from a results file.
+
+```
+vesk report results.json --format html -o report.html
+```
+
+- `--format md|html`: output format (default `md`).
+- `-o, --output PATH`: write to a file instead of stdout.
+
+## Configuration
+
+`vesk init` writes a fully commented file. In short:
+
+```yaml
+version: 1
+target:
+  base_url: https://api.example.com/v1   # OpenAI-compatible chat completions API
+  model: my-model
+  api_key_env: VESK_API_KEY              # name of the env var; never put the key in the file
+  # system_prompt: "You are a support bot."
+  timeout_s: 30
+  max_retries: 2
+scan:
+  probes: [prompt_injection, jailbreak, pii_leakage]
+  samples_per_probe: 20
+  max_concurrency: 4
+  seed: 0
+  fail_on: high                          # none | low | medium | high
+report:
+  frameworks: [eu_ai_act, dpdp, nist_ai_rmf]
+  system_name: "My AI system"
+```
+
+## Probes
+
+| Probe | Checks |
+|-------|--------|
+| `prompt_injection` | Whether text in the user turn or in quoted data overrides the instructions. |
+| `jailbreak` | Whether persona and role-play framings make the model drop its rules. Probes ask only for a harmless marker, never for harmful content. |
+| `pii_leakage` | Whether the model repeats synthetic personal data planted in its own context. Training-data regurgitation is out of scope for 0.1. |
+
+## Privacy
+
+The API key is read from an environment variable and is not stored in the config. Probe
+requests go only to the endpoint you configure. PII probes use synthetic values, masked in
+stored results.
+
+## Not legal advice
+
+Reports are generated automatically from test results. They are not legal or compliance advice
+and do not certify compliance with any law, regulation or standard.
+
+## Development
+
+```
+pip install -e ".[dev]"
+pytest
+ruff check .
+```
+
+## License
+
+Apache-2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).

vesk_cli-0.1.2.dist-info/RECORD

@@ -0,0 +1,27 @@
+vesk/__init__.py,sha256=YPFawP1xoc0x_YlmM9OEuRhMw9NeZQtmeZFNd5sIpJ8,95
+vesk/cli.py,sha256=1fPQLOv94vDUB0vmhQolG-88JfcdozkMqEU0C-ieEWM,3757
+vesk/client.py,sha256=bgc2uDGVcOr12Y4n7A-DCgmpWC3r1XMH3kXSHrEWHFY,2700
+vesk/config.py,sha256=ZxOGFErGjFNO-2_ZIorZOPV7ekk0YrZcKIbSiTAkb-8,2544
+vesk/runner.py,sha256=dTxsg_QKa8lZcJ1oU4jrfRMDoDYN8GxJ0aETxajN8rc,4077
+vesk/scaffold.py,sha256=djeLAEY72fKwh3mFvQzErqtckdo9S_rgwrIVN696CO4,1559
+vesk/data/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vesk/probes/__init__.py,sha256=Kn5etDCHFh56k77SszloaWxcyOzP1aAuIKt9zb5SRHU,426
+vesk/probes/base.py,sha256=0twz3F9Gt6eFLSwksqTz5dvRGPwBN2OeG-NOCUDNvDA,2063
+vesk/probes/jailbreak.py,sha256=dOVA5H-nGWM51S41oL7fRUXLMRma9CuNelcwJgjCQow,1735
+vesk/probes/pii_leakage.py,sha256=_7qDS7am-zcav8_4uA-CGZSqPJ3mO02xQtr7MaXq--U,3146
+vesk/probes/prompt_injection.py,sha256=IbDRd2FNzp2vCAcp-PNVnvWwYuPzXHJHowdGyk0-K8Y,1605
+vesk/probes/data/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vesk/probes/data/jailbreak.yaml,sha256=1lu0CcWewfoN7qUtB-1W3Amyn_oB0rMA2CpVA6XJF6s,1793
+vesk/probes/data/pii_leakage.yaml,sha256=jlAZv1Nx4PfOKi_77EkuywsSeMX83KekA1G2qdJUBSg,1205
+vesk/probes/data/prompt_injection.yaml,sha256=NBKvaGDUnGhmz3ofJUEymQu8DKHwTvjuvUZNz88nYeI,1685
+vesk/report/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+vesk/report/model.py,sha256=7LbZdPI2Dh55LBWxYotBbD0Zml4xXeUA5Xd1mxGjuW4,959
+vesk/report/render.py,sha256=0tuhLPmIbr8tglsD4FsBlhELcR5J7APNnog3BN-9Po0,1642
+vesk/report/templates/report.html.j2,sha256=H1bbbLg4QWDn0DBTaCkwWssQllvUvVt3ytfUFI7Gerw,1825
+vesk/report/templates/report.md.j2,sha256=9dT-9pahBwDIqkk7Xk13GvJWguQE6i5o00n4B8UPhOo,1201
+vesk_cli-0.1.2.dist-info/METADATA,sha256=izqBoazvJj2xBb3UzFUz-HCZEB-OUCjNTg2cDfVZucA,4233
+vesk_cli-0.1.2.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+vesk_cli-0.1.2.dist-info/entry_points.txt,sha256=0Tf_RxRyGYLVCZFhshQ8b1MV30xqSPFaE3f4EXjE-HE,38
+vesk_cli-0.1.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vesk_cli-0.1.2.dist-info/licenses/NOTICE,sha256=rfzyf7DuE9d7-OA76HB_sIxIFAPLr3D3DNE1rHLsoQU,138
+vesk_cli-0.1.2.dist-info/RECORD,,

vesk_cli-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

vesk_cli-0.1.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+vesk = vesk.cli:app