aijscan 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aij/__init__.py +34 -0
- aij/auth.py +339 -0
- aij/cli.py +817 -0
- aij/config.py +284 -0
- aij/corpus.py +156 -0
- aij/credentials.py +278 -0
- aij/data/__init__.py +1 -0
- aij/data/objectives/__init__.py +1 -0
- aij/data/objectives/content.yaml +256 -0
- aij/data/objectives/cyber.yaml +402 -0
- aij/data/objectives/harm.yaml +344 -0
- aij/data/objectives/privacy.yaml +253 -0
- aij/detectors/__init__.py +15 -0
- aij/detectors/base.py +26 -0
- aij/detectors/heuristic.py +307 -0
- aij/detectors/judge.py +201 -0
- aij/engine/__init__.py +27 -0
- aij/engine/budget.py +110 -0
- aij/engine/scanner.py +491 -0
- aij/engine/store.py +270 -0
- aij/errors.py +62 -0
- aij/models.py +320 -0
- aij/providers/__init__.py +32 -0
- aij/providers/anthropic.py +90 -0
- aij/providers/base.py +206 -0
- aij/providers/gemini.py +104 -0
- aij/providers/ollama.py +61 -0
- aij/providers/openai.py +126 -0
- aij/providers/pricing.py +46 -0
- aij/report/__init__.py +451 -0
- aij/techniques/__init__.py +157 -0
- aij/techniques/base.py +84 -0
- aij/techniques/multi.py +173 -0
- aij/techniques/single.py +406 -0
- aijscan-0.1.0.dist-info/METADATA +360 -0
- aijscan-0.1.0.dist-info/RECORD +39 -0
- aijscan-0.1.0.dist-info/WHEEL +4 -0
- aijscan-0.1.0.dist-info/entry_points.txt +3 -0
- aijscan-0.1.0.dist-info/licenses/LICENSE +21 -0
aij/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""AIJ — async jailbreak-resistance scanner for LLM endpoints."""
|
|
2
|
+
|
|
3
|
+
from aij.models import (
|
|
4
|
+
AttackPlan,
|
|
5
|
+
Detection,
|
|
6
|
+
Message,
|
|
7
|
+
ModelResponse,
|
|
8
|
+
Objective,
|
|
9
|
+
ResultRecord,
|
|
10
|
+
ScanSummary,
|
|
11
|
+
Severity,
|
|
12
|
+
TechniqueSpec,
|
|
13
|
+
TurnRecord,
|
|
14
|
+
Usage,
|
|
15
|
+
Verdict,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__version__ = "0.1.0"
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"AttackPlan",
|
|
22
|
+
"Detection",
|
|
23
|
+
"Message",
|
|
24
|
+
"ModelResponse",
|
|
25
|
+
"Objective",
|
|
26
|
+
"ResultRecord",
|
|
27
|
+
"ScanSummary",
|
|
28
|
+
"Severity",
|
|
29
|
+
"TechniqueSpec",
|
|
30
|
+
"TurnRecord",
|
|
31
|
+
"Usage",
|
|
32
|
+
"Verdict",
|
|
33
|
+
"__version__",
|
|
34
|
+
]
|
aij/auth.py
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"""The ``aij auth`` command group: save, inspect, and test endpoints."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
from rich.console import Console
|
|
11
|
+
from rich.markup import escape
|
|
12
|
+
from rich.table import Table
|
|
13
|
+
|
|
14
|
+
from aij.credentials import (
|
|
15
|
+
BASE_URL_ENV_VARS,
|
|
16
|
+
KEY_ENV_VARS,
|
|
17
|
+
CredentialStore,
|
|
18
|
+
mask_secret,
|
|
19
|
+
)
|
|
20
|
+
from aij.errors import AIJError, ConfigError
|
|
21
|
+
from aij.models import Message, Role
|
|
22
|
+
from aij.providers import build_provider
|
|
23
|
+
|
|
24
|
+
auth_app = typer.Typer(
|
|
25
|
+
name="auth",
|
|
26
|
+
help="Save API keys and endpoints so scans run without repeating flags.",
|
|
27
|
+
no_args_is_help=True,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
console = Console()
|
|
31
|
+
err_console = Console(stderr=True)
|
|
32
|
+
|
|
33
|
+
PROVIDER_KINDS = ["openai", "anthropic", "gemini", "ollama", "openai-compatible"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _choose_provider() -> str:
|
|
37
|
+
"""Interactive provider picker.
|
|
38
|
+
|
|
39
|
+
``typer.prompt(type=...)`` changed shape across Typer versions, so the
|
|
40
|
+
choices are validated here instead of relying on it.
|
|
41
|
+
"""
|
|
42
|
+
while True:
|
|
43
|
+
raw = typer.prompt(f"Provider ({'/'.join(PROVIDER_KINDS)})", default="openai").strip()
|
|
44
|
+
if raw in PROVIDER_KINDS:
|
|
45
|
+
return raw
|
|
46
|
+
console.print(f"[red]unknown provider[/red] {raw!r}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@auth_app.command("add")
|
|
50
|
+
def add_command(
|
|
51
|
+
name: Annotated[str, typer.Argument(help="Profile name, e.g. 'openai' or 'corp-gateway'.")],
|
|
52
|
+
provider: Annotated[
|
|
53
|
+
str | None,
|
|
54
|
+
typer.Option(
|
|
55
|
+
"--provider",
|
|
56
|
+
"-p",
|
|
57
|
+
help="openai | anthropic | gemini | ollama | openai-compatible",
|
|
58
|
+
),
|
|
59
|
+
] = None,
|
|
60
|
+
model: Annotated[str | None, typer.Option("--model", "-m", help="Default model.")] = None,
|
|
61
|
+
api_key: Annotated[
|
|
62
|
+
str | None,
|
|
63
|
+
typer.Option("--api-key", "-k", help="API key. Omit to be prompted (hidden input)."),
|
|
64
|
+
] = None,
|
|
65
|
+
base_url: Annotated[
|
|
66
|
+
str | None, typer.Option("--base-url", "-u", help="Endpoint base URL or gateway root.")
|
|
67
|
+
] = None,
|
|
68
|
+
api_key_env: Annotated[
|
|
69
|
+
str | None,
|
|
70
|
+
typer.Option("--from-env", help="Read the key from this env var instead of storing it."),
|
|
71
|
+
] = None,
|
|
72
|
+
make_default: Annotated[
|
|
73
|
+
bool, typer.Option("--default", help="Use this profile when none is named.")
|
|
74
|
+
] = False,
|
|
75
|
+
force: Annotated[bool, typer.Option("--force", help="Overwrite an existing profile.")] = False,
|
|
76
|
+
store_path: Annotated[
|
|
77
|
+
Path | None, typer.Option("--store", help="Override the credentials file location.")
|
|
78
|
+
] = None,
|
|
79
|
+
) -> None:
|
|
80
|
+
"""Save an endpoint profile.
|
|
81
|
+
|
|
82
|
+
With no flags this prompts for everything, which is the intended first-run
|
|
83
|
+
path. The key is read with hidden input so it never lands in shell history.
|
|
84
|
+
"""
|
|
85
|
+
store = CredentialStore(store_path)
|
|
86
|
+
|
|
87
|
+
if store.exists() and name in store.load() and not force:
|
|
88
|
+
err_console.print(
|
|
89
|
+
f"[yellow]profile {escape(name)!r} already exists[/yellow]; pass --force to overwrite"
|
|
90
|
+
)
|
|
91
|
+
raise typer.Exit(1)
|
|
92
|
+
|
|
93
|
+
if not provider:
|
|
94
|
+
provider = _choose_provider()
|
|
95
|
+
if provider not in PROVIDER_KINDS:
|
|
96
|
+
err_console.print(
|
|
97
|
+
f"[red]unknown provider[/red] {escape(provider)!r}; "
|
|
98
|
+
f"expected {', '.join(PROVIDER_KINDS)}"
|
|
99
|
+
)
|
|
100
|
+
raise typer.Exit(2)
|
|
101
|
+
|
|
102
|
+
if not model:
|
|
103
|
+
model = typer.prompt("Default model", default={"ollama": "llama3.1"}.get(provider, ""))
|
|
104
|
+
if not model:
|
|
105
|
+
err_console.print("[red]a model is required[/red]")
|
|
106
|
+
raise typer.Exit(2)
|
|
107
|
+
|
|
108
|
+
resolved_key: str | None = None
|
|
109
|
+
if api_key_env:
|
|
110
|
+
import os
|
|
111
|
+
|
|
112
|
+
resolved_key = os.environ.get(api_key_env)
|
|
113
|
+
if not resolved_key:
|
|
114
|
+
err_console.print(f"[red]{escape(api_key_env)} is not set[/red] in this shell")
|
|
115
|
+
raise typer.Exit(2)
|
|
116
|
+
elif api_key:
|
|
117
|
+
resolved_key = api_key
|
|
118
|
+
elif provider != "ollama":
|
|
119
|
+
env_name = KEY_ENV_VARS.get(provider, "")
|
|
120
|
+
from_env = __import__("os").environ.get(env_name) if env_name else None
|
|
121
|
+
if from_env:
|
|
122
|
+
use_it = typer.confirm(f"Use {env_name} from the current environment?", default=True)
|
|
123
|
+
if use_it:
|
|
124
|
+
resolved_key = from_env
|
|
125
|
+
if not resolved_key:
|
|
126
|
+
resolved_key = typer.prompt(
|
|
127
|
+
f"{provider} API key", hide_input=True, default="", show_default=False
|
|
128
|
+
)
|
|
129
|
+
if not resolved_key:
|
|
130
|
+
err_console.print("[red]an API key is required[/red] (or use --from-env)")
|
|
131
|
+
raise typer.Exit(2)
|
|
132
|
+
|
|
133
|
+
if not base_url and provider in {"openai-compatible", "ollama"}:
|
|
134
|
+
# These two have no useful public default, so ask.
|
|
135
|
+
base_url = (
|
|
136
|
+
typer.prompt(
|
|
137
|
+
"Base URL",
|
|
138
|
+
default="http://localhost:11434" if provider == "ollama" else "",
|
|
139
|
+
show_default=provider == "ollama",
|
|
140
|
+
)
|
|
141
|
+
or None
|
|
142
|
+
)
|
|
143
|
+
elif not base_url and provider not in PROVIDER_KINDS:
|
|
144
|
+
base_url = None
|
|
145
|
+
|
|
146
|
+
try:
|
|
147
|
+
path = store.add(
|
|
148
|
+
name,
|
|
149
|
+
kind=provider,
|
|
150
|
+
model=model,
|
|
151
|
+
api_key=resolved_key,
|
|
152
|
+
base_url=base_url,
|
|
153
|
+
make_default=make_default,
|
|
154
|
+
)
|
|
155
|
+
except ConfigError as exc:
|
|
156
|
+
err_console.print(f"[red]error[/red] {escape(str(exc))}")
|
|
157
|
+
raise typer.Exit(2) from exc
|
|
158
|
+
|
|
159
|
+
console.print(f"[green]saved[/green] profile [cyan]{escape(name)}[/cyan] → {escape(str(path))}")
|
|
160
|
+
console.print(
|
|
161
|
+
f" {escape(provider)} / {escape(model)}"
|
|
162
|
+
+ (f" · {escape(base_url)}" if base_url else "")
|
|
163
|
+
+ f" · key {escape(mask_secret(resolved_key))}"
|
|
164
|
+
)
|
|
165
|
+
console.print(f" next: [bold]aij scan --profile {escape(name)} --dry-run[/bold]")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@auth_app.command("list")
|
|
169
|
+
def list_command(
|
|
170
|
+
store_path: Annotated[
|
|
171
|
+
Path | None, typer.Option("--store", help="Override the credentials file location.")
|
|
172
|
+
] = None,
|
|
173
|
+
show_keys: Annotated[
|
|
174
|
+
bool, typer.Option("--show-keys", help="Print masked keys and base URLs.")
|
|
175
|
+
] = False,
|
|
176
|
+
) -> None:
|
|
177
|
+
"""List saved profiles."""
|
|
178
|
+
store = CredentialStore(store_path)
|
|
179
|
+
profiles = store.load()
|
|
180
|
+
if not profiles:
|
|
181
|
+
console.print(f"[dim]no profiles saved[/dim] ({escape(str(store.path))})")
|
|
182
|
+
console.print("add one with [bold]aij auth add openai[/bold]")
|
|
183
|
+
return
|
|
184
|
+
|
|
185
|
+
table = Table(title=f"profiles ({len(profiles)})")
|
|
186
|
+
table.add_column("name", style="cyan")
|
|
187
|
+
table.add_column("provider")
|
|
188
|
+
table.add_column("model")
|
|
189
|
+
table.add_column("default", justify="center")
|
|
190
|
+
if show_keys:
|
|
191
|
+
table.add_column("base URL")
|
|
192
|
+
table.add_column("key")
|
|
193
|
+
for name in sorted(profiles):
|
|
194
|
+
values = profiles[name]
|
|
195
|
+
row = [
|
|
196
|
+
escape(name),
|
|
197
|
+
escape(str(values.get("kind", "?"))),
|
|
198
|
+
escape(str(values.get("model", "?"))),
|
|
199
|
+
"●" if values.get("default") else "",
|
|
200
|
+
]
|
|
201
|
+
if show_keys:
|
|
202
|
+
row.append(escape(str(values.get("base_url") or "(default)")))
|
|
203
|
+
row.append(escape(mask_secret(values.get("api_key"))))
|
|
204
|
+
table.add_row(*row)
|
|
205
|
+
console.print(table)
|
|
206
|
+
console.print(f"[dim]{escape(str(store.path))}[/dim]")
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@auth_app.command("remove")
|
|
210
|
+
def remove_command(
|
|
211
|
+
name: Annotated[str, typer.Argument(help="Profile to delete.")],
|
|
212
|
+
yes: Annotated[bool, typer.Option("--yes", "-y", help="Skip the confirmation.")] = False,
|
|
213
|
+
store_path: Annotated[
|
|
214
|
+
Path | None, typer.Option("--store", help="Override the credentials file location.")
|
|
215
|
+
] = None,
|
|
216
|
+
) -> None:
|
|
217
|
+
"""Delete a saved profile."""
|
|
218
|
+
store = CredentialStore(store_path)
|
|
219
|
+
if name not in store.load():
|
|
220
|
+
err_console.print(f"[red]no profile named[/red] {escape(name)!r}")
|
|
221
|
+
raise typer.Exit(1)
|
|
222
|
+
if not yes and not typer.confirm(f"Delete profile {name!r}?"):
|
|
223
|
+
raise typer.Exit(0)
|
|
224
|
+
store.remove(name)
|
|
225
|
+
console.print(f"[green]removed[/green] {escape(name)}")
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@auth_app.command("default")
|
|
229
|
+
def default_command(
|
|
230
|
+
name: Annotated[str, typer.Argument(help="Profile to use when none is named.")],
|
|
231
|
+
store_path: Annotated[
|
|
232
|
+
Path | None, typer.Option("--store", help="Override the credentials file location.")
|
|
233
|
+
] = None,
|
|
234
|
+
) -> None:
|
|
235
|
+
"""Mark a profile as the default."""
|
|
236
|
+
store = CredentialStore(store_path)
|
|
237
|
+
try:
|
|
238
|
+
store.set_default(name)
|
|
239
|
+
except ConfigError as exc:
|
|
240
|
+
err_console.print(f"[red]error[/red] {escape(str(exc))}")
|
|
241
|
+
raise typer.Exit(1) from exc
|
|
242
|
+
console.print(f"[green]default[/green] → {escape(name)}")
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@auth_app.command("test")
|
|
246
|
+
def test_command(
|
|
247
|
+
name: Annotated[
|
|
248
|
+
str | None, typer.Argument(help="Profile to test. Defaults to the default profile.")
|
|
249
|
+
] = None,
|
|
250
|
+
store_path: Annotated[
|
|
251
|
+
Path | None, typer.Option("--store", help="Override the credentials file location.")
|
|
252
|
+
] = None,
|
|
253
|
+
timeout: Annotated[float, typer.Option("--timeout", help="Seconds to wait.")] = 30.0,
|
|
254
|
+
) -> None:
|
|
255
|
+
"""Send one real request to verify the profile works.
|
|
256
|
+
|
|
257
|
+
This is the check that catches the failure modes a saved key alone cannot:
|
|
258
|
+
wrong key, unpaid account, missing model entitlement, wrong base URL path.
|
|
259
|
+
"""
|
|
260
|
+
store = CredentialStore(store_path)
|
|
261
|
+
try:
|
|
262
|
+
resolved = store.resolve(name)
|
|
263
|
+
except ConfigError as exc:
|
|
264
|
+
err_console.print(f"[red]error[/red] {escape(str(exc))}")
|
|
265
|
+
raise typer.Exit(2) from exc
|
|
266
|
+
if resolved is None:
|
|
267
|
+
err_console.print(
|
|
268
|
+
f"[red]no profiles saved[/red] in {escape(str(store.path))}; run 'aij auth add' first"
|
|
269
|
+
)
|
|
270
|
+
raise typer.Exit(2)
|
|
271
|
+
|
|
272
|
+
profile_name, values = resolved
|
|
273
|
+
try:
|
|
274
|
+
config = store.to_provider_config(values)
|
|
275
|
+
except ConfigError as exc:
|
|
276
|
+
err_console.print(f"[red]profile {escape(profile_name)}[/red] {escape(str(exc))}")
|
|
277
|
+
raise typer.Exit(2) from exc
|
|
278
|
+
|
|
279
|
+
async def check() -> None:
|
|
280
|
+
import httpx
|
|
281
|
+
|
|
282
|
+
provider = build_provider(config)
|
|
283
|
+
console.print(
|
|
284
|
+
f"testing [cyan]{escape(profile_name)}[/cyan] → "
|
|
285
|
+
f"{escape(config.kind)}/{escape(config.model)}"
|
|
286
|
+
)
|
|
287
|
+
try:
|
|
288
|
+
async with httpx.AsyncClient(timeout=timeout) as client:
|
|
289
|
+
response = await provider.complete(
|
|
290
|
+
[Message(role=Role.USER, content="Reply with the single word: pong")],
|
|
291
|
+
client=client,
|
|
292
|
+
max_retries=0,
|
|
293
|
+
)
|
|
294
|
+
except AIJError as exc:
|
|
295
|
+
status = getattr(exc, "status", None)
|
|
296
|
+
err_console.print(f"[red]failed[/red] {escape(str(exc))}")
|
|
297
|
+
if status in (401, 403):
|
|
298
|
+
err_console.print(
|
|
299
|
+
" The key was rejected or the account is not entitled to this model. "
|
|
300
|
+
"Check the key, the model name, and your billing."
|
|
301
|
+
)
|
|
302
|
+
elif status == 404:
|
|
303
|
+
err_console.print(
|
|
304
|
+
" Model or URL path not found. Check --model and the base URL "
|
|
305
|
+
f"({escape(str(config.base_url or 'default'))})."
|
|
306
|
+
)
|
|
307
|
+
elif status is None:
|
|
308
|
+
err_console.print(
|
|
309
|
+
f" Could not reach {escape(str(config.base_url or 'the default endpoint'))}. "
|
|
310
|
+
"Check the base URL and your network."
|
|
311
|
+
)
|
|
312
|
+
raise typer.Exit(1) from exc
|
|
313
|
+
|
|
314
|
+
if response.blocked:
|
|
315
|
+
# A safety block still proves the key, model, and URL all work.
|
|
316
|
+
console.print(
|
|
317
|
+
f"[green]ok[/green] reached the endpoint (it returned a safety block: "
|
|
318
|
+
f"{escape(response.block_reason or 'filtered')})"
|
|
319
|
+
)
|
|
320
|
+
else:
|
|
321
|
+
console.print(f"[green]ok[/green] reply: {escape(repr(response.text.strip()[:80]))}")
|
|
322
|
+
console.print(f" {response.usage.total_tokens} tokens · {response.latency_ms:.0f}ms")
|
|
323
|
+
|
|
324
|
+
asyncio.run(check())
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@auth_app.command("path")
|
|
328
|
+
def path_command(
|
|
329
|
+
store_path: Annotated[
|
|
330
|
+
Path | None, typer.Option("--store", help="Override the credentials file location.")
|
|
331
|
+
] = None,
|
|
332
|
+
) -> None:
|
|
333
|
+
"""Print where credentials are stored."""
|
|
334
|
+
store = CredentialStore(store_path)
|
|
335
|
+
state = "exists" if store.exists() else "not created yet"
|
|
336
|
+
console.print(f"{escape(str(store.path))} [dim]({state})[/dim]")
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
__all__ = ["BASE_URL_ENV_VARS", "KEY_ENV_VARS", "auth_app"]
|