giskard-scan 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. giskard_scan-1.0.0/.gitignore +191 -0
  2. giskard_scan-1.0.0/PKG-INFO +183 -0
  3. giskard_scan-1.0.0/README.md +165 -0
  4. giskard_scan-1.0.0/pyproject.toml +62 -0
  5. giskard_scan-1.0.0/src/giskard/scan/__init__.py +69 -0
  6. giskard_scan-1.0.0/src/giskard/scan/catalog.py +114 -0
  7. giskard_scan-1.0.0/src/giskard/scan/generators/__init__.py +35 -0
  8. giskard_scan-1.0.0/src/giskard/scan/generators/adversarial.py +226 -0
  9. giskard_scan-1.0.0/src/giskard/scan/generators/base.py +281 -0
  10. giskard_scan-1.0.0/src/giskard/scan/generators/crescendo.py +138 -0
  11. giskard_scan-1.0.0/src/giskard/scan/generators/data/prompt_injection.jsonl +18 -0
  12. giskard_scan-1.0.0/src/giskard/scan/generators/gcg.py +110 -0
  13. giskard_scan-1.0.0/src/giskard/scan/generators/goat.py +228 -0
  14. giskard_scan-1.0.0/src/giskard/scan/generators/huggingface.py +164 -0
  15. giskard_scan-1.0.0/src/giskard/scan/generators/knowledge_base/__init__.py +31 -0
  16. giskard_scan-1.0.0/src/giskard/scan/generators/knowledge_base/base.py +210 -0
  17. giskard_scan-1.0.0/src/giskard/scan/generators/knowledge_base/hallucination.py +24 -0
  18. giskard_scan-1.0.0/src/giskard/scan/generators/knowledge_base/multi_topic.py +115 -0
  19. giskard_scan-1.0.0/src/giskard/scan/generators/knowledge_base/out_of_scope.py +247 -0
  20. giskard_scan-1.0.0/src/giskard/scan/generators/knowledge_base/split_questions.py +45 -0
  21. giskard_scan-1.0.0/src/giskard/scan/generators/knowledge_base/sycophancy.py +20 -0
  22. giskard_scan-1.0.0/src/giskard/scan/generators/prompt_injection.py +20 -0
  23. giskard_scan-1.0.0/src/giskard/scan/integrations/__init__.py +6 -0
  24. giskard_scan-1.0.0/src/giskard/scan/integrations/_entry_point.py +112 -0
  25. giskard_scan-1.0.0/src/giskard/scan/integrations/_listing.py +57 -0
  26. giskard_scan-1.0.0/src/giskard/scan/integrations/_shared.py +86 -0
  27. giskard_scan-1.0.0/src/giskard/scan/integrations/deepteam/__init__.py +11 -0
  28. giskard_scan-1.0.0/src/giskard/scan/integrations/deepteam/_adapter.py +291 -0
  29. giskard_scan-1.0.0/src/giskard/scan/integrations/deepteam/_bridge.py +93 -0
  30. giskard_scan-1.0.0/src/giskard/scan/integrations/deepteam/_judge_generator.py +63 -0
  31. giskard_scan-1.0.0/src/giskard/scan/integrations/deepteam/_selection.py +128 -0
  32. giskard_scan-1.0.0/src/giskard/scan/integrations/garak/__init__.py +10 -0
  33. giskard_scan-1.0.0/src/giskard/scan/integrations/garak/_adapter.py +593 -0
  34. giskard_scan-1.0.0/src/giskard/scan/integrations/garak/_bridge.py +9 -0
  35. giskard_scan-1.0.0/src/giskard/scan/integrations/garak/_generator.py +86 -0
  36. giskard_scan-1.0.0/src/giskard/scan/integrations/garak/_judge_generator.py +98 -0
  37. giskard_scan-1.0.0/src/giskard/scan/integrations/lidar/__init__.py +7 -0
  38. giskard_scan-1.0.0/src/giskard/scan/integrations/lidar/_adapter.py +274 -0
  39. giskard_scan-1.0.0/src/giskard/scan/integrations/lidar/_target.py +103 -0
  40. giskard_scan-1.0.0/src/giskard/scan/prompts/generate_suite/generation_rules.j2 +55 -0
  41. giskard_scan-1.0.0/src/giskard/scan/prompts/generate_suite/out_of_scope_candidate.j2 +49 -0
  42. giskard_scan-1.0.0/src/giskard/scan/prompts/generate_suite/out_of_scope_validation.j2 +40 -0
  43. giskard_scan-1.0.0/src/giskard/scan/prompts/generate_suite/quality_recommendation.j2 +36 -0
  44. giskard_scan-1.0.0/src/giskard/scan/prompts/judges/crescendo_eval.j2 +38 -0
  45. giskard_scan-1.0.0/src/giskard/scan/prompts/judges/goat_eval.j2 +45 -0
  46. giskard_scan-1.0.0/src/giskard/scan/prompts/judges/harmbench_safety.j2 +48 -0
  47. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/adversarial.j2 +66 -0
  48. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/crescendo_attack.j2 +95 -0
  49. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/goat_attack.j2 +92 -0
  50. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/knowledge_base_multi_topic.j2 +54 -0
  51. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/knowledge_base_out_of_scope.j2 +72 -0
  52. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/knowledge_base_question.j2 +59 -0
  53. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/knowledge_base_split_question.j2 +100 -0
  54. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/knowledge_base_sycophancy.j2 +94 -0
  55. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/llm01_indirect_injection.j2 +36 -0
  56. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/llm01_injection.j2 +46 -0
  57. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/llm01_jailbreak.j2 +54 -0
  58. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/llm01_multilingual_injection.j2 +45 -0
  59. giskard_scan-1.0.0/src/giskard/scan/prompts/scenarios/llm01_refusal_suppression.j2 +47 -0
  60. giskard_scan-1.0.0/src/giskard/scan/py.typed +0 -0
  61. giskard_scan-1.0.0/src/giskard/scan/quality.py +163 -0
  62. giskard_scan-1.0.0/src/giskard/scan/registry.py +49 -0
  63. giskard_scan-1.0.0/src/giskard/scan/types.py +128 -0
  64. giskard_scan-1.0.0/src/giskard/scan/utils/__init__.py +9 -0
  65. giskard_scan-1.0.0/src/giskard/scan/utils/knowledge_base.py +216 -0
  66. giskard_scan-1.0.0/src/giskard/scan/utils/recommendation.py +54 -0
  67. giskard_scan-1.0.0/src/giskard/scan/vulnerability.py +126 -0
@@ -0,0 +1,191 @@
1
+ enron_with_categories
2
+ .history
3
+ .mypy_cache
4
+ docker-stack.yml
5
+ local_report.md
6
+ .run
7
+ .DS_Store
8
+ .claude/worktrees/
9
+
10
+ # Byte-compiled / optimized / DLL files
11
+ __pycache__/
12
+ *.py[cod]
13
+ *$py.class
14
+
15
+ # C extensions
16
+ *.so
17
+
18
+ # Distribution / packaging
19
+ .Python
20
+ build/
21
+ develop-eggs/
22
+ dist/
23
+ downloads/
24
+ eggs/
25
+ .eggs/
26
+ lib/
27
+ lib64/
28
+ parts/
29
+ sdist/
30
+ var/
31
+ wheels/
32
+ share/python-wheels/
33
+ *.egg-info/
34
+ .installed.cfg
35
+ *.egg
36
+ MANIFEST
37
+
38
+ # PyInstaller
39
+ # Usually these files are written by a python script from a template
40
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
41
+ *.manifest
42
+ *.spec
43
+
44
+ # Installer logs
45
+ pip-log.txt
46
+ pip-delete-this-directory.txt
47
+
48
+ # Unit test / coverage reports
49
+ htmlcov/
50
+ .tox/
51
+ .nox/
52
+ .coverage
53
+ .coverage.*
54
+ .cache
55
+ nosetests.xml
56
+ coverage.xml
57
+ *.cover
58
+ *.py,cover
59
+ .hypothesis/
60
+ .pytest_cache/
61
+ cover/
62
+
63
+ # Translations
64
+ *.mo
65
+ *.pot
66
+
67
+ # Django stuff:
68
+ *.log
69
+ local_settings.py
70
+ db.sqlite3
71
+ db.sqlite3-journal
72
+
73
+ # Flask stuff:
74
+ instance/
75
+ .webassets-cache
76
+
77
+ # Scrapy stuff:
78
+ .scrapy
79
+
80
+ # PyBuilder
81
+ .pybuilder/
82
+ target/
83
+
84
+ # Jupyter Notebook
85
+ .ipynb_checkpoints
86
+
87
+ # IPython
88
+ profile_default/
89
+ ipython_config.py
90
+
91
+ # pyenv
92
+ # For a library or package, you might want to ignore these files since the code is
93
+ # intended to run in multiple environments; otherwise, check them in:
94
+ # .python-version
95
+
96
+ # pipenv
97
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
98
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
99
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
100
+ # install all needed dependencies.
101
+ #Pipfile.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ #poetry.lock
109
+
110
+ # pdm
111
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
112
+ #pdm.lock
113
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
114
+ # in version control.
115
+ # https://pdm.fming.dev/#use-with-ide
116
+ .pdm.toml
117
+ .pdm-python
118
+
119
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
120
+ __pypackages__/
121
+
122
+ # Celery stuff
123
+ celerybeat-schedule
124
+ celerybeat.pid
125
+
126
+ # SageMath parsed files
127
+ *.sage.py
128
+
129
+ # Environments
130
+ .env
131
+ .venv
132
+ env/
133
+ venv/
134
+ ENV/
135
+ env.bak/
136
+ venv.bak/
137
+
138
+ # Spyder project settings
139
+ .spyderproject
140
+ .spyproject
141
+
142
+ # Rope project settings
143
+ .ropeproject
144
+
145
+ # mkdocs documentation
146
+ /site
147
+
148
+ # mypy
149
+ .mypy_cache/
150
+ .dmypy.json
151
+ dmypy.json
152
+
153
+ # Pyre type checker
154
+ .pyre/
155
+
156
+ # pytype static type analyzer
157
+ .pytype/
158
+
159
+ # Cython debug symbols
160
+ cython_debug/
161
+
162
+ # PyCharm
163
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
164
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
165
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
166
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
167
+ .idea/
168
+ catboost_info
169
+ .scannerwork
170
+ *.iml.pdm-python
171
+ *pb2*.py
172
+ generated
173
+ # npm
174
+ node_modules/
175
+ mlruns
176
+
177
+ # VSCode
178
+ .vscode/
179
+ .github/CREDENTIALS.md
180
+
181
+ # Local development
182
+ local/
183
+ *.local
184
+ .claude/settings.local.json
185
+
186
+ # Stale v2 package tree at repo root (namespace lives under libs/*/src/giskard)
187
+ /giskard/
188
+
189
+ # Agent tooling state (generated, machine-local)
190
+ .deepeval/
191
+ .superpowers/
@@ -0,0 +1,183 @@
1
+ Metadata-Version: 2.5
2
+ Name: giskard-scan
3
+ Version: 1.0.0
4
+ Summary: Agent vulnerability scanner — red teaming, prompt injection, adversarial scenario generation
5
+ Author-email: Giskard R&D <rnd@giskard.ai>
6
+ License: Apache Software License 2.0
7
+ Requires-Python: >=3.12
8
+ Requires-Dist: giskard-agents<2,>=1.0.2
9
+ Requires-Dist: giskard-checks<2,>=1.0.2
10
+ Requires-Dist: huggingface-hub<2,>=1.11.0
11
+ Requires-Dist: numpy<3,>=2.4.1
12
+ Provides-Extra: deepteam
13
+ Requires-Dist: deepteam<2,>=1.0.7; extra == 'deepteam'
14
+ Requires-Dist: standard-nntplib>=3.13; (python_version >= '3.13') and extra == 'deepteam'
15
+ Provides-Extra: garak
16
+ Requires-Dist: garak<1,>=0.15; extra == 'garak'
17
+ Description-Content-Type: text/markdown
18
+
19
+ # giskard-scan
20
+
21
+ Agent vulnerability scanner — red teaming, prompt injection, adversarial scenario generation.
22
+
23
+ ## Scan entrypoints
24
+
25
+ `quality_scan` and `vulnerability_scan` share the same explicit execution options.
26
+ Pass shared settings by keyword on either entrypoint; each scan adds scan-specific
27
+ options (`knowledge_base` for quality, `commercial_use` for vulnerability).
28
+
29
+ ```python
30
+ from giskard.scan import quality_scan, vulnerability_scan
31
+
32
+
33
+ async def echo(inputs: str) -> str:
34
+ return inputs
35
+
36
+
37
+ vulnerability_result = await vulnerability_scan(
38
+ target=echo,
39
+ description="Customer support chatbot for an e-commerce store.",
40
+ languages=["en"],
41
+ max_scenarios=20,
42
+ seed=42,
43
+ group_by="threat-type",
44
+ parallel=True,
45
+ max_concurrency=8,
46
+ return_exception=False,
47
+ target_mode="multiturn",
48
+ commercial_use=False,
49
+ )
50
+
51
+ quality_result = await quality_scan(
52
+ target=echo,
53
+ description="Customer support chatbot for an e-commerce store.",
54
+ languages=["en"],
55
+ knowledge_base=["Paris is the capital of France."],
56
+ max_scenarios=20,
57
+ seed=42,
58
+ group_by="component",
59
+ parallel=True,
60
+ max_concurrency=8,
61
+ return_exception=False,
62
+ target_mode="multiturn",
63
+ )
64
+ ```
65
+
66
+ ## Shared defaults
67
+
68
+ `DEFAULT_TARGET_MODE` (`"multiturn"`) is shared by `generate_suite`,
69
+ `vulnerability_scan`, `quality_scan`, and `third_party_scan`. Pass
70
+ `target_mode="singleturn"` to skip multi-turn generators/attacks and cap turn
71
+ budgets to 1.
72
+
73
+ Discover selectable items for any tool with `list_scan_items`:
74
+
75
+ ```python
76
+ from giskard.scan import list_scan_items
77
+
78
+ list_scan_items("giskard") # scenario generator class names
79
+ list_scan_items("garak") # active garak probe names
80
+ list_scan_items("deepteam") # vulnerability + attack names
81
+ ```
82
+
83
+ ## Third-party scanners (experimental)
84
+
85
+ `third_party_scan` runs an external security scanner against a Giskard target and
86
+ returns a `SuiteResult`. Two scanners are supported, each shipping as an optional
87
+ extra: [garak](https://github.com/NVIDIA/garak) and
88
+ [deepteam](https://github.com/confident-ai/deepteam).
89
+
90
+ ```bash
91
+ pip install giskard-scan[garak]
92
+ pip install giskard-scan[deepteam]
93
+ ```
94
+
95
+ ### garak
96
+
97
+ ```python
98
+ import asyncio
99
+
100
+ from giskard.scan import list_scan_items, third_party_scan
101
+
102
+
103
+ def target(inputs: str) -> str:
104
+ # Your agent / model call. Structured (BaseModel) inputs also work.
105
+ return call_my_agent(inputs)
106
+
107
+
108
+ result = asyncio.run(
109
+ third_party_scan(
110
+ target,
111
+ tool="garak",
112
+ description="A helpful assistant", # required by the API; garak ignores it
113
+ # probes=None -> curated default set; probes="all" -> every active probe
114
+ probes=["probes.goodside.ThreatenJSON"],
115
+ # target_mode defaults to "multiturn"; pass "singleturn" to drop iterative probes
116
+ )
117
+ )
118
+
119
+ print(result)
120
+ print(list_scan_items("garak")[:5])
121
+ ```
122
+
123
+ Probes run in parallel; the target is invoked concurrently, so it must be safe to
124
+ call from multiple threads (per-conversation state is tracked in the `Trace`, not on
125
+ the target).
126
+
127
+ Omitting `probes` runs a small curated default set aligned with DeepTeam themes
128
+ (Bias/Toxicity, PII, Misinformation, PromptLeakage, jailbreak/injection, data
129
+ exfil) — not the full garak catalog. Pass `probes="all"` for every active probe,
130
+ or an explicit name list.
131
+
132
+ Unknown, inactive, or unloadable probe names are logged and emitted as
133
+ ``CheckResult.skip`` scenarios rather than raising.
134
+
135
+ ### deepteam
136
+
137
+ Deepteam generates adversarial attacks with an LLM and judges the responses with an
138
+ LLM, so it needs a working Giskard default generator (see
139
+ `giskard.checks.get_default_generator()`) — there is no keyless mode.
140
+
141
+ ```python
142
+ result = asyncio.run(
143
+ third_party_scan(
144
+ target,
145
+ tool="deepteam",
146
+ description="A helpful assistant", # becomes deepteam's target_purpose
147
+ vulnerabilities=["Bias", "Toxicity"], # omit for a curated default set
148
+ attacks=["PromptInjection", "LinearJailbreaking"], # omit for defaults
149
+ attacks_per_vulnerability_type=1, # default; each vuln subtype × this many
150
+ # target_mode defaults to "multiturn" (shared with native Giskard scans)
151
+ )
152
+ )
153
+ ```
154
+
155
+ `vulnerabilities` accepts `Bias`, `Toxicity`, `PIILeakage`, `PromptLeakage`, and
156
+ `Misinformation`. Instantiating those classes without subtypes runs **all** of
157
+ their types (for example Bias → race, gender, politics, religion), so cost scales
158
+ with types × `attacks_per_vulnerability_type` × attacks.
159
+
160
+ `attacks` accepts the single-turn `PromptInjection`, `Roleplay`, `Leetspeak`, and
161
+ `ROT13`, plus the multi-turn `LinearJailbreaking`, `CrescendoJailbreaking`,
162
+ `TreeJailbreaking`, `SequentialJailbreak`, and `BadLikertJudge`. Unrecognized
163
+ names are logged and emitted as skip scenarios (valid names in the same call still
164
+ run).
165
+
166
+ `target_mode` defaults to the shared `DEFAULT_TARGET_MODE` (`"multiturn"`).
167
+ Pass `"singleturn"` to drop multi-turn attacks (they surface as skip scenarios).
168
+ Unknown names also skip — the suite is never an empty "everything passed"
169
+ result when you asked for attacks that could not run.
170
+
171
+ ### API keys and LLM-judge detectors
172
+
173
+ Some garak detectors need an LLM or a third-party API to score a probe:
174
+
175
+ - **LLM-judge detectors** (garak's `judge.*`, e.g. refusal detection) normally require
176
+ their own OpenAI key. Instead, they are automatically backed by Giskard's default
177
+ generator (`giskard.checks.get_default_generator()`), so they run with the same
178
+ credentials as the rest of Giskard — no separate OpenAI key needed.
179
+ - **Detectors that need a third-party API key** you have not set (for example
180
+ `perspective.*`, which needs `PERSPECTIVE_API_KEY`) are **skipped** rather than
181
+ silently dropping the whole probe. Each skipped detector surfaces as a skip result
182
+ (`CheckResult.skip`) in the returned `SuiteResult`, with the missing key named in the
183
+ message, so the rest of the probe's detectors still run.
@@ -0,0 +1,165 @@
1
+ # giskard-scan
2
+
3
+ Agent vulnerability scanner — red teaming, prompt injection, adversarial scenario generation.
4
+
5
+ ## Scan entrypoints
6
+
7
+ `quality_scan` and `vulnerability_scan` share the same explicit execution options.
8
+ Pass shared settings by keyword on either entrypoint; each scan adds scan-specific
9
+ options (`knowledge_base` for quality, `commercial_use` for vulnerability).
10
+
11
+ ```python
12
+ from giskard.scan import quality_scan, vulnerability_scan
13
+
14
+
15
+ async def echo(inputs: str) -> str:
16
+ return inputs
17
+
18
+
19
+ vulnerability_result = await vulnerability_scan(
20
+ target=echo,
21
+ description="Customer support chatbot for an e-commerce store.",
22
+ languages=["en"],
23
+ max_scenarios=20,
24
+ seed=42,
25
+ group_by="threat-type",
26
+ parallel=True,
27
+ max_concurrency=8,
28
+ return_exception=False,
29
+ target_mode="multiturn",
30
+ commercial_use=False,
31
+ )
32
+
33
+ quality_result = await quality_scan(
34
+ target=echo,
35
+ description="Customer support chatbot for an e-commerce store.",
36
+ languages=["en"],
37
+ knowledge_base=["Paris is the capital of France."],
38
+ max_scenarios=20,
39
+ seed=42,
40
+ group_by="component",
41
+ parallel=True,
42
+ max_concurrency=8,
43
+ return_exception=False,
44
+ target_mode="multiturn",
45
+ )
46
+ ```
47
+
48
+ ## Shared defaults
49
+
50
+ `DEFAULT_TARGET_MODE` (`"multiturn"`) is shared by `generate_suite`,
51
+ `vulnerability_scan`, `quality_scan`, and `third_party_scan`. Pass
52
+ `target_mode="singleturn"` to skip multi-turn generators/attacks and cap turn
53
+ budgets to 1.
54
+
55
+ Discover selectable items for any tool with `list_scan_items`:
56
+
57
+ ```python
58
+ from giskard.scan import list_scan_items
59
+
60
+ list_scan_items("giskard") # scenario generator class names
61
+ list_scan_items("garak") # active garak probe names
62
+ list_scan_items("deepteam") # vulnerability + attack names
63
+ ```
64
+
65
+ ## Third-party scanners (experimental)
66
+
67
+ `third_party_scan` runs an external security scanner against a Giskard target and
68
+ returns a `SuiteResult`. Two scanners are supported, each shipping as an optional
69
+ extra: [garak](https://github.com/NVIDIA/garak) and
70
+ [deepteam](https://github.com/confident-ai/deepteam).
71
+
72
+ ```bash
73
+ pip install giskard-scan[garak]
74
+ pip install giskard-scan[deepteam]
75
+ ```
76
+
77
+ ### garak
78
+
79
+ ```python
80
+ import asyncio
81
+
82
+ from giskard.scan import list_scan_items, third_party_scan
83
+
84
+
85
+ def target(inputs: str) -> str:
86
+ # Your agent / model call. Structured (BaseModel) inputs also work.
87
+ return call_my_agent(inputs)
88
+
89
+
90
+ result = asyncio.run(
91
+ third_party_scan(
92
+ target,
93
+ tool="garak",
94
+ description="A helpful assistant", # required by the API; garak ignores it
95
+ # probes=None -> curated default set; probes="all" -> every active probe
96
+ probes=["probes.goodside.ThreatenJSON"],
97
+ # target_mode defaults to "multiturn"; pass "singleturn" to drop iterative probes
98
+ )
99
+ )
100
+
101
+ print(result)
102
+ print(list_scan_items("garak")[:5])
103
+ ```
104
+
105
+ Probes run in parallel; the target is invoked concurrently, so it must be safe to
106
+ call from multiple threads (per-conversation state is tracked in the `Trace`, not on
107
+ the target).
108
+
109
+ Omitting `probes` runs a small curated default set aligned with DeepTeam themes
110
+ (Bias/Toxicity, PII, Misinformation, PromptLeakage, jailbreak/injection, data
111
+ exfil) — not the full garak catalog. Pass `probes="all"` for every active probe,
112
+ or an explicit name list.
113
+
114
+ Unknown, inactive, or unloadable probe names are logged and emitted as
115
+ ``CheckResult.skip`` scenarios rather than raising.
116
+
117
+ ### deepteam
118
+
119
+ Deepteam generates adversarial attacks with an LLM and judges the responses with an
120
+ LLM, so it needs a working Giskard default generator (see
121
+ `giskard.checks.get_default_generator()`) — there is no keyless mode.
122
+
123
+ ```python
124
+ result = asyncio.run(
125
+ third_party_scan(
126
+ target,
127
+ tool="deepteam",
128
+ description="A helpful assistant", # becomes deepteam's target_purpose
129
+ vulnerabilities=["Bias", "Toxicity"], # omit for a curated default set
130
+ attacks=["PromptInjection", "LinearJailbreaking"], # omit for defaults
131
+ attacks_per_vulnerability_type=1, # default; each vuln subtype × this many
132
+ # target_mode defaults to "multiturn" (shared with native Giskard scans)
133
+ )
134
+ )
135
+ ```
136
+
137
+ `vulnerabilities` accepts `Bias`, `Toxicity`, `PIILeakage`, `PromptLeakage`, and
138
+ `Misinformation`. Instantiating those classes without subtypes runs **all** of
139
+ their types (for example Bias → race, gender, politics, religion), so cost scales
140
+ with types × `attacks_per_vulnerability_type` × attacks.
141
+
142
+ `attacks` accepts the single-turn `PromptInjection`, `Roleplay`, `Leetspeak`, and
143
+ `ROT13`, plus the multi-turn `LinearJailbreaking`, `CrescendoJailbreaking`,
144
+ `TreeJailbreaking`, `SequentialJailbreak`, and `BadLikertJudge`. Unrecognized
145
+ names are logged and emitted as skip scenarios (valid names in the same call still
146
+ run).
147
+
148
+ `target_mode` defaults to the shared `DEFAULT_TARGET_MODE` (`"multiturn"`).
149
+ Pass `"singleturn"` to drop multi-turn attacks (they surface as skip scenarios).
150
+ Unknown names also skip — the suite is never an empty "everything passed"
151
+ result when you asked for attacks that could not run.
152
+
153
+ ### API keys and LLM-judge detectors
154
+
155
+ Some garak detectors need an LLM or a third-party API to score a probe:
156
+
157
+ - **LLM-judge detectors** (garak's `judge.*`, e.g. refusal detection) normally require
158
+ their own OpenAI key. Instead, they are automatically backed by Giskard's default
159
+ generator (`giskard.checks.get_default_generator()`), so they run with the same
160
+ credentials as the rest of Giskard — no separate OpenAI key needed.
161
+ - **Detectors that need a third-party API key** you have not set (for example
162
+ `perspective.*`, which needs `PERSPECTIVE_API_KEY`) are **skipped** rather than
163
+ silently dropping the whole probe. Each skipped detector surfaces as a skip result
164
+ (`CheckResult.skip`) in the returned `SuiteResult`, with the missing key named in the
165
+ message, so the rest of the probe's detectors still run.
@@ -0,0 +1,62 @@
1
+ [project]
2
+ name = "giskard-scan"
3
+ version = "1.0.0"
4
+ description = "Agent vulnerability scanner — red teaming, prompt injection, adversarial scenario generation"
5
+ readme = "README.md"
6
+ license = { text = "Apache Software License 2.0" }
7
+ authors = [{ name = "Giskard R&D", email = "rnd@giskard.ai" }]
8
+ requires-python = ">=3.12"
9
+ dependencies = [
10
+ "giskard-checks>=1.0.2,<2",
11
+ "giskard-agents>=1.0.2,<2",
12
+ "numpy>=2.4.1,<3",
13
+ "huggingface-hub>=1.11.0,<2",
14
+ ]
15
+
16
+ [project.optional-dependencies]
17
+ garak = ["garak>=0.15,<1"]
18
+ deepteam = [
19
+ "deepteam>=1.0.7,<2",
20
+ # deepteam imports stdlib nntplib, removed in 3.13+ (PEP 594)
21
+ "standard-nntplib>=3.13; python_version >= '3.13'",
22
+ ]
23
+
24
+
25
+ [build-system]
26
+ requires = ["hatchling>=1.25.0"]
27
+ build-backend = "hatchling.build"
28
+
29
+ [tool.hatch.build.targets.wheel]
30
+ packages = ["src/giskard"]
31
+
32
+ [tool.hatch.build.targets.sdist]
33
+ include = [
34
+ "src/giskard/scan",
35
+ "README.md",
36
+ "pyproject.toml",
37
+ ]
38
+
39
+ [tool.pytest.ini_options]
40
+ testpaths = ["tests"]
41
+ # Optional-extra src trees are excluded by conftest.py's collect_ignore_glob, not by
42
+ # relative --ignore flags here: pytest resolves those against the invocation CWD, so they
43
+ # silently missed whenever pytest ran from the repo root. See conftest.py.
44
+ addopts = "--doctest-modules"
45
+ asyncio_mode = "auto"
46
+ pythonpath = ["src"]
47
+ markers = [
48
+ "integration: integration tests that are skipped unless --run-integration is provided",
49
+ "functional: functional tests that exercise a real external tool (skipped when it is not installed)",
50
+ ]
51
+
52
+ [tool.ruff.lint]
53
+ select = ["E", "W", "I", "F"]
54
+ ignore = ["E501"]
55
+
56
+ [tool.ruff.lint.isort.sections]
57
+ "my_packages" = ["giskard"]
58
+
59
+ [tool.pyright]
60
+ failOnWarnings = false
61
+ typeCheckingMode = "recommended"
62
+ reportImportCycles = "warning"
@@ -0,0 +1,69 @@
1
+ """Public package exports for giskard.scan."""
2
+
3
+ from pathlib import Path
4
+
5
+ from giskard.agents import add_prompts_path
6
+ from giskard.core.utils import get_lib_version
7
+
8
+ from .catalog import generate_suite
9
+ from .generators.adversarial import AdversarialScenarioGenerator
10
+ from .generators.base import (
11
+ DEFAULT_TARGET_MODE,
12
+ LocalDatasetScenarioGenerator,
13
+ ScenarioGenerator,
14
+ TargetMode,
15
+ )
16
+ from .generators.crescendo import CrescendoAttackScenarioGenerator
17
+ from .generators.gcg import GCGInjectionScenarioGenerator
18
+ from .generators.goat import GOATAttackScenarioGenerator
19
+ from .generators.huggingface import HuggingFaceDatasetScenarioGenerator
20
+ from .generators.knowledge_base import (
21
+ HallucinationScenarioGenerator,
22
+ KnowledgeBaseScenarioGenerator,
23
+ MultiTopicScenarioGenerator,
24
+ OutOfScopeScenarioGenerator,
25
+ SplitQuestionsScenarioGenerator,
26
+ SycophancyScenarioGenerator,
27
+ )
28
+ from .generators.prompt_injection import PromptInjectionScenarioGenerator
29
+ from .integrations import ScanTool, list_scan_items, third_party_scan
30
+ from .quality import quality_scan, quality_suite_generator_registry
31
+ from .registry import SuiteGeneratorRegistry
32
+ from .types import ScanOptions
33
+ from .utils.knowledge_base import Document, KnowledgeBase
34
+ from .vulnerability import vulnerability_scan, vulnerability_suite_generator_registry
35
+
36
+ add_prompts_path(str(Path(__file__).parent / "prompts"), "giskard.scan")
37
+
38
+ __version__ = get_lib_version("giskard-scan")
39
+
40
+ __all__ = [
41
+ "DEFAULT_TARGET_MODE",
42
+ "TargetMode",
43
+ "generate_suite",
44
+ "ScanTool",
45
+ "list_scan_items",
46
+ "ScenarioGenerator",
47
+ "LocalDatasetScenarioGenerator",
48
+ "AdversarialScenarioGenerator",
49
+ "CrescendoAttackScenarioGenerator",
50
+ "GCGInjectionScenarioGenerator",
51
+ "GOATAttackScenarioGenerator",
52
+ "HuggingFaceDatasetScenarioGenerator",
53
+ "PromptInjectionScenarioGenerator",
54
+ "KnowledgeBaseScenarioGenerator",
55
+ "HallucinationScenarioGenerator",
56
+ "MultiTopicScenarioGenerator",
57
+ "OutOfScopeScenarioGenerator",
58
+ "SplitQuestionsScenarioGenerator",
59
+ "SycophancyScenarioGenerator",
60
+ "Document",
61
+ "KnowledgeBase",
62
+ "ScanOptions",
63
+ "SuiteGeneratorRegistry",
64
+ "quality_suite_generator_registry",
65
+ "quality_scan",
66
+ "third_party_scan",
67
+ "vulnerability_suite_generator_registry",
68
+ "vulnerability_scan",
69
+ ]