neuraltrust-haystack 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- neuraltrust_haystack-0.1.0/.gitignore +23 -0
- neuraltrust_haystack-0.1.0/CHANGELOG.md +26 -0
- neuraltrust_haystack-0.1.0/LICENSE +21 -0
- neuraltrust_haystack-0.1.0/PKG-INFO +255 -0
- neuraltrust_haystack-0.1.0/README.md +226 -0
- neuraltrust_haystack-0.1.0/examples/chat_pipeline.py +58 -0
- neuraltrust_haystack-0.1.0/examples/text_pipeline.py +44 -0
- neuraltrust_haystack-0.1.0/pyproject.toml +88 -0
- neuraltrust_haystack-0.1.0/scripts/check_release_permissions.py +126 -0
- neuraltrust_haystack-0.1.0/scripts/release_version.py +114 -0
- neuraltrust_haystack-0.1.0/src/haystack_integrations/components/guardrails/neuraltrust/__init__.py +25 -0
- neuraltrust_haystack-0.1.0/src/haystack_integrations/components/guardrails/neuraltrust/_base.py +247 -0
- neuraltrust_haystack-0.1.0/src/haystack_integrations/components/guardrails/neuraltrust/_client.py +374 -0
- neuraltrust_haystack-0.1.0/src/haystack_integrations/components/guardrails/neuraltrust/_version.py +3 -0
- neuraltrust_haystack-0.1.0/src/haystack_integrations/components/guardrails/neuraltrust/chat_guard.py +99 -0
- neuraltrust_haystack-0.1.0/src/haystack_integrations/components/guardrails/neuraltrust/errors.py +41 -0
- neuraltrust_haystack-0.1.0/src/haystack_integrations/components/guardrails/neuraltrust/guard.py +62 -0
- neuraltrust_haystack-0.1.0/src/haystack_integrations/components/guardrails/neuraltrust/py.typed +0 -0
- neuraltrust_haystack-0.1.0/tests/conftest.py +63 -0
- neuraltrust_haystack-0.1.0/tests/live/test_production.py +143 -0
- neuraltrust_haystack-0.1.0/tests/test_client_lifecycle.py +446 -0
- neuraltrust_haystack-0.1.0/tests/test_guards.py +732 -0
- neuraltrust_haystack-0.1.0/tests/test_pipelines.py +132 -0
- neuraltrust_haystack-0.1.0/tests/test_release_permissions.py +117 -0
- neuraltrust_haystack-0.1.0/tests/test_release_version.py +101 -0
- neuraltrust_haystack-0.1.0/tests/test_snapshot.py +59 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
.venv*/
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
.pytest_cache/
|
|
5
|
+
.mypy_cache/
|
|
6
|
+
.ruff_cache/
|
|
7
|
+
.coverage*
|
|
8
|
+
coverage.xml
|
|
9
|
+
htmlcov/
|
|
10
|
+
dist/
|
|
11
|
+
build/
|
|
12
|
+
*.egg-info/
|
|
13
|
+
.env
|
|
14
|
+
.env.*
|
|
15
|
+
!.env.example
|
|
16
|
+
.creds.json
|
|
17
|
+
*credentials*.json
|
|
18
|
+
.local/
|
|
19
|
+
.DS_Store
|
|
20
|
+
|
|
21
|
+
# Integration guides live in NeuralTrust/docs; local research/drafts stay ignored.
|
|
22
|
+
/docs/
|
|
23
|
+
/integrations/
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
## [v0.1.0] — 2026-09-15
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- `NeuralTrustGuard` and `NeuralTrustChatGuard` for synchronous and asynchronous evaluation of text and text-only chat in input and output policy phases.
|
|
10
|
+
- Handling for `allow`, `report`, `transform`, `block`, and `ask` verdicts, with strict transformation validation and fail-closed errors. Route mode prevents blocked content from reaching required downstream inputs.
|
|
11
|
+
- Haystack Secret configuration, pipeline serialization, request context, and bounded retries for transient failures.
|
|
12
|
+
- Reusable HTTP connection pools, explicit synchronous/asynchronous cleanup, and asynchronous client/TLS setup that does not block the event loop.
|
|
13
|
+
- Support for Python 3.10+ and Haystack 2.31/3.x, with offline component, pipeline, compatibility, packaging, and opt-in production tests.
|
|
14
|
+
- [Official Haystack integration documentation](https://docs.neuraltrust.ai/integrations/haystack), runnable text/chat examples, and contributor guidance.
|
|
15
|
+
- Shared NeuralTrust CI and automatic release versioning, trusted PyPI publishing, and unique development artifacts. Release checks validate token permissions, tags, source versions, and wheel/source-distribution metadata before publication.
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
|
|
19
|
+
- Run shared lint once and test all ten Python/Haystack combinations in isolated environments, with individual results in the job summary. Pull requests no longer receive duplicate CI runs from `develop` pushes.
|
|
20
|
+
- Run PR metadata validation separately so title and description edits do not repeat the full test matrix.
|
|
21
|
+
- Use Node24 artifact actions and standard package installation instructions, with documentation and changelog links in package metadata.
|
|
22
|
+
|
|
23
|
+
### Fixed
|
|
24
|
+
|
|
25
|
+
- Prevent PR metadata edits from cancelling active code checks and eliminate malformed names from unused shared lint/test jobs.
|
|
26
|
+
- Recognize GitHub's `exempt` release-token permission mode when validating access through inherited branch rules.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 NeuralTrust
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: neuraltrust-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: NeuralTrust TrustGuard security components for Haystack pipelines
|
|
5
|
+
Project-URL: Homepage, https://neuraltrust.ai
|
|
6
|
+
Project-URL: Documentation, https://docs.neuraltrust.ai/integrations/haystack
|
|
7
|
+
Project-URL: Changelog, https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/CHANGELOG.md
|
|
8
|
+
Project-URL: Repository, https://github.com/NeuralTrust/neuraltrust-haystack
|
|
9
|
+
Project-URL: Issues, https://github.com/NeuralTrust/neuraltrust-haystack/issues
|
|
10
|
+
Author: NeuralTrust
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: guardrails,haystack,neuraltrust,security,trustguard
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Security
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Requires-Dist: haystack-ai<4,>=2.31.0
|
|
27
|
+
Requires-Dist: httpx<1,>=0.27
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# neuraltrust-haystack
|
|
31
|
+
|
|
32
|
+
Add [NeuralTrust TrustGuard](https://neuraltrust.ai) evaluation to Haystack text and chat pipelines. Screen user input before a model runs, or inspect completed assistant replies before returning them to your application.
|
|
33
|
+
|
|
34
|
+
Read the [official Haystack integration guide](https://docs.neuraltrust.ai/integrations/haystack) for setup and usage documentation.
|
|
35
|
+
|
|
36
|
+
## Installation
|
|
37
|
+
|
|
38
|
+
Requires Python 3.10+ and Haystack 2.31 or 3.x (`haystack-ai>=2.31.0,<4`).
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install neuraltrust-haystack
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
For installation from source and development checks, see the [contributing guide](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/CONTRIBUTING.md).
|
|
45
|
+
|
|
46
|
+
## Connect to TrustGuard
|
|
47
|
+
|
|
48
|
+
Create or select a TrustGuard collector with the policy you want to evaluate, then set its API key in your environment:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
export TRUSTGUARD_API_KEY="your-collector-api-key"
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The default API origin is `https://trustguard.neuraltrust.ai`. Pass `api_base` for the public HTTPS origin of a regional or self-hosted deployment. The component sends evaluation requests to `/v1/evaluate`.
|
|
55
|
+
|
|
56
|
+
The API key selects the collector and its policy. The input/output direction selects the policy phase. An `allow` result only reflects the configured policy; a collector without applicable checks does not establish that content was scanned for every threat.
|
|
57
|
+
|
|
58
|
+
## Components
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from haystack_integrations.components.guardrails.neuraltrust import (
|
|
62
|
+
NeuralTrustChatGuard,
|
|
63
|
+
NeuralTrustGuard,
|
|
64
|
+
)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
| Component | Required run input | Passing output |
|
|
68
|
+
| --- | --- | --- |
|
|
69
|
+
| `NeuralTrustGuard` | `text: str` | `text: str`, `verdict: dict` |
|
|
70
|
+
| `NeuralTrustChatGuard` | `messages: list[ChatMessage]` | `messages: list[ChatMessage]`, `verdict: dict` |
|
|
71
|
+
|
|
72
|
+
Both components implement `run`, `run_async`, `to_dict`, and `from_dict`.
|
|
73
|
+
|
|
74
|
+
### Screen text
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
78
|
+
|
|
79
|
+
with NeuralTrustGuard() as guard:
|
|
80
|
+
result = guard.run(text="What is the capital of France?")
|
|
81
|
+
print(result["text"])
|
|
82
|
+
print(result["verdict"]["status"])
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The default `on_violation="raise"` stops execution with `NeuralTrustBlockedError` for a `block` or `ask` verdict. API and response errors also stop execution.
|
|
86
|
+
|
|
87
|
+
### Route a pipeline
|
|
88
|
+
|
|
89
|
+
Use `on_violation="route"` when the application should handle denied requests through the verdict output:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from haystack import Pipeline, component
|
|
93
|
+
|
|
94
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@component
|
|
98
|
+
class AcceptText:
|
|
99
|
+
@component.output_types(accepted=str)
|
|
100
|
+
def run(self, text: str) -> dict[str, str]:
|
|
101
|
+
return {"accepted": text}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
with NeuralTrustGuard(on_violation="route") as guard:
|
|
105
|
+
pipeline = Pipeline()
|
|
106
|
+
pipeline.add_component("guard", guard)
|
|
107
|
+
pipeline.add_component("accept", AcceptText())
|
|
108
|
+
pipeline.connect("guard.text", "accept.text")
|
|
109
|
+
|
|
110
|
+
result = pipeline.run(
|
|
111
|
+
{"guard": {"text": "What is the capital of France?"}},
|
|
112
|
+
include_outputs_from={"guard"},
|
|
113
|
+
)
|
|
114
|
+
print(result["guard"]["verdict"]["status"])
|
|
115
|
+
if "accept" in result:
|
|
116
|
+
print(result["accept"]["accepted"])
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
On `block` or `ask`, the guard emits only `verdict`. The required `accept.text` input receives no value, so that component does not run. The guard omits the passing socket entirely: emitting an empty string or empty list would still supply a value to a downstream component. Keep guarded content connected through the guard's output, and use required inputs for the protected downstream step.
|
|
120
|
+
|
|
121
|
+
From a source checkout, the [text example](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/examples/text_pipeline.py) runs this pattern from the command line:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
uv run python examples/text_pipeline.py "What is the capital of France?"
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Screen completed chat replies
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from haystack.dataclasses import ChatMessage
|
|
131
|
+
|
|
132
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustChatGuard
|
|
133
|
+
|
|
134
|
+
with NeuralTrustChatGuard(direction="output") as guard:
|
|
135
|
+
result = guard.run(messages=[ChatMessage.from_assistant("Paris is the capital of France.")])
|
|
136
|
+
print(result["messages"][0].text)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Connect `chat_generator.replies` to `guard.messages` to evaluate completed generator replies. In a source checkout, the [chat example](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/examples/chat_pipeline.py) uses a local component that produces a fixed assistant reply:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
uv run python examples/chat_pipeline.py
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
The chat guard accepts a nonempty list of `system`, `user`, and `assistant` messages, each with exactly one nonempty text part, and preserves message names and metadata. Multiple content parts, reasoning, multimodal content, tool calls, and tool results are rejected. Transformed responses must map unambiguously to the original messages; incompatible message counts, roles, or content fail closed. The text guard also rejects empty or whitespace-only input.
|
|
146
|
+
|
|
147
|
+
### Async execution
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
import asyncio
|
|
151
|
+
|
|
152
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
async def main() -> None:
|
|
156
|
+
async with NeuralTrustGuard() as guard:
|
|
157
|
+
result = await guard.run_async(text="What is the capital of France?")
|
|
158
|
+
print(result["verdict"]["status"])
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
asyncio.run(main())
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
For async pipelines, Haystack 3.x uses `await Pipeline.run_async(...)`; Haystack 2.31 uses `await AsyncPipeline.run_async(...)` with `AsyncPipeline` imported from `haystack`. Synchronous and asynchronous calls use the same component inputs, verdict handling, and error behavior.
|
|
165
|
+
|
|
166
|
+
### Client lifetime
|
|
167
|
+
|
|
168
|
+
Reuse guard instances across evaluations to reuse HTTP connections. Synchronous calls share a pool; asynchronous calls use a separate pool for each event loop. Async client and TLS setup runs off the event loop and is shared by concurrent initial calls. Credentials still resolve on every evaluation.
|
|
169
|
+
|
|
170
|
+
Use `with guard` for synchronous work or `async with guard` around the lifetime of an asynchronous pipeline. At application shutdown, `guard.close()` drains the synchronous pool. `await guard.aclose()` drains both the synchronous pool and the current loop's asynchronous pool. Call it in each owning event loop before that loop stops. Cleanup is idempotent; subsequent evaluations can create a fresh pool. Network clients and locks are excluded from serialization and component copies.
|
|
171
|
+
|
|
172
|
+
## Configuration
|
|
173
|
+
|
|
174
|
+
All constructor arguments are keyword-only.
|
|
175
|
+
|
|
176
|
+
| Argument | Default | Purpose |
|
|
177
|
+
| --- | --- | --- |
|
|
178
|
+
| `api_key` | `Secret.from_env_var("TRUSTGUARD_API_KEY")` | Haystack Secret containing the evaluation credential. |
|
|
179
|
+
| `api_base` | `https://trustguard.neuraltrust.ai` | Public HTTPS API origin. |
|
|
180
|
+
| `direction` | `"input"` | Policy phase: `"input"` or `"output"`. |
|
|
181
|
+
| `on_violation` | `"raise"` | `"raise"` stops with an exception; `"route"` returns only the verdict for `block`/`ask`. |
|
|
182
|
+
| `timeout` | `5.0` | Positive HTTP timeout in seconds for each network operation, not an overall retry deadline. |
|
|
183
|
+
| `max_retries` | `2` | Additional attempts for eligible transient failures; integer from 0 to 10. |
|
|
184
|
+
| `collector_key` | `None` | Optional collector identifier when using a service token. This is not an API credential. |
|
|
185
|
+
|
|
186
|
+
The optional keyword-only run arguments `session_id`, `consumer_id`, and `attributes` attach request context. `attributes` must contain JSON-compatible values. `consumer_id` can select a policy override configured in TrustGuard.
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
with NeuralTrustGuard() as guard:
|
|
190
|
+
result = guard.run(
|
|
191
|
+
text="What is the capital of France?",
|
|
192
|
+
session_id="example-session",
|
|
193
|
+
consumer_id="example-consumer",
|
|
194
|
+
attributes={"source": {"application": "haystack-example"}},
|
|
195
|
+
)
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Configure policies in TrustGuard. These components do not accept a per-request policy ID or detector ID.
|
|
199
|
+
|
|
200
|
+
## Verdicts and errors
|
|
201
|
+
|
|
202
|
+
| TrustGuard status | Component behavior |
|
|
203
|
+
| --- | --- |
|
|
204
|
+
| `allow` | Forward original content. |
|
|
205
|
+
| `report` | Forward original content and return findings in `verdict`. |
|
|
206
|
+
| `transform` | Forward validated transformed content. |
|
|
207
|
+
| `block` | Raise or omit the passing output, according to `on_violation`. |
|
|
208
|
+
| `ask` | Stop like `block`, preserving the `ask` status. This package does not grant approval. |
|
|
209
|
+
|
|
210
|
+
The verdict contains `status` and, when provided, `findings`, `trace_id`, and `request_id`. Findings can contain sensitive evidence from evaluated content. Select the fields your application needs and apply its normal access and retention controls; avoid dumping the full verdict into logs.
|
|
211
|
+
|
|
212
|
+
Import the exceptions from the same public component namespace:
|
|
213
|
+
|
|
214
|
+
| Exception | Meaning |
|
|
215
|
+
| --- | --- |
|
|
216
|
+
| `NeuralTrustBlockedError` | `block` or `ask`; exposes `status` and a verdict limited to status and validated correlation IDs. |
|
|
217
|
+
| `NeuralTrustAuthenticationError` | Missing/invalid credentials or HTTP 401/403. |
|
|
218
|
+
| `NeuralTrustUnavailableError` | Retryable failure exhausted the configured attempts. |
|
|
219
|
+
| `NeuralTrustRequestError` | Rejected request or non-retryable transport failure. |
|
|
220
|
+
| `NeuralTrustInvalidResponseError` | Malformed verdict or unusable transformation. |
|
|
221
|
+
| `NeuralTrustError` | Base class for the errors above. |
|
|
222
|
+
|
|
223
|
+
Exceptions use sanitized messages. A Haystack pipeline may wrap component failures in its own execution exception; inspect the chained cause when handling a specific NeuralTrust error at the pipeline boundary.
|
|
224
|
+
|
|
225
|
+
Retries cover timeouts, connection failures, and HTTP 429/502/504. TLS failures, authentication failures, other HTTP errors, and invalid verdicts are not converted into passing content. Retry delays are bounded and honor supported `Retry-After` values up to five seconds. There is no fail-open mode; `on_violation="route"` changes only the handling of valid `block` and `ask` verdicts.
|
|
226
|
+
|
|
227
|
+
## Save and restore pipelines
|
|
228
|
+
|
|
229
|
+
```python
|
|
230
|
+
from haystack import Pipeline
|
|
231
|
+
|
|
232
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
233
|
+
|
|
234
|
+
pipeline = Pipeline()
|
|
235
|
+
pipeline.add_component("guard", NeuralTrustGuard())
|
|
236
|
+
serialized = pipeline.dumps()
|
|
237
|
+
restored = Pipeline.loads(serialized)
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
Environment-based Secrets serialize the variable name, never its resolved value. Set the credential in the restoring process before running the pipeline. `Secret.from_token(...)` is supported for direct use, but Haystack intentionally refuses to serialize token-based Secrets. The canonical `haystack_integrations` namespace also works with Haystack 3.x's default deserialization allowlist.
|
|
241
|
+
|
|
242
|
+
## Scope
|
|
243
|
+
|
|
244
|
+
- Text and text-only chat are supported. Document batches, tools, multimodal data, and native Agent lifecycle hooks are outside the components' supported interface.
|
|
245
|
+
- A guard before/after an Agent covers its pipeline input/output. It does not intercept the Agent's internal model calls or tool actions.
|
|
246
|
+
- Output evaluation happens after a completed reply. Tokens already delivered through a streaming callback cannot be withheld by a later pipeline component. Buffer replies when they must pass evaluation before delivery.
|
|
247
|
+
- Detection and transformation depend on the collector policy, its direction, and the TrustGuard service. Local validation does not establish detection accuracy for every policy or input.
|
|
248
|
+
|
|
249
|
+
See the [Haystack integration guide](https://docs.neuraltrust.ai/integrations/haystack) for usage documentation and the [contributing guide](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/CONTRIBUTING.md) for development checks.
|
|
250
|
+
|
|
251
|
+
Release history is recorded in the [changelog](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/CHANGELOG.md).
|
|
252
|
+
|
|
253
|
+
## License
|
|
254
|
+
|
|
255
|
+
This package is distributed under the [MIT License](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/LICENSE).
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# neuraltrust-haystack
|
|
2
|
+
|
|
3
|
+
Add [NeuralTrust TrustGuard](https://neuraltrust.ai) evaluation to Haystack text and chat pipelines. Screen user input before a model runs, or inspect completed assistant replies before returning them to your application.
|
|
4
|
+
|
|
5
|
+
Read the [official Haystack integration guide](https://docs.neuraltrust.ai/integrations/haystack) for setup and usage documentation.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
Requires Python 3.10+ and Haystack 2.31 or 3.x (`haystack-ai>=2.31.0,<4`).
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install neuraltrust-haystack
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
For installation from source and development checks, see the [contributing guide](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/CONTRIBUTING.md).
|
|
16
|
+
|
|
17
|
+
## Connect to TrustGuard
|
|
18
|
+
|
|
19
|
+
Create or select a TrustGuard collector with the policy you want to evaluate, then set its API key in your environment:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
export TRUSTGUARD_API_KEY="your-collector-api-key"
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The default API origin is `https://trustguard.neuraltrust.ai`. Pass `api_base` for the public HTTPS origin of a regional or self-hosted deployment. The component sends evaluation requests to `/v1/evaluate`.
|
|
26
|
+
|
|
27
|
+
The API key selects the collector and its policy. The input/output direction selects the policy phase. An `allow` result only reflects the configured policy; a collector without applicable checks does not establish that content was scanned for every threat.
|
|
28
|
+
|
|
29
|
+
## Components
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from haystack_integrations.components.guardrails.neuraltrust import (
|
|
33
|
+
NeuralTrustChatGuard,
|
|
34
|
+
NeuralTrustGuard,
|
|
35
|
+
)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
| Component | Required run input | Passing output |
|
|
39
|
+
| --- | --- | --- |
|
|
40
|
+
| `NeuralTrustGuard` | `text: str` | `text: str`, `verdict: dict` |
|
|
41
|
+
| `NeuralTrustChatGuard` | `messages: list[ChatMessage]` | `messages: list[ChatMessage]`, `verdict: dict` |
|
|
42
|
+
|
|
43
|
+
Both components implement `run`, `run_async`, `to_dict`, and `from_dict`.
|
|
44
|
+
|
|
45
|
+
### Screen text
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
49
|
+
|
|
50
|
+
with NeuralTrustGuard() as guard:
|
|
51
|
+
result = guard.run(text="What is the capital of France?")
|
|
52
|
+
print(result["text"])
|
|
53
|
+
print(result["verdict"]["status"])
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The default `on_violation="raise"` stops execution with `NeuralTrustBlockedError` for a `block` or `ask` verdict. API and response errors also stop execution.
|
|
57
|
+
|
|
58
|
+
### Route a pipeline
|
|
59
|
+
|
|
60
|
+
Use `on_violation="route"` when the application should handle denied requests through the verdict output:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from haystack import Pipeline, component
|
|
64
|
+
|
|
65
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@component
|
|
69
|
+
class AcceptText:
|
|
70
|
+
@component.output_types(accepted=str)
|
|
71
|
+
def run(self, text: str) -> dict[str, str]:
|
|
72
|
+
return {"accepted": text}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
with NeuralTrustGuard(on_violation="route") as guard:
|
|
76
|
+
pipeline = Pipeline()
|
|
77
|
+
pipeline.add_component("guard", guard)
|
|
78
|
+
pipeline.add_component("accept", AcceptText())
|
|
79
|
+
pipeline.connect("guard.text", "accept.text")
|
|
80
|
+
|
|
81
|
+
result = pipeline.run(
|
|
82
|
+
{"guard": {"text": "What is the capital of France?"}},
|
|
83
|
+
include_outputs_from={"guard"},
|
|
84
|
+
)
|
|
85
|
+
print(result["guard"]["verdict"]["status"])
|
|
86
|
+
if "accept" in result:
|
|
87
|
+
print(result["accept"]["accepted"])
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
On `block` or `ask`, the guard emits only `verdict`. The required `accept.text` input receives no value, so that component does not run. The guard omits the passing socket entirely: emitting an empty string or empty list would still supply a value to a downstream component. Keep guarded content connected through the guard's output, and use required inputs for the protected downstream step.
|
|
91
|
+
|
|
92
|
+
From a source checkout, the [text example](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/examples/text_pipeline.py) runs this pattern from the command line:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
uv run python examples/text_pipeline.py "What is the capital of France?"
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### Screen completed chat replies
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from haystack.dataclasses import ChatMessage
|
|
102
|
+
|
|
103
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustChatGuard
|
|
104
|
+
|
|
105
|
+
with NeuralTrustChatGuard(direction="output") as guard:
|
|
106
|
+
result = guard.run(messages=[ChatMessage.from_assistant("Paris is the capital of France.")])
|
|
107
|
+
print(result["messages"][0].text)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Connect `chat_generator.replies` to `guard.messages` to evaluate completed generator replies. In a source checkout, the [chat example](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/examples/chat_pipeline.py) uses a local component that produces a fixed assistant reply:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
uv run python examples/chat_pipeline.py
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The chat guard accepts a nonempty list of `system`, `user`, and `assistant` messages, each with exactly one nonempty text part, and preserves message names and metadata. Multiple content parts, reasoning, multimodal content, tool calls, and tool results are rejected. Transformed responses must map unambiguously to the original messages; incompatible message counts, roles, or content fail closed. The text guard also rejects empty or whitespace-only input.
|
|
117
|
+
|
|
118
|
+
### Async execution
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
import asyncio
|
|
122
|
+
|
|
123
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
async def main() -> None:
|
|
127
|
+
async with NeuralTrustGuard() as guard:
|
|
128
|
+
result = await guard.run_async(text="What is the capital of France?")
|
|
129
|
+
print(result["verdict"]["status"])
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
asyncio.run(main())
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
For async pipelines, Haystack 3.x uses `await Pipeline.run_async(...)`; Haystack 2.31 uses `await AsyncPipeline.run_async(...)` with `AsyncPipeline` imported from `haystack`. Synchronous and asynchronous calls use the same component inputs, verdict handling, and error behavior.
|
|
136
|
+
|
|
137
|
+
### Client lifetime
|
|
138
|
+
|
|
139
|
+
Reuse guard instances across evaluations to reuse HTTP connections. Synchronous calls share a pool; asynchronous calls use a separate pool for each event loop. Async client and TLS setup runs off the event loop and is shared by concurrent initial calls. Credentials still resolve on every evaluation.
|
|
140
|
+
|
|
141
|
+
Use `with guard` for synchronous work or `async with guard` around the lifetime of an asynchronous pipeline. At application shutdown, `guard.close()` drains the synchronous pool. `await guard.aclose()` drains both the synchronous pool and the current loop's asynchronous pool. Call it in each owning event loop before that loop stops. Cleanup is idempotent; subsequent evaluations can create a fresh pool. Network clients and locks are excluded from serialization and component copies.
|
|
142
|
+
|
|
143
|
+
## Configuration
|
|
144
|
+
|
|
145
|
+
All constructor arguments are keyword-only.
|
|
146
|
+
|
|
147
|
+
| Argument | Default | Purpose |
|
|
148
|
+
| --- | --- | --- |
|
|
149
|
+
| `api_key` | `Secret.from_env_var("TRUSTGUARD_API_KEY")` | Haystack Secret containing the evaluation credential. |
|
|
150
|
+
| `api_base` | `https://trustguard.neuraltrust.ai` | Public HTTPS API origin. |
|
|
151
|
+
| `direction` | `"input"` | Policy phase: `"input"` or `"output"`. |
|
|
152
|
+
| `on_violation` | `"raise"` | `"raise"` stops with an exception; `"route"` returns only the verdict for `block`/`ask`. |
|
|
153
|
+
| `timeout` | `5.0` | Positive HTTP timeout in seconds for each network operation, not an overall retry deadline. |
|
|
154
|
+
| `max_retries` | `2` | Additional attempts for eligible transient failures; integer from 0 to 10. |
|
|
155
|
+
| `collector_key` | `None` | Optional collector identifier when using a service token. This is not an API credential. |
|
|
156
|
+
|
|
157
|
+
The optional keyword-only run arguments `session_id`, `consumer_id`, and `attributes` attach request context. `attributes` must contain JSON-compatible values. `consumer_id` can select a policy override configured in TrustGuard.
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
with NeuralTrustGuard() as guard:
|
|
161
|
+
result = guard.run(
|
|
162
|
+
text="What is the capital of France?",
|
|
163
|
+
session_id="example-session",
|
|
164
|
+
consumer_id="example-consumer",
|
|
165
|
+
attributes={"source": {"application": "haystack-example"}},
|
|
166
|
+
)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Configure policies in TrustGuard. These components do not accept a per-request policy ID or detector ID.
|
|
170
|
+
|
|
171
|
+
## Verdicts and errors
|
|
172
|
+
|
|
173
|
+
| TrustGuard status | Component behavior |
|
|
174
|
+
| --- | --- |
|
|
175
|
+
| `allow` | Forward original content. |
|
|
176
|
+
| `report` | Forward original content and return findings in `verdict`. |
|
|
177
|
+
| `transform` | Forward validated transformed content. |
|
|
178
|
+
| `block` | Raise or omit the passing output, according to `on_violation`. |
|
|
179
|
+
| `ask` | Stop like `block`, preserving the `ask` status. This package does not grant approval. |
|
|
180
|
+
|
|
181
|
+
The verdict contains `status` and, when provided, `findings`, `trace_id`, and `request_id`. Findings can contain sensitive evidence from evaluated content. Select the fields your application needs and apply its normal access and retention controls; avoid dumping the full verdict into logs.
|
|
182
|
+
|
|
183
|
+
Import the exceptions from the same public component namespace:
|
|
184
|
+
|
|
185
|
+
| Exception | Meaning |
|
|
186
|
+
| --- | --- |
|
|
187
|
+
| `NeuralTrustBlockedError` | `block` or `ask`; exposes `status` and a verdict limited to status and validated correlation IDs. |
|
|
188
|
+
| `NeuralTrustAuthenticationError` | Missing/invalid credentials or HTTP 401/403. |
|
|
189
|
+
| `NeuralTrustUnavailableError` | Retryable failure exhausted the configured attempts. |
|
|
190
|
+
| `NeuralTrustRequestError` | Rejected request or non-retryable transport failure. |
|
|
191
|
+
| `NeuralTrustInvalidResponseError` | Malformed verdict or unusable transformation. |
|
|
192
|
+
| `NeuralTrustError` | Base class for the errors above. |
|
|
193
|
+
|
|
194
|
+
Exceptions use sanitized messages. A Haystack pipeline may wrap component failures in its own execution exception; inspect the chained cause when handling a specific NeuralTrust error at the pipeline boundary.
|
|
195
|
+
|
|
196
|
+
Retries cover timeouts, connection failures, and HTTP 429/502/504. TLS failures, authentication failures, other HTTP errors, and invalid verdicts are not converted into passing content. Retry delays are bounded and honor supported `Retry-After` values up to five seconds. There is no fail-open mode; `on_violation="route"` changes only the handling of valid `block` and `ask` verdicts.
|
|
197
|
+
|
|
198
|
+
## Save and restore pipelines
|
|
199
|
+
|
|
200
|
+
```python
|
|
201
|
+
from haystack import Pipeline
|
|
202
|
+
|
|
203
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
204
|
+
|
|
205
|
+
pipeline = Pipeline()
|
|
206
|
+
pipeline.add_component("guard", NeuralTrustGuard())
|
|
207
|
+
serialized = pipeline.dumps()
|
|
208
|
+
restored = Pipeline.loads(serialized)
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Environment-based Secrets serialize the variable name, never its resolved value. Set the credential in the restoring process before running the pipeline. `Secret.from_token(...)` is supported for direct use, but Haystack intentionally refuses to serialize token-based Secrets. The canonical `haystack_integrations` namespace also works with Haystack 3.x's default deserialization allowlist.
|
|
212
|
+
|
|
213
|
+
## Scope
|
|
214
|
+
|
|
215
|
+
- Text and text-only chat are supported. Document batches, tools, multimodal data, and native Agent lifecycle hooks are outside the components' supported interface.
|
|
216
|
+
- A guard before/after an Agent covers its pipeline input/output. It does not intercept the Agent's internal model calls or tool actions.
|
|
217
|
+
- Output evaluation happens after a completed reply. Tokens already delivered through a streaming callback cannot be withheld by a later pipeline component. Buffer replies when they must pass evaluation before delivery.
|
|
218
|
+
- Detection and transformation depend on the collector policy, its direction, and the TrustGuard service. Local validation does not establish detection accuracy for every policy or input.
|
|
219
|
+
|
|
220
|
+
See the [Haystack integration guide](https://docs.neuraltrust.ai/integrations/haystack) for usage documentation and the [contributing guide](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/CONTRIBUTING.md) for development checks.
|
|
221
|
+
|
|
222
|
+
Release history is recorded in the [changelog](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/CHANGELOG.md).
|
|
223
|
+
|
|
224
|
+
## License
|
|
225
|
+
|
|
226
|
+
This package is distributed under the [MIT License](https://github.com/NeuralTrust/neuraltrust-haystack/blob/main/LICENSE).
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Evaluate a completed assistant reply with the TrustGuard output policy phase."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
|
|
5
|
+
from haystack import Pipeline, component
|
|
6
|
+
from haystack.dataclasses import ChatMessage
|
|
7
|
+
|
|
8
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustChatGuard
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@component
|
|
12
|
+
class ExampleAnswer:
|
|
13
|
+
"""Produce a fixed assistant reply locally."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, text: str) -> None:
|
|
16
|
+
self.text = text
|
|
17
|
+
|
|
18
|
+
@component.output_types(replies=list[ChatMessage])
|
|
19
|
+
def run(self) -> dict[str, list[ChatMessage]]:
|
|
20
|
+
"""Create the completed reply that the output guard will evaluate."""
|
|
21
|
+
return {"replies": [ChatMessage.from_assistant(self.text, meta={"source": "local-example"})]}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@component
|
|
25
|
+
class AcceptMessages:
|
|
26
|
+
"""Accept messages only through the guard's passing output."""
|
|
27
|
+
|
|
28
|
+
@component.output_types(accepted=list[ChatMessage])
|
|
29
|
+
def run(self, messages: list[ChatMessage]) -> dict[str, list[ChatMessage]]:
|
|
30
|
+
"""Receive evaluated messages through a required input socket."""
|
|
31
|
+
return {"accepted": messages}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def main() -> None:
|
|
35
|
+
"""Run output evaluation with TRUSTGUARD_API_KEY and a local reply producer."""
|
|
36
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
37
|
+
parser.add_argument("text", nargs="?", default="Paris is the capital of France.")
|
|
38
|
+
args = parser.parse_args()
|
|
39
|
+
|
|
40
|
+
with NeuralTrustChatGuard(direction="output", on_violation="route") as guard:
|
|
41
|
+
pipeline = Pipeline()
|
|
42
|
+
pipeline.add_component("answer", ExampleAnswer(args.text))
|
|
43
|
+
pipeline.add_component("guard", guard)
|
|
44
|
+
pipeline.add_component("accept", AcceptMessages())
|
|
45
|
+
pipeline.connect("answer.replies", "guard.messages")
|
|
46
|
+
pipeline.connect("guard.messages", "accept.messages")
|
|
47
|
+
|
|
48
|
+
result = pipeline.run({}, include_outputs_from={"guard"})
|
|
49
|
+
print(f"TrustGuard output verdict: {result['guard']['verdict']['status']}")
|
|
50
|
+
if "accept" in result:
|
|
51
|
+
for message in result["accept"]["accepted"]:
|
|
52
|
+
print(f"Accepted reply: {message.text}")
|
|
53
|
+
else:
|
|
54
|
+
print("The protected downstream component did not run.")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
if __name__ == "__main__":
|
|
58
|
+
main()
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Evaluate input text with TrustGuard before a local pipeline sink runs."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
|
|
5
|
+
from haystack import Pipeline, component
|
|
6
|
+
|
|
7
|
+
from haystack_integrations.components.guardrails.neuraltrust import NeuralTrustGuard
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@component
|
|
11
|
+
class AcceptText:
|
|
12
|
+
"""Return the text supplied through the guard's passing output."""
|
|
13
|
+
|
|
14
|
+
@component.output_types(accepted=str)
|
|
15
|
+
def run(self, text: str) -> dict[str, str]:
|
|
16
|
+
"""Accept evaluated text through a required input socket."""
|
|
17
|
+
return {"accepted": text}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def main() -> None:
|
|
21
|
+
"""Run a real Haystack pipeline using TRUSTGUARD_API_KEY."""
|
|
22
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
23
|
+
parser.add_argument("text", nargs="?", default="What is the capital of France?")
|
|
24
|
+
args = parser.parse_args()
|
|
25
|
+
|
|
26
|
+
with NeuralTrustGuard(on_violation="route") as guard:
|
|
27
|
+
pipeline = Pipeline()
|
|
28
|
+
pipeline.add_component("guard", guard)
|
|
29
|
+
pipeline.add_component("accept", AcceptText())
|
|
30
|
+
pipeline.connect("guard.text", "accept.text")
|
|
31
|
+
|
|
32
|
+
result = pipeline.run(
|
|
33
|
+
{"guard": {"text": args.text}},
|
|
34
|
+
include_outputs_from={"guard"},
|
|
35
|
+
)
|
|
36
|
+
print(f"TrustGuard input verdict: {result['guard']['verdict']['status']}")
|
|
37
|
+
if "accept" in result:
|
|
38
|
+
print(f"Accepted text: {result['accept']['accepted']}")
|
|
39
|
+
else:
|
|
40
|
+
print("The protected downstream component did not run.")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
main()
|