nace-haystack 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- haystack_integrations/common/nace/__init__.py +3 -0
- haystack_integrations/common/nace/client.py +171 -0
- haystack_integrations/common/nace/concurrency.py +49 -0
- haystack_integrations/common/nace/errors.py +138 -0
- haystack_integrations/common/nace/images.py +204 -0
- haystack_integrations/common/nace/request.py +144 -0
- haystack_integrations/common/py.typed +0 -0
- haystack_integrations/components/classifiers/nace/__init__.py +7 -0
- haystack_integrations/components/classifiers/nace/document_classifier.py +390 -0
- haystack_integrations/components/classifiers/py.typed +0 -0
- haystack_integrations/components/evaluators/nace/__init__.py +13 -0
- haystack_integrations/components/evaluators/nace/_base.py +247 -0
- haystack_integrations/components/evaluators/nace/context_relevance.py +139 -0
- haystack_integrations/components/evaluators/nace/faithfulness.py +169 -0
- haystack_integrations/components/evaluators/py.typed +0 -0
- haystack_integrations/components/routers/nace/__init__.py +7 -0
- haystack_integrations/components/routers/nace/text_router.py +316 -0
- haystack_integrations/components/routers/py.typed +0 -0
- nace_haystack-0.1.0.dist-info/METADATA +230 -0
- nace_haystack-0.1.0.dist-info/RECORD +22 -0
- nace_haystack-0.1.0.dist-info/WHEEL +4 -0
- nace_haystack-0.1.0.dist-info/licenses/LICENSE.txt +201 -0
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026-present Nace AI
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
|
|
7
|
+
# Mirrors deepset's TypeSafeTextRouter (typesafe-haystack) so a Jev pipeline switches to Drex by changing the import,
|
|
8
|
+
# the class name and the API key.
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import nace_sdk
|
|
12
|
+
from haystack import component, default_to_dict
|
|
13
|
+
from haystack.utils import Secret
|
|
14
|
+
|
|
15
|
+
from haystack_integrations.common.nace.client import (
|
|
16
|
+
DEFAULT_MODEL,
|
|
17
|
+
client_options,
|
|
18
|
+
is_openrouter,
|
|
19
|
+
make_async_client,
|
|
20
|
+
make_client,
|
|
21
|
+
provider_name,
|
|
22
|
+
resolve_base_url,
|
|
23
|
+
)
|
|
24
|
+
from haystack_integrations.common.nace.errors import readable_error
|
|
25
|
+
from haystack_integrations.common.nace.images import ImageInput, image_types
|
|
26
|
+
from haystack_integrations.common.nace.request import DrexResponse, build_call
|
|
27
|
+
|
|
28
|
+
_LOW_CONFIDENCE = "low_confidence"
|
|
29
|
+
_PROBABILITIES = "probabilities"
|
|
30
|
+
_QUESTION_ID = "route"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@component
|
|
34
|
+
class NaceTextRouter:
|
|
35
|
+
"""
|
|
36
|
+
Routes a text to the connection of the label a Drex decision model picks for it.
|
|
37
|
+
|
|
38
|
+
The model answers a single `choice` question over `labels` and returns calibrated probabilities. The component
|
|
39
|
+
has one output per label. When `min_confidence` is set, it also has a `low_confidence` output that receives the
|
|
40
|
+
text whenever the answer's `confidence` is below the threshold, so uncertain texts can go to a fallback such as
|
|
41
|
+
an LLM or a human. With `include_probabilities=True`, a `probabilities` output carries the full answer, so
|
|
42
|
+
thresholds stay in your pipeline.
|
|
43
|
+
|
|
44
|
+
Images can be routed too: pass them to `run` and the default model, `drex-max-v1.1`, reads them alongside the text,
|
|
45
|
+
as "Image 1", "Image 2" and so on. Images too big for one request are shrunk to fit.
|
|
46
|
+
|
|
47
|
+
The component also works with OpenRouter and other TypeSafe-compatible hosts: set `api_base_url` to the server
|
|
48
|
+
and `model` to one of its models.
|
|
49
|
+
|
|
50
|
+
### Usage example
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from haystack_integrations.components.routers.nace import NaceTextRouter
|
|
54
|
+
|
|
55
|
+
router = NaceTextRouter(
|
|
56
|
+
labels={
|
|
57
|
+
"billing": "Payments, invoices, refunds and charges",
|
|
58
|
+
"technical": "Bugs, crashes and errors in the product",
|
|
59
|
+
},
|
|
60
|
+
instructions="Which team should handle this support request?",
|
|
61
|
+
min_confidence=0.6,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
print(router.run(text="I was charged twice for my subscription."))
|
|
65
|
+
# {'billing': 'I was charged twice for my subscription.'}
|
|
66
|
+
```
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
def __init__(
|
|
70
|
+
self,
|
|
71
|
+
labels: list[str] | dict[str, str | None],
|
|
72
|
+
*,
|
|
73
|
+
instructions: str = "Which category does this text belong to?",
|
|
74
|
+
model: str = DEFAULT_MODEL,
|
|
75
|
+
api_key: Secret = Secret.from_env_var("NACE_API_KEY"),
|
|
76
|
+
api_base_url: str | None = None,
|
|
77
|
+
min_confidence: float | None = None,
|
|
78
|
+
include_probabilities: bool = False,
|
|
79
|
+
timeout: float | None = None,
|
|
80
|
+
max_retries: int | None = None,
|
|
81
|
+
) -> None:
|
|
82
|
+
"""
|
|
83
|
+
Creates a NaceTextRouter.
|
|
84
|
+
|
|
85
|
+
:param labels:
|
|
86
|
+
The labels to route between, each becoming an output connection. Pass a dict of label to description to
|
|
87
|
+
tell the model what each label means.
|
|
88
|
+
:param instructions:
|
|
89
|
+
The question the model answers by picking one of the labels.
|
|
90
|
+
:param model:
|
|
91
|
+
The Drex model to use. Defaults to the newest Drex model, which reads images; pin a model such as
|
|
92
|
+
`"drex-max-v1.1"` or `"drex-v2.0"` in production. On OpenRouter, use its model IDs, such as
|
|
93
|
+
`"nace-ai/drex-max"`.
|
|
94
|
+
:param api_key:
|
|
95
|
+
The Nace API key.
|
|
96
|
+
:param api_base_url:
|
|
97
|
+
Base URL of the API. If `None`, the `NACE_BASE_URL` environment variable is used, or
|
|
98
|
+
`https://console.nace.ai` if it is unset.
|
|
99
|
+
:param min_confidence:
|
|
100
|
+
Threshold between 0 and 1 for the answer's `confidence`. Texts below it are sent to the `low_confidence`
|
|
101
|
+
output. If `None`, every text goes to a label output.
|
|
102
|
+
:param include_probabilities:
|
|
103
|
+
If `True`, a `probabilities` output also receives the full answer: the picked label, its `confidence` and
|
|
104
|
+
every label's probability.
|
|
105
|
+
:param timeout:
|
|
106
|
+
Timeout in seconds for each request attempt. If `None`, 60 seconds are used, because Drex may take up to
|
|
107
|
+
55 seconds to answer.
|
|
108
|
+
:param max_retries:
|
|
109
|
+
Maximum number of retries after a failed request attempt. If `None`, the SDK default of 2 is used.
|
|
110
|
+
:raises ValueError:
|
|
111
|
+
If `labels` is empty, contains a reserved output name or a described label named `type`, or if
|
|
112
|
+
`min_confidence` is outside 0 to 1.
|
|
113
|
+
"""
|
|
114
|
+
if not labels:
|
|
115
|
+
msg = "`labels` must contain at least one label."
|
|
116
|
+
raise ValueError(msg)
|
|
117
|
+
if _LOW_CONFIDENCE in labels:
|
|
118
|
+
msg = f"`{_LOW_CONFIDENCE}` is reserved for the low confidence output and can't be used as a label."
|
|
119
|
+
raise ValueError(msg)
|
|
120
|
+
if include_probabilities and _PROBABILITIES in labels:
|
|
121
|
+
msg = f"`{_PROBABILITIES}` is reserved for the probabilities output and can't be used as a label."
|
|
122
|
+
raise ValueError(msg)
|
|
123
|
+
if isinstance(labels, dict) and "type" in labels:
|
|
124
|
+
# Haystack reads a saved dict with a "type" key as a serialized object, so the pipeline couldn't be loaded
|
|
125
|
+
msg = (
|
|
126
|
+
"A label named `type` can't have a description, because a saved pipeline couldn't be loaded. "
|
|
127
|
+
"Rename the label, or pass `labels` as a list."
|
|
128
|
+
)
|
|
129
|
+
raise ValueError(msg)
|
|
130
|
+
if min_confidence is not None and not 0 <= min_confidence <= 1:
|
|
131
|
+
msg = f"`min_confidence` must be between 0 and 1, got {min_confidence}."
|
|
132
|
+
raise ValueError(msg)
|
|
133
|
+
|
|
134
|
+
self.labels = labels
|
|
135
|
+
self.instructions = instructions
|
|
136
|
+
self.model = model
|
|
137
|
+
self.api_key = api_key
|
|
138
|
+
self.api_base_url = api_base_url
|
|
139
|
+
self.min_confidence = min_confidence
|
|
140
|
+
self.include_probabilities = include_probabilities
|
|
141
|
+
self.timeout = timeout
|
|
142
|
+
self.max_retries = max_retries
|
|
143
|
+
self._client: nace_sdk.NaceClient | None = None
|
|
144
|
+
self._async_client: nace_sdk.AsyncNaceClient | None = None
|
|
145
|
+
self._async_client_loop: asyncio.AbstractEventLoop | None = None
|
|
146
|
+
|
|
147
|
+
# Drex takes choice criteria as a dict of label to description
|
|
148
|
+
criteria = labels if isinstance(labels, dict) else dict.fromkeys(labels)
|
|
149
|
+
self._question = nace_sdk.Choice(instructions=instructions, criteria=criteria)
|
|
150
|
+
|
|
151
|
+
outputs: dict[str, Any] = dict.fromkeys(labels, str)
|
|
152
|
+
if min_confidence is not None:
|
|
153
|
+
outputs[_LOW_CONFIDENCE] = str
|
|
154
|
+
if include_probabilities:
|
|
155
|
+
outputs[_PROBABILITIES] = dict[str, Any]
|
|
156
|
+
component.set_output_types(self, **outputs)
|
|
157
|
+
|
|
158
|
+
def _client_options(self) -> dict[str, Any]:
|
|
159
|
+
return client_options(self.api_key, self.api_base_url, self.timeout, self.max_retries)
|
|
160
|
+
|
|
161
|
+
def warm_up(self) -> None:
|
|
162
|
+
"""
|
|
163
|
+
Creates the synchronous Nace client.
|
|
164
|
+
"""
|
|
165
|
+
if self._client is None:
|
|
166
|
+
self._client = make_client(self._client_options())
|
|
167
|
+
|
|
168
|
+
async def warm_up_async(self) -> None:
|
|
169
|
+
"""
|
|
170
|
+
Creates the asynchronous Nace client for the running event loop.
|
|
171
|
+
"""
|
|
172
|
+
loop = asyncio.get_running_loop()
|
|
173
|
+
if self._async_client_loop is not None and self._async_client_loop is not loop:
|
|
174
|
+
# An async HTTP client belongs to the event loop it first ran on, so a new `asyncio.run` (a second
|
|
175
|
+
# notebook cell, a second script run) needs a new client. The old one can't be closed from this loop.
|
|
176
|
+
self._async_client = None
|
|
177
|
+
if self._async_client is None:
|
|
178
|
+
self._async_client = make_async_client(self._client_options())
|
|
179
|
+
self._async_client_loop = loop
|
|
180
|
+
|
|
181
|
+
def close(self) -> None:
|
|
182
|
+
"""
|
|
183
|
+
Releases the synchronous Nace client.
|
|
184
|
+
"""
|
|
185
|
+
if self._client is not None:
|
|
186
|
+
self._client.close()
|
|
187
|
+
self._client = None
|
|
188
|
+
|
|
189
|
+
async def close_async(self) -> None:
|
|
190
|
+
"""
|
|
191
|
+
Releases the asynchronous Nace client.
|
|
192
|
+
"""
|
|
193
|
+
if self._async_client is not None:
|
|
194
|
+
await self._async_client.aclose()
|
|
195
|
+
self._async_client = None
|
|
196
|
+
self._async_client_loop = None
|
|
197
|
+
|
|
198
|
+
def to_dict(self) -> dict[str, Any]:
|
|
199
|
+
"""
|
|
200
|
+
Serializes the component to a dictionary.
|
|
201
|
+
|
|
202
|
+
:returns:
|
|
203
|
+
Dictionary with serialized data.
|
|
204
|
+
"""
|
|
205
|
+
return default_to_dict(
|
|
206
|
+
self,
|
|
207
|
+
labels=self.labels,
|
|
208
|
+
instructions=self.instructions,
|
|
209
|
+
model=self.model,
|
|
210
|
+
api_key=self.api_key,
|
|
211
|
+
api_base_url=self.api_base_url,
|
|
212
|
+
min_confidence=self.min_confidence,
|
|
213
|
+
include_probabilities=self.include_probabilities,
|
|
214
|
+
timeout=self.timeout,
|
|
215
|
+
max_retries=self.max_retries,
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
def _call(self, text: str, images: list[ImageInput] | None) -> dict[str, Any]:
|
|
219
|
+
if not isinstance(text, str):
|
|
220
|
+
msg = "NaceTextRouter expects a str as input."
|
|
221
|
+
raise TypeError(msg)
|
|
222
|
+
return build_call(
|
|
223
|
+
text,
|
|
224
|
+
{_QUESTION_ID: self._question},
|
|
225
|
+
model=self.model,
|
|
226
|
+
images=images,
|
|
227
|
+
openrouter=is_openrouter(resolve_base_url(self.api_base_url)),
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
def _error(self, error: nace_sdk.NaceError, images: list[ImageInput] | None) -> nace_sdk.NaceError:
|
|
231
|
+
return readable_error(
|
|
232
|
+
error, provider_name(resolve_base_url(self.api_base_url)), image_types=image_types(images or [])
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
def _route(self, text: str, response: DrexResponse) -> dict[str, Any]:
|
|
236
|
+
"""
|
|
237
|
+
Picks the output for the text from the Drex response.
|
|
238
|
+
|
|
239
|
+
:param text:
|
|
240
|
+
The routed text.
|
|
241
|
+
:param response:
|
|
242
|
+
The Drex response for the text.
|
|
243
|
+
:returns:
|
|
244
|
+
A dictionary mapping `low_confidence` to the text if `min_confidence` is set and the answer's `confidence`
|
|
245
|
+
is below it, otherwise mapping the picked label to the text. With `include_probabilities`, it also maps
|
|
246
|
+
`probabilities` to the full answer.
|
|
247
|
+
"""
|
|
248
|
+
answer = response.answers[_QUESTION_ID]
|
|
249
|
+
if answer["choice"] not in self._question.criteria:
|
|
250
|
+
msg = (
|
|
251
|
+
f"{provider_name(resolve_base_url(self.api_base_url))} answered {answer['choice']!r}, which isn't one "
|
|
252
|
+
f"of the labels {list(self._question.criteria)}."
|
|
253
|
+
)
|
|
254
|
+
raise ValueError(msg)
|
|
255
|
+
if self.min_confidence is not None and answer["confidence"] < self.min_confidence:
|
|
256
|
+
routed: dict[str, Any] = {_LOW_CONFIDENCE: text}
|
|
257
|
+
else:
|
|
258
|
+
routed = {answer["choice"]: text}
|
|
259
|
+
if self.include_probabilities:
|
|
260
|
+
routed[_PROBABILITIES] = answer
|
|
261
|
+
return routed
|
|
262
|
+
|
|
263
|
+
def run(self, text: str, images: list[ImageInput] | None = None) -> dict[str, Any]:
|
|
264
|
+
"""
|
|
265
|
+
Routes the text to the output of the label the model picks.
|
|
266
|
+
|
|
267
|
+
:param text:
|
|
268
|
+
The text to route.
|
|
269
|
+
:param images:
|
|
270
|
+
Optional images the model reads with the text, referred to as "Image 1", "Image 2" and so on:
|
|
271
|
+
`ImageContent`, `ByteStream`, or files uploaded to Drex with nace-sdk.
|
|
272
|
+
:returns:
|
|
273
|
+
A dictionary with the picked label or `low_confidence` mapped to the input text, and `probabilities`
|
|
274
|
+
mapped to the full answer if `include_probabilities` is `True`.
|
|
275
|
+
:raises TypeError:
|
|
276
|
+
If `text` is not a string.
|
|
277
|
+
:raises ValueError:
|
|
278
|
+
If images are passed to a model that reads text only.
|
|
279
|
+
"""
|
|
280
|
+
call = self._call(text, images)
|
|
281
|
+
self.warm_up()
|
|
282
|
+
assert self._client is not None # noqa: S101 # mypy: the client is created by warm_up above
|
|
283
|
+
|
|
284
|
+
try:
|
|
285
|
+
response = self._client.system_one(**call)
|
|
286
|
+
except nace_sdk.NaceError as error:
|
|
287
|
+
raise self._error(error, images) from error
|
|
288
|
+
return self._route(text=text, response=response)
|
|
289
|
+
|
|
290
|
+
async def run_async(self, text: str, images: list[ImageInput] | None = None) -> dict[str, Any]:
|
|
291
|
+
"""
|
|
292
|
+
Asynchronously routes the text to the output of the label the model picks.
|
|
293
|
+
|
|
294
|
+
This is the asynchronous version of the `run` method with the same parameters and return values.
|
|
295
|
+
|
|
296
|
+
:param text:
|
|
297
|
+
The text to route.
|
|
298
|
+
:param images:
|
|
299
|
+
Optional images the model reads with the text, referred to as "Image 1", "Image 2" and so on.
|
|
300
|
+
:returns:
|
|
301
|
+
A dictionary with the picked label or `low_confidence` mapped to the input text, and `probabilities`
|
|
302
|
+
mapped to the full answer if `include_probabilities` is `True`.
|
|
303
|
+
:raises TypeError:
|
|
304
|
+
If `text` is not a string.
|
|
305
|
+
:raises ValueError:
|
|
306
|
+
If images are passed to a model that reads text only.
|
|
307
|
+
"""
|
|
308
|
+
call = self._call(text, images)
|
|
309
|
+
await self.warm_up_async()
|
|
310
|
+
assert self._async_client is not None # noqa: S101 # mypy: the client is created by warm_up_async above
|
|
311
|
+
|
|
312
|
+
try:
|
|
313
|
+
response = await self._async_client.system_one(**call)
|
|
314
|
+
except nace_sdk.NaceError as error:
|
|
315
|
+
raise self._error(error, images) from error
|
|
316
|
+
return self._route(text=text, response=response)
|
|
File without changes
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: nace-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Haystack integration for Drex decision models by Nace
|
|
5
|
+
Project-URL: Documentation, https://console.nace.ai/docs
|
|
6
|
+
Project-URL: Issues, https://github.com/nace-ai/nace-haystack/issues
|
|
7
|
+
Project-URL: Source, https://github.com/nace-ai/nace-haystack
|
|
8
|
+
Author-email: Nace AI <hello@nace.ai>
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
License-File: LICENSE.txt
|
|
11
|
+
Keywords: Drex,Haystack,Nace,classification,decision model,evaluation,routing
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Requires-Dist: filetype>=1.0.0
|
|
23
|
+
Requires-Dist: haystack-ai>=3.0.0
|
|
24
|
+
Requires-Dist: httpx<1.0,>=0.27
|
|
25
|
+
Requires-Dist: nace-sdk<1.0.0,>=0.3.0
|
|
26
|
+
Requires-Dist: pydantic>=2.0
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# nace-haystack
|
|
30
|
+
|
|
31
|
+
Haystack components for [Drex](https://console.nace.ai/docs), Nace's decision models.
|
|
32
|
+
|
|
33
|
+
Drex models don't generate text. They answer typed questions about a text, JSON or images in one request and return
|
|
34
|
+
calibrated probabilities:
|
|
35
|
+
|
|
36
|
+
- `choice` picks one label from a set of options,
|
|
37
|
+
- `score` rates the input on ordered levels,
|
|
38
|
+
- `noul` gives the probability that a yes/no statement is true.
|
|
39
|
+
|
|
40
|
+
This package brings them to Haystack pipelines:
|
|
41
|
+
|
|
42
|
+
| Component | What it does |
|
|
43
|
+
| --- | --- |
|
|
44
|
+
| `NaceTextRouter` | Routes a text (and optional images) to one output per label, with a `low_confidence` fallback |
|
|
45
|
+
| `NaceDocumentClassifier` | Answers questions about each document and stores the answers in its metadata |
|
|
46
|
+
| `NaceFaithfulnessEvaluator` | Judges whether answers are supported by their contexts, like Haystack's `FaithfulnessEvaluator` |
|
|
47
|
+
| `NaceContextRelevanceEvaluator` | Judges whether retrieved contexts help answer the question, like `ContextRelevanceEvaluator` |
|
|
48
|
+
|
|
49
|
+
The router and classifier mirror deepset's `typesafe-haystack` components for Jev, so a Jev pipeline switches to Drex
|
|
50
|
+
with a few edits (see [Switching from TypeSafe](#switching-from-typesafe)).
|
|
51
|
+
|
|
52
|
+
## Installation
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install nace-haystack
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Create an API key at [console.nace.ai](https://console.nace.ai) and set it as `NACE_API_KEY`.
|
|
59
|
+
|
|
60
|
+
## Usage
|
|
61
|
+
|
|
62
|
+
### Route text
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from haystack_integrations.components.routers.nace import NaceTextRouter
|
|
66
|
+
|
|
67
|
+
router = NaceTextRouter(
|
|
68
|
+
labels={
|
|
69
|
+
"billing": "Payments, invoices, refunds and charges",
|
|
70
|
+
"technical": "Bugs, crashes and errors in the product",
|
|
71
|
+
},
|
|
72
|
+
instructions="Which team should handle this support request?",
|
|
73
|
+
min_confidence=0.6,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
print(router.run(text="I was charged twice for my subscription."))
|
|
77
|
+
# {'billing': 'I was charged twice for my subscription.'}
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Texts whose answer is less confident than `min_confidence` go to the `low_confidence` output, for a fallback such
|
|
81
|
+
as an LLM or a person. With `include_probabilities=True`, a `probabilities` output carries the full answer, so
|
|
82
|
+
thresholds stay in your pipeline.
|
|
83
|
+
|
|
84
|
+
### Classify documents
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from haystack import Document
|
|
88
|
+
|
|
89
|
+
from haystack_integrations.components.classifiers.nace import NaceDocumentClassifier
|
|
90
|
+
|
|
91
|
+
classifier = NaceDocumentClassifier(
|
|
92
|
+
questions={
|
|
93
|
+
"department": {
|
|
94
|
+
"type": "choice",
|
|
95
|
+
"instructions": "Which department should handle this ticket?",
|
|
96
|
+
"criteria": {"billing": None, "technical": None, "sales": None},
|
|
97
|
+
},
|
|
98
|
+
"urgency": {
|
|
99
|
+
"type": "score",
|
|
100
|
+
"instructions": "How urgent is this ticket?",
|
|
101
|
+
"criteria": ["can wait", "this week", "today"],
|
|
102
|
+
},
|
|
103
|
+
"refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"},
|
|
104
|
+
},
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
result = classifier.run(documents=[Document(content="I was charged twice, please refund me today.")])
|
|
108
|
+
print(result["documents"][0].meta["nace"]["department"])
|
|
109
|
+
# {'type': 'choice', 'choice': 'billing', 'confidence': 0.97, 'probabilities': {...}}
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Each document is one request carrying every question, with up to `max_workers` requests (default 3) at once.
|
|
113
|
+
Documents that can't be classified are returned in `failed_documents` with the reason in
|
|
114
|
+
`meta["classification_error"]`.
|
|
115
|
+
|
|
116
|
+
### Evaluate RAG answers
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from haystack_integrations.components.evaluators.nace import NaceFaithfulnessEvaluator
|
|
120
|
+
|
|
121
|
+
evaluator = NaceFaithfulnessEvaluator()
|
|
122
|
+
result = evaluator.run(
|
|
123
|
+
questions=["Who won the race?"],
|
|
124
|
+
contexts=[["Ada won the 2026 city race."]],
|
|
125
|
+
predicted_answers=["Ada won the race."],
|
|
126
|
+
)
|
|
127
|
+
print(result["score"], result["individual_scores"])
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Both evaluators take the same inputs and return the same outputs (`score`, `individual_scores`, `results`, `meta`) as
|
|
131
|
+
Haystack's built-in LLM evaluators. Each item is one Drex request, and its score is the probability that the answer
|
|
132
|
+
is faithful (or that the contexts are relevant).
|
|
133
|
+
|
|
134
|
+
## Models
|
|
135
|
+
|
|
136
|
+
The default model is `drex-max-v1.1` (Drex Max v1.1), which reads text, JSON and images. The default moves to the
|
|
137
|
+
newest Drex model in package releases, and a new model can shift probabilities, so **pin `model=` in production** if
|
|
138
|
+
your pipeline relies on thresholds.
|
|
139
|
+
|
|
140
|
+
`drex-v2.0` reads text only and bills fewer tokens on short texts: for a short email with three questions, we
|
|
141
|
+
measured 74 input tokens on `drex-v2.0` and 165 on `drex-max-v1.1` (2026-10-29). See
|
|
142
|
+
[Drex's limits](https://console.nace.ai/docs/drex/reference/limits) for questions per request and state sizes.
|
|
143
|
+
Any model ID is sent as given, so new Drex models work without a new release of this package.
|
|
144
|
+
|
|
145
|
+
## Images
|
|
146
|
+
|
|
147
|
+
The router and the evaluators take images: `ImageContent`, `ByteStream`, or files uploaded to Drex with nace-sdk
|
|
148
|
+
(`client.documents.upload(...)`). Prompts refer to them as "Image 1", "Image 2" and so on.
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
from haystack.dataclasses import ImageContent
|
|
152
|
+
|
|
153
|
+
router = NaceTextRouter(
|
|
154
|
+
labels={"invoice": "A bill or invoice", "receipt": "A proof of payment", "other": None},
|
|
155
|
+
instructions="What kind of document is Image 1?",
|
|
156
|
+
)
|
|
157
|
+
router.run(text="A customer attached a scan.", images=[ImageContent.from_file_path("scan.jpg")])
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Images too big to send in one request are shrunk automatically, with a logged warning. Drex Max reads images at about
|
|
161
|
+
1.6 megapixels (1,638,400 pixels), and an image is never shrunk below about 1 megapixel: if the text leaves too little
|
|
162
|
+
room for the images, you get a clear error instead of a blurry image. Photos are turned upright using their EXIF
|
|
163
|
+
orientation before shrinking. Shrinking needs `pip install pillow`; to skip it, downscale earlier with
|
|
164
|
+
`ImageFileToImageContent(size=(2048, 2048))` or `PDFToImageContent(size=(2048, 2048))`. PDFs must be converted to
|
|
165
|
+
images first.
|
|
166
|
+
|
|
167
|
+
## Other backends
|
|
168
|
+
|
|
169
|
+
Set `api_base_url` to use Drex through [OpenRouter](https://openrouter.ai) (`https://openrouter.ai/api/v1`, with
|
|
170
|
+
model IDs such as `nace-ai/drex-max`) or another TypeSafe-compatible host. OpenRouter takes inline images only.
|
|
171
|
+
|
|
172
|
+
## Errors and retries
|
|
173
|
+
|
|
174
|
+
Requests that hit a rate limit (429) or an overloaded model (529) are retried, waiting as long as Drex asks. Other
|
|
175
|
+
errors are raised as nace-sdk's error classes with a readable message, for example:
|
|
176
|
+
|
|
177
|
+
```text
|
|
178
|
+
422 from Drex: the request is invalid
|
|
179
|
+
questions.department.criteria: must have at least 2 labels
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
## Switching from TypeSafe
|
|
183
|
+
|
|
184
|
+
The router and classifier take the same parameters and produce the same outputs as `TypeSafeTextRouter` and
|
|
185
|
+
`TypeSafeDocumentClassifier`.
|
|
186
|
+
|
|
187
|
+
In Python code that uses Jev's default model, change the import, the class name and the API key:
|
|
188
|
+
|
|
189
|
+
```diff
|
|
190
|
+
-from haystack_integrations.components.routers.typesafe import TypeSafeTextRouter
|
|
191
|
+
+from haystack_integrations.components.routers.nace import NaceTextRouter
|
|
192
|
+
|
|
193
|
+
-router = TypeSafeTextRouter(labels=..., min_confidence=0.6)
|
|
194
|
+
+router = NaceTextRouter(labels=..., min_confidence=0.6)
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
```diff
|
|
198
|
+
-export TYPESAFE_API_KEY=...
|
|
199
|
+
+export NACE_API_KEY=...
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
The classifier writes answers to `meta["nace"]` by default. If your code reads `meta["typesafe"]` (in a filter or a
|
|
203
|
+
`MetadataRouter` rule, for example), pass `metadata_field="typesafe"` to keep the old key.
|
|
204
|
+
|
|
205
|
+
Questions built with `typesafe_sdk` objects (`Choice`, `Score`, `Noul`) work as they are; you can switch them to
|
|
206
|
+
`nace_sdk`'s classes of the same names when convenient. Where your code catches `TypeSafeError`, catch
|
|
207
|
+
`nace_sdk.NaceError` instead.
|
|
208
|
+
|
|
209
|
+
In a saved YAML pipeline, make three edits per component:
|
|
210
|
+
|
|
211
|
+
```diff
|
|
212
|
+
- type: haystack_integrations.components.routers.typesafe.text_router.TypeSafeTextRouter
|
|
213
|
+
+ type: haystack_integrations.components.routers.nace.text_router.NaceTextRouter
|
|
214
|
+
...
|
|
215
|
+
- - TYPESAFE_API_KEY
|
|
216
|
+
+ - NACE_API_KEY
|
|
217
|
+
...
|
|
218
|
+
- model: jev-latest
|
|
219
|
+
+ model: drex-max-v1.1
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
The classifier's type becomes `haystack_integrations.components.classifiers.nace.document_classifier.NaceDocumentClassifier`.
|
|
223
|
+
A saved pipeline keeps `metadata_field: typesafe`, so answers stay where your filters read them. Model names aren't
|
|
224
|
+
translated: Drex rejects `jev-latest` with a message listing its models.
|
|
225
|
+
|
|
226
|
+
Drex's own guide covers the API side: [Migrate from TypeSafe](https://console.nace.ai/docs/drex/guides/migrate-from-typesafe).
|
|
227
|
+
|
|
228
|
+
## License
|
|
229
|
+
|
|
230
|
+
Apache-2.0
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
haystack_integrations/common/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
haystack_integrations/common/nace/__init__.py,sha256=QolspE1wblU-W4W7Pu_vsjugoVpVLj_cjvWbXxymbkY,87
|
|
3
|
+
haystack_integrations/common/nace/client.py,sha256=ySzvabuUFZ9vSHqLCa59l4Ab6700nH3A856SttBY8Aw,6599
|
|
4
|
+
haystack_integrations/common/nace/concurrency.py,sha256=M1ba-1OBlzwhmYrZbvvH6K1bTOBeumpKQri9kTiw800,2024
|
|
5
|
+
haystack_integrations/common/nace/errors.py,sha256=zujZc6rHx9Z2SBtlA5ICP7geJ-KVu_zoU_x93qHcm1w,5896
|
|
6
|
+
haystack_integrations/common/nace/images.py,sha256=ZjAyj5ISYJOrQpMFHVRmiBcHFyOrt0PmMmwxLyMpVsg,9032
|
|
7
|
+
haystack_integrations/common/nace/request.py,sha256=ISeMV8T35HH0JOnav7Q02QfEELVVB81rd30KPREXDHE,6173
|
|
8
|
+
haystack_integrations/components/classifiers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
haystack_integrations/components/classifiers/nace/__init__.py,sha256=7zsTQdJqrASGK8PPlvNcv8vHQA-NEhief6FTnbqsXTE,182
|
|
10
|
+
haystack_integrations/components/classifiers/nace/document_classifier.py,sha256=2UeMax_D68_Jnojxcxdqt5UbGlV8Of235bjdAt8YAXQ,17582
|
|
11
|
+
haystack_integrations/components/evaluators/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
haystack_integrations/components/evaluators/nace/__init__.py,sha256=kcWH4m89kQzQdF4IolK71lTpU71DYDfOPNwBUfiAYIg,416
|
|
13
|
+
haystack_integrations/components/evaluators/nace/_base.py,sha256=QH1xgZhVusVatLJsz_Jqk1anZkwDuYQwCBkdtu-BaVU,9854
|
|
14
|
+
haystack_integrations/components/evaluators/nace/context_relevance.py,sha256=CpgAQ7MMPyRCPYQDo5v-zDQTrUfdY_w_MmJgmhKaL9k,5528
|
|
15
|
+
haystack_integrations/components/evaluators/nace/faithfulness.py,sha256=4rTX5RuydvjgsolxvL9TYddm86Zu--_-YMsQTXIpuzE,6513
|
|
16
|
+
haystack_integrations/components/routers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
haystack_integrations/components/routers/nace/__init__.py,sha256=s-lbJqrePJgDwjBsPcpLhP08ax8fImuWu14F6B-YsVY,158
|
|
18
|
+
haystack_integrations/components/routers/nace/text_router.py,sha256=N_4Ktpu4b8U3k2T7jQW4mIGbt_Jcql41n1Z7tmkmB5Q,13458
|
|
19
|
+
nace_haystack-0.1.0.dist-info/METADATA,sha256=oETyIzCbc3Hqf3Lzu3INf3qEYgMW5lVRH-PgAS9qpx0,9421
|
|
20
|
+
nace_haystack-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
21
|
+
nace_haystack-0.1.0.dist-info/licenses/LICENSE.txt,sha256=VuvjWkPRc52LhptIM3ddscFplogwRiT9Qx4QgMm7s_8,11345
|
|
22
|
+
nace_haystack-0.1.0.dist-info/RECORD,,
|