mistral-haystack 1.4.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/CHANGELOG.md +23 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/LICENSE.txt +1 -1
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/PKG-INFO +3 -3
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/pyproject.toml +2 -1
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/converters/mistral/ocr_document_converter.py +19 -2
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/embedders/mistral/document_embedder.py +0 -3
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/embedders/mistral/text_embedder.py +0 -3
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/generators/mistral/chat/chat_generator.py +14 -10
- mistral_haystack-2.0.0/tests/conftest.py +11 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/tests/test_mistral_chat_generator.py +31 -97
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/tests/test_mistral_chat_generator_async.py +13 -2
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/tests/test_ocr_document_converter.py +237 -170
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/.gitignore +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/README.md +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/examples/chat_generator_with_structured_output.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/examples/indexing_ocr_pipeline.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/examples/indexing_pipeline.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/examples/streaming_chat_with_rag.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/pydoc/config_docusaurus.yml +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/converters/mistral/__init__.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/converters/py.typed +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/embedders/mistral/__init__.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/embedders/py.typed +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/generators/mistral/__init__.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/generators/mistral/chat/__init__.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/src/haystack_integrations/components/generators/py.typed +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/tests/__init__.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/tests/test_mistral_document_embedder.py +0 -0
- {mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/tests/test_mistral_text_embedder.py +0 -0
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/mistral-v1.5.0] - 2026-07-06
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- Support Haystack 3.0's lazy OpenAI client initialization in OpenAI-inheriting generators (#3536)
|
|
8
|
+
|
|
9
|
+
### 🧪 Testing
|
|
10
|
+
|
|
11
|
+
- Trust test modules under Haystack 3.0's deserialization allowlist (#3537)
|
|
12
|
+
- Make Tool/Agent serialization assertions version-agnostic for Haystack 2.x/3.x (#3533)
|
|
13
|
+
- Guard ToolInvoker imports so chat-generator tests run under Haystack 3.0 (#3535)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## [integrations/mistral-v1.4.0] - 2026-06-05
|
|
17
|
+
|
|
18
|
+
### 🚀 Features
|
|
19
|
+
|
|
20
|
+
- Mistral - accept str as ChatGenerator input (#3397)
|
|
21
|
+
|
|
22
|
+
### 🌀 Miscellaneous
|
|
23
|
+
|
|
24
|
+
- Exclude mistralai 2.4.6 from dependencies (#3305)
|
|
25
|
+
|
|
3
26
|
## [integrations/mistral-v1.3.0] - 2026-05-14
|
|
4
27
|
|
|
5
28
|
### 🚀 Features
|
|
@@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work.
|
|
|
58
58
|
|
|
59
59
|
To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
|
|
60
60
|
|
|
61
|
-
Copyright
|
|
61
|
+
Copyright 2023-present deepset GmbH
|
|
62
62
|
|
|
63
63
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
64
64
|
you may not use this file except in compliance with the License.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: mistral-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/mistral#readme
|
|
5
5
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
6
6
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/mistral
|
|
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
18
18
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
19
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
20
|
Requires-Python: >=3.10
|
|
21
|
-
Requires-Dist: haystack-ai>=
|
|
21
|
+
Requires-Dist: haystack-ai>=3.0.0
|
|
22
22
|
Requires-Dist: mistralai!=2.4.6,>=2.0.0
|
|
23
23
|
Description-Content-Type: text/markdown
|
|
24
24
|
|
|
@@ -24,7 +24,7 @@ classifiers = [
|
|
|
24
24
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
25
25
|
]
|
|
26
26
|
# mistralai 2.4.6 is excluded because of security issue https://github.com/mistralai/client-python/security/advisories/GHSA-wx9m-wx4f-4cmg
|
|
27
|
-
dependencies = ["haystack-ai>=
|
|
27
|
+
dependencies = ["haystack-ai>=3.0.0", "mistralai>=2.0.0,!=2.4.6"]
|
|
28
28
|
|
|
29
29
|
[project.urls]
|
|
30
30
|
Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/mistral#readme"
|
|
@@ -130,6 +130,7 @@ ignore = [
|
|
|
130
130
|
"PLR0912",
|
|
131
131
|
"PLR0913",
|
|
132
132
|
"PLR0915",
|
|
133
|
+
"PLR0917",
|
|
133
134
|
# Misc
|
|
134
135
|
"B008",
|
|
135
136
|
"S101",
|
|
@@ -149,8 +149,18 @@ class MistralOCRDocumentConverter:
|
|
|
149
149
|
self.image_min_size = image_min_size
|
|
150
150
|
self.cleanup_uploaded_files = cleanup_uploaded_files
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
|
|
152
|
+
self.client: Mistral | None = None
|
|
153
|
+
|
|
154
|
+
def warm_up(self) -> None:
|
|
155
|
+
"""Initialize the Mistral client."""
|
|
156
|
+
if self.client is None:
|
|
157
|
+
self.client = Mistral(api_key=self.api_key.resolve_value())
|
|
158
|
+
|
|
159
|
+
def close(self) -> None:
|
|
160
|
+
"""Close the Mistral client."""
|
|
161
|
+
if self.client is not None:
|
|
162
|
+
self.client.__exit__(None, None, None)
|
|
163
|
+
self.client = None
|
|
154
164
|
|
|
155
165
|
def to_dict(self) -> dict[str, Any]:
|
|
156
166
|
"""
|
|
@@ -229,6 +239,9 @@ class MistralOCRDocumentConverter:
|
|
|
229
239
|
List of dictionaries containing raw OCR responses from Mistral API (one per source).
|
|
230
240
|
Each response includes per-page details, images, annotations, and usage info.
|
|
231
241
|
"""
|
|
242
|
+
self.warm_up()
|
|
243
|
+
assert self.client is not None
|
|
244
|
+
|
|
232
245
|
# Convert Pydantic models to Mistral ResponseFormat schemas
|
|
233
246
|
bbox_annotation_format = (
|
|
234
247
|
response_format_from_pydantic_model(bbox_annotation_schema) if bbox_annotation_schema else None
|
|
@@ -294,6 +307,7 @@ class MistralOCRDocumentConverter:
|
|
|
294
307
|
A tuple of (Document|None, raw_response_dict|None, uploaded_file_id|None).
|
|
295
308
|
Returns (None, None, uploaded_file_id) if processing fails but file was uploaded.
|
|
296
309
|
"""
|
|
310
|
+
assert self.client is not None
|
|
297
311
|
uploaded_file_id = None
|
|
298
312
|
try:
|
|
299
313
|
chunk = self._convert_source_to_chunk(source)
|
|
@@ -333,6 +347,7 @@ class MistralOCRDocumentConverter:
|
|
|
333
347
|
if not self.cleanup_uploaded_files or not file_ids:
|
|
334
348
|
return
|
|
335
349
|
|
|
350
|
+
assert self.client is not None
|
|
336
351
|
for file_id in file_ids:
|
|
337
352
|
try:
|
|
338
353
|
self.client.files.delete(file_id=file_id)
|
|
@@ -363,6 +378,8 @@ class MistralOCRDocumentConverter:
|
|
|
363
378
|
if isinstance(source, (DocumentURLChunk, FileChunk, ImageURLChunk)):
|
|
364
379
|
return source
|
|
365
380
|
|
|
381
|
+
assert self.client is not None
|
|
382
|
+
|
|
366
383
|
# Convert str/Path/ByteStream to ByteStream
|
|
367
384
|
bytestream = get_bytestream_from_source(source=source)
|
|
368
385
|
|
|
@@ -105,9 +105,6 @@ class MistralDocumentEmbedder(OpenAIDocumentEmbedder):
|
|
|
105
105
|
max_retries=max_retries,
|
|
106
106
|
http_client_kwargs=http_client_kwargs,
|
|
107
107
|
)
|
|
108
|
-
# We add these since they were only added in Haystack 2.14.0
|
|
109
|
-
self.timeout = timeout
|
|
110
|
-
self.max_retries = max_retries
|
|
111
108
|
|
|
112
109
|
def to_dict(self) -> dict[str, Any]:
|
|
113
110
|
"""
|
|
@@ -87,9 +87,6 @@ class MistralTextEmbedder(OpenAITextEmbedder):
|
|
|
87
87
|
max_retries=max_retries,
|
|
88
88
|
http_client_kwargs=http_client_kwargs,
|
|
89
89
|
)
|
|
90
|
-
# We add these since they were only added in Haystack 2.14.0
|
|
91
|
-
self.timeout = timeout
|
|
92
|
-
self.max_retries = max_retries
|
|
93
90
|
|
|
94
91
|
def to_dict(self) -> dict[str, Any]:
|
|
95
92
|
"""
|
|
@@ -343,8 +343,9 @@ class MistralChatGenerator(OpenAIChatGenerator):
|
|
|
343
343
|
:param streaming_callback:
|
|
344
344
|
A callback function that is called when a new token is received from the stream.
|
|
345
345
|
:param generation_kwargs:
|
|
346
|
-
Additional keyword arguments for text generation. These
|
|
347
|
-
|
|
346
|
+
Additional keyword arguments for text generation. These are merged per key with the
|
|
347
|
+
`generation_kwargs` passed at initialization: keys provided here take precedence, keys set only
|
|
348
|
+
at initialization are kept.
|
|
348
349
|
For details on Mistral API parameters, see
|
|
349
350
|
[Mistral docs](https://docs.mistral.ai/api/).
|
|
350
351
|
:param tools: A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
|
|
@@ -357,8 +358,8 @@ class MistralChatGenerator(OpenAIChatGenerator):
|
|
|
357
358
|
- `replies`: A list containing the generated responses as ChatMessage instances.
|
|
358
359
|
"""
|
|
359
360
|
messages = _normalize_messages(messages)
|
|
360
|
-
|
|
361
|
-
|
|
361
|
+
self.warm_up()
|
|
362
|
+
assert self.client is not None
|
|
362
363
|
|
|
363
364
|
if len(messages) == 0:
|
|
364
365
|
return {"replies": []}
|
|
@@ -417,7 +418,9 @@ class MistralChatGenerator(OpenAIChatGenerator):
|
|
|
417
418
|
A callback function that is called when a new token is received from the stream.
|
|
418
419
|
Must be a coroutine.
|
|
419
420
|
:param generation_kwargs:
|
|
420
|
-
Additional keyword arguments for text generation.
|
|
421
|
+
Additional keyword arguments for text generation. These are merged per key with the
|
|
422
|
+
`generation_kwargs` passed at initialization: keys provided here take precedence, keys set only
|
|
423
|
+
at initialization are kept.
|
|
421
424
|
:param tools: A list of Tool and/or Toolset objects, or a single Toolset.
|
|
422
425
|
:param tools_strict:
|
|
423
426
|
Whether to enable strict schema adherence for tool calls.
|
|
@@ -427,8 +430,8 @@ class MistralChatGenerator(OpenAIChatGenerator):
|
|
|
427
430
|
- `replies`: A list containing the generated responses as ChatMessage instances.
|
|
428
431
|
"""
|
|
429
432
|
messages = _normalize_messages(messages)
|
|
430
|
-
|
|
431
|
-
|
|
433
|
+
await self.warm_up_async()
|
|
434
|
+
assert self.async_client is not None
|
|
432
435
|
|
|
433
436
|
if len(messages) == 0:
|
|
434
437
|
return {"replies": []}
|
|
@@ -459,9 +462,10 @@ class MistralChatGenerator(OpenAIChatGenerator):
|
|
|
459
462
|
chat_completion = await getattr(self.async_client.chat.completions, openai_endpoint)(**api_args)
|
|
460
463
|
completions = await self._handle_async_stream_response(chat_completion, streaming_callback)
|
|
461
464
|
else:
|
|
462
|
-
raw_response = await getattr(
|
|
463
|
-
|
|
464
|
-
|
|
465
|
+
raw_response = await getattr(
|
|
466
|
+
self.async_client.chat.completions.with_raw_response,
|
|
467
|
+
openai_endpoint,
|
|
468
|
+
)(**api_args)
|
|
465
469
|
completions = _convert_mistral_response_to_chat_messages(raw_response.text)
|
|
466
470
|
|
|
467
471
|
for message in completions:
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@pytest.fixture(autouse=True)
|
|
5
|
+
def allow_deserialization_of_test_modules(monkeypatch):
|
|
6
|
+
"""
|
|
7
|
+
haystack-ai >= 3.0 refuses to deserialize classes and callables from modules outside its
|
|
8
|
+
trusted-module allowlist. Tools and callbacks defined in the test modules live outside that
|
|
9
|
+
allowlist, so trust them explicitly; haystack-ai 2.x ignores this environment variable.
|
|
10
|
+
"""
|
|
11
|
+
monkeypatch.setenv("HAYSTACK_DESERIALIZATION_ALLOWLIST", "tests,test_*")
|
|
@@ -6,7 +6,6 @@ from unittest.mock import ANY, AsyncMock, patch
|
|
|
6
6
|
import pytest
|
|
7
7
|
from haystack import Pipeline
|
|
8
8
|
from haystack.components.generators.utils import print_streaming_chunk
|
|
9
|
-
from haystack.components.tools import ToolInvoker
|
|
10
9
|
from haystack.dataclasses import (
|
|
11
10
|
ChatMessage,
|
|
12
11
|
ChatRole,
|
|
@@ -133,16 +132,24 @@ class TestMistralChatGenerator:
|
|
|
133
132
|
def test_init_default(self, monkeypatch):
|
|
134
133
|
monkeypatch.setenv("MISTRAL_API_KEY", "test-api-key")
|
|
135
134
|
component = MistralChatGenerator()
|
|
136
|
-
assert component.
|
|
135
|
+
assert component.api_key.resolve_value() == "test-api-key"
|
|
137
136
|
assert component.model == "mistral-small-latest"
|
|
138
137
|
assert component.api_base_url == "https://api.mistral.ai/v1"
|
|
139
138
|
assert component.streaming_callback is None
|
|
140
139
|
assert not component.generation_kwargs
|
|
141
140
|
|
|
142
|
-
def
|
|
141
|
+
def test_warm_up(self, monkeypatch):
|
|
142
|
+
monkeypatch.setenv("MISTRAL_API_KEY", "test-api-key")
|
|
143
|
+
component = MistralChatGenerator()
|
|
144
|
+
component.warm_up()
|
|
145
|
+
assert component.client.api_key == "test-api-key"
|
|
146
|
+
assert component.client is not None
|
|
147
|
+
|
|
148
|
+
def test_warm_up_fail_wo_api_key(self, monkeypatch):
|
|
143
149
|
monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
|
|
144
150
|
with pytest.raises(ValueError, match=r"None of the .* environment variables are set"):
|
|
145
|
-
MistralChatGenerator()
|
|
151
|
+
component = MistralChatGenerator()
|
|
152
|
+
component.warm_up()
|
|
146
153
|
|
|
147
154
|
def test_init_with_parameters(self):
|
|
148
155
|
component = MistralChatGenerator(
|
|
@@ -152,7 +159,7 @@ class TestMistralChatGenerator:
|
|
|
152
159
|
api_base_url="test-base-url",
|
|
153
160
|
generation_kwargs={"max_tokens": 10, "some_test_param": "test-params"},
|
|
154
161
|
)
|
|
155
|
-
assert component.
|
|
162
|
+
assert component.api_key.resolve_value() == "test-api-key"
|
|
156
163
|
assert component.model == "mistral-small"
|
|
157
164
|
assert component.streaming_callback is print_streaming_chunk
|
|
158
165
|
assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"}
|
|
@@ -256,21 +263,6 @@ class TestMistralChatGenerator:
|
|
|
256
263
|
assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"}
|
|
257
264
|
assert component.api_key == Secret.from_env_var("MISTRAL_API_KEY")
|
|
258
265
|
|
|
259
|
-
def test_from_dict_fail_wo_env_var(self, monkeypatch):
|
|
260
|
-
monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
|
|
261
|
-
data = {
|
|
262
|
-
"type": "haystack_integrations.components.generators.mistral.chat.chat_generator.MistralChatGenerator",
|
|
263
|
-
"init_parameters": {
|
|
264
|
-
"api_key": {"env_vars": ["MISTRAL_API_KEY"], "strict": True, "type": "env_var"},
|
|
265
|
-
"model": "mistral-small",
|
|
266
|
-
"api_base_url": "test-base-url",
|
|
267
|
-
"streaming_callback": "haystack.components.generators.utils.print_streaming_chunk",
|
|
268
|
-
"generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"},
|
|
269
|
-
},
|
|
270
|
-
}
|
|
271
|
-
with pytest.raises(ValueError, match=r"None of the .* environment variables are set"):
|
|
272
|
-
MistralChatGenerator.from_dict(data)
|
|
273
|
-
|
|
274
266
|
def test_init_with_mixed_tools(self, monkeypatch):
|
|
275
267
|
monkeypatch.setenv("MISTRAL_API_KEY", "test-api-key")
|
|
276
268
|
|
|
@@ -293,6 +285,8 @@ class TestMistralChatGenerator:
|
|
|
293
285
|
assert component.tools == [weather_tool, toolset]
|
|
294
286
|
|
|
295
287
|
def test_handle_stream_response(self):
|
|
288
|
+
streamed_usage = CompletionUsage(completion_tokens=35, prompt_tokens=77, total_tokens=112)
|
|
289
|
+
|
|
296
290
|
mistral_chunks = [
|
|
297
291
|
ChatCompletionChunk(
|
|
298
292
|
id="76535283139540de943bc2036121d4c5",
|
|
@@ -328,11 +322,7 @@ class TestMistralChatGenerator:
|
|
|
328
322
|
created=1750076261,
|
|
329
323
|
model="mistral-small-latest",
|
|
330
324
|
object="chat.completion.chunk",
|
|
331
|
-
usage=
|
|
332
|
-
completion_tokens=35,
|
|
333
|
-
prompt_tokens=77,
|
|
334
|
-
total_tokens=112,
|
|
335
|
-
),
|
|
325
|
+
usage=streamed_usage,
|
|
336
326
|
),
|
|
337
327
|
]
|
|
338
328
|
|
|
@@ -377,13 +367,7 @@ class TestMistralChatGenerator:
|
|
|
377
367
|
],
|
|
378
368
|
"finish_reason": "tool_calls",
|
|
379
369
|
"received_at": ANY,
|
|
380
|
-
"usage":
|
|
381
|
-
"completion_tokens": 35,
|
|
382
|
-
"prompt_tokens": 77,
|
|
383
|
-
"total_tokens": 112,
|
|
384
|
-
"completion_tokens_details": None,
|
|
385
|
-
"prompt_tokens_details": None,
|
|
386
|
-
},
|
|
370
|
+
"usage": streamed_usage.model_dump(),
|
|
387
371
|
},
|
|
388
372
|
component_info=ComponentInfo(
|
|
389
373
|
type="haystack_integrations.components.generators.mistral.chat.chat_generator.MistralChatGenerator",
|
|
@@ -415,13 +399,7 @@ class TestMistralChatGenerator:
|
|
|
415
399
|
assert result.meta["finish_reason"] == "tool_calls"
|
|
416
400
|
assert result.meta["index"] == 0
|
|
417
401
|
assert result.meta["completion_start_time"] is not None
|
|
418
|
-
assert result.meta["usage"] ==
|
|
419
|
-
"completion_tokens": 35,
|
|
420
|
-
"prompt_tokens": 77,
|
|
421
|
-
"total_tokens": 112,
|
|
422
|
-
"completion_tokens_details": None,
|
|
423
|
-
"prompt_tokens_details": None,
|
|
424
|
-
}
|
|
402
|
+
assert result.meta["usage"] == streamed_usage.model_dump()
|
|
425
403
|
|
|
426
404
|
def test_run(self, chat_messages, mock_chat_completion, monkeypatch): # noqa: ARG002
|
|
427
405
|
monkeypatch.setenv("MISTRAL_API_KEY", "fake-api-key")
|
|
@@ -501,6 +479,15 @@ class TestMistralChatGenerator:
|
|
|
501
479
|
assert len(response["replies"]) == 1
|
|
502
480
|
assert [isinstance(reply, ChatMessage) for reply in response["replies"]]
|
|
503
481
|
|
|
482
|
+
def test_run_with_generation_kwargs(self, chat_messages, mock_chat_completion, monkeypatch):
|
|
483
|
+
monkeypatch.setenv("MISTRAL_API_KEY", "fake-api-key")
|
|
484
|
+
component = MistralChatGenerator(generation_kwargs={"max_tokens": 10, "temperature": 0.5})
|
|
485
|
+
component.run(chat_messages, generation_kwargs={"temperature": 0.9})
|
|
486
|
+
|
|
487
|
+
_, kwargs = mock_chat_completion.call_args
|
|
488
|
+
assert kwargs["max_tokens"] == 10
|
|
489
|
+
assert kwargs["temperature"] == 0.9
|
|
490
|
+
|
|
504
491
|
@pytest.mark.skipif(
|
|
505
492
|
not os.environ.get("MISTRAL_API_KEY", None),
|
|
506
493
|
reason="Export an env var called MISTRAL_API_KEY containing the Mistral API key to run this test.",
|
|
@@ -727,35 +714,6 @@ class TestMistralChatGenerator:
|
|
|
727
714
|
assert sorted(arguments, key=lambda x: x["city"]) == [{"city": "Berlin"}, {"city": "Paris"}]
|
|
728
715
|
assert tool_message.meta["finish_reason"] == "tool_calls"
|
|
729
716
|
|
|
730
|
-
@pytest.mark.skipif(
|
|
731
|
-
not os.environ.get("MISTRAL_API_KEY", None),
|
|
732
|
-
reason="Export an env var called MISTRAL_API_KEY containing the OpenAI API key to run this test.",
|
|
733
|
-
)
|
|
734
|
-
@pytest.mark.integration
|
|
735
|
-
def test_pipeline_with_mistral_chat_generator(self, tools):
|
|
736
|
-
"""
|
|
737
|
-
Test that the MistralChatGenerator component can be used in a pipeline
|
|
738
|
-
"""
|
|
739
|
-
pipeline = Pipeline()
|
|
740
|
-
pipeline.add_component("generator", MistralChatGenerator(tools=tools))
|
|
741
|
-
pipeline.add_component("tool_invoker", ToolInvoker(tools=tools))
|
|
742
|
-
|
|
743
|
-
pipeline.connect("generator", "tool_invoker")
|
|
744
|
-
|
|
745
|
-
results = pipeline.run(
|
|
746
|
-
data={
|
|
747
|
-
"generator": {
|
|
748
|
-
"messages": [ChatMessage.from_user("What's the weather like in Paris?")],
|
|
749
|
-
"generation_kwargs": {"tool_choice": "any"},
|
|
750
|
-
}
|
|
751
|
-
}
|
|
752
|
-
)
|
|
753
|
-
|
|
754
|
-
assert (
|
|
755
|
-
"The weather in Paris is sunny and 32°C"
|
|
756
|
-
== results["tool_invoker"]["tool_messages"][0].tool_call_result.result
|
|
757
|
-
)
|
|
758
|
-
|
|
759
717
|
def test_serde_in_pipeline(self, monkeypatch):
|
|
760
718
|
"""
|
|
761
719
|
Test serialization/deserialization of MistralChatGenerator in a Pipeline,
|
|
@@ -786,6 +744,11 @@ class TestMistralChatGenerator:
|
|
|
786
744
|
|
|
787
745
|
# Get pipeline dictionary and verify its structure
|
|
788
746
|
pipeline_dict = pipeline.to_dict()
|
|
747
|
+
|
|
748
|
+
# the Tool serialization format is owned by haystack-ai and varies across its versions; the
|
|
749
|
+
# dumps/loads round-trip below covers the tools, so exclude them from the pinned-dict comparison
|
|
750
|
+
tools_entries = pipeline_dict["components"]["generator"]["init_parameters"].pop("tools")
|
|
751
|
+
assert len(tools_entries) == 1
|
|
789
752
|
expected_dict = {
|
|
790
753
|
"metadata": {},
|
|
791
754
|
"max_runs_per_component": 100,
|
|
@@ -799,17 +762,6 @@ class TestMistralChatGenerator:
|
|
|
799
762
|
"streaming_callback": "haystack.components.generators.utils.print_streaming_chunk",
|
|
800
763
|
"api_base_url": "https://api.mistral.ai/v1",
|
|
801
764
|
"generation_kwargs": {"temperature": 0.7},
|
|
802
|
-
"tools": [
|
|
803
|
-
{
|
|
804
|
-
"type": "haystack.tools.tool.Tool",
|
|
805
|
-
"data": {
|
|
806
|
-
"name": "weather",
|
|
807
|
-
"description": "useful to determine the weather in a given location",
|
|
808
|
-
"parameters": {"city": {"type": "string"}},
|
|
809
|
-
"function": "tests.test_mistral_chat_generator.weather",
|
|
810
|
-
},
|
|
811
|
-
}
|
|
812
|
-
],
|
|
813
765
|
"http_client_kwargs": None,
|
|
814
766
|
"timeout": None,
|
|
815
767
|
"max_retries": None,
|
|
@@ -819,24 +771,6 @@ class TestMistralChatGenerator:
|
|
|
819
771
|
"connections": [],
|
|
820
772
|
}
|
|
821
773
|
|
|
822
|
-
if not hasattr(pipeline, "_connection_type_validation"):
|
|
823
|
-
expected_dict.pop("connection_type_validation")
|
|
824
|
-
|
|
825
|
-
# add outputs_to_string, inputs_from_state and outputs_to_state tool parameters for compatibility with
|
|
826
|
-
# haystack-ai>=2.12.0
|
|
827
|
-
if hasattr(tool, "outputs_to_string"):
|
|
828
|
-
expected_dict["components"]["generator"]["init_parameters"]["tools"][0]["data"]["outputs_to_string"] = (
|
|
829
|
-
tool.outputs_to_string
|
|
830
|
-
)
|
|
831
|
-
if hasattr(tool, "inputs_from_state"):
|
|
832
|
-
expected_dict["components"]["generator"]["init_parameters"]["tools"][0]["data"]["inputs_from_state"] = (
|
|
833
|
-
tool.inputs_from_state
|
|
834
|
-
)
|
|
835
|
-
if hasattr(tool, "outputs_to_state"):
|
|
836
|
-
expected_dict["components"]["generator"]["init_parameters"]["tools"][0]["data"]["outputs_to_state"] = (
|
|
837
|
-
tool.outputs_to_state
|
|
838
|
-
)
|
|
839
|
-
|
|
840
774
|
assert pipeline_dict == expected_dict
|
|
841
775
|
|
|
842
776
|
# Test YAML serialization/deserialization
|
{mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/tests/test_mistral_chat_generator_async.py
RENAMED
|
@@ -78,14 +78,15 @@ def mock_async_chat_completion():
|
|
|
78
78
|
|
|
79
79
|
|
|
80
80
|
class TestMistralChatGeneratorAsync:
|
|
81
|
-
|
|
81
|
+
@pytest.mark.asyncio
|
|
82
|
+
async def test_warm_up_async(self, monkeypatch):
|
|
82
83
|
monkeypatch.setenv("MISTRAL_API_KEY", "test-api-key")
|
|
83
84
|
component = MistralChatGenerator()
|
|
85
|
+
await component.warm_up_async()
|
|
84
86
|
|
|
85
87
|
assert isinstance(component.async_client, AsyncOpenAI)
|
|
86
88
|
assert component.async_client.api_key == "test-api-key"
|
|
87
89
|
assert component.async_client.base_url == "https://api.mistral.ai/v1/"
|
|
88
|
-
assert not component.generation_kwargs
|
|
89
90
|
|
|
90
91
|
@pytest.mark.asyncio
|
|
91
92
|
async def test_run_async(self, chat_messages, mock_async_chat_completion, monkeypatch): # noqa: ARG002
|
|
@@ -118,6 +119,16 @@ class TestMistralChatGeneratorAsync:
|
|
|
118
119
|
assert len(response["replies"]) == 1
|
|
119
120
|
assert [isinstance(reply, ChatMessage) for reply in response["replies"]]
|
|
120
121
|
|
|
122
|
+
@pytest.mark.asyncio
|
|
123
|
+
async def test_run_async_with_generation_kwargs(self, chat_messages, mock_async_chat_completion, monkeypatch):
|
|
124
|
+
monkeypatch.setenv("MISTRAL_API_KEY", "fake-api-key")
|
|
125
|
+
component = MistralChatGenerator(generation_kwargs={"max_tokens": 10, "temperature": 0.5})
|
|
126
|
+
await component.run_async(chat_messages, generation_kwargs={"temperature": 0.9})
|
|
127
|
+
|
|
128
|
+
_, kwargs = mock_async_chat_completion.call_args
|
|
129
|
+
assert kwargs["max_tokens"] == 10
|
|
130
|
+
assert kwargs["temperature"] == 0.9
|
|
131
|
+
|
|
121
132
|
@pytest.mark.skipif(
|
|
122
133
|
not os.environ.get("MISTRAL_API_KEY", None),
|
|
123
134
|
reason="Export an env var called MISTRAL_API_KEY containing the OpenAI API key to run this test.",
|
|
@@ -16,11 +16,7 @@ from haystack_integrations.components.converters.mistral import (
|
|
|
16
16
|
)
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
class
|
|
20
|
-
CLASS_TYPE = (
|
|
21
|
-
"haystack_integrations.components.converters.mistral.ocr_document_converter.MistralOCRDocumentConverter"
|
|
22
|
-
)
|
|
23
|
-
|
|
19
|
+
class TestInitialization:
|
|
24
20
|
def test_supported_models(self) -> None:
|
|
25
21
|
"""SUPPORTED_MODELS is a non-empty list of strings."""
|
|
26
22
|
models = MistralOCRDocumentConverter.SUPPORTED_MODELS
|
|
@@ -38,6 +34,7 @@ class TestMistralOCRDocumentConverter:
|
|
|
38
34
|
assert converter.pages is None
|
|
39
35
|
assert converter.image_limit is None
|
|
40
36
|
assert converter.image_min_size is None
|
|
37
|
+
assert converter.client is None
|
|
41
38
|
|
|
42
39
|
def test_init_with_all_optional_parameters(self):
|
|
43
40
|
converter = MistralOCRDocumentConverter(
|
|
@@ -56,6 +53,12 @@ class TestMistralOCRDocumentConverter:
|
|
|
56
53
|
assert converter.image_limit == 10
|
|
57
54
|
assert converter.image_min_size == 100
|
|
58
55
|
|
|
56
|
+
|
|
57
|
+
class TestSerialization:
|
|
58
|
+
CLASS_TYPE = (
|
|
59
|
+
"haystack_integrations.components.converters.mistral.ocr_document_converter.MistralOCRDocumentConverter"
|
|
60
|
+
)
|
|
61
|
+
|
|
59
62
|
def test_to_dict(self, monkeypatch):
|
|
60
63
|
monkeypatch.setenv("MISTRAL_API_KEY", "test-api-key")
|
|
61
64
|
converter = MistralOCRDocumentConverter()
|
|
@@ -164,6 +167,52 @@ class TestMistralOCRDocumentConverter:
|
|
|
164
167
|
assert converter.image_min_size == 100
|
|
165
168
|
assert converter.cleanup_uploaded_files is False
|
|
166
169
|
|
|
170
|
+
|
|
171
|
+
class TestComponentLifecycle:
|
|
172
|
+
@patch("haystack_integrations.components.converters.mistral.ocr_document_converter.Mistral")
|
|
173
|
+
def test_key_resolved_at_warm_up_not_init(self, mock_mistral, monkeypatch):
|
|
174
|
+
monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
|
|
175
|
+
|
|
176
|
+
converter = MistralOCRDocumentConverter()
|
|
177
|
+
|
|
178
|
+
assert converter.client is None
|
|
179
|
+
with pytest.raises(ValueError, match=r"None of the .* environment variables are set"):
|
|
180
|
+
converter.warm_up()
|
|
181
|
+
mock_mistral.assert_not_called()
|
|
182
|
+
|
|
183
|
+
@patch("haystack_integrations.components.converters.mistral.ocr_document_converter.Mistral")
|
|
184
|
+
def test_sync_lifecycle(self, mock_mistral):
|
|
185
|
+
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"))
|
|
186
|
+
client = mock_mistral.return_value
|
|
187
|
+
|
|
188
|
+
converter.warm_up()
|
|
189
|
+
assert converter.client is client
|
|
190
|
+
|
|
191
|
+
converter.close()
|
|
192
|
+
client.__exit__.assert_called_once_with(None, None, None)
|
|
193
|
+
assert converter.client is None
|
|
194
|
+
|
|
195
|
+
converter.warm_up()
|
|
196
|
+
assert mock_mistral.call_count == 2
|
|
197
|
+
|
|
198
|
+
@patch("haystack_integrations.components.converters.mistral.ocr_document_converter.Mistral")
|
|
199
|
+
def test_warm_up_is_idempotent(self, mock_mistral):
|
|
200
|
+
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"))
|
|
201
|
+
|
|
202
|
+
converter.warm_up()
|
|
203
|
+
converter.warm_up()
|
|
204
|
+
|
|
205
|
+
mock_mistral.assert_called_once_with(api_key="test-api-key")
|
|
206
|
+
|
|
207
|
+
def test_close_is_safe_without_warm_up(self):
|
|
208
|
+
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"))
|
|
209
|
+
|
|
210
|
+
converter.close()
|
|
211
|
+
|
|
212
|
+
assert converter.client is None
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class TestRun:
|
|
167
216
|
@pytest.fixture
|
|
168
217
|
def mock_ocr_response(self):
|
|
169
218
|
"""Create a mock OCR response"""
|
|
@@ -216,16 +265,18 @@ class TestMistralOCRDocumentConverter:
|
|
|
216
265
|
"""Test processing with remote chunk types (DocumentURLChunk, FileChunk, ImageURLChunk)"""
|
|
217
266
|
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"))
|
|
218
267
|
|
|
219
|
-
|
|
220
|
-
|
|
268
|
+
converter.client = MagicMock()
|
|
269
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
270
|
+
|
|
271
|
+
result = converter.run(sources=[source])
|
|
221
272
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
273
|
+
assert len(result["documents"]) == 1
|
|
274
|
+
assert isinstance(result["documents"][0], Document)
|
|
275
|
+
assert result["documents"][0].content == "# Sample Document\n\nThis is page 1."
|
|
276
|
+
# Metadata assertions apply to all chunk types
|
|
277
|
+
if isinstance(source, DocumentURLChunk):
|
|
278
|
+
assert result["documents"][0].meta["source_page_count"] == 1
|
|
279
|
+
assert result["documents"][0].meta["source_total_images"] == 0
|
|
229
280
|
|
|
230
281
|
@pytest.mark.parametrize(
|
|
231
282
|
"source_type",
|
|
@@ -251,16 +302,17 @@ class TestMistralOCRDocumentConverter:
|
|
|
251
302
|
mock_uploaded_file = MagicMock()
|
|
252
303
|
mock_uploaded_file.id = "uploaded-file-123"
|
|
253
304
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
305
|
+
converter.client = MagicMock()
|
|
306
|
+
converter.client.files.upload.return_value = mock_uploaded_file
|
|
307
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
308
|
+
|
|
309
|
+
result = converter.run(sources=[source])
|
|
258
310
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
311
|
+
assert len(result["documents"]) == 1
|
|
312
|
+
assert isinstance(result["documents"][0], Document)
|
|
313
|
+
# Verify file was uploaded for local sources
|
|
314
|
+
if source_type == "file_path_str":
|
|
315
|
+
converter.client.files.upload.assert_called_once()
|
|
264
316
|
|
|
265
317
|
def test_run_with_multiple_sources(self, mock_ocr_response, tmp_path):
|
|
266
318
|
"""Test processing with multiple mixed source types"""
|
|
@@ -273,18 +325,19 @@ class TestMistralOCRDocumentConverter:
|
|
|
273
325
|
mock_uploaded_file = MagicMock()
|
|
274
326
|
mock_uploaded_file.id = "uploaded-file-123"
|
|
275
327
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
sources = [
|
|
280
|
-
DocumentURLChunk(document_url="https://example.com/doc.pdf"),
|
|
281
|
-
FileChunk(file_id="file-123"),
|
|
282
|
-
str(test_file),
|
|
283
|
-
]
|
|
284
|
-
result = converter.run(sources=sources)
|
|
328
|
+
converter.client = MagicMock()
|
|
329
|
+
converter.client.files.upload.return_value = mock_uploaded_file
|
|
330
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
285
331
|
|
|
286
|
-
|
|
287
|
-
|
|
332
|
+
sources = [
|
|
333
|
+
DocumentURLChunk(document_url="https://example.com/doc.pdf"),
|
|
334
|
+
FileChunk(file_id="file-123"),
|
|
335
|
+
str(test_file),
|
|
336
|
+
]
|
|
337
|
+
result = converter.run(sources=sources)
|
|
338
|
+
|
|
339
|
+
assert len(result["documents"]) == 3
|
|
340
|
+
assert all(isinstance(doc, Document) for doc in result["documents"])
|
|
288
341
|
|
|
289
342
|
def test_run_with_bbox_annotations(self):
|
|
290
343
|
"""Test processing with bbox annotation schema"""
|
|
@@ -311,13 +364,15 @@ class TestMistralOCRDocumentConverter:
|
|
|
311
364
|
"document_annotation": None,
|
|
312
365
|
}
|
|
313
366
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
367
|
+
converter.client = MagicMock()
|
|
368
|
+
converter.client.ocr.process.return_value = mock_response
|
|
369
|
+
|
|
370
|
+
sources = [DocumentURLChunk(document_url="https://example.com/doc.pdf")]
|
|
371
|
+
result = converter.run(sources=sources, bbox_annotation_schema=ImageAnnotation)
|
|
317
372
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
373
|
+
assert len(result["documents"]) == 1
|
|
374
|
+
# Check that image annotation was enriched in content
|
|
375
|
+
assert "Image Annotation:" in result["documents"][0].content
|
|
321
376
|
|
|
322
377
|
def test_run_with_document_annotations(self):
|
|
323
378
|
"""Test processing with document annotation schema"""
|
|
@@ -341,14 +396,16 @@ class TestMistralOCRDocumentConverter:
|
|
|
341
396
|
"document_annotation": '{"language": "en", "topics": ["AI", "ML"]}',
|
|
342
397
|
}
|
|
343
398
|
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
result = converter.run(sources=sources, document_annotation_schema=DocumentAnnotation)
|
|
399
|
+
converter.client = MagicMock()
|
|
400
|
+
converter.client.ocr.process.return_value = mock_response
|
|
347
401
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
402
|
+
sources = [DocumentURLChunk(document_url="https://example.com/doc.pdf")]
|
|
403
|
+
result = converter.run(sources=sources, document_annotation_schema=DocumentAnnotation)
|
|
404
|
+
|
|
405
|
+
assert len(result["documents"]) == 1
|
|
406
|
+
# Check that document annotations are in metadata
|
|
407
|
+
assert result["documents"][0].meta["source_language"] == "en"
|
|
408
|
+
assert result["documents"][0].meta["source_topics"] == ["AI", "ML"]
|
|
352
409
|
|
|
353
410
|
def test_run_with_both_annotations(self):
|
|
354
411
|
"""Test processing with both bbox and document annotation schemas"""
|
|
@@ -377,114 +434,123 @@ class TestMistralOCRDocumentConverter:
|
|
|
377
434
|
"document_annotation": '{"language": "en"}',
|
|
378
435
|
}
|
|
379
436
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
result = converter.run(
|
|
383
|
-
sources=sources,
|
|
384
|
-
bbox_annotation_schema=ImageAnnotation,
|
|
385
|
-
document_annotation_schema=DocumentAnnotation,
|
|
386
|
-
)
|
|
437
|
+
converter.client = MagicMock()
|
|
438
|
+
converter.client.ocr.process.return_value = mock_response
|
|
387
439
|
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
440
|
+
sources = [DocumentURLChunk(document_url="https://example.com/doc.pdf")]
|
|
441
|
+
result = converter.run(
|
|
442
|
+
sources=sources,
|
|
443
|
+
bbox_annotation_schema=ImageAnnotation,
|
|
444
|
+
document_annotation_schema=DocumentAnnotation,
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
assert len(result["documents"]) == 1
|
|
448
|
+
assert "Image Annotation:" in result["documents"][0].content
|
|
449
|
+
assert result["documents"][0].meta["source_language"] == "en"
|
|
391
450
|
|
|
392
451
|
def test_run_with_pages_parameter(self, mock_ocr_response):
|
|
393
452
|
"""Test that pages parameter is passed to API"""
|
|
394
453
|
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"), pages=[0, 1])
|
|
395
454
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
455
|
+
converter.client = MagicMock()
|
|
456
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
457
|
+
|
|
458
|
+
sources = [DocumentURLChunk(document_url="https://example.com/doc.pdf")]
|
|
459
|
+
result = converter.run(sources=sources)
|
|
399
460
|
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
461
|
+
# Verify pages parameter was passed
|
|
462
|
+
call_args = converter.client.ocr.process.call_args
|
|
463
|
+
assert call_args.kwargs["pages"] == [0, 1]
|
|
464
|
+
assert len(result["documents"]) == 1
|
|
404
465
|
|
|
405
466
|
def test_run_handles_api_error(self, mock_ocr_response):
|
|
406
467
|
"""Test error handling when API fails"""
|
|
407
468
|
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"))
|
|
408
469
|
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
result = converter.run(sources=sources)
|
|
470
|
+
converter.client = MagicMock()
|
|
471
|
+
converter.client.ocr.process.side_effect = [
|
|
472
|
+
mock_ocr_response,
|
|
473
|
+
Exception("API Error"),
|
|
474
|
+
mock_ocr_response,
|
|
475
|
+
]
|
|
476
|
+
|
|
477
|
+
sources = [
|
|
478
|
+
DocumentURLChunk(document_url="https://example.com/doc1.pdf"),
|
|
479
|
+
DocumentURLChunk(document_url="https://example.com/doc2.pdf"),
|
|
480
|
+
DocumentURLChunk(document_url="https://example.com/doc3.pdf"),
|
|
481
|
+
]
|
|
482
|
+
result = converter.run(sources=sources)
|
|
423
483
|
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
484
|
+
# Should only return 2 documents (failed source skipped)
|
|
485
|
+
assert len(result["documents"]) == 2
|
|
486
|
+
assert len(result["raw_mistral_response"]) == 2
|
|
427
487
|
|
|
428
488
|
def test_run_with_meta_single_dict(self, mock_ocr_response):
|
|
429
489
|
"""Test that meta parameter with single dict is applied to all documents"""
|
|
430
490
|
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"))
|
|
431
491
|
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
492
|
+
converter.client = MagicMock()
|
|
493
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
494
|
+
|
|
495
|
+
sources = [
|
|
496
|
+
DocumentURLChunk(document_url="https://example.com/doc1.pdf"),
|
|
497
|
+
DocumentURLChunk(document_url="https://example.com/doc2.pdf"),
|
|
498
|
+
]
|
|
499
|
+
result = converter.run(sources=sources, meta={"department": "engineering", "year": 2024})
|
|
438
500
|
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
501
|
+
assert len(result["documents"]) == 2
|
|
502
|
+
# Both documents should have the same metadata
|
|
503
|
+
for doc in result["documents"]:
|
|
504
|
+
assert doc.meta["department"] == "engineering"
|
|
505
|
+
assert doc.meta["year"] == 2024
|
|
506
|
+
# Automatic metadata should still be present
|
|
507
|
+
assert "source_page_count" in doc.meta
|
|
508
|
+
assert "source_total_images" in doc.meta
|
|
447
509
|
|
|
448
510
|
def test_run_with_meta_list_of_dicts(self, mock_ocr_response):
|
|
449
511
|
"""Test that meta parameter with list of dicts applies each dict to corresponding document"""
|
|
450
512
|
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"))
|
|
451
513
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
DocumentURLChunk(document_url="https://example.com/doc1.pdf"),
|
|
455
|
-
DocumentURLChunk(document_url="https://example.com/doc2.pdf"),
|
|
456
|
-
]
|
|
457
|
-
result = converter.run(
|
|
458
|
-
sources=sources,
|
|
459
|
-
meta=[
|
|
460
|
-
{"author": "Alice", "category": "report"},
|
|
461
|
-
{"author": "Bob", "category": "invoice"},
|
|
462
|
-
],
|
|
463
|
-
)
|
|
514
|
+
converter.client = MagicMock()
|
|
515
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
464
516
|
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
517
|
+
sources = [
|
|
518
|
+
DocumentURLChunk(document_url="https://example.com/doc1.pdf"),
|
|
519
|
+
DocumentURLChunk(document_url="https://example.com/doc2.pdf"),
|
|
520
|
+
]
|
|
521
|
+
result = converter.run(
|
|
522
|
+
sources=sources,
|
|
523
|
+
meta=[
|
|
524
|
+
{"author": "Alice", "category": "report"},
|
|
525
|
+
{"author": "Bob", "category": "invoice"},
|
|
526
|
+
],
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
assert len(result["documents"]) == 2
|
|
530
|
+
# First document
|
|
531
|
+
assert result["documents"][0].meta["author"] == "Alice"
|
|
532
|
+
assert result["documents"][0].meta["category"] == "report"
|
|
533
|
+
# Second document
|
|
534
|
+
assert result["documents"][1].meta["author"] == "Bob"
|
|
535
|
+
assert result["documents"][1].meta["category"] == "invoice"
|
|
536
|
+
# Automatic metadata should still be present in both
|
|
537
|
+
assert "source_page_count" in result["documents"][0].meta
|
|
538
|
+
assert "source_page_count" in result["documents"][1].meta
|
|
475
539
|
|
|
476
540
|
def test_run_with_meta_none(self, mock_ocr_response):
|
|
477
541
|
"""Test that meta parameter with None works correctly"""
|
|
478
542
|
converter = MistralOCRDocumentConverter(api_key=Secret.from_token("test-api-key"))
|
|
479
543
|
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
544
|
+
converter.client = MagicMock()
|
|
545
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
546
|
+
|
|
547
|
+
sources = [DocumentURLChunk(document_url="https://example.com/doc.pdf")]
|
|
548
|
+
result = converter.run(sources=sources, meta=None)
|
|
483
549
|
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
550
|
+
assert len(result["documents"]) == 1
|
|
551
|
+
# Only automatic metadata should be present
|
|
552
|
+
assert "source_page_count" in result["documents"][0].meta
|
|
553
|
+
assert "source_total_images" in result["documents"][0].meta
|
|
488
554
|
|
|
489
555
|
def test_run_with_meta_list_length_mismatch(self):
|
|
490
556
|
"""Test that meta parameter with list length mismatch raises ValueError"""
|
|
@@ -548,16 +614,17 @@ class TestMistralOCRDocumentConverter:
|
|
|
548
614
|
mock_uploaded_file = MagicMock()
|
|
549
615
|
mock_uploaded_file.id = "uploaded-file-123"
|
|
550
616
|
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
617
|
+
converter.client = MagicMock()
|
|
618
|
+
converter.client.files.upload.return_value = mock_uploaded_file
|
|
619
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
620
|
+
|
|
621
|
+
sources = [str(test_file)]
|
|
622
|
+
result = converter.run(sources=sources)
|
|
556
623
|
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
624
|
+
# Verify file was uploaded but NOT deleted
|
|
625
|
+
assert len(result["documents"]) == 1
|
|
626
|
+
converter.client.files.upload.assert_called_once()
|
|
627
|
+
converter.client.files.delete.assert_not_called()
|
|
561
628
|
|
|
562
629
|
def test_run_cleanup_happens_on_ocr_failure(self, tmp_path):
|
|
563
630
|
"""Test that cleanup happens even when OCR processing fails"""
|
|
@@ -570,16 +637,17 @@ class TestMistralOCRDocumentConverter:
|
|
|
570
637
|
mock_uploaded_file = MagicMock()
|
|
571
638
|
mock_uploaded_file.id = "uploaded-file-123"
|
|
572
639
|
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
640
|
+
converter.client = MagicMock()
|
|
641
|
+
converter.client.files.upload.return_value = mock_uploaded_file
|
|
642
|
+
converter.client.ocr.process.side_effect = Exception("OCR failed")
|
|
643
|
+
|
|
644
|
+
sources = [str(test_file)]
|
|
645
|
+
result = converter.run(sources=sources)
|
|
578
646
|
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
647
|
+
# Verify no documents returned due to failure
|
|
648
|
+
assert len(result["documents"]) == 0
|
|
649
|
+
# But file should still be deleted
|
|
650
|
+
converter.client.files.delete.assert_called_once_with(file_id="uploaded-file-123")
|
|
583
651
|
|
|
584
652
|
def test_run_cleanup_failure_does_not_break_flow(self, mock_ocr_response, tmp_path):
|
|
585
653
|
"""Test that cleanup failures don't break the main flow"""
|
|
@@ -592,20 +660,18 @@ class TestMistralOCRDocumentConverter:
|
|
|
592
660
|
mock_uploaded_file = MagicMock()
|
|
593
661
|
mock_uploaded_file.id = "uploaded-file-123"
|
|
594
662
|
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
assert len(result["documents"]) == 1
|
|
608
|
-
assert isinstance(result["documents"][0], Document)
|
|
663
|
+
converter.client = MagicMock()
|
|
664
|
+
converter.client.files.upload.return_value = mock_uploaded_file
|
|
665
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
666
|
+
converter.client.files.delete.side_effect = Exception("Delete failed")
|
|
667
|
+
|
|
668
|
+
sources = [str(test_file)]
|
|
669
|
+
# Should not raise an exception
|
|
670
|
+
result = converter.run(sources=sources)
|
|
671
|
+
|
|
672
|
+
# Verify document was still processed successfully
|
|
673
|
+
assert len(result["documents"]) == 1
|
|
674
|
+
assert isinstance(result["documents"][0], Document)
|
|
609
675
|
|
|
610
676
|
def test_run_mixed_sources_only_uploaded_files_deleted(self, mock_ocr_response, tmp_path):
|
|
611
677
|
"""Test that only uploaded files are deleted, not user-provided chunks"""
|
|
@@ -618,20 +684,21 @@ class TestMistralOCRDocumentConverter:
|
|
|
618
684
|
mock_uploaded_file = MagicMock()
|
|
619
685
|
mock_uploaded_file.id = "uploaded-file-123"
|
|
620
686
|
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
687
|
+
converter.client = MagicMock()
|
|
688
|
+
converter.client.files.upload.return_value = mock_uploaded_file
|
|
689
|
+
converter.client.ocr.process.return_value = mock_ocr_response
|
|
690
|
+
|
|
691
|
+
sources = [
|
|
692
|
+
str(test_file), # This will be uploaded
|
|
693
|
+
FileChunk(file_id="user-file-123"), # User-provided
|
|
694
|
+
DocumentURLChunk(document_url="https://example.com/doc.pdf"), # URL
|
|
695
|
+
]
|
|
696
|
+
result = converter.run(sources=sources)
|
|
697
|
+
|
|
698
|
+
# Verify all sources processed
|
|
699
|
+
assert len(result["documents"]) == 3
|
|
700
|
+
# Only the uploaded file should be deleted
|
|
701
|
+
converter.client.files.delete.assert_called_once_with(file_id="uploaded-file-123")
|
|
635
702
|
|
|
636
703
|
@pytest.mark.skipif(
|
|
637
704
|
not os.environ.get("MISTRAL_API_KEY"),
|
|
File without changes
|
|
File without changes
|
{mistral_haystack-1.4.0 → mistral_haystack-2.0.0}/examples/chat_generator_with_structured_output.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|