langchain-google-genai 2.1.11__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of langchain-google-genai might be problematic. Click here for more details.
- langchain_google_genai/__init__.py +3 -3
- langchain_google_genai/_common.py +29 -17
- langchain_google_genai/_compat.py +286 -0
- langchain_google_genai/_function_utils.py +77 -59
- langchain_google_genai/_genai_extension.py +60 -27
- langchain_google_genai/_image_utils.py +10 -9
- langchain_google_genai/chat_models.py +803 -297
- langchain_google_genai/embeddings.py +17 -24
- langchain_google_genai/genai_aqa.py +29 -18
- langchain_google_genai/google_vector_store.py +45 -25
- langchain_google_genai/llms.py +8 -7
- {langchain_google_genai-2.1.11.dist-info → langchain_google_genai-3.0.0.dist-info}/METADATA +43 -30
- langchain_google_genai-3.0.0.dist-info/RECORD +18 -0
- langchain_google_genai-2.1.11.dist-info/RECORD +0 -17
- {langchain_google_genai-2.1.11.dist-info → langchain_google_genai-3.0.0.dist-info}/WHEEL +0 -0
- {langchain_google_genai-2.1.11.dist-info → langchain_google_genai-3.0.0.dist-info}/entry_points.txt +0 -0
- {langchain_google_genai-2.1.11.dist-info → langchain_google_genai-3.0.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -10,30 +10,24 @@ import time
|
|
|
10
10
|
import uuid
|
|
11
11
|
import warnings
|
|
12
12
|
import wave
|
|
13
|
+
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
|
13
14
|
from difflib import get_close_matches
|
|
14
15
|
from operator import itemgetter
|
|
15
16
|
from typing import (
|
|
16
17
|
Any,
|
|
17
|
-
AsyncIterator,
|
|
18
18
|
Callable,
|
|
19
19
|
Dict,
|
|
20
|
-
Iterator,
|
|
21
20
|
List,
|
|
22
21
|
Literal,
|
|
23
|
-
Mapping,
|
|
24
22
|
Optional,
|
|
25
|
-
Sequence,
|
|
26
23
|
Tuple,
|
|
27
24
|
Type,
|
|
28
25
|
Union,
|
|
29
26
|
cast,
|
|
30
27
|
)
|
|
31
28
|
|
|
32
|
-
import filetype # type: ignore[import]
|
|
33
|
-
import
|
|
34
|
-
|
|
35
|
-
# TODO: remove ignore once the Google package is published with types
|
|
36
|
-
import proto # type: ignore[import]
|
|
29
|
+
import filetype # type: ignore[import-untyped]
|
|
30
|
+
import proto # type: ignore[import-untyped]
|
|
37
31
|
from google.ai.generativelanguage_v1beta import (
|
|
38
32
|
GenerativeServiceAsyncClient as v1betaGenerativeServiceAsyncClient,
|
|
39
33
|
)
|
|
@@ -57,12 +51,24 @@ from google.ai.generativelanguage_v1beta.types import (
|
|
|
57
51
|
VideoMetadata,
|
|
58
52
|
)
|
|
59
53
|
from google.ai.generativelanguage_v1beta.types import Tool as GoogleTool
|
|
54
|
+
from google.api_core.exceptions import (
|
|
55
|
+
FailedPrecondition,
|
|
56
|
+
GoogleAPIError,
|
|
57
|
+
InvalidArgument,
|
|
58
|
+
ResourceExhausted,
|
|
59
|
+
ServiceUnavailable,
|
|
60
|
+
)
|
|
61
|
+
from google.protobuf.json_format import MessageToDict
|
|
60
62
|
from langchain_core.callbacks.manager import (
|
|
61
63
|
AsyncCallbackManagerForLLMRun,
|
|
62
64
|
CallbackManagerForLLMRun,
|
|
63
65
|
)
|
|
64
|
-
from langchain_core.language_models import
|
|
65
|
-
|
|
66
|
+
from langchain_core.language_models import (
|
|
67
|
+
LangSmithParams,
|
|
68
|
+
LanguageModelInput,
|
|
69
|
+
is_openai_data_block,
|
|
70
|
+
)
|
|
71
|
+
from langchain_core.language_models.chat_models import BaseChatModel
|
|
66
72
|
from langchain_core.messages import (
|
|
67
73
|
AIMessage,
|
|
68
74
|
AIMessageChunk,
|
|
@@ -73,6 +79,7 @@ from langchain_core.messages import (
|
|
|
73
79
|
ToolMessage,
|
|
74
80
|
is_data_content_block,
|
|
75
81
|
)
|
|
82
|
+
from langchain_core.messages import content as types
|
|
76
83
|
from langchain_core.messages.ai import UsageMetadata, add_usage, subtract_usage
|
|
77
84
|
from langchain_core.messages.tool import invalid_tool_call, tool_call, tool_call_chunk
|
|
78
85
|
from langchain_core.output_parsers import JsonOutputParser, PydanticOutputParser
|
|
@@ -109,6 +116,9 @@ from langchain_google_genai._common import (
|
|
|
109
116
|
_BaseGoogleGenerativeAI,
|
|
110
117
|
get_client_info,
|
|
111
118
|
)
|
|
119
|
+
from langchain_google_genai._compat import (
|
|
120
|
+
_convert_from_v1_to_generativelanguage_v1beta,
|
|
121
|
+
)
|
|
112
122
|
from langchain_google_genai._function_utils import (
|
|
113
123
|
_dict_to_gapic_schema,
|
|
114
124
|
_tool_choice_to_tool_config,
|
|
@@ -139,12 +149,11 @@ _FunctionDeclarationType = Union[
|
|
|
139
149
|
|
|
140
150
|
|
|
141
151
|
class ChatGoogleGenerativeAIError(GoogleGenerativeAIError):
|
|
142
|
-
"""
|
|
143
|
-
Custom exception class for errors associated with the `Google GenAI` API.
|
|
152
|
+
"""Custom exception class for errors associated with the `Google GenAI` API.
|
|
144
153
|
|
|
145
|
-
This exception is raised when there are specific issues related to the
|
|
146
|
-
|
|
147
|
-
|
|
154
|
+
This exception is raised when there are specific issues related to the Google genai
|
|
155
|
+
API usage in the ChatGoogleGenerativeAI class, such as unsupported message types or
|
|
156
|
+
roles.
|
|
148
157
|
"""
|
|
149
158
|
|
|
150
159
|
|
|
@@ -154,12 +163,11 @@ def _create_retry_decorator(
|
|
|
154
163
|
wait_exponential_min: float = 1.0,
|
|
155
164
|
wait_exponential_max: float = 60.0,
|
|
156
165
|
) -> Callable[[Any], Any]:
|
|
157
|
-
"""
|
|
158
|
-
Creates and returns a preconfigured tenacity retry decorator.
|
|
166
|
+
"""Creates and returns a preconfigured tenacity retry decorator.
|
|
159
167
|
|
|
160
|
-
The retry decorator is configured to handle specific Google API exceptions
|
|
161
|
-
|
|
162
|
-
|
|
168
|
+
The retry decorator is configured to handle specific Google API exceptions such as
|
|
169
|
+
ResourceExhausted and ServiceUnavailable. It uses an exponential backoff strategy
|
|
170
|
+
for retries.
|
|
163
171
|
|
|
164
172
|
Returns:
|
|
165
173
|
Callable[[Any], Any]: A retry decorator configured for handling specific
|
|
@@ -174,21 +182,20 @@ def _create_retry_decorator(
|
|
|
174
182
|
max=wait_exponential_max,
|
|
175
183
|
),
|
|
176
184
|
retry=(
|
|
177
|
-
retry_if_exception_type(
|
|
178
|
-
| retry_if_exception_type(
|
|
179
|
-
| retry_if_exception_type(
|
|
185
|
+
retry_if_exception_type(ResourceExhausted)
|
|
186
|
+
| retry_if_exception_type(ServiceUnavailable)
|
|
187
|
+
| retry_if_exception_type(GoogleAPIError)
|
|
180
188
|
),
|
|
181
189
|
before_sleep=before_sleep_log(logger, logging.WARNING),
|
|
182
190
|
)
|
|
183
191
|
|
|
184
192
|
|
|
185
193
|
def _chat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
|
|
186
|
-
"""
|
|
187
|
-
Executes a chat generation method with retry logic using tenacity.
|
|
194
|
+
"""Executes a chat generation method with retry logic using tenacity.
|
|
188
195
|
|
|
189
|
-
This function is a wrapper that applies a retry mechanism to a provided
|
|
190
|
-
|
|
191
|
-
|
|
196
|
+
This function is a wrapper that applies a retry mechanism to a provided chat
|
|
197
|
+
generation function. It is useful for handling intermittent issues like network
|
|
198
|
+
errors or temporary service unavailability.
|
|
192
199
|
|
|
193
200
|
Args:
|
|
194
201
|
generation_method (Callable): The chat generation method to be executed.
|
|
@@ -208,7 +215,7 @@ def _chat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
|
|
|
208
215
|
def _chat_with_retry(**kwargs: Any) -> Any:
|
|
209
216
|
try:
|
|
210
217
|
return generation_method(**kwargs)
|
|
211
|
-
except
|
|
218
|
+
except FailedPrecondition as exc:
|
|
212
219
|
if "location is not supported" in exc.message:
|
|
213
220
|
error_msg = (
|
|
214
221
|
"Your location is not supported by google-generativeai "
|
|
@@ -217,19 +224,18 @@ def _chat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
|
|
|
217
224
|
)
|
|
218
225
|
raise ValueError(error_msg)
|
|
219
226
|
|
|
220
|
-
except
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
except google.api_core.exceptions.ResourceExhausted as e:
|
|
227
|
+
except InvalidArgument as e:
|
|
228
|
+
msg = f"Invalid argument provided to Gemini: {e}"
|
|
229
|
+
raise ChatGoogleGenerativeAIError(msg) from e
|
|
230
|
+
except ResourceExhausted as e:
|
|
225
231
|
# Handle quota-exceeded error with recommended retry delay
|
|
226
|
-
if hasattr(e, "retry_after") and e
|
|
232
|
+
if hasattr(e, "retry_after") and getattr(e, "retry_after", 0) < kwargs.get(
|
|
227
233
|
"wait_exponential_max", 60.0
|
|
228
234
|
):
|
|
229
|
-
time.sleep(e
|
|
230
|
-
raise
|
|
231
|
-
except Exception
|
|
232
|
-
raise
|
|
235
|
+
time.sleep(getattr(e, "retry_after"))
|
|
236
|
+
raise
|
|
237
|
+
except Exception:
|
|
238
|
+
raise
|
|
233
239
|
|
|
234
240
|
params = (
|
|
235
241
|
{k: v for k, v in kwargs.items() if k in _allowed_params_prediction_service}
|
|
@@ -242,12 +248,11 @@ def _chat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
|
|
|
242
248
|
|
|
243
249
|
|
|
244
250
|
async def _achat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
|
|
245
|
-
"""
|
|
246
|
-
Executes a chat generation method with retry logic using tenacity.
|
|
251
|
+
"""Executes a chat generation method with retry logic using tenacity.
|
|
247
252
|
|
|
248
|
-
This function is a wrapper that applies a retry mechanism to a provided
|
|
249
|
-
|
|
250
|
-
|
|
253
|
+
This function is a wrapper that applies a retry mechanism to a provided chat
|
|
254
|
+
generation function. It is useful for handling intermittent issues like network
|
|
255
|
+
errors or temporary service unavailability.
|
|
251
256
|
|
|
252
257
|
Args:
|
|
253
258
|
generation_method (Callable): The chat generation method to be executed.
|
|
@@ -256,8 +261,12 @@ async def _achat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
|
|
|
256
261
|
Returns:
|
|
257
262
|
Any: The result from the chat generation method.
|
|
258
263
|
"""
|
|
259
|
-
retry_decorator = _create_retry_decorator(
|
|
260
|
-
|
|
264
|
+
retry_decorator = _create_retry_decorator(
|
|
265
|
+
max_retries=kwargs.get("max_retries", 6),
|
|
266
|
+
wait_exponential_multiplier=kwargs.get("wait_exponential_multiplier", 2.0),
|
|
267
|
+
wait_exponential_min=kwargs.get("wait_exponential_min", 1.0),
|
|
268
|
+
wait_exponential_max=kwargs.get("wait_exponential_max", 60.0),
|
|
269
|
+
)
|
|
261
270
|
|
|
262
271
|
@retry_decorator
|
|
263
272
|
async def _achat_with_retry(**kwargs: Any) -> Any:
|
|
@@ -265,11 +274,17 @@ async def _achat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
|
|
|
265
274
|
return await generation_method(**kwargs)
|
|
266
275
|
except InvalidArgument as e:
|
|
267
276
|
# Do not retry for these errors.
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
277
|
+
msg = f"Invalid argument provided to Gemini: {e}"
|
|
278
|
+
raise ChatGoogleGenerativeAIError(msg) from e
|
|
279
|
+
except ResourceExhausted as e:
|
|
280
|
+
# Handle quota-exceeded error with recommended retry delay
|
|
281
|
+
if hasattr(e, "retry_after") and getattr(e, "retry_after", 0) < kwargs.get(
|
|
282
|
+
"wait_exponential_max", 60.0
|
|
283
|
+
):
|
|
284
|
+
time.sleep(getattr(e, "retry_after"))
|
|
285
|
+
raise
|
|
286
|
+
except Exception:
|
|
287
|
+
raise
|
|
273
288
|
|
|
274
289
|
params = (
|
|
275
290
|
{k: v for k, v in kwargs.items() if k in _allowed_params_prediction_service}
|
|
@@ -281,55 +296,78 @@ async def _achat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
|
|
|
281
296
|
return await _achat_with_retry(**params)
|
|
282
297
|
|
|
283
298
|
|
|
284
|
-
def _is_lc_content_block(part: dict) -> bool:
|
|
285
|
-
return "type" in part
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
def _is_openai_image_block(block: dict) -> bool:
|
|
289
|
-
"""Check if the block contains image data in OpenAI Chat Completions format."""
|
|
290
|
-
if block.get("type") == "image_url":
|
|
291
|
-
if (
|
|
292
|
-
(set(block.keys()) <= {"type", "image_url", "detail"})
|
|
293
|
-
and (image_url := block.get("image_url"))
|
|
294
|
-
and isinstance(image_url, dict)
|
|
295
|
-
):
|
|
296
|
-
url = image_url.get("url")
|
|
297
|
-
if isinstance(url, str):
|
|
298
|
-
return True
|
|
299
|
-
else:
|
|
300
|
-
return False
|
|
301
|
-
|
|
302
|
-
return False
|
|
303
|
-
|
|
304
|
-
|
|
305
299
|
def _convert_to_parts(
|
|
306
300
|
raw_content: Union[str, Sequence[Union[str, dict]]],
|
|
307
301
|
) -> List[Part]:
|
|
308
|
-
"""Converts
|
|
309
|
-
|
|
302
|
+
"""Converts LangChain message content into generativelanguage_v1beta parts.
|
|
303
|
+
|
|
304
|
+
Used when preparing Human, System and AI messages for sending to the API.
|
|
305
|
+
|
|
306
|
+
Handles both legacy (pre-v1) dict-based content blocks and v1 ContentBlock objects.
|
|
307
|
+
"""
|
|
310
308
|
content = [raw_content] if isinstance(raw_content, str) else raw_content
|
|
311
309
|
image_loader = ImageBytesLoader()
|
|
310
|
+
|
|
311
|
+
parts = []
|
|
312
|
+
# Iterate over each item in the content list, constructing a list of Parts
|
|
312
313
|
for part in content:
|
|
313
314
|
if isinstance(part, str):
|
|
314
315
|
parts.append(Part(text=part))
|
|
315
316
|
elif isinstance(part, Mapping):
|
|
316
|
-
if
|
|
317
|
+
if "type" in part:
|
|
317
318
|
if part["type"] == "text":
|
|
318
|
-
|
|
319
|
+
# Either old dict-style CC text block or new TextContentBlock
|
|
320
|
+
# Check if there's a signature attached to this text block
|
|
321
|
+
thought_sig = None
|
|
322
|
+
if "extras" in part and isinstance(part["extras"], dict):
|
|
323
|
+
sig = part["extras"].get("signature")
|
|
324
|
+
if sig and isinstance(sig, str):
|
|
325
|
+
# Decode base64-encoded signature back to bytes
|
|
326
|
+
thought_sig = base64.b64decode(sig)
|
|
327
|
+
if thought_sig:
|
|
328
|
+
parts.append(
|
|
329
|
+
Part(text=part["text"], thought_signature=thought_sig)
|
|
330
|
+
)
|
|
331
|
+
else:
|
|
332
|
+
parts.append(Part(text=part["text"]))
|
|
319
333
|
elif is_data_content_block(part):
|
|
320
|
-
|
|
334
|
+
# Handle both legacy LC blocks (with `source_type`) and blocks >= v1
|
|
335
|
+
|
|
336
|
+
if "source_type" in part:
|
|
337
|
+
# Catch legacy v0 formats
|
|
338
|
+
# Safe since v1 content blocks don't have `source_type` key
|
|
339
|
+
if part["source_type"] == "url":
|
|
340
|
+
bytes_ = image_loader._bytes_from_url(part["url"])
|
|
341
|
+
elif part["source_type"] == "base64":
|
|
342
|
+
bytes_ = base64.b64decode(part["data"])
|
|
343
|
+
else:
|
|
344
|
+
# Unable to support IDContentBlock
|
|
345
|
+
msg = "source_type must be url or base64."
|
|
346
|
+
raise ValueError(msg)
|
|
347
|
+
elif "url" in part:
|
|
348
|
+
# v1 multimodal block w/ URL
|
|
321
349
|
bytes_ = image_loader._bytes_from_url(part["url"])
|
|
322
|
-
elif
|
|
323
|
-
|
|
350
|
+
elif "base64" in part:
|
|
351
|
+
# v1 multimodal block w/ base64
|
|
352
|
+
bytes_ = base64.b64decode(part["base64"])
|
|
324
353
|
else:
|
|
325
|
-
|
|
354
|
+
msg = (
|
|
355
|
+
"Data content block must contain 'url', 'base64', or "
|
|
356
|
+
"'data' field."
|
|
357
|
+
)
|
|
358
|
+
raise ValueError(msg)
|
|
326
359
|
inline_data: dict = {"data": bytes_}
|
|
327
360
|
if "mime_type" in part:
|
|
328
361
|
inline_data["mime_type"] = part["mime_type"]
|
|
329
362
|
else:
|
|
330
|
-
|
|
363
|
+
# Guess mime type based on data field if not provided
|
|
364
|
+
source = cast(
|
|
365
|
+
"str",
|
|
366
|
+
part.get("url") or part.get("base64") or part.get("data"),
|
|
367
|
+
)
|
|
331
368
|
mime_type, _ = mimetypes.guess_type(source)
|
|
332
369
|
if not mime_type:
|
|
370
|
+
# Last resort - try to guess based on file bytes
|
|
333
371
|
kind = filetype.guess(bytes_)
|
|
334
372
|
if kind:
|
|
335
373
|
mime_type = kind.mime
|
|
@@ -337,56 +375,127 @@ def _convert_to_parts(
|
|
|
337
375
|
inline_data["mime_type"] = mime_type
|
|
338
376
|
parts.append(Part(inline_data=inline_data))
|
|
339
377
|
elif part["type"] == "image_url":
|
|
378
|
+
# Chat Completions image format
|
|
340
379
|
img_url = part["image_url"]
|
|
341
380
|
if isinstance(img_url, dict):
|
|
342
381
|
if "url" not in img_url:
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
)
|
|
382
|
+
msg = f"Unrecognized message image format: {img_url}"
|
|
383
|
+
raise ValueError(msg)
|
|
346
384
|
img_url = img_url["url"]
|
|
347
385
|
parts.append(image_loader.load_part(img_url))
|
|
348
|
-
# Handle media type like LangChain.js
|
|
349
|
-
# https://github.com/langchain-ai/langchainjs/blob/e536593e2585f1dd7b0afc187de4d07cb40689ba/libs/langchain-google-common/src/utils/gemini.ts#L93-L106
|
|
350
386
|
elif part["type"] == "media":
|
|
387
|
+
# Handle `media` following pattern established in LangChain.js
|
|
388
|
+
# https://github.com/langchain-ai/langchainjs/blob/e536593e2585f1dd7b0afc187de4d07cb40689ba/libs/langchain-google-common/src/utils/gemini.ts#L93-L106
|
|
351
389
|
if "mime_type" not in part:
|
|
352
|
-
|
|
390
|
+
msg = f"Missing mime_type in media part: {part}"
|
|
391
|
+
raise ValueError(msg)
|
|
353
392
|
mime_type = part["mime_type"]
|
|
354
393
|
media_part = Part()
|
|
355
394
|
|
|
356
395
|
if "data" in part:
|
|
396
|
+
# Embedded media
|
|
357
397
|
media_part.inline_data = Blob(
|
|
358
398
|
data=part["data"], mime_type=mime_type
|
|
359
399
|
)
|
|
360
400
|
elif "file_uri" in part:
|
|
401
|
+
# Referenced files (e.g. stored in GCS)
|
|
361
402
|
media_part.file_data = FileData(
|
|
362
403
|
file_uri=part["file_uri"], mime_type=mime_type
|
|
363
404
|
)
|
|
364
405
|
else:
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
)
|
|
406
|
+
msg = f"Media part must have either data or file_uri: {part}"
|
|
407
|
+
raise ValueError(msg)
|
|
368
408
|
if "video_metadata" in part:
|
|
369
409
|
metadata = VideoMetadata(part["video_metadata"])
|
|
370
410
|
media_part.video_metadata = metadata
|
|
371
411
|
parts.append(media_part)
|
|
412
|
+
elif part["type"] == "function_call_signature":
|
|
413
|
+
# Signature for function_call Part - skip it here as it should be
|
|
414
|
+
# attached to the actual function_call Part
|
|
415
|
+
# This is handled separately in the history parsing logic
|
|
416
|
+
pass
|
|
417
|
+
elif part["type"] == "thinking":
|
|
418
|
+
# Pre-existing thinking block format that we continue to store as
|
|
419
|
+
thought_sig = None
|
|
420
|
+
if "signature" in part:
|
|
421
|
+
sig = part["signature"]
|
|
422
|
+
if sig and isinstance(sig, str):
|
|
423
|
+
# Decode base64-encoded signature back to bytes
|
|
424
|
+
thought_sig = base64.b64decode(sig)
|
|
425
|
+
parts.append(
|
|
426
|
+
Part(
|
|
427
|
+
text=part["thinking"],
|
|
428
|
+
thought=True,
|
|
429
|
+
thought_signature=thought_sig,
|
|
430
|
+
)
|
|
431
|
+
)
|
|
432
|
+
elif part["type"] == "reasoning":
|
|
433
|
+
# ReasoningContentBlock (when output_version = "v1")
|
|
434
|
+
extras = part.get("extras", {}) or {}
|
|
435
|
+
sig = extras.get("signature")
|
|
436
|
+
thought_sig = None
|
|
437
|
+
if sig and isinstance(sig, str):
|
|
438
|
+
# Decode base64-encoded signature back to bytes
|
|
439
|
+
thought_sig = base64.b64decode(sig)
|
|
440
|
+
parts.append(
|
|
441
|
+
Part(
|
|
442
|
+
text=part["reasoning"],
|
|
443
|
+
thought=True,
|
|
444
|
+
thought_signature=thought_sig,
|
|
445
|
+
)
|
|
446
|
+
)
|
|
447
|
+
elif part["type"] == "server_tool_call":
|
|
448
|
+
if part.get("name") == "code_interpreter":
|
|
449
|
+
args = part.get("args", {})
|
|
450
|
+
code = args.get("code", "")
|
|
451
|
+
language = args.get("language", "python")
|
|
452
|
+
executable_code_part = Part(
|
|
453
|
+
executable_code=ExecutableCode(language=language, code=code)
|
|
454
|
+
)
|
|
455
|
+
parts.append(executable_code_part)
|
|
456
|
+
else:
|
|
457
|
+
warnings.warn(
|
|
458
|
+
f"Server tool call with name '{part.get('name')}' is not "
|
|
459
|
+
"currently supported by Google GenAI. Only "
|
|
460
|
+
"'code_interpreter' is supported.",
|
|
461
|
+
stacklevel=2,
|
|
462
|
+
)
|
|
372
463
|
elif part["type"] == "executable_code":
|
|
464
|
+
# Legacy executable_code format (backward compat)
|
|
373
465
|
if "executable_code" not in part or "language" not in part:
|
|
374
|
-
|
|
466
|
+
msg = (
|
|
375
467
|
"Executable code part must have 'code' and 'language' "
|
|
376
468
|
f"keys, got {part}"
|
|
377
469
|
)
|
|
470
|
+
raise ValueError(msg)
|
|
378
471
|
executable_code_part = Part(
|
|
379
472
|
executable_code=ExecutableCode(
|
|
380
473
|
language=part["language"], code=part["executable_code"]
|
|
381
474
|
)
|
|
382
475
|
)
|
|
383
476
|
parts.append(executable_code_part)
|
|
477
|
+
elif part["type"] == "server_tool_result":
|
|
478
|
+
output = part.get("output", "")
|
|
479
|
+
status = part.get("status", "success")
|
|
480
|
+
# Map status to outcome: success → 1 (OUTCOME_OK), error → 2
|
|
481
|
+
outcome = 1 if status == "success" else 2
|
|
482
|
+
# Check extras for original outcome if available
|
|
483
|
+
if "extras" in part and "outcome" in part["extras"]:
|
|
484
|
+
outcome = part["extras"]["outcome"]
|
|
485
|
+
code_execution_result_part = Part(
|
|
486
|
+
code_execution_result=CodeExecutionResult(
|
|
487
|
+
output=str(output), outcome=outcome
|
|
488
|
+
)
|
|
489
|
+
)
|
|
490
|
+
parts.append(code_execution_result_part)
|
|
384
491
|
elif part["type"] == "code_execution_result":
|
|
492
|
+
# Legacy code_execution_result format (backward compat)
|
|
385
493
|
if "code_execution_result" not in part:
|
|
386
|
-
|
|
494
|
+
msg = (
|
|
387
495
|
"Code execution result part must have "
|
|
388
496
|
f"'code_execution_result', got {part}"
|
|
389
497
|
)
|
|
498
|
+
raise ValueError(msg)
|
|
390
499
|
if "outcome" in part:
|
|
391
500
|
outcome = part["outcome"]
|
|
392
501
|
else:
|
|
@@ -398,25 +507,18 @@ def _convert_to_parts(
|
|
|
398
507
|
)
|
|
399
508
|
)
|
|
400
509
|
parts.append(code_execution_result_part)
|
|
401
|
-
elif part["type"] == "thinking":
|
|
402
|
-
parts.append(Part(text=part["thinking"], thought=True))
|
|
403
510
|
else:
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
f"image_url, and media types are supported."
|
|
407
|
-
)
|
|
511
|
+
msg = f"Unrecognized message part type: {part['type']}."
|
|
512
|
+
raise ValueError(msg)
|
|
408
513
|
else:
|
|
409
|
-
# Yolo
|
|
514
|
+
# Yolo. The input message content doesn't have a `type` key
|
|
410
515
|
logger.warning(
|
|
411
516
|
"Unrecognized message part format. Assuming it's a text part."
|
|
412
517
|
)
|
|
413
518
|
parts.append(Part(text=str(part)))
|
|
414
519
|
else:
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
raise ChatGoogleGenerativeAIError(
|
|
418
|
-
"Gemini only supports text and inline_data parts."
|
|
419
|
-
)
|
|
520
|
+
msg = "Unknown error occurred while converting LC message content to parts."
|
|
521
|
+
raise ChatGoogleGenerativeAIError(msg)
|
|
420
522
|
return parts
|
|
421
523
|
|
|
422
524
|
|
|
@@ -433,7 +535,7 @@ def _convert_tool_message_to_parts(
|
|
|
433
535
|
other_blocks = []
|
|
434
536
|
for block in message.content:
|
|
435
537
|
if isinstance(block, dict) and (
|
|
436
|
-
is_data_content_block(block) or
|
|
538
|
+
is_data_content_block(block) or is_openai_data_block(block)
|
|
437
539
|
):
|
|
438
540
|
media_blocks.append(block)
|
|
439
541
|
else:
|
|
@@ -463,14 +565,15 @@ def _convert_tool_message_to_parts(
|
|
|
463
565
|
def _get_ai_message_tool_messages_parts(
|
|
464
566
|
tool_messages: Sequence[ToolMessage], ai_message: AIMessage
|
|
465
567
|
) -> list[Part]:
|
|
466
|
-
"""
|
|
467
|
-
|
|
468
|
-
|
|
568
|
+
"""Conversion.
|
|
569
|
+
|
|
570
|
+
Finds relevant tool messages for the AI message and converts them to a single list
|
|
571
|
+
of Parts.
|
|
469
572
|
"""
|
|
470
573
|
# We are interested only in the tool messages that are part of the AI message
|
|
471
574
|
tool_calls_ids = {tool_call["id"]: tool_call for tool_call in ai_message.tool_calls}
|
|
472
575
|
parts = []
|
|
473
|
-
for
|
|
576
|
+
for _i, message in enumerate(tool_messages):
|
|
474
577
|
if not tool_calls_ids:
|
|
475
578
|
break
|
|
476
579
|
if message.tool_call_id in tool_calls_ids:
|
|
@@ -487,7 +590,20 @@ def _get_ai_message_tool_messages_parts(
|
|
|
487
590
|
def _parse_chat_history(
|
|
488
591
|
input_messages: Sequence[BaseMessage], convert_system_message_to_human: bool = False
|
|
489
592
|
) -> Tuple[Optional[Content], List[Content]]:
|
|
490
|
-
|
|
593
|
+
"""Parses sequence of `BaseMessage` into system instruction and formatted messages.
|
|
594
|
+
|
|
595
|
+
Args:
|
|
596
|
+
input_messages: Sequence of `BaseMessage` objects representing the chat history.
|
|
597
|
+
convert_system_message_to_human: Whether to convert the first system message
|
|
598
|
+
into a human message. Deprecated, use system instructions instead.
|
|
599
|
+
|
|
600
|
+
Returns:
|
|
601
|
+
A tuple containing:
|
|
602
|
+
- An optional `google.ai.generativelanguage_v1beta.types.Content` representing
|
|
603
|
+
the system instruction (if any).
|
|
604
|
+
- A list of `google.ai.generativelanguage_v1beta.types.Content` representing the
|
|
605
|
+
formatted messages.
|
|
606
|
+
"""
|
|
491
607
|
|
|
492
608
|
if convert_system_message_to_human:
|
|
493
609
|
warnings.warn(
|
|
@@ -496,6 +612,28 @@ def _parse_chat_history(
|
|
|
496
612
|
DeprecationWarning,
|
|
497
613
|
stacklevel=2,
|
|
498
614
|
)
|
|
615
|
+
input_messages = list(input_messages) # Make a mutable copy
|
|
616
|
+
|
|
617
|
+
# Case where content was serialized to v1 format
|
|
618
|
+
for idx, message in enumerate(input_messages):
|
|
619
|
+
if (
|
|
620
|
+
isinstance(message, AIMessage)
|
|
621
|
+
and message.response_metadata.get("output_version") == "v1"
|
|
622
|
+
):
|
|
623
|
+
# Unpack known v1 content to v1beta format for the request
|
|
624
|
+
#
|
|
625
|
+
# Old content types and any previously serialized messages passed back in to
|
|
626
|
+
# history will skip this, but hit and processed in `_convert_to_parts`
|
|
627
|
+
input_messages[idx] = message.model_copy(
|
|
628
|
+
update={
|
|
629
|
+
"content": _convert_from_v1_to_generativelanguage_v1beta(
|
|
630
|
+
cast(list[types.ContentBlock], message.content),
|
|
631
|
+
message.response_metadata.get("model_provider"),
|
|
632
|
+
)
|
|
633
|
+
}
|
|
634
|
+
)
|
|
635
|
+
|
|
636
|
+
formatted_messages: List[Content] = []
|
|
499
637
|
|
|
500
638
|
system_instruction: Optional[Content] = None
|
|
501
639
|
messages_without_tool_messages = [
|
|
@@ -514,25 +652,49 @@ def _parse_chat_history(
|
|
|
514
652
|
else:
|
|
515
653
|
pass
|
|
516
654
|
continue
|
|
517
|
-
|
|
655
|
+
if isinstance(message, AIMessage):
|
|
518
656
|
role = "model"
|
|
519
657
|
if message.tool_calls:
|
|
520
658
|
ai_message_parts = []
|
|
521
|
-
|
|
659
|
+
# Extract any function_call_signature blocks from content
|
|
660
|
+
function_call_sigs: dict[int, bytes] = {}
|
|
661
|
+
if isinstance(message.content, list):
|
|
662
|
+
for idx, item in enumerate(message.content):
|
|
663
|
+
if (
|
|
664
|
+
isinstance(item, dict)
|
|
665
|
+
and item.get("type") == "function_call_signature"
|
|
666
|
+
):
|
|
667
|
+
sig_str = item.get("signature", "")
|
|
668
|
+
if sig_str and isinstance(sig_str, str):
|
|
669
|
+
# Decode base64-encoded signature back to bytes
|
|
670
|
+
sig_bytes = base64.b64decode(sig_str)
|
|
671
|
+
function_call_sigs[idx] = sig_bytes
|
|
672
|
+
|
|
673
|
+
for tool_call_idx, tool_call in enumerate(message.tool_calls):
|
|
522
674
|
function_call = FunctionCall(
|
|
523
675
|
{
|
|
524
676
|
"name": tool_call["name"],
|
|
525
677
|
"args": tool_call["args"],
|
|
526
678
|
}
|
|
527
679
|
)
|
|
528
|
-
|
|
680
|
+
# Check if there's a signature for this function call
|
|
681
|
+
# (We use the index to match signature to function call)
|
|
682
|
+
sig = function_call_sigs.get(tool_call_idx)
|
|
683
|
+
if sig:
|
|
684
|
+
ai_message_parts.append(
|
|
685
|
+
Part(function_call=function_call, thought_signature=sig)
|
|
686
|
+
)
|
|
687
|
+
else:
|
|
688
|
+
ai_message_parts.append(Part(function_call=function_call))
|
|
529
689
|
tool_messages_parts = _get_ai_message_tool_messages_parts(
|
|
530
690
|
tool_messages=tool_messages, ai_message=message
|
|
531
691
|
)
|
|
532
|
-
|
|
533
|
-
|
|
692
|
+
formatted_messages.append(Content(role=role, parts=ai_message_parts))
|
|
693
|
+
formatted_messages.append(
|
|
694
|
+
Content(role="user", parts=tool_messages_parts)
|
|
695
|
+
)
|
|
534
696
|
continue
|
|
535
|
-
|
|
697
|
+
if raw_function_call := message.additional_kwargs.get("function_call"):
|
|
536
698
|
function_call = FunctionCall(
|
|
537
699
|
{
|
|
538
700
|
"name": raw_function_call["name"],
|
|
@@ -541,23 +703,30 @@ def _parse_chat_history(
|
|
|
541
703
|
)
|
|
542
704
|
parts = [Part(function_call=function_call)]
|
|
543
705
|
else:
|
|
544
|
-
|
|
706
|
+
if message.response_metadata.get("output_version") == "v1":
|
|
707
|
+
# Already converted to v1beta format above
|
|
708
|
+
parts = message.content # type: ignore[assignment]
|
|
709
|
+
else:
|
|
710
|
+
# Prepare request content parts from message.content field
|
|
711
|
+
parts = _convert_to_parts(message.content)
|
|
545
712
|
elif isinstance(message, HumanMessage):
|
|
546
713
|
role = "user"
|
|
547
714
|
parts = _convert_to_parts(message.content)
|
|
548
715
|
if i == 1 and convert_system_message_to_human and system_instruction:
|
|
549
|
-
parts =
|
|
716
|
+
parts = list(system_instruction.parts) + parts
|
|
550
717
|
system_instruction = None
|
|
551
718
|
elif isinstance(message, FunctionMessage):
|
|
552
719
|
role = "user"
|
|
553
720
|
parts = _convert_tool_message_to_parts(message)
|
|
554
721
|
else:
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
)
|
|
722
|
+
msg = f"Unexpected message with type {type(message)} at the position {i}."
|
|
723
|
+
raise ValueError(msg)
|
|
558
724
|
|
|
559
|
-
|
|
560
|
-
|
|
725
|
+
# Final step; assemble the Content object to pass to the API
|
|
726
|
+
# If version = "v1", the parts are already in v1beta format and will be
|
|
727
|
+
# automatically converted using protobuf's auto-conversion
|
|
728
|
+
formatted_messages.append(Content(role=role, parts=parts))
|
|
729
|
+
return system_instruction, formatted_messages
|
|
561
730
|
|
|
562
731
|
|
|
563
732
|
# Helper function to append content consistently
|
|
@@ -567,27 +736,34 @@ def _append_to_content(
|
|
|
567
736
|
"""Appends a new item to the content, handling different initial content types."""
|
|
568
737
|
if current_content is None and isinstance(new_item, str):
|
|
569
738
|
return new_item
|
|
570
|
-
|
|
739
|
+
if current_content is None:
|
|
571
740
|
return [new_item]
|
|
572
|
-
|
|
741
|
+
if isinstance(current_content, str):
|
|
573
742
|
return [current_content, new_item]
|
|
574
|
-
|
|
743
|
+
if isinstance(current_content, list):
|
|
575
744
|
current_content.append(new_item)
|
|
576
745
|
return current_content
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
746
|
+
# This case should ideally not be reached with proper type checking,
|
|
747
|
+
# but it catches any unexpected types that might slip through.
|
|
748
|
+
msg = f"Unexpected content type: {type(current_content)}"
|
|
749
|
+
raise TypeError(msg)
|
|
581
750
|
|
|
582
751
|
|
|
583
752
|
def _parse_response_candidate(
|
|
584
|
-
response_candidate: Candidate,
|
|
753
|
+
response_candidate: Candidate,
|
|
754
|
+
streaming: bool = False,
|
|
755
|
+
model_name: Optional[str] = None,
|
|
585
756
|
) -> AIMessage:
|
|
586
757
|
content: Union[None, str, List[Union[str, dict]]] = None
|
|
587
758
|
additional_kwargs: Dict[str, Any] = {}
|
|
759
|
+
response_metadata: Dict[str, Any] = {"model_provider": "google_genai"}
|
|
760
|
+
if model_name:
|
|
761
|
+
response_metadata["model_name"] = model_name
|
|
588
762
|
tool_calls = []
|
|
589
763
|
invalid_tool_calls = []
|
|
590
764
|
tool_call_chunks = []
|
|
765
|
+
# Track function call signatures separately to handle them conditionally
|
|
766
|
+
function_call_signatures: List[dict] = []
|
|
591
767
|
|
|
592
768
|
for part in response_candidate.content.parts:
|
|
593
769
|
text: Optional[str] = None
|
|
@@ -600,37 +776,71 @@ def _parse_response_candidate(
|
|
|
600
776
|
except AttributeError:
|
|
601
777
|
pass
|
|
602
778
|
|
|
779
|
+
# Extract thought signature if present (can be on any Part type)
|
|
780
|
+
# Signatures are binary data, encode to base64 string for JSON serialization
|
|
781
|
+
thought_sig: Optional[str] = None
|
|
782
|
+
if hasattr(part, "thought_signature") and part.thought_signature:
|
|
783
|
+
try:
|
|
784
|
+
# Encode binary signature to base64 string
|
|
785
|
+
thought_sig = base64.b64encode(part.thought_signature).decode("ascii")
|
|
786
|
+
if not thought_sig: # Empty string
|
|
787
|
+
thought_sig = None
|
|
788
|
+
except (AttributeError, TypeError):
|
|
789
|
+
thought_sig = None
|
|
790
|
+
|
|
603
791
|
if hasattr(part, "thought") and part.thought:
|
|
604
792
|
thinking_message = {
|
|
605
793
|
"type": "thinking",
|
|
606
794
|
"thinking": part.text,
|
|
607
795
|
}
|
|
796
|
+
# Include signature if present
|
|
797
|
+
if thought_sig:
|
|
798
|
+
thinking_message["signature"] = thought_sig
|
|
608
799
|
content = _append_to_content(content, thinking_message)
|
|
609
800
|
elif text is not None and text:
|
|
610
|
-
|
|
801
|
+
# Check if this text Part has a signature attached
|
|
802
|
+
if thought_sig:
|
|
803
|
+
# Text with signature needs structured block to preserve signature
|
|
804
|
+
# We use a v1 TextContentBlock
|
|
805
|
+
text_with_sig = {
|
|
806
|
+
"type": "text",
|
|
807
|
+
"text": text,
|
|
808
|
+
"extras": {"signature": thought_sig},
|
|
809
|
+
}
|
|
810
|
+
content = _append_to_content(content, text_with_sig)
|
|
811
|
+
else:
|
|
812
|
+
content = _append_to_content(content, text)
|
|
611
813
|
|
|
612
814
|
if hasattr(part, "executable_code") and part.executable_code is not None:
|
|
613
815
|
if part.executable_code.code and part.executable_code.language:
|
|
816
|
+
code_id = str(uuid.uuid4()) # Generate ID if not present, needed later
|
|
614
817
|
code_message = {
|
|
615
818
|
"type": "executable_code",
|
|
616
819
|
"executable_code": part.executable_code.code,
|
|
617
820
|
"language": part.executable_code.language,
|
|
821
|
+
"id": code_id,
|
|
618
822
|
}
|
|
619
823
|
content = _append_to_content(content, code_message)
|
|
620
824
|
|
|
621
825
|
if (
|
|
622
826
|
hasattr(part, "code_execution_result")
|
|
623
827
|
and part.code_execution_result is not None
|
|
624
|
-
):
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
828
|
+
) and part.code_execution_result.output:
|
|
829
|
+
# outcome: 1 = OUTCOME_OK (success), else = error
|
|
830
|
+
outcome = part.code_execution_result.outcome
|
|
831
|
+
execution_result = {
|
|
832
|
+
"type": "code_execution_result",
|
|
833
|
+
"code_execution_result": part.code_execution_result.output,
|
|
834
|
+
"outcome": outcome,
|
|
835
|
+
"tool_call_id": "", # Linked via block translator
|
|
836
|
+
}
|
|
837
|
+
content = _append_to_content(content, execution_result)
|
|
632
838
|
|
|
633
|
-
if
|
|
839
|
+
if (
|
|
840
|
+
hasattr(part, "inline_data")
|
|
841
|
+
and part.inline_data
|
|
842
|
+
and part.inline_data.mime_type.startswith("audio/")
|
|
843
|
+
):
|
|
634
844
|
buffer = io.BytesIO()
|
|
635
845
|
|
|
636
846
|
with wave.open(buffer, "wb") as wf:
|
|
@@ -640,9 +850,17 @@ def _parse_response_candidate(
|
|
|
640
850
|
wf.setframerate(24000)
|
|
641
851
|
wf.writeframes(part.inline_data.data)
|
|
642
852
|
|
|
643
|
-
|
|
853
|
+
audio_data = buffer.getvalue()
|
|
854
|
+
additional_kwargs["audio"] = audio_data
|
|
855
|
+
|
|
856
|
+
# For backwards compatibility, audio stays in additional_kwargs by default
|
|
857
|
+
# and is accessible via .content_blocks property
|
|
644
858
|
|
|
645
|
-
if
|
|
859
|
+
if (
|
|
860
|
+
hasattr(part, "inline_data")
|
|
861
|
+
and part.inline_data
|
|
862
|
+
and part.inline_data.mime_type.startswith("image/")
|
|
863
|
+
):
|
|
646
864
|
image_format = part.inline_data.mime_type[6:]
|
|
647
865
|
image_message = {
|
|
648
866
|
"type": "image_url",
|
|
@@ -701,6 +919,23 @@ def _parse_response_candidate(
|
|
|
701
919
|
id=tool_call_dict.get("id", str(uuid.uuid4())),
|
|
702
920
|
)
|
|
703
921
|
)
|
|
922
|
+
|
|
923
|
+
# If this function_call Part has a signature, track it separately
|
|
924
|
+
# We'll add it to content only if there's other content present
|
|
925
|
+
if thought_sig:
|
|
926
|
+
sig_block = {
|
|
927
|
+
"type": "function_call_signature",
|
|
928
|
+
"signature": thought_sig,
|
|
929
|
+
}
|
|
930
|
+
function_call_signatures.append(sig_block)
|
|
931
|
+
|
|
932
|
+
# Add function call signatures to content only if there's already other content
|
|
933
|
+
# This preserves backward compatibility where content is "" for
|
|
934
|
+
# function-only responses
|
|
935
|
+
if function_call_signatures and content is not None:
|
|
936
|
+
for sig_block in function_call_signatures:
|
|
937
|
+
content = _append_to_content(content, sig_block)
|
|
938
|
+
|
|
704
939
|
if content is None:
|
|
705
940
|
content = ""
|
|
706
941
|
if isinstance(content, list) and any(
|
|
@@ -708,29 +943,107 @@ def _parse_response_candidate(
|
|
|
708
943
|
):
|
|
709
944
|
warnings.warn(
|
|
710
945
|
"""
|
|
711
|
-
|
|
712
|
-
- 'executable_code': Always present.
|
|
713
|
-
- 'execution_result' & 'image_url': May be absent for some queries.
|
|
946
|
+
Warning: Output may vary each run.
|
|
947
|
+
- 'executable_code': Always present.
|
|
948
|
+
- 'execution_result' & 'image_url': May be absent for some queries.
|
|
714
949
|
|
|
715
950
|
Validate before using in production.
|
|
716
951
|
"""
|
|
717
952
|
)
|
|
718
|
-
|
|
719
953
|
if streaming:
|
|
720
954
|
return AIMessageChunk(
|
|
721
|
-
content=
|
|
955
|
+
content=content,
|
|
722
956
|
additional_kwargs=additional_kwargs,
|
|
957
|
+
response_metadata=response_metadata,
|
|
723
958
|
tool_call_chunks=tool_call_chunks,
|
|
724
959
|
)
|
|
725
960
|
|
|
726
961
|
return AIMessage(
|
|
727
|
-
content=
|
|
962
|
+
content=content,
|
|
728
963
|
additional_kwargs=additional_kwargs,
|
|
964
|
+
response_metadata=response_metadata,
|
|
729
965
|
tool_calls=tool_calls,
|
|
730
966
|
invalid_tool_calls=invalid_tool_calls,
|
|
731
967
|
)
|
|
732
968
|
|
|
733
969
|
|
|
970
|
+
def _extract_grounding_metadata(candidate: Any) -> Dict[str, Any]:
|
|
971
|
+
"""Extract grounding metadata from candidate.
|
|
972
|
+
|
|
973
|
+
core's block translator converts this metadata into citation annotations.
|
|
974
|
+
|
|
975
|
+
Uses `MessageToDict` for complete unfiltered extraction.
|
|
976
|
+
|
|
977
|
+
Falls back to custom field extraction in cases of failure for robustness.
|
|
978
|
+
"""
|
|
979
|
+
if not hasattr(candidate, "grounding_metadata") or not candidate.grounding_metadata:
|
|
980
|
+
return {}
|
|
981
|
+
|
|
982
|
+
grounding_metadata = candidate.grounding_metadata
|
|
983
|
+
|
|
984
|
+
try:
|
|
985
|
+
# proto-plus wraps protobuf messages - access ._pb to get the raw protobuf
|
|
986
|
+
# message that MessageToDict expects
|
|
987
|
+
pb_message = (
|
|
988
|
+
grounding_metadata._pb
|
|
989
|
+
if hasattr(grounding_metadata, "_pb")
|
|
990
|
+
else grounding_metadata
|
|
991
|
+
)
|
|
992
|
+
|
|
993
|
+
return MessageToDict( # type: ignore[call-arg]
|
|
994
|
+
pb_message,
|
|
995
|
+
preserving_proto_field_name=True,
|
|
996
|
+
always_print_fields_with_no_presence=True,
|
|
997
|
+
# type stub issue - ensures that protobuf fields with default values
|
|
998
|
+
# (like start_index=0) are included in the output
|
|
999
|
+
)
|
|
1000
|
+
except (AttributeError, TypeError, ImportError):
|
|
1001
|
+
# Attempt manual extraction of known fields
|
|
1002
|
+
result: Dict[str, Any] = {}
|
|
1003
|
+
|
|
1004
|
+
# Grounding chunks
|
|
1005
|
+
if hasattr(grounding_metadata, "grounding_chunks"):
|
|
1006
|
+
grounding_chunks = []
|
|
1007
|
+
for chunk in grounding_metadata.grounding_chunks:
|
|
1008
|
+
chunk_data: Dict[str, Any] = {}
|
|
1009
|
+
if hasattr(chunk, "web") and chunk.web:
|
|
1010
|
+
chunk_data["web"] = {
|
|
1011
|
+
"uri": chunk.web.uri if hasattr(chunk.web, "uri") else "",
|
|
1012
|
+
"title": chunk.web.title if hasattr(chunk.web, "title") else "",
|
|
1013
|
+
}
|
|
1014
|
+
grounding_chunks.append(chunk_data)
|
|
1015
|
+
result["grounding_chunks"] = grounding_chunks
|
|
1016
|
+
|
|
1017
|
+
# Grounding supports
|
|
1018
|
+
if hasattr(grounding_metadata, "grounding_supports"):
|
|
1019
|
+
grounding_supports = []
|
|
1020
|
+
for support in grounding_metadata.grounding_supports:
|
|
1021
|
+
support_data: Dict[str, Any] = {}
|
|
1022
|
+
if hasattr(support, "segment") and support.segment:
|
|
1023
|
+
support_data["segment"] = {
|
|
1024
|
+
"start_index": getattr(support.segment, "start_index", 0),
|
|
1025
|
+
"end_index": getattr(support.segment, "end_index", 0),
|
|
1026
|
+
"text": getattr(support.segment, "text", ""),
|
|
1027
|
+
"part_index": getattr(support.segment, "part_index", 0),
|
|
1028
|
+
}
|
|
1029
|
+
if hasattr(support, "grounding_chunk_indices"):
|
|
1030
|
+
support_data["grounding_chunk_indices"] = list(
|
|
1031
|
+
support.grounding_chunk_indices
|
|
1032
|
+
)
|
|
1033
|
+
if hasattr(support, "confidence_scores"):
|
|
1034
|
+
support_data["confidence_scores"] = [
|
|
1035
|
+
round(score, 6) for score in support.confidence_scores
|
|
1036
|
+
]
|
|
1037
|
+
grounding_supports.append(support_data)
|
|
1038
|
+
result["grounding_supports"] = grounding_supports
|
|
1039
|
+
|
|
1040
|
+
# Web search queries
|
|
1041
|
+
if hasattr(grounding_metadata, "web_search_queries"):
|
|
1042
|
+
result["web_search_queries"] = list(grounding_metadata.web_search_queries)
|
|
1043
|
+
|
|
1044
|
+
return result
|
|
1045
|
+
|
|
1046
|
+
|
|
734
1047
|
def _response_to_result(
|
|
735
1048
|
response: GenerateContentResponse,
|
|
736
1049
|
stream: bool = False,
|
|
@@ -793,19 +1106,20 @@ def _response_to_result(
|
|
|
793
1106
|
proto.Message.to_dict(safety_rating, use_integers_for_enums=False)
|
|
794
1107
|
for safety_rating in candidate.safety_ratings
|
|
795
1108
|
]
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
generation_info["grounding_metadata"] = proto.Message.to_dict(
|
|
799
|
-
candidate.grounding_metadata
|
|
800
|
-
)
|
|
801
|
-
except AttributeError:
|
|
802
|
-
pass
|
|
1109
|
+
grounding_metadata = _extract_grounding_metadata(candidate)
|
|
1110
|
+
generation_info["grounding_metadata"] = grounding_metadata
|
|
803
1111
|
message = _parse_response_candidate(candidate, streaming=stream)
|
|
1112
|
+
|
|
804
1113
|
message.usage_metadata = lc_usage
|
|
1114
|
+
|
|
1115
|
+
if not hasattr(message, "response_metadata"):
|
|
1116
|
+
message.response_metadata = {}
|
|
1117
|
+
message.response_metadata["grounding_metadata"] = grounding_metadata
|
|
1118
|
+
|
|
805
1119
|
if stream:
|
|
806
1120
|
generations.append(
|
|
807
1121
|
ChatGenerationChunk(
|
|
808
|
-
message=cast(AIMessageChunk, message),
|
|
1122
|
+
message=cast("AIMessageChunk", message),
|
|
809
1123
|
generation_info=generation_info,
|
|
810
1124
|
)
|
|
811
1125
|
)
|
|
@@ -849,13 +1163,14 @@ def _is_event_loop_running() -> bool:
|
|
|
849
1163
|
|
|
850
1164
|
|
|
851
1165
|
class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
852
|
-
"""`Google AI` chat models integration.
|
|
1166
|
+
r"""`Google AI` chat models integration.
|
|
853
1167
|
|
|
854
1168
|
Instantiation:
|
|
855
1169
|
To use, you must have either:
|
|
856
1170
|
|
|
857
1171
|
1. The ``GOOGLE_API_KEY`` environment variable set with your API key, or
|
|
858
|
-
2. Pass your API key using the ``google_api_key`` kwarg to the
|
|
1172
|
+
2. Pass your API key using the ``google_api_key`` kwarg to the
|
|
1173
|
+
ChatGoogleGenerativeAI constructor.
|
|
859
1174
|
|
|
860
1175
|
.. code-block:: python
|
|
861
1176
|
|
|
@@ -877,9 +1192,38 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
877
1192
|
|
|
878
1193
|
AIMessage(
|
|
879
1194
|
content="J'adore programmer. \\n",
|
|
880
|
-
response_metadata={
|
|
881
|
-
|
|
882
|
-
|
|
1195
|
+
response_metadata={
|
|
1196
|
+
"prompt_feedback": {"block_reason": 0, "safety_ratings": []},
|
|
1197
|
+
"finish_reason": "STOP",
|
|
1198
|
+
"safety_ratings": [
|
|
1199
|
+
{
|
|
1200
|
+
"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
|
|
1201
|
+
"probability": "NEGLIGIBLE",
|
|
1202
|
+
"blocked": False,
|
|
1203
|
+
},
|
|
1204
|
+
{
|
|
1205
|
+
"category": "HARM_CATEGORY_HATE_SPEECH",
|
|
1206
|
+
"probability": "NEGLIGIBLE",
|
|
1207
|
+
"blocked": False,
|
|
1208
|
+
},
|
|
1209
|
+
{
|
|
1210
|
+
"category": "HARM_CATEGORY_HARASSMENT",
|
|
1211
|
+
"probability": "NEGLIGIBLE",
|
|
1212
|
+
"blocked": False,
|
|
1213
|
+
},
|
|
1214
|
+
{
|
|
1215
|
+
"category": "HARM_CATEGORY_DANGEROUS_CONTENT",
|
|
1216
|
+
"probability": "NEGLIGIBLE",
|
|
1217
|
+
"blocked": False,
|
|
1218
|
+
},
|
|
1219
|
+
],
|
|
1220
|
+
},
|
|
1221
|
+
id="run-56cecc34-2e54-4b52-a974-337e47008ad2-0",
|
|
1222
|
+
usage_metadata={
|
|
1223
|
+
"input_tokens": 18,
|
|
1224
|
+
"output_tokens": 5,
|
|
1225
|
+
"total_tokens": 23,
|
|
1226
|
+
},
|
|
883
1227
|
)
|
|
884
1228
|
|
|
885
1229
|
Stream:
|
|
@@ -890,8 +1234,50 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
890
1234
|
|
|
891
1235
|
.. code-block:: python
|
|
892
1236
|
|
|
893
|
-
AIMessageChunk(
|
|
894
|
-
|
|
1237
|
+
AIMessageChunk(
|
|
1238
|
+
content="J",
|
|
1239
|
+
response_metadata={"finish_reason": "STOP", "safety_ratings": []},
|
|
1240
|
+
id="run-e905f4f4-58cb-4a10-a960-448a2bb649e3",
|
|
1241
|
+
usage_metadata={
|
|
1242
|
+
"input_tokens": 18,
|
|
1243
|
+
"output_tokens": 1,
|
|
1244
|
+
"total_tokens": 19,
|
|
1245
|
+
},
|
|
1246
|
+
)
|
|
1247
|
+
AIMessageChunk(
|
|
1248
|
+
content="'adore programmer. \\n",
|
|
1249
|
+
response_metadata={
|
|
1250
|
+
"finish_reason": "STOP",
|
|
1251
|
+
"safety_ratings": [
|
|
1252
|
+
{
|
|
1253
|
+
"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
|
|
1254
|
+
"probability": "NEGLIGIBLE",
|
|
1255
|
+
"blocked": False,
|
|
1256
|
+
},
|
|
1257
|
+
{
|
|
1258
|
+
"category": "HARM_CATEGORY_HATE_SPEECH",
|
|
1259
|
+
"probability": "NEGLIGIBLE",
|
|
1260
|
+
"blocked": False,
|
|
1261
|
+
},
|
|
1262
|
+
{
|
|
1263
|
+
"category": "HARM_CATEGORY_HARASSMENT",
|
|
1264
|
+
"probability": "NEGLIGIBLE",
|
|
1265
|
+
"blocked": False,
|
|
1266
|
+
},
|
|
1267
|
+
{
|
|
1268
|
+
"category": "HARM_CATEGORY_DANGEROUS_CONTENT",
|
|
1269
|
+
"probability": "NEGLIGIBLE",
|
|
1270
|
+
"blocked": False,
|
|
1271
|
+
},
|
|
1272
|
+
],
|
|
1273
|
+
},
|
|
1274
|
+
id="run-e905f4f4-58cb-4a10-a960-448a2bb649e3",
|
|
1275
|
+
usage_metadata={
|
|
1276
|
+
"input_tokens": 18,
|
|
1277
|
+
"output_tokens": 5,
|
|
1278
|
+
"total_tokens": 23,
|
|
1279
|
+
},
|
|
1280
|
+
)
|
|
895
1281
|
|
|
896
1282
|
.. code-block:: python
|
|
897
1283
|
|
|
@@ -905,9 +1291,37 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
905
1291
|
|
|
906
1292
|
AIMessageChunk(
|
|
907
1293
|
content="J'adore programmer. \\n",
|
|
908
|
-
response_metadata={
|
|
909
|
-
|
|
910
|
-
|
|
1294
|
+
response_metadata={
|
|
1295
|
+
"finish_reason": "STOPSTOP",
|
|
1296
|
+
"safety_ratings": [
|
|
1297
|
+
{
|
|
1298
|
+
"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
|
|
1299
|
+
"probability": "NEGLIGIBLE",
|
|
1300
|
+
"blocked": False,
|
|
1301
|
+
},
|
|
1302
|
+
{
|
|
1303
|
+
"category": "HARM_CATEGORY_HATE_SPEECH",
|
|
1304
|
+
"probability": "NEGLIGIBLE",
|
|
1305
|
+
"blocked": False,
|
|
1306
|
+
},
|
|
1307
|
+
{
|
|
1308
|
+
"category": "HARM_CATEGORY_HARASSMENT",
|
|
1309
|
+
"probability": "NEGLIGIBLE",
|
|
1310
|
+
"blocked": False,
|
|
1311
|
+
},
|
|
1312
|
+
{
|
|
1313
|
+
"category": "HARM_CATEGORY_DANGEROUS_CONTENT",
|
|
1314
|
+
"probability": "NEGLIGIBLE",
|
|
1315
|
+
"blocked": False,
|
|
1316
|
+
},
|
|
1317
|
+
],
|
|
1318
|
+
},
|
|
1319
|
+
id="run-3ce13a42-cd30-4ad7-a684-f1f0b37cdeec",
|
|
1320
|
+
usage_metadata={
|
|
1321
|
+
"input_tokens": 36,
|
|
1322
|
+
"output_tokens": 6,
|
|
1323
|
+
"total_tokens": 42,
|
|
1324
|
+
},
|
|
911
1325
|
)
|
|
912
1326
|
|
|
913
1327
|
Async:
|
|
@@ -922,9 +1336,10 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
922
1336
|
# await llm.abatch([messages])
|
|
923
1337
|
|
|
924
1338
|
Context Caching:
|
|
925
|
-
Context caching allows you to store and reuse content (e.g., PDFs, images) for
|
|
926
|
-
The ``cached_content`` parameter accepts a cache name created
|
|
927
|
-
Below are two examples: caching a single file
|
|
1339
|
+
Context caching allows you to store and reuse content (e.g., PDFs, images) for
|
|
1340
|
+
faster processing. The ``cached_content`` parameter accepts a cache name created
|
|
1341
|
+
via the Google Generative AI API. Below are two examples: caching a single file
|
|
1342
|
+
directly and caching multiple files using ``Part``.
|
|
928
1343
|
|
|
929
1344
|
Single File Example:
|
|
930
1345
|
This caches a single file and queries it.
|
|
@@ -941,23 +1356,23 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
941
1356
|
|
|
942
1357
|
# Upload file
|
|
943
1358
|
file = client.files.upload(file="./example_file")
|
|
944
|
-
while file.state.name ==
|
|
1359
|
+
while file.state.name == "PROCESSING":
|
|
945
1360
|
time.sleep(2)
|
|
946
1361
|
file = client.files.get(name=file.name)
|
|
947
1362
|
|
|
948
1363
|
# Create cache
|
|
949
|
-
model =
|
|
1364
|
+
model = "models/gemini-2.5-flash"
|
|
950
1365
|
cache = client.caches.create(
|
|
951
1366
|
model=model,
|
|
952
1367
|
config=types.CreateCachedContentConfig(
|
|
953
|
-
display_name=
|
|
1368
|
+
display_name="Cached Content",
|
|
954
1369
|
system_instruction=(
|
|
955
|
-
|
|
956
|
-
|
|
1370
|
+
"You are an expert content analyzer, and your job is to answer "
|
|
1371
|
+
"the user's query based on the file you have access to."
|
|
957
1372
|
),
|
|
958
1373
|
contents=[file],
|
|
959
1374
|
ttl="300s",
|
|
960
|
-
)
|
|
1375
|
+
),
|
|
961
1376
|
)
|
|
962
1377
|
|
|
963
1378
|
# Query with LangChain
|
|
@@ -983,12 +1398,12 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
983
1398
|
|
|
984
1399
|
# Upload files
|
|
985
1400
|
file_1 = client.files.upload(file="./file1")
|
|
986
|
-
while file_1.state.name ==
|
|
1401
|
+
while file_1.state.name == "PROCESSING":
|
|
987
1402
|
time.sleep(2)
|
|
988
1403
|
file_1 = client.files.get(name=file_1.name)
|
|
989
1404
|
|
|
990
1405
|
file_2 = client.files.upload(file="./file2")
|
|
991
|
-
while file_2.state.name ==
|
|
1406
|
+
while file_2.state.name == "PROCESSING":
|
|
992
1407
|
time.sleep(2)
|
|
993
1408
|
file_2 = client.files.get(name=file_2.name)
|
|
994
1409
|
|
|
@@ -1002,18 +1417,18 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1002
1417
|
],
|
|
1003
1418
|
)
|
|
1004
1419
|
]
|
|
1005
|
-
model = "gemini-
|
|
1420
|
+
model = "gemini-2.5-flash"
|
|
1006
1421
|
cache = client.caches.create(
|
|
1007
1422
|
model=model,
|
|
1008
1423
|
config=CreateCachedContentConfig(
|
|
1009
|
-
display_name=
|
|
1424
|
+
display_name="Cached Contents",
|
|
1010
1425
|
system_instruction=(
|
|
1011
|
-
|
|
1012
|
-
|
|
1426
|
+
"You are an expert content analyzer, and your job is to answer "
|
|
1427
|
+
"the user's query based on the files you have access to."
|
|
1013
1428
|
),
|
|
1014
1429
|
contents=contents,
|
|
1015
1430
|
ttl="300s",
|
|
1016
|
-
)
|
|
1431
|
+
),
|
|
1017
1432
|
)
|
|
1018
1433
|
|
|
1019
1434
|
# Query with LangChain
|
|
@@ -1021,7 +1436,9 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1021
1436
|
model=model,
|
|
1022
1437
|
cached_content=cache.name,
|
|
1023
1438
|
)
|
|
1024
|
-
message = HumanMessage(
|
|
1439
|
+
message = HumanMessage(
|
|
1440
|
+
content="Provide a summary of the key information across both files."
|
|
1441
|
+
)
|
|
1025
1442
|
llm.invoke([message])
|
|
1026
1443
|
|
|
1027
1444
|
Tool calling:
|
|
@@ -1054,23 +1471,34 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1054
1471
|
|
|
1055
1472
|
.. code-block:: python
|
|
1056
1473
|
|
|
1057
|
-
[
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1474
|
+
[
|
|
1475
|
+
{
|
|
1476
|
+
"name": "GetWeather",
|
|
1477
|
+
"args": {"location": "Los Angeles, CA"},
|
|
1478
|
+
"id": "c186c99f-f137-4d52-947f-9e3deabba6f6",
|
|
1479
|
+
},
|
|
1480
|
+
{
|
|
1481
|
+
"name": "GetWeather",
|
|
1482
|
+
"args": {"location": "New York City, NY"},
|
|
1483
|
+
"id": "cebd4a5d-e800-4fa5-babd-4aa286af4f31",
|
|
1484
|
+
},
|
|
1485
|
+
{
|
|
1486
|
+
"name": "GetPopulation",
|
|
1487
|
+
"args": {"location": "Los Angeles, CA"},
|
|
1488
|
+
"id": "4f92d897-f5e4-4d34-a3bc-93062c92591e",
|
|
1489
|
+
},
|
|
1490
|
+
{
|
|
1491
|
+
"name": "GetPopulation",
|
|
1492
|
+
"args": {"location": "New York City, NY"},
|
|
1493
|
+
"id": "634582de-5186-4e4b-968b-f192f0a93678",
|
|
1494
|
+
},
|
|
1495
|
+
]
|
|
1069
1496
|
|
|
1070
1497
|
Use Search with Gemini 2:
|
|
1071
1498
|
.. code-block:: python
|
|
1072
1499
|
|
|
1073
1500
|
from google.ai.generativelanguage_v1beta.types import Tool as GenAITool
|
|
1501
|
+
|
|
1074
1502
|
llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash")
|
|
1075
1503
|
resp = llm.invoke(
|
|
1076
1504
|
"When is the next total solar eclipse in US?",
|
|
@@ -1090,20 +1518,38 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1090
1518
|
|
|
1091
1519
|
setup: str = Field(description="The setup of the joke")
|
|
1092
1520
|
punchline: str = Field(description="The punchline to the joke")
|
|
1093
|
-
rating: Optional[int] = Field(
|
|
1521
|
+
rating: Optional[int] = Field(
|
|
1522
|
+
description="How funny the joke is, from 1 to 10"
|
|
1523
|
+
)
|
|
1094
1524
|
|
|
1095
1525
|
|
|
1526
|
+
# Default method uses function calling
|
|
1096
1527
|
structured_llm = llm.with_structured_output(Joke)
|
|
1097
|
-
|
|
1528
|
+
|
|
1529
|
+
# For more reliable output, use json_schema with native responseSchema
|
|
1530
|
+
structured_llm_json = llm.with_structured_output(Joke, method="json_schema")
|
|
1531
|
+
structured_llm_json.invoke("Tell me a joke about cats")
|
|
1098
1532
|
|
|
1099
1533
|
.. code-block:: python
|
|
1100
1534
|
|
|
1101
1535
|
Joke(
|
|
1102
|
-
setup=
|
|
1103
|
-
punchline=
|
|
1104
|
-
rating=None
|
|
1536
|
+
setup="Why are cats so good at video games?",
|
|
1537
|
+
punchline="They have nine lives on the internet",
|
|
1538
|
+
rating=None,
|
|
1105
1539
|
)
|
|
1106
1540
|
|
|
1541
|
+
Two methods are supported for structured output:
|
|
1542
|
+
|
|
1543
|
+
* ``method="function_calling"`` (default): Uses tool calling to extract
|
|
1544
|
+
structured data. Compatible with all models.
|
|
1545
|
+
* ``method="json_schema"``: Uses Gemini's native structured output with
|
|
1546
|
+
responseSchema. More reliable but requires Gemini 1.5+ models.
|
|
1547
|
+
``method="json_mode"`` also works for backwards compatibility but is a misnomer.
|
|
1548
|
+
|
|
1549
|
+
The ``json_schema`` method is recommended for better reliability as it
|
|
1550
|
+
constrains the model's generation process directly rather than relying on
|
|
1551
|
+
post-processing tool calls.
|
|
1552
|
+
|
|
1107
1553
|
Image input:
|
|
1108
1554
|
.. code-block:: python
|
|
1109
1555
|
|
|
@@ -1127,7 +1573,10 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1127
1573
|
|
|
1128
1574
|
.. code-block:: python
|
|
1129
1575
|
|
|
1130
|
-
|
|
1576
|
+
"The weather in this image appears to be sunny and pleasant. The sky is a
|
|
1577
|
+
bright blue with scattered white clouds, suggesting fair weather. The lush
|
|
1578
|
+
green grass and trees indicate a warm and possibly slightly breezy day.
|
|
1579
|
+
There are no signs of rain or storms."
|
|
1131
1580
|
|
|
1132
1581
|
PDF input:
|
|
1133
1582
|
.. code-block:: python
|
|
@@ -1135,8 +1584,8 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1135
1584
|
import base64
|
|
1136
1585
|
from langchain_core.messages import HumanMessage
|
|
1137
1586
|
|
|
1138
|
-
pdf_bytes = open("/path/to/your/test.pdf",
|
|
1139
|
-
pdf_base64 = base64.b64encode(pdf_bytes).decode(
|
|
1587
|
+
pdf_bytes = open("/path/to/your/test.pdf", "rb").read()
|
|
1588
|
+
pdf_base64 = base64.b64encode(pdf_bytes).decode("utf-8")
|
|
1140
1589
|
|
|
1141
1590
|
message = HumanMessage(
|
|
1142
1591
|
content=[
|
|
@@ -1144,9 +1593,9 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1144
1593
|
{
|
|
1145
1594
|
"type": "file",
|
|
1146
1595
|
"source_type": "base64",
|
|
1147
|
-
"mime_type":"application/pdf",
|
|
1148
|
-
"data": pdf_base64
|
|
1149
|
-
}
|
|
1596
|
+
"mime_type": "application/pdf",
|
|
1597
|
+
"data": pdf_base64,
|
|
1598
|
+
},
|
|
1150
1599
|
]
|
|
1151
1600
|
)
|
|
1152
1601
|
ai_msg = llm.invoke([message])
|
|
@@ -1154,7 +1603,11 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1154
1603
|
|
|
1155
1604
|
.. code-block:: python
|
|
1156
1605
|
|
|
1157
|
-
|
|
1606
|
+
"This research paper describes a system developed for SemEval-2025 Task 9,
|
|
1607
|
+
which aims to automate the detection of food hazards from recall reports,
|
|
1608
|
+
addressing the class imbalance problem by leveraging LLM-based data
|
|
1609
|
+
augmentation techniques and transformer-based models to improve
|
|
1610
|
+
performance."
|
|
1158
1611
|
|
|
1159
1612
|
Video input:
|
|
1160
1613
|
.. code-block:: python
|
|
@@ -1162,18 +1615,21 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1162
1615
|
import base64
|
|
1163
1616
|
from langchain_core.messages import HumanMessage
|
|
1164
1617
|
|
|
1165
|
-
video_bytes = open("/path/to/your/video.mp4",
|
|
1166
|
-
video_base64 = base64.b64encode(video_bytes).decode(
|
|
1618
|
+
video_bytes = open("/path/to/your/video.mp4", "rb").read()
|
|
1619
|
+
video_base64 = base64.b64encode(video_bytes).decode("utf-8")
|
|
1167
1620
|
|
|
1168
1621
|
message = HumanMessage(
|
|
1169
1622
|
content=[
|
|
1170
|
-
{
|
|
1623
|
+
{
|
|
1624
|
+
"type": "text",
|
|
1625
|
+
"text": "describe what's in this video in a sentence",
|
|
1626
|
+
},
|
|
1171
1627
|
{
|
|
1172
1628
|
"type": "file",
|
|
1173
1629
|
"source_type": "base64",
|
|
1174
1630
|
"mime_type": "video/mp4",
|
|
1175
|
-
"data": video_base64
|
|
1176
|
-
}
|
|
1631
|
+
"data": video_base64,
|
|
1632
|
+
},
|
|
1177
1633
|
]
|
|
1178
1634
|
)
|
|
1179
1635
|
ai_msg = llm.invoke([message])
|
|
@@ -1181,7 +1637,9 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1181
1637
|
|
|
1182
1638
|
.. code-block:: python
|
|
1183
1639
|
|
|
1184
|
-
|
|
1640
|
+
"Tom and Jerry, along with a turkey, engage in a chaotic Thanksgiving-themed
|
|
1641
|
+
adventure involving a corn-on-the-cob chase, maze antics, and a disastrous
|
|
1642
|
+
attempt to prepare a turkey dinner."
|
|
1185
1643
|
|
|
1186
1644
|
You can also pass YouTube URLs directly:
|
|
1187
1645
|
|
|
@@ -1196,7 +1654,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1196
1654
|
"type": "media",
|
|
1197
1655
|
"file_uri": "https://www.youtube.com/watch?v=9hE5-98ZeCg",
|
|
1198
1656
|
"mime_type": "video/mp4",
|
|
1199
|
-
}
|
|
1657
|
+
},
|
|
1200
1658
|
]
|
|
1201
1659
|
)
|
|
1202
1660
|
ai_msg = llm.invoke([message])
|
|
@@ -1204,7 +1662,10 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1204
1662
|
|
|
1205
1663
|
.. code-block:: python
|
|
1206
1664
|
|
|
1207
|
-
|
|
1665
|
+
"The video is a demo of multimodal live streaming in Gemini 2.0. The
|
|
1666
|
+
narrator is sharing his screen in AI Studio and asks if the AI can see it.
|
|
1667
|
+
The AI then reads text that is highlighted on the screen, defines the word
|
|
1668
|
+
“multimodal,” and summarizes everything that was seen and heard."
|
|
1208
1669
|
|
|
1209
1670
|
Audio input:
|
|
1210
1671
|
.. code-block:: python
|
|
@@ -1212,8 +1673,8 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1212
1673
|
import base64
|
|
1213
1674
|
from langchain_core.messages import HumanMessage
|
|
1214
1675
|
|
|
1215
|
-
audio_bytes = open("/path/to/your/audio.mp3",
|
|
1216
|
-
audio_base64 = base64.b64encode(audio_bytes).decode(
|
|
1676
|
+
audio_bytes = open("/path/to/your/audio.mp3", "rb").read()
|
|
1677
|
+
audio_base64 = base64.b64encode(audio_bytes).decode("utf-8")
|
|
1217
1678
|
|
|
1218
1679
|
message = HumanMessage(
|
|
1219
1680
|
content=[
|
|
@@ -1221,9 +1682,9 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1221
1682
|
{
|
|
1222
1683
|
"type": "file",
|
|
1223
1684
|
"source_type": "base64",
|
|
1224
|
-
"mime_type":"audio/mp3",
|
|
1225
|
-
"data": audio_base64
|
|
1226
|
-
}
|
|
1685
|
+
"mime_type": "audio/mp3",
|
|
1686
|
+
"data": audio_base64,
|
|
1687
|
+
},
|
|
1227
1688
|
]
|
|
1228
1689
|
)
|
|
1229
1690
|
ai_msg = llm.invoke([message])
|
|
@@ -1231,7 +1692,11 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1231
1692
|
|
|
1232
1693
|
.. code-block:: python
|
|
1233
1694
|
|
|
1234
|
-
"In this episode of the Made by Google podcast, Stephen Johnson and Simon
|
|
1695
|
+
"In this episode of the Made by Google podcast, Stephen Johnson and Simon
|
|
1696
|
+
Tokumine discuss NotebookLM, a tool designed to help users understand
|
|
1697
|
+
complex material in various modalities, with a focus on its unexpected uses,
|
|
1698
|
+
the development of audio overviews, and the implementation of new features
|
|
1699
|
+
like mind maps and source discovery."
|
|
1235
1700
|
|
|
1236
1701
|
File upload (URI-based):
|
|
1237
1702
|
You can also upload files to Google's servers and reference them by URI.
|
|
@@ -1265,7 +1730,10 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1265
1730
|
|
|
1266
1731
|
.. code-block:: python
|
|
1267
1732
|
|
|
1268
|
-
"This research paper assesses and mitigates multi-turn jailbreak
|
|
1733
|
+
"This research paper assesses and mitigates multi-turn jailbreak
|
|
1734
|
+
vulnerabilities in large language models using the Crescendo attack study,
|
|
1735
|
+
evaluating attack success rates and mitigation strategies like prompt
|
|
1736
|
+
hardening and LLM-as-guardrail."
|
|
1269
1737
|
|
|
1270
1738
|
Token usage:
|
|
1271
1739
|
.. code-block:: python
|
|
@@ -1275,7 +1743,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1275
1743
|
|
|
1276
1744
|
.. code-block:: python
|
|
1277
1745
|
|
|
1278
|
-
{
|
|
1746
|
+
{"input_tokens": 18, "output_tokens": 5, "total_tokens": 23}
|
|
1279
1747
|
|
|
1280
1748
|
|
|
1281
1749
|
Response metadata
|
|
@@ -1287,9 +1755,30 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1287
1755
|
.. code-block:: python
|
|
1288
1756
|
|
|
1289
1757
|
{
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1758
|
+
"prompt_feedback": {"block_reason": 0, "safety_ratings": []},
|
|
1759
|
+
"finish_reason": "STOP",
|
|
1760
|
+
"safety_ratings": [
|
|
1761
|
+
{
|
|
1762
|
+
"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
|
|
1763
|
+
"probability": "NEGLIGIBLE",
|
|
1764
|
+
"blocked": False,
|
|
1765
|
+
},
|
|
1766
|
+
{
|
|
1767
|
+
"category": "HARM_CATEGORY_HATE_SPEECH",
|
|
1768
|
+
"probability": "NEGLIGIBLE",
|
|
1769
|
+
"blocked": False,
|
|
1770
|
+
},
|
|
1771
|
+
{
|
|
1772
|
+
"category": "HARM_CATEGORY_HARASSMENT",
|
|
1773
|
+
"probability": "NEGLIGIBLE",
|
|
1774
|
+
"blocked": False,
|
|
1775
|
+
},
|
|
1776
|
+
{
|
|
1777
|
+
"category": "HARM_CATEGORY_DANGEROUS_CONTENT",
|
|
1778
|
+
"probability": "NEGLIGIBLE",
|
|
1779
|
+
"blocked": False,
|
|
1780
|
+
},
|
|
1781
|
+
],
|
|
1293
1782
|
}
|
|
1294
1783
|
|
|
1295
1784
|
""" # noqa: E501
|
|
@@ -1303,32 +1792,33 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1303
1792
|
convert_system_message_to_human: bool = False
|
|
1304
1793
|
"""Whether to merge any leading SystemMessage into the following HumanMessage.
|
|
1305
1794
|
|
|
1306
|
-
Gemini does not support system messages; any unsupported messages will
|
|
1307
|
-
|
|
1795
|
+
Gemini does not support system messages; any unsupported messages will raise an
|
|
1796
|
+
error.
|
|
1797
|
+
"""
|
|
1308
1798
|
|
|
1309
1799
|
response_mime_type: Optional[str] = None
|
|
1310
1800
|
"""Optional. Output response mimetype of the generated candidate text. Only
|
|
1311
1801
|
supported in Gemini 1.5 and later models.
|
|
1312
|
-
|
|
1802
|
+
|
|
1313
1803
|
Supported mimetype:
|
|
1314
1804
|
* ``'text/plain'``: (default) Text output.
|
|
1315
1805
|
* ``'application/json'``: JSON response in the candidates.
|
|
1316
1806
|
* ``'text/x.enum'``: Enum in plain text.
|
|
1317
|
-
|
|
1807
|
+
|
|
1318
1808
|
The model also needs to be prompted to output the appropriate response
|
|
1319
1809
|
type, otherwise the behavior is undefined. This is a preview feature.
|
|
1320
1810
|
"""
|
|
1321
1811
|
|
|
1322
1812
|
response_schema: Optional[Dict[str, Any]] = None
|
|
1323
|
-
""" Optional. Enforce an schema to the output.
|
|
1324
|
-
|
|
1813
|
+
""" Optional. Enforce an schema to the output. The format of the dictionary should
|
|
1814
|
+
follow Open API schema.
|
|
1325
1815
|
"""
|
|
1326
1816
|
|
|
1327
1817
|
cached_content: Optional[str] = None
|
|
1328
|
-
"""The name of the cached content used as context to serve the prediction.
|
|
1818
|
+
"""The name of the cached content used as context to serve the prediction.
|
|
1329
1819
|
|
|
1330
|
-
Note: only used in explicit caching, where users can have control over caching
|
|
1331
|
-
(e.g. what content to cache) and enjoy guaranteed cost savings. Format:
|
|
1820
|
+
Note: only used in explicit caching, where users can have control over caching
|
|
1821
|
+
(e.g. what content to cache) and enjoy guaranteed cost savings. Format:
|
|
1332
1822
|
``cachedContents/{cachedContent}``.
|
|
1333
1823
|
"""
|
|
1334
1824
|
|
|
@@ -1384,7 +1874,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1384
1874
|
)
|
|
1385
1875
|
|
|
1386
1876
|
@classmethod
|
|
1387
|
-
def is_lc_serializable(
|
|
1877
|
+
def is_lc_serializable(cls) -> bool:
|
|
1388
1878
|
return True
|
|
1389
1879
|
|
|
1390
1880
|
@model_validator(mode="before")
|
|
@@ -1392,20 +1882,22 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1392
1882
|
def build_extra(cls, values: dict[str, Any]) -> Any:
|
|
1393
1883
|
"""Build extra kwargs from additional params that were passed in."""
|
|
1394
1884
|
all_required_field_names = get_pydantic_field_names(cls)
|
|
1395
|
-
|
|
1396
|
-
return values
|
|
1885
|
+
return _build_model_kwargs(values, all_required_field_names)
|
|
1397
1886
|
|
|
1398
1887
|
@model_validator(mode="after")
|
|
1399
1888
|
def validate_environment(self) -> Self:
|
|
1400
1889
|
"""Validates params and passes them to google-generativeai package."""
|
|
1401
1890
|
if self.temperature is not None and not 0 <= self.temperature <= 2.0:
|
|
1402
|
-
|
|
1891
|
+
msg = "temperature must be in the range [0.0, 2.0]"
|
|
1892
|
+
raise ValueError(msg)
|
|
1403
1893
|
|
|
1404
1894
|
if self.top_p is not None and not 0 <= self.top_p <= 1:
|
|
1405
|
-
|
|
1895
|
+
msg = "top_p must be in the range [0.0, 1.0]"
|
|
1896
|
+
raise ValueError(msg)
|
|
1406
1897
|
|
|
1407
1898
|
if self.top_k is not None and self.top_k <= 0:
|
|
1408
|
-
|
|
1899
|
+
msg = "top_k must be positive"
|
|
1900
|
+
raise ValueError(msg)
|
|
1409
1901
|
|
|
1410
1902
|
if not any(self.model.startswith(prefix) for prefix in ("models/",)):
|
|
1411
1903
|
self.model = f"models/{self.model}"
|
|
@@ -1480,31 +1972,29 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1480
1972
|
code_execution: Optional[bool] = None,
|
|
1481
1973
|
stop: Optional[list[str]] = None,
|
|
1482
1974
|
**kwargs: Any,
|
|
1483
|
-
) ->
|
|
1484
|
-
"""
|
|
1485
|
-
Enable code execution. Supported on: gemini-1.5-pro, gemini-1.5-flash,
|
|
1486
|
-
gemini-2.0-flash, and gemini-2.0-pro. When enabled, the model can execute
|
|
1487
|
-
code to solve problems.
|
|
1488
|
-
"""
|
|
1975
|
+
) -> AIMessage:
|
|
1976
|
+
"""Override invoke to add code_execution parameter.
|
|
1489
1977
|
|
|
1490
|
-
|
|
1978
|
+
Supported on: gemini-1.5-pro, gemini-1.5-flash, gemini-2.0-flash, and
|
|
1979
|
+
gemini-2.0-pro. When enabled, the model can execute code to solve problems.
|
|
1980
|
+
"""
|
|
1491
1981
|
|
|
1492
1982
|
if code_execution is not None:
|
|
1493
1983
|
if not self._supports_code_execution:
|
|
1494
|
-
|
|
1984
|
+
msg = (
|
|
1495
1985
|
f"Code execution is only supported on Gemini 1.5 Pro, \
|
|
1496
1986
|
Gemini 1.5 Flash, "
|
|
1497
1987
|
f"Gemini 2.0 Flash, and Gemini 2.0 Pro models. \
|
|
1498
1988
|
Current model: {self.model}"
|
|
1499
1989
|
)
|
|
1990
|
+
raise ValueError(msg)
|
|
1500
1991
|
if "tools" not in kwargs:
|
|
1501
1992
|
code_execution_tool = GoogleTool(code_execution=CodeExecution())
|
|
1502
1993
|
kwargs["tools"] = [code_execution_tool]
|
|
1503
1994
|
|
|
1504
1995
|
else:
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
)
|
|
1996
|
+
msg = "Tools are already defined.code_execution tool can't be defined"
|
|
1997
|
+
raise ValueError(msg)
|
|
1508
1998
|
|
|
1509
1999
|
return super().invoke(input, config, stop=stop, **kwargs)
|
|
1510
2000
|
|
|
@@ -1616,6 +2106,10 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1616
2106
|
tool_choice=tool_choice,
|
|
1617
2107
|
**kwargs,
|
|
1618
2108
|
)
|
|
2109
|
+
if self.timeout is not None and "timeout" not in kwargs:
|
|
2110
|
+
kwargs["timeout"] = self.timeout
|
|
2111
|
+
if "max_retries" not in kwargs:
|
|
2112
|
+
kwargs["max_retries"] = self.max_retries
|
|
1619
2113
|
response: GenerateContentResponse = _chat_with_retry(
|
|
1620
2114
|
request=request,
|
|
1621
2115
|
**kwargs,
|
|
@@ -1642,13 +2136,11 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1642
2136
|
if not self.async_client:
|
|
1643
2137
|
updated_kwargs = {
|
|
1644
2138
|
**kwargs,
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
"generation_config": generation_config,
|
|
1651
|
-
},
|
|
2139
|
+
"tools": tools,
|
|
2140
|
+
"functions": functions,
|
|
2141
|
+
"safety_settings": safety_settings,
|
|
2142
|
+
"tool_config": tool_config,
|
|
2143
|
+
"generation_config": generation_config,
|
|
1652
2144
|
}
|
|
1653
2145
|
return await super()._agenerate(
|
|
1654
2146
|
messages, stop, run_manager, **updated_kwargs
|
|
@@ -1666,6 +2158,10 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1666
2158
|
tool_choice=tool_choice,
|
|
1667
2159
|
**kwargs,
|
|
1668
2160
|
)
|
|
2161
|
+
if self.timeout is not None and "timeout" not in kwargs:
|
|
2162
|
+
kwargs["timeout"] = self.timeout
|
|
2163
|
+
if "max_retries" not in kwargs:
|
|
2164
|
+
kwargs["max_retries"] = self.max_retries
|
|
1669
2165
|
response: GenerateContentResponse = await _achat_with_retry(
|
|
1670
2166
|
request=request,
|
|
1671
2167
|
**kwargs,
|
|
@@ -1701,6 +2197,10 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1701
2197
|
tool_choice=tool_choice,
|
|
1702
2198
|
**kwargs,
|
|
1703
2199
|
)
|
|
2200
|
+
if self.timeout is not None and "timeout" not in kwargs:
|
|
2201
|
+
kwargs["timeout"] = self.timeout
|
|
2202
|
+
if "max_retries" not in kwargs:
|
|
2203
|
+
kwargs["max_retries"] = self.max_retries
|
|
1704
2204
|
response: GenerateContentResponse = _chat_with_retry(
|
|
1705
2205
|
request=request,
|
|
1706
2206
|
generation_method=self.client.stream_generate_content,
|
|
@@ -1713,8 +2213,8 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1713
2213
|
_chat_result = _response_to_result(
|
|
1714
2214
|
chunk, stream=True, prev_usage=prev_usage_metadata
|
|
1715
2215
|
)
|
|
1716
|
-
gen = cast(ChatGenerationChunk, _chat_result.generations[0])
|
|
1717
|
-
message = cast(AIMessageChunk, gen.message)
|
|
2216
|
+
gen = cast("ChatGenerationChunk", _chat_result.generations[0])
|
|
2217
|
+
message = cast("AIMessageChunk", gen.message)
|
|
1718
2218
|
|
|
1719
2219
|
prev_usage_metadata = (
|
|
1720
2220
|
message.usage_metadata
|
|
@@ -1744,13 +2244,11 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1744
2244
|
if not self.async_client:
|
|
1745
2245
|
updated_kwargs = {
|
|
1746
2246
|
**kwargs,
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
"generation_config": generation_config,
|
|
1753
|
-
},
|
|
2247
|
+
"tools": tools,
|
|
2248
|
+
"functions": functions,
|
|
2249
|
+
"safety_settings": safety_settings,
|
|
2250
|
+
"tool_config": tool_config,
|
|
2251
|
+
"generation_config": generation_config,
|
|
1754
2252
|
}
|
|
1755
2253
|
async for value in super()._astream(
|
|
1756
2254
|
messages, stop, run_manager, **updated_kwargs
|
|
@@ -1769,6 +2267,10 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1769
2267
|
tool_choice=tool_choice,
|
|
1770
2268
|
**kwargs,
|
|
1771
2269
|
)
|
|
2270
|
+
if self.timeout is not None and "timeout" not in kwargs:
|
|
2271
|
+
kwargs["timeout"] = self.timeout
|
|
2272
|
+
if "max_retries" not in kwargs:
|
|
2273
|
+
kwargs["max_retries"] = self.max_retries
|
|
1772
2274
|
prev_usage_metadata: UsageMetadata | None = None # cumulative usage
|
|
1773
2275
|
async for chunk in await _achat_with_retry(
|
|
1774
2276
|
request=request,
|
|
@@ -1779,8 +2281,8 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1779
2281
|
_chat_result = _response_to_result(
|
|
1780
2282
|
chunk, stream=True, prev_usage=prev_usage_metadata
|
|
1781
2283
|
)
|
|
1782
|
-
gen = cast(ChatGenerationChunk, _chat_result.generations[0])
|
|
1783
|
-
message = cast(AIMessageChunk, gen.message)
|
|
2284
|
+
gen = cast("ChatGenerationChunk", _chat_result.generations[0])
|
|
2285
|
+
message = cast("AIMessageChunk", gen.message)
|
|
1784
2286
|
|
|
1785
2287
|
prev_usage_metadata = (
|
|
1786
2288
|
message.usage_metadata
|
|
@@ -1807,10 +2309,11 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1807
2309
|
**kwargs: Any,
|
|
1808
2310
|
) -> GenerateContentRequest:
|
|
1809
2311
|
if tool_choice and tool_config:
|
|
1810
|
-
|
|
2312
|
+
msg = (
|
|
1811
2313
|
"Must specify at most one of tool_choice and tool_config, received "
|
|
1812
2314
|
f"both:\n\n{tool_choice=}\n\n{tool_config=}"
|
|
1813
2315
|
)
|
|
2316
|
+
raise ValueError(msg)
|
|
1814
2317
|
|
|
1815
2318
|
formatted_tools = None
|
|
1816
2319
|
code_execution_tool = GoogleTool(code_execution=CodeExecution())
|
|
@@ -1848,16 +2351,15 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1848
2351
|
all_names: List[str] = []
|
|
1849
2352
|
for t in formatted_tools:
|
|
1850
2353
|
if hasattr(t, "function_declarations"):
|
|
1851
|
-
t_with_declarations = cast(Any, t)
|
|
2354
|
+
t_with_declarations = cast("Any", t)
|
|
1852
2355
|
all_names.extend(
|
|
1853
2356
|
f.name for f in t_with_declarations.function_declarations
|
|
1854
2357
|
)
|
|
1855
2358
|
elif isinstance(t, GoogleTool) and hasattr(t, "code_execution"):
|
|
1856
2359
|
continue
|
|
1857
2360
|
else:
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
)
|
|
2361
|
+
msg = f"Tool {t} doesn't have function_declarations attribute"
|
|
2362
|
+
raise TypeError(msg)
|
|
1861
2363
|
|
|
1862
2364
|
tool_config = _tool_choice_to_tool_config(tool_choice, all_names)
|
|
1863
2365
|
|
|
@@ -1874,7 +2376,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1874
2376
|
]
|
|
1875
2377
|
request = GenerateContentRequest(
|
|
1876
2378
|
model=self.model,
|
|
1877
|
-
contents=history,
|
|
2379
|
+
contents=history, # google.ai.generativelanguage_v1beta.types.Content
|
|
1878
2380
|
tools=formatted_tools,
|
|
1879
2381
|
tool_config=formatted_tool_config,
|
|
1880
2382
|
safety_settings=formatted_safety_settings,
|
|
@@ -1891,7 +2393,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1891
2393
|
return request
|
|
1892
2394
|
|
|
1893
2395
|
def get_num_tokens(self, text: str) -> int:
|
|
1894
|
-
"""Get the number of tokens present in the text.
|
|
2396
|
+
"""Get the number of tokens present in the text. Uses the model's tokenizer.
|
|
1895
2397
|
|
|
1896
2398
|
Useful for checking if an input will fit in a model's context window.
|
|
1897
2399
|
|
|
@@ -1909,18 +2411,22 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1909
2411
|
def with_structured_output(
|
|
1910
2412
|
self,
|
|
1911
2413
|
schema: Union[Dict, Type[BaseModel]],
|
|
1912
|
-
method: Optional[
|
|
2414
|
+
method: Optional[
|
|
2415
|
+
Literal["function_calling", "json_mode", "json_schema"]
|
|
2416
|
+
] = "function_calling",
|
|
1913
2417
|
*,
|
|
1914
2418
|
include_raw: bool = False,
|
|
1915
2419
|
**kwargs: Any,
|
|
1916
2420
|
) -> Runnable[LanguageModelInput, Union[Dict, BaseModel]]:
|
|
1917
2421
|
_ = kwargs.pop("strict", None)
|
|
1918
2422
|
if kwargs:
|
|
1919
|
-
|
|
2423
|
+
msg = f"Received unsupported arguments {kwargs}"
|
|
2424
|
+
raise ValueError(msg)
|
|
1920
2425
|
|
|
1921
2426
|
parser: OutputParserLike
|
|
1922
2427
|
|
|
1923
|
-
|
|
2428
|
+
# `json_schema` preferred, but `json_mode` kept for backwards compatibility
|
|
2429
|
+
if method in ("json_mode", "json_schema"):
|
|
1924
2430
|
if isinstance(schema, type) and is_basemodel_subclass(schema):
|
|
1925
2431
|
if issubclass(schema, BaseModelV1):
|
|
1926
2432
|
schema_json = schema.schema()
|
|
@@ -1933,7 +2439,8 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1933
2439
|
elif isinstance(schema, dict):
|
|
1934
2440
|
schema_json = schema
|
|
1935
2441
|
else:
|
|
1936
|
-
|
|
2442
|
+
msg = f"Unsupported schema type {type(schema)}"
|
|
2443
|
+
raise ValueError(msg)
|
|
1937
2444
|
parser = JsonOutputParser()
|
|
1938
2445
|
|
|
1939
2446
|
# Resolve refs in schema because they are not supported
|
|
@@ -1976,8 +2483,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1976
2483
|
exception_key="parsing_error",
|
|
1977
2484
|
)
|
|
1978
2485
|
return {"raw": llm} | parser_with_fallback
|
|
1979
|
-
|
|
1980
|
-
return llm | parser
|
|
2486
|
+
return llm | parser
|
|
1981
2487
|
|
|
1982
2488
|
def bind_tools(
|
|
1983
2489
|
self,
|
|
@@ -1988,27 +2494,28 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
|
|
|
1988
2494
|
*,
|
|
1989
2495
|
tool_choice: Optional[Union[_ToolChoiceType, bool]] = None,
|
|
1990
2496
|
**kwargs: Any,
|
|
1991
|
-
) -> Runnable[LanguageModelInput,
|
|
2497
|
+
) -> Runnable[LanguageModelInput, AIMessage]:
|
|
1992
2498
|
"""Bind tool-like objects to this chat model.
|
|
1993
2499
|
|
|
1994
2500
|
Assumes model is compatible with google-generativeAI tool-calling API.
|
|
1995
2501
|
|
|
1996
2502
|
Args:
|
|
1997
2503
|
tools: A list of tool definitions to bind to this chat model.
|
|
1998
|
-
Can be a pydantic model, callable, or BaseTool. Pydantic
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2504
|
+
Can be a pydantic model, callable, or BaseTool. Pydantic models,
|
|
2505
|
+
callables, and BaseTools will be automatically converted to their schema
|
|
2506
|
+
dictionary representation. Tools with Union types in their arguments are
|
|
2507
|
+
now supported and converted to `anyOf` schemas.
|
|
2002
2508
|
**kwargs: Any additional parameters to pass to the
|
|
2003
2509
|
:class:`~langchain.runnable.Runnable` constructor.
|
|
2004
2510
|
"""
|
|
2005
2511
|
if tool_choice and tool_config:
|
|
2006
|
-
|
|
2512
|
+
msg = (
|
|
2007
2513
|
"Must specify at most one of tool_choice and tool_config, received "
|
|
2008
2514
|
f"both:\n\n{tool_choice=}\n\n{tool_config=}"
|
|
2009
2515
|
)
|
|
2516
|
+
raise ValueError(msg)
|
|
2010
2517
|
try:
|
|
2011
|
-
formatted_tools: list = [convert_to_openai_tool(tool) for tool in tools]
|
|
2518
|
+
formatted_tools: list = [convert_to_openai_tool(tool) for tool in tools]
|
|
2012
2519
|
except Exception:
|
|
2013
2520
|
formatted_tools = [
|
|
2014
2521
|
tool_to_dict(convert_to_genai_function_declarations(tools))
|
|
@@ -2035,9 +2542,8 @@ def _get_tool_name(
|
|
|
2035
2542
|
) -> str:
|
|
2036
2543
|
try:
|
|
2037
2544
|
genai_tool = tool_to_dict(convert_to_genai_function_declarations([tool]))
|
|
2038
|
-
return
|
|
2039
|
-
except ValueError
|
|
2545
|
+
return next(f["name"] for f in genai_tool["function_declarations"]) # type: ignore[index]
|
|
2546
|
+
except ValueError: # other TypedDict
|
|
2040
2547
|
if is_typeddict(tool):
|
|
2041
|
-
return convert_to_openai_tool(cast(Dict, tool))["function"]["name"]
|
|
2042
|
-
|
|
2043
|
-
raise e
|
|
2548
|
+
return convert_to_openai_tool(cast("Dict", tool))["function"]["name"]
|
|
2549
|
+
raise
|