langchain-google-genai 2.1.4__py3-none-any.whl → 2.1.6__py3-none-any.whl

langchain_google_genai/_common.py

@@ -84,6 +84,11 @@ Supported examples:
         default=None, description="Indicates the thinking budget in tokens."
     )
 
+    include_thoughts: Optional[bool] = Field(
+        default=None,
+        description="Indicates whether to include thoughts in the response.",
+    )
+
     safety_settings: Optional[Dict[HarmCategory, HarmBlockThreshold]] = None
     """The default safety settings to use for all generations. 
     

langchain_google_genai/_function_utils.py

@@ -30,6 +30,7 @@ from langchain_core.utils.function_calling import (
 from langchain_core.utils.json_schema import dereference_refs
 from pydantic import BaseModel
 from pydantic.v1 import BaseModel as BaseModelV1
+from typing_extensions import NotRequired
 
 logger = logging.getLogger(__name__)
 
@@ -65,11 +66,15 @@ _GoogleSearchRetrievalLike = Union[
     gapic.GoogleSearchRetrieval,
     Dict[str, Any],
 ]
+_GoogleSearchLike = Union[gapic.Tool.GoogleSearch, Dict[str, Any]]
+_CodeExecutionLike = Union[gapic.CodeExecution, Dict[str, Any]]
 
 
 class _ToolDict(TypedDict):
     function_declarations: Sequence[_FunctionDeclarationLike]
     google_search_retrieval: Optional[_GoogleSearchRetrievalLike]
+    google_search: NotRequired[_GoogleSearchLike]
+    code_execution: NotRequired[_CodeExecutionLike]
 
 
 # Info: This means one tool=Sequence of FunctionDeclaration
@@ -158,6 +163,8 @@ def convert_to_genai_function_declarations(
                 for f in [
                     "function_declarations",
                     "google_search_retrieval",
+                    "google_search",
+                    "code_execution",
                 ]
             ):
                 fd = _format_to_gapic_function_declaration(tool)  # type: ignore[arg-type]
@@ -184,6 +191,12 @@ def convert_to_genai_function_declarations(
                 gapic_tool.google_search_retrieval = gapic.GoogleSearchRetrieval(
                     tool["google_search_retrieval"]
                 )
+            if "google_search" in tool:
+                gapic_tool.google_search = gapic.Tool.GoogleSearch(
+                    tool["google_search"]
+                )
+            if "code_execution" in tool:
+                gapic_tool.code_execution = gapic.CodeExecution(tool["code_execution"])
         else:
             fd = _format_to_gapic_function_declaration(tool)  # type: ignore[arg-type]
             gapic_tool.function_declarations.append(fd)
@@ -520,3 +533,60 @@ def safe_import(module_name: str, attribute_name: str = "") -> bool:
         return True
     except ImportError:
         return False
+
+
+def replace_defs_in_schema(original_schema: dict, defs: Optional[dict] = None) -> dict:
+    """Given an OpenAPI schema with a property '$defs' replaces all occurrences of
+    referenced items in the dictionary.
+
+    Args:
+        original_schema: Schema generated by `BaseModel.model_schema_json`
+        defs: Definitions for recursive calls.
+
+    Returns:
+        Schema with refs replaced.
+    """
+
+    new_defs = defs or original_schema.get("$defs")
+
+    if new_defs is None or not isinstance(new_defs, dict):
+        return original_schema.copy()
+
+    resulting_schema = {}
+
+    for key, value in original_schema.items():
+        if key == "$defs":
+            continue
+
+        if not isinstance(value, dict):
+            resulting_schema[key] = value
+        else:
+            if "$ref" in value:
+                new_value = value.copy()
+
+                path = new_value.pop("$ref")
+                def_key = _get_def_key_from_schema_path(path)
+                new_item = new_defs.get(def_key)
+
+                assert isinstance(new_item, dict)
+                new_value.update(new_item)
+
+                resulting_schema[key] = replace_defs_in_schema(new_value, defs=new_defs)
+            else:
+                resulting_schema[key] = replace_defs_in_schema(value, defs=new_defs)
+
+    return resulting_schema
+
+
+def _get_def_key_from_schema_path(schema_path: str) -> str:
+    error_message = f"Malformed schema reference path {schema_path}"
+
+    if not isinstance(schema_path, str) or not schema_path.startswith("#/$defs/"):
+        raise ValueError(error_message)
+
+    # Schema has to have only one extra level.
+    parts = schema_path.split("/")
+    if len(parts) != 3:
+        raise ValueError(error_message)
+
+    return parts[-1]

langchain_google_genai/chat_models.py

@@ -2,11 +2,13 @@ from __future__ import annotations
 
 import asyncio
 import base64
+import io
 import json
 import logging
 import mimetypes
 import uuid
 import warnings
+import wave
 from difflib import get_close_matches
 from operator import itemgetter
 from typing import (
@@ -16,6 +18,7 @@ from typing import (
     Dict,
     Iterator,
     List,
+    Literal,
     Mapping,
     Optional,
     Sequence,
@@ -37,7 +40,9 @@ from google.ai.generativelanguage_v1beta.types import (
     Blob,
     Candidate,
     CodeExecution,
+    CodeExecutionResult,
     Content,
+    ExecutableCode,
     FileData,
     FunctionCall,
     FunctionDeclaration,
@@ -69,6 +74,7 @@ from langchain_core.messages import (
 )
 from langchain_core.messages.ai import UsageMetadata
 from langchain_core.messages.tool import invalid_tool_call, tool_call, tool_call_chunk
+from langchain_core.output_parsers import JsonOutputParser, PydanticOutputParser
 from langchain_core.output_parsers.base import OutputParserLike
 from langchain_core.output_parsers.openai_tools import (
     JsonOutputKeyToolsParser,
@@ -79,7 +85,11 @@ from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResu
 from langchain_core.runnables import Runnable, RunnableConfig, RunnablePassthrough
 from langchain_core.tools import BaseTool
 from langchain_core.utils import get_pydantic_field_names
-from langchain_core.utils.function_calling import convert_to_openai_tool
+from langchain_core.utils.function_calling import (
+    convert_to_json_schema,
+    convert_to_openai_tool,
+)
+from langchain_core.utils.pydantic import is_basemodel_subclass
 from langchain_core.utils.utils import _build_model_kwargs
 from pydantic import (
     BaseModel,
@@ -88,6 +98,7 @@ from pydantic import (
     SecretStr,
     model_validator,
 )
+from pydantic.v1 import BaseModel as BaseModelV1
 from tenacity import (
     before_sleep_log,
     retry,
@@ -104,12 +115,14 @@ from langchain_google_genai._common import (
     get_client_info,
 )
 from langchain_google_genai._function_utils import (
+    _dict_to_gapic_schema,
     _tool_choice_to_tool_config,
     _ToolChoiceType,
     _ToolConfigDict,
     _ToolDict,
     convert_to_genai_function_declarations,
     is_basemodel_subclass_safe,
+    replace_defs_in_schema,
     tool_to_dict,
 )
 from langchain_google_genai._image_utils import (
@@ -121,6 +134,7 @@ from . import _genai_extension as genaix
 
 logger = logging.getLogger(__name__)
 
+_allowed_params_prediction_service = ["request", "timeout", "metadata", "labels"]
 
 _FunctionDeclarationType = Union[
     FunctionDeclaration,
@@ -207,7 +221,14 @@ def _chat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
         except Exception as e:
             raise e
 
-    return _chat_with_retry(**kwargs)
+    params = (
+        {k: v for k, v in kwargs.items() if k in _allowed_params_prediction_service}
+        if (request := kwargs.get("request"))
+        and hasattr(request, "model")
+        and "gemini" in request.model
+        else kwargs
+    )
+    return _chat_with_retry(**params)
 
 
 async def _achat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
@@ -240,13 +261,37 @@ async def _achat_with_retry(generation_method: Callable, **kwargs: Any) -> Any:
         except Exception as e:
             raise e
 
-    return await _achat_with_retry(**kwargs)
+    params = (
+        {k: v for k, v in kwargs.items() if k in _allowed_params_prediction_service}
+        if (request := kwargs.get("request"))
+        and hasattr(request, "model")
+        and "gemini" in request.model
+        else kwargs
+    )
+    return await _achat_with_retry(**params)
 
 
 def _is_lc_content_block(part: dict) -> bool:
     return "type" in part
 
 
+def _is_openai_image_block(block: dict) -> bool:
+    """Check if the block contains image data in OpenAI Chat Completions format."""
+    if block.get("type") == "image_url":
+        if (
+            (set(block.keys()) <= {"type", "image_url", "detail"})
+            and (image_url := block.get("image_url"))
+            and isinstance(image_url, dict)
+        ):
+            url = image_url.get("url")
+            if isinstance(url, str):
+                return True
+    else:
+        return False
+
+    return False
+
+
 def _convert_to_parts(
     raw_content: Union[str, Sequence[Union[str, dict]]],
 ) -> List[Part]:
@@ -314,6 +359,37 @@ def _convert_to_parts(
                         metadata = VideoMetadata(part["video_metadata"])
                         media_part.video_metadata = metadata
                     parts.append(media_part)
+                elif part["type"] == "executable_code":
+                    if "executable_code" not in part or "language" not in part:
+                        raise ValueError(
+                            "Executable code part must have 'code' and 'language' "
+                            f"keys, got {part}"
+                        )
+                    executable_code_part = Part(
+                        executable_code=ExecutableCode(
+                            language=part["language"], code=part["executable_code"]
+                        )
+                    )
+                    parts.append(executable_code_part)
+                elif part["type"] == "code_execution_result":
+                    if "code_execution_result" not in part:
+                        raise ValueError(
+                            "Code execution result part must have "
+                            f"'code_execution_result', got {part}"
+                        )
+                    if "outcome" in part:
+                        outcome = part["outcome"]
+                    else:
+                        # Backward compatibility
+                        outcome = 1  # Default to success if not specified
+                    code_execution_result_part = Part(
+                        code_execution_result=CodeExecutionResult(
+                            output=part["code_execution_result"], outcome=outcome
+                        )
+                    )
+                    parts.append(code_execution_result_part)
+                elif part["type"] == "thinking":
+                    parts.append(Part(text=part["thinking"], thought=True))
                 else:
                     raise ValueError(
                         f"Unrecognized message part type: {part['type']}. Only text, "
@@ -334,14 +410,28 @@ def _convert_to_parts(
     return parts
 
 
-def _convert_tool_message_to_part(
+def _convert_tool_message_to_parts(
     message: ToolMessage | FunctionMessage, name: Optional[str] = None
-) -> Part:
+) -> list[Part]:
     """Converts a tool or function message to a google part."""
     # Legacy agent stores tool name in message.additional_kwargs instead of message.name
     name = message.name or name or message.additional_kwargs.get("name")
     response: Any
-    if not isinstance(message.content, str):
+    parts: list[Part] = []
+    if isinstance(message.content, list):
+        media_blocks = []
+        other_blocks = []
+        for block in message.content:
+            if isinstance(block, dict) and (
+                is_data_content_block(block) or _is_openai_image_block(block)
+            ):
+                media_blocks.append(block)
+            else:
+                other_blocks.append(block)
+        parts.extend(_convert_to_parts(media_blocks))
+        response = other_blocks
+
+    elif not isinstance(message.content, str):
         response = message.content
     else:
         try:
@@ -356,7 +446,8 @@ def _convert_tool_message_to_part(
             ),
         )
     )
-    return part
+    parts.append(part)
+    return parts
 
 
 def _get_ai_message_tool_messages_parts(
@@ -374,8 +465,10 @@ def _get_ai_message_tool_messages_parts(
             break
         if message.tool_call_id in tool_calls_ids:
             tool_call = tool_calls_ids[message.tool_call_id]
-            part = _convert_tool_message_to_part(message, name=tool_call.get("name"))
-            parts.append(part)
+            message_parts = _convert_tool_message_to_parts(
+                message, name=tool_call.get("name")
+            )
+            parts.extend(message_parts)
             # remove the id from the dict, so that we do not iterate over it again
             tool_calls_ids.pop(message.tool_call_id)
     return parts
@@ -442,7 +535,7 @@ def _parse_chat_history(
                 system_instruction = None
         elif isinstance(message, FunctionMessage):
             role = "user"
-            parts = [_convert_tool_message_to_part(message)]
+            parts = _convert_tool_message_to_parts(message)
         else:
             raise ValueError(
                 f"Unexpected message with type {type(message)} at the position {i}."
@@ -452,33 +545,54 @@ def _parse_chat_history(
     return system_instruction, messages
 
 
+# Helper function to append content consistently
+def _append_to_content(
+    current_content: Union[str, List[Any], None], new_item: Any
+) -> Union[str, List[Any]]:
+    """Appends a new item to the content, handling different initial content types."""
+    if current_content is None and isinstance(new_item, str):
+        return new_item
+    elif current_content is None:
+        return [new_item]
+    elif isinstance(current_content, str):
+        return [current_content, new_item]
+    elif isinstance(current_content, list):
+        current_content.append(new_item)
+        return current_content
+    else:
+        # This case should ideally not be reached with proper type checking,
+        # but it catches any unexpected types that might slip through.
+        raise TypeError(f"Unexpected content type: {type(current_content)}")
+
+
 def _parse_response_candidate(
     response_candidate: Candidate, streaming: bool = False
 ) -> AIMessage:
     content: Union[None, str, List[Union[str, dict]]] = None
-    additional_kwargs = {}
+    additional_kwargs: Dict[str, Any] = {}
     tool_calls = []
     invalid_tool_calls = []
     tool_call_chunks = []
 
     for part in response_candidate.content.parts:
+        text: Optional[str] = None
         try:
-            text: Optional[str] = part.text
-            # Remove erroneous newline character if present
-            if not streaming and text is not None:
-                text = text.rstrip("\n")
+            if hasattr(part, "text") and part.text is not None:
+                text = part.text
+                # Remove erroneous newline character if present
+                if not streaming:
+                    text = text.rstrip("\n")
         except AttributeError:
-            text = None
-
-        if text is not None:
-            if not content:
-                content = text
-            elif isinstance(content, str) and text:
-                content = [content, text]
-            elif isinstance(content, list) and text:
-                content.append(text)
-            elif text:
-                raise Exception("Unexpected content type")
+            pass
+
+        if hasattr(part, "thought") and part.thought:
+            thinking_message = {
+                "type": "thinking",
+                "thinking": part.text,
+            }
+            content = _append_to_content(content, thinking_message)
+        elif text is not None and text:
+            content = _append_to_content(content, text)
 
         if hasattr(part, "executable_code") and part.executable_code is not None:
             if part.executable_code.code and part.executable_code.language:
@@ -487,14 +601,7 @@ def _parse_response_candidate(
                     "executable_code": part.executable_code.code,
                     "language": part.executable_code.language,
                 }
-                if not content:
-                    content = [code_message]
-                elif isinstance(content, str):
-                    content = [content, code_message]
-                elif isinstance(content, list):
-                    content.append(code_message)
-                else:
-                    raise Exception("Unexpected content type")
+                content = _append_to_content(content, code_message)
 
         if (
             hasattr(part, "code_execution_result")
@@ -504,20 +611,25 @@ def _parse_response_candidate(
                 execution_result = {
                     "type": "code_execution_result",
                     "code_execution_result": part.code_execution_result.output,
+                    "outcome": part.code_execution_result.outcome,
                 }
+                content = _append_to_content(content, execution_result)
 
-                if not content:
-                    content = [execution_result]
-                elif isinstance(content, str):
-                    content = [content, execution_result]
-                elif isinstance(content, list):
-                    content.append(execution_result)
-                else:
-                    raise Exception("Unexpected content type")
+        if part.inline_data.mime_type.startswith("audio/"):
+            buffer = io.BytesIO()
+
+            with wave.open(buffer, "wb") as wf:
+                wf.setnchannels(1)
+                wf.setsampwidth(2)
+                # TODO: Read Sample Rate from MIME content type.
+                wf.setframerate(24000)
+                wf.writeframes(part.inline_data.data)
+
+            additional_kwargs["audio"] = buffer.getvalue()
 
         if part.inline_data.mime_type.startswith("image/"):
             image_format = part.inline_data.mime_type[6:]
-            message = {
+            image_message = {
                 "type": "image_url",
                 "image_url": {
                     "url": image_bytes_to_b64_string(
@@ -525,15 +637,7 @@ def _parse_response_candidate(
                     )
                 },
             }
-
-            if not content:
-                content = [message]
-            elif isinstance(content, str) and message:
-                content = [content, message]
-            elif isinstance(content, list) and message:
-                content.append(message)
-            elif message:
-                raise Exception("Unexpected content type")
+            content = _append_to_content(content, image_message)
 
         if part.function_call:
             function_call = {"name": part.function_call.name}
@@ -658,6 +762,13 @@ def _response_to_result(
             proto.Message.to_dict(safety_rating, use_integers_for_enums=False)
             for safety_rating in candidate.safety_ratings
         ]
+        try:
+            if candidate.grounding_metadata:
+                generation_info["grounding_metadata"] = proto.Message.to_dict(
+                    candidate.grounding_metadata
+                )
+        except AttributeError:
+            pass
         message = _parse_response_candidate(candidate, streaming=stream)
         message.usage_metadata = lc_usage
         if stream:
@@ -712,7 +823,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
 
             from langchain_google_genai import ChatGoogleGenerativeAI
 
-            llm = ChatGoogleGenerativeAI(model="gemini-1.5-pro")
+            llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash-001")
             llm.invoke("Write me a ballad about LangChain")
 
     Invoke:
@@ -797,7 +908,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
                 file = client.files.get(name=file.name)
 
             # Create cache
-            model = 'models/gemini-1.5-flash-001'
+            model = 'models/gemini-1.5-flash-latest'
             cache = client.caches.create(
                 model=model,
                 config=types.CreateCachedContentConfig(
@@ -853,7 +964,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
                     ],
                 )
             ]
-            model = "gemini-1.5-flash-001"
+            model = "gemini-1.5-flash-latest"
             cache = client.caches.create(
                 model=model,
                 config=CreateCachedContentConfig(
@@ -1019,6 +1130,21 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
     Gemini does not support system messages; any unsupported messages will
     raise an error."""
 
+    response_mime_type: Optional[str] = None
+    """Optional. Output response mimetype of the generated candidate text. Only
+        supported in Gemini 1.5 and later models. Supported mimetype:
+            * "text/plain": (default) Text output.
+            * "application/json": JSON response in the candidates.
+            * "text/x.enum": Enum in plain text.
+       The model also needs to be prompted to output the appropriate response
+       type, otherwise the behavior is undefined. This is a preview feature.
+    """
+
+    response_schema: Optional[Dict[str, Any]] = None
+    """ Optional. Enforce an schema to the output.
+        The format of the dictionary should follow Open API schema.
+    """
+
     cached_content: Optional[str] = None
     """The name of the cached content used as context to serve the prediction. 
 
@@ -1034,7 +1160,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
         """Needed for arg validation."""
         # Get all valid field names, including aliases
         valid_fields = set()
-        for field_name, field_info in self.model_fields.items():
+        for field_name, field_info in self.__class__.model_fields.items():
             valid_fields.add(field_name)
             if hasattr(field_info, "alias") and field_info.alias is not None:
                 valid_fields.add(field_info.alias)
@@ -1160,6 +1286,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
             "safety_settings": self.safety_settings,
             "response_modalities": self.response_modalities,
             "thinking_budget": self.thinking_budget,
+            "include_thoughts": self.include_thoughts,
         }
 
     def invoke(
@@ -1225,6 +1352,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
         self,
         stop: Optional[List[str]],
         generation_config: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
     ) -> GenerationConfig:
         gen_config = {
             k: v
@@ -1236,14 +1364,43 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
                 "top_k": self.top_k,
                 "top_p": self.top_p,
                 "response_modalities": self.response_modalities,
-                "thinking_config": {"thinking_budget": self.thinking_budget}
-                if self.thinking_budget is not None
+                "thinking_config": (
+                    (
+                        {"thinking_budget": self.thinking_budget}
+                        if self.thinking_budget is not None
+                        else {}
+                    )
+                    | (
+                        {"include_thoughts": self.include_thoughts}
+                        if self.include_thoughts is not None
+                        else {}
+                    )
+                )
+                if self.thinking_budget is not None or self.include_thoughts is not None
                 else None,
             }.items()
             if v is not None
         }
         if generation_config:
             gen_config = {**gen_config, **generation_config}
+
+        response_mime_type = kwargs.get("response_mime_type", self.response_mime_type)
+        if response_mime_type is not None:
+            gen_config["response_mime_type"] = response_mime_type
+
+        response_schema = kwargs.get("response_schema", self.response_schema)
+        if response_schema is not None:
+            allowed_mime_types = ("application/json", "text/x.enum")
+            if response_mime_type not in allowed_mime_types:
+                error_message = (
+                    "`response_schema` is only supported when "
+                    f"`response_mime_type` is set to one of {allowed_mime_types}"
+                )
+                raise ValueError(error_message)
+
+            gapic_response_schema = _dict_to_gapic_schema(response_schema)
+            if gapic_response_schema is not None:
+                gen_config["response_schema"] = gapic_response_schema
         return GenerationConfig(**gen_config)
 
     def _generate(
@@ -1271,6 +1428,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
             generation_config=generation_config,
             cached_content=cached_content or self.cached_content,
             tool_choice=tool_choice,
+            **kwargs,
         )
         response: GenerateContentResponse = _chat_with_retry(
             request=request,
@@ -1320,6 +1478,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
             generation_config=generation_config,
             cached_content=cached_content or self.cached_content,
             tool_choice=tool_choice,
+            **kwargs,
         )
         response: GenerateContentResponse = await _achat_with_retry(
             request=request,
@@ -1354,6 +1513,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
             generation_config=generation_config,
             cached_content=cached_content or self.cached_content,
             tool_choice=tool_choice,
+            **kwargs,
         )
         response: GenerateContentResponse = _chat_with_retry(
             request=request,
@@ -1432,6 +1592,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
                 generation_config=generation_config,
                 cached_content=cached_content or self.cached_content,
                 tool_choice=tool_choice,
+                **kwargs,
             )
             prev_usage_metadata: UsageMetadata | None = None
             async for chunk in await _achat_with_retry(
@@ -1479,6 +1640,7 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
         tool_choice: Optional[Union[_ToolChoiceType, bool]] = None,
         generation_config: Optional[Dict[str, Any]] = None,
         cached_content: Optional[str] = None,
+        **kwargs: Any,
     ) -> Tuple[GenerateContentRequest, Dict[str, Any]]:
         if tool_choice and tool_config:
             raise ValueError(
@@ -1550,7 +1712,9 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
             tool_config=formatted_tool_config,
             safety_settings=formatted_safety_settings,
             generation_config=self._prepare_params(
-                stop, generation_config=generation_config
+                stop,
+                generation_config=generation_config,
+                **kwargs,
             ),
             cached_content=cached_content,
         )
@@ -1578,33 +1742,65 @@ class ChatGoogleGenerativeAI(_BaseGoogleGenerativeAI, BaseChatModel):
     def with_structured_output(
         self,
         schema: Union[Dict, Type[BaseModel]],
+        method: Optional[Literal["function_calling", "json_mode"]] = "function_calling",
         *,
         include_raw: bool = False,
         **kwargs: Any,
     ) -> Runnable[LanguageModelInput, Union[Dict, BaseModel]]:
-        _ = kwargs.pop("method", None)
         _ = kwargs.pop("strict", None)
         if kwargs:
             raise ValueError(f"Received unsupported arguments {kwargs}")
-        tool_name = _get_tool_name(schema)  # type: ignore[arg-type]
-        if isinstance(schema, type) and is_basemodel_subclass_safe(schema):
-            parser: OutputParserLike = PydanticToolsParser(
-                tools=[schema], first_tool_only=True
-            )
-        else:
-            parser = JsonOutputKeyToolsParser(key_name=tool_name, first_tool_only=True)
-        tool_choice = tool_name if self._supports_tool_choice else None
-        try:
-            llm = self.bind_tools(
-                [schema],
-                tool_choice=tool_choice,
+
+        parser: OutputParserLike
+
+        if method == "json_mode":
+            if isinstance(schema, type) and is_basemodel_subclass(schema):
+                if issubclass(schema, BaseModelV1):
+                    schema_json = schema.schema()
+                else:
+                    schema_json = schema.model_json_schema()
+                parser = PydanticOutputParser(pydantic_object=schema)
+            else:
+                if is_typeddict(schema):
+                    schema_json = convert_to_json_schema(schema)
+                elif isinstance(schema, dict):
+                    schema_json = schema
+                else:
+                    raise ValueError(f"Unsupported schema type {type(schema)}")
+                parser = JsonOutputParser()
+
+            # Resolve refs in schema because they are not supported
+            # by the Gemini API.
+            schema_json = replace_defs_in_schema(schema_json)
+
+            llm = self.bind(
+                response_mime_type="application/json",
+                response_schema=schema_json,
                 ls_structured_output_format={
-                    "kwargs": {"method": "function_calling"},
-                    "schema": convert_to_openai_tool(schema),
+                    "kwargs": {"method": method},
+                    "schema": schema_json,
                 },
             )
-        except Exception:
-            llm = self.bind_tools([schema], tool_choice=tool_choice)
+        else:
+            tool_name = _get_tool_name(schema)  # type: ignore[arg-type]
+            if isinstance(schema, type) and is_basemodel_subclass_safe(schema):
+                parser = PydanticToolsParser(tools=[schema], first_tool_only=True)
+            else:
+                parser = JsonOutputKeyToolsParser(
+                    key_name=tool_name, first_tool_only=True
+                )
+            tool_choice = tool_name if self._supports_tool_choice else None
+            try:
+                llm = self.bind_tools(
+                    [schema],
+                    tool_choice=tool_choice,
+                    ls_structured_output_format={
+                        "kwargs": {"method": "function_calling"},
+                        "schema": convert_to_openai_tool(schema),
+                    },
+                )
+            except Exception:
+                llm = self.bind_tools([schema], tool_choice=tool_choice)
         if include_raw:
             parser_with_fallback = RunnablePassthrough.assign(
                 parsed=itemgetter("raw") | parser, parsing_error=lambda _: None

langchain_google_genai/embeddings.py

@@ -98,6 +98,7 @@ class GoogleGenerativeAIEmbeddings(BaseModel, Embeddings):
             api_key=google_api_key,
             client_info=client_info,
             client_options=self.client_options,
+            transport=self.transport,
         )
         return self
 
@@ -253,7 +254,9 @@ class GoogleGenerativeAIEmbeddings(BaseModel, Embeddings):
         Returns:
             Embedding for the text.
         """
-        task_type = self.task_type or "RETRIEVAL_QUERY"
+        task_type_to_use = task_type if task_type else self.task_type
+        if task_type_to_use is None:
+            task_type_to_use = "RETRIEVAL_QUERY"  # Default to RETRIEVAL_QUERY
         try:
             request: EmbedContentRequest = self._prepare_request(
                 text=text,

langchain_google_genai-2.1.6.dist-info/METADATA

@@ -0,0 +1,260 @@
+Metadata-Version: 2.1
+Name: langchain-google-genai
+Version: 2.1.6
+Summary: An integration package connecting Google's genai package and LangChain
+Home-page: https://github.com/langchain-ai/langchain-google
+License: MIT
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: filetype (>=1.2.0,<2.0.0)
+Requires-Dist: google-ai-generativelanguage (>=0.6.18,<0.7.0)
+Requires-Dist: langchain-core (>=0.3.66,<0.4.0)
+Requires-Dist: pydantic (>=2,<3)
+Project-URL: Repository, https://github.com/langchain-ai/langchain-google
+Project-URL: Source Code, https://github.com/langchain-ai/langchain-google/tree/main/libs/genai
+Description-Content-Type: text/markdown
+
+# langchain-google-genai
+
+**LangChain integration for Google Gemini models using the `generative-ai` SDK**
+
+This package enables seamless access to Google Gemini's chat, vision, embeddings, and retrieval-augmented generation (RAG) features within the LangChain ecosystem.
+
+---
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Installation](#installation)
+- [Quickstart](#quickstart)
+- [Chat Models](#chat-models)
+  - [Multimodal Inputs](#multimodal-inputs)
+  - [Multimodal Outputs](#multimodal-outputs)
+  - [Multimodal Outputs in Chains](#multimodal-outputs-in-chains)
+  - [Thinking Support](#thinking-support)
+- [Embeddings](#embeddings)
+- [Semantic Retrieval (RAG)](#semantic-retrieval-rag)
+
+---
+
+## Overview
+
+This package provides LangChain support for Google Gemini models (via the official [Google Generative AI SDK](https://googleapis.github.io/python-genai/)). It supports:
+
+- Text and vision-based chat models
+- Embeddings for semantic search
+- Multimodal inputs and outputs
+- Retrieval-Augmented Generation (RAG)
+- Thought tracing with reasoning tokens
+
+---
+
+## Installation
+
+```bash
+pip install -U langchain-google-genai
+````
+
+---
+
+## Quickstart
+
+Set up your environment variable with your Gemini API key:
+
+```bash
+export GOOGLE_API_KEY=your-api-key
+```
+
+Then use the `ChatGoogleGenerativeAI` interface:
+
+```python
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+llm = ChatGoogleGenerativeAI(model="gemini-pro")
+response = llm.invoke("Sing a ballad of LangChain.")
+print(response.content)
+```
+
+---
+
+## Chat Models
+
+The main interface for Gemini chat models is `ChatGoogleGenerativeAI`.
+
+### Multimodal Inputs
+
+Gemini vision models support image inputs in single messages.
+
+```python
+from langchain_core.messages import HumanMessage
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+llm = ChatGoogleGenerativeAI(model="gemini-pro-vision")
+
+message = HumanMessage(
+    content=[
+        {"type": "text", "text": "What's in this image?"},
+        {"type": "image_url", "image_url": "https://picsum.photos/seed/picsum/200/300"},
+    ]
+)
+
+response = llm.invoke([message])
+print(response.content)
+```
+
+✅ `image_url` can be:
+
+* A public image URL
+* A Google Cloud Storage path (`gcs://...`)
+* A base64-encoded image (e.g., `data:image/png;base64,...`)
+
+---
+
+### Multimodal Outputs
+
+The Gemini 2.0 Flash Experimental model supports both text and inline image outputs.
+
+```python
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+llm = ChatGoogleGenerativeAI(model="models/gemini-2.0-flash-exp-image-generation")
+
+response = llm.invoke(
+    "Generate an image of a cat and say meow",
+    generation_config=dict(response_modalities=["TEXT", "IMAGE"]),
+)
+
+image_base64 = response.content[0].get("image_url").get("url").split(",")[-1]
+meow_text = response.content[1]
+print(meow_text)
+```
+
+---
+
+### Audio Output
+
+```
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+llm = ChatGoogleGenerativeAI(model="models/gemini-2.5-flash-preview-tts")
+# example
+response = llm.invoke(
+    "Please say The quick brown fox jumps over the lazy dog",
+    generation_config=dict(response_modalities=["AUDIO"]),
+)
+
+# Base64 encoded binary data of the image
+wav_data = response.additional_kwargs.get("audio")
+with open("output.wav", "wb") as f:
+    f.write(wav_data)
+```
+
+---
+
+### Multimodal Outputs in Chains
+
+You can use Gemini models in a LangChain chain:
+
+```python
+from langchain_core.runnables import RunnablePassthrough
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_google_genai import ChatGoogleGenerativeAI, Modality
+
+llm = ChatGoogleGenerativeAI(
+    model="models/gemini-2.0-flash-exp-image-generation",
+    response_modalities=[Modality.TEXT, Modality.IMAGE],
+)
+
+prompt = ChatPromptTemplate.from_messages([
+    ("human", "Generate an image of {animal} and tell me the sound it makes.")
+])
+
+chain = {"animal": RunnablePassthrough()} | prompt | llm
+response = chain.invoke("cat")
+```
+
+---
+
+### Thinking Support
+
+Gemini 2.5 Flash Preview supports internal reasoning ("thoughts").
+
+```python
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+llm = ChatGoogleGenerativeAI(
+    model="models/gemini-2.5-flash-preview-04-17",
+    thinking_budget=1024
+)
+
+response = llm.invoke("How many O's are in Google? How did you verify your answer?")
+reasoning_score = response.usage_metadata["output_token_details"]["reasoning"]
+
+print("Response:", response.content)
+print("Reasoning tokens used:", reasoning_score)
+```
+
+---
+
+## Embeddings
+
+You can use Gemini embeddings in LangChain:
+
+```python
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+
+embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+vector = embeddings.embed_query("hello, world!")
+print(vector)
+```
+
+---
+
+## Semantic Retrieval (RAG)
+
+Use Gemini with RAG to retrieve relevant documents from your knowledge base.
+
+```python
+from langchain_google_genai.vectorstores import GoogleVectorStore
+from langchain_text_splitters import CharacterTextSplitter
+from langchain_community.document_loaders import DirectoryLoader
+
+# Create a corpus (collection of documents)
+corpus_store = GoogleVectorStore.create_corpus(display_name="My Corpus")
+
+# Create a document under that corpus
+document_store = GoogleVectorStore.create_document(
+    corpus_id=corpus_store.corpus_id, display_name="My Document"
+)
+
+# Load and upload documents
+text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=0)
+for file in DirectoryLoader(path="data/").load():
+    chunks = text_splitter.split_documents([file])
+    document_store.add_documents(chunks)
+
+# Query the document corpus
+aqa = corpus_store.as_aqa()
+response = aqa.invoke("What is the meaning of life?")
+
+print("Answer:", response.answer)
+print("Passages:", response.attributed_passages)
+print("Answerable probability:", response.answerable_probability)
+```
+
+---
+
+
+## Resources
+
+* [LangChain Documentation](https://docs.langchain.com/)
+* [Google Generative AI SDK](https://googleapis.github.io/python-genai/)
+* [Gemini Model Documentation](https://ai.google.dev/)
+
+
+

{langchain_google_genai-2.1.4.dist-info → langchain_google_genai-2.1.6.dist-info}/RECORD

@@ -1,16 +1,16 @@
 langchain_google_genai/__init__.py,sha256=IsTvA3UcECLDckt3zWxK6u-n3MEa5KeEQpqsS-Z8shM,2784
-langchain_google_genai/_common.py,sha256=BwEKrMqa4-yp8H57HTZJdDjEUZcGq9MSxT9bXtzQ6Yg,5783
+langchain_google_genai/_common.py,sha256=TeAmAwcRPk_dTXiIQQubamHAhuKNb_XsWy7yllkKW4w,5936
 langchain_google_genai/_enums.py,sha256=Zj3BXXLlkm_UybegCi6fLsfFhriJCt_LAJvgatgPWQ0,252
-langchain_google_genai/_function_utils.py,sha256=KO5KGNcPZo17qksQNYBsLRWjms8zOKcNhGRIuSHeyFU,19257
+langchain_google_genai/_function_utils.py,sha256=CkQyxfUVGqHfmZQwf4JtMXBsS9SOzk99Pa2TytnuLbE,21648
 langchain_google_genai/_genai_extension.py,sha256=81a4ly5ZHlqMf37uJfdB8K41qE6J5ujLnbUypIfFf2o,20775
 langchain_google_genai/_image_utils.py,sha256=tPrQyMvVmO8xkuow1SvA91omxUEv9ZUy1EMHNGjMAKY,5202
-langchain_google_genai/chat_models.py,sha256=SbIQkJnmNtvZxaiKUiT7VIniHNIrp8YXUXIk25zcWic,65478
-langchain_google_genai/embeddings.py,sha256=syN-GXcLAeuHEnF8Yqp2AQPD7rKEaR9l29jSLmt9dwM,10468
+langchain_google_genai/chat_models.py,sha256=WUt41wshRAKS1p7NjGCod_q3Vhry7CWIrXTFEPvqc-E,73558
+langchain_google_genai/embeddings.py,sha256=IwM6PBnuweABvYmzjgShOxrVffKbQnnIzPfKBFf8xms,10636
 langchain_google_genai/genai_aqa.py,sha256=qB6h3-BSXqe0YLR3eeVllYzmNKK6ofI6xJLdBahUVZo,4300
 langchain_google_genai/google_vector_store.py,sha256=4wvhIiOmc3Fo046FyafPmT9NBCLek-9bgluvuTfrbpQ,16148
 langchain_google_genai/llms.py,sha256=ASjrEk2T_1hUXVNJlfPB8PKC4PbhPe00H3_UHunMc_Q,5334
 langchain_google_genai/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-langchain_google_genai-2.1.4.dist-info/LICENSE,sha256=DppmdYJVSc1jd0aio6ptnMUn5tIHrdAhQ12SclEBfBg,1072
-langchain_google_genai-2.1.4.dist-info/METADATA,sha256=TlT4wlGgAZtj9P1-FUElJsEiEoqFnrAeCoddDsmZZYw,5165
-langchain_google_genai-2.1.4.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-langchain_google_genai-2.1.4.dist-info/RECORD,,
+langchain_google_genai-2.1.6.dist-info/LICENSE,sha256=DppmdYJVSc1jd0aio6ptnMUn5tIHrdAhQ12SclEBfBg,1072
+langchain_google_genai-2.1.6.dist-info/METADATA,sha256=An9rP3RLphq82natJt0lhu4sjG-OKx2g-I1bx4YfN_Q,6995
+langchain_google_genai-2.1.6.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+langchain_google_genai-2.1.6.dist-info/RECORD,,

langchain_google_genai-2.1.4.dist-info/METADATA

@@ -1,174 +0,0 @@
-Metadata-Version: 2.1
-Name: langchain-google-genai
-Version: 2.1.4
-Summary: An integration package connecting Google's genai package and LangChain
-Home-page: https://github.com/langchain-ai/langchain-google
-License: MIT
-Requires-Python: >=3.9,<4.0
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: filetype (>=1.2.0,<2.0.0)
-Requires-Dist: google-ai-generativelanguage (>=0.6.18,<0.7.0)
-Requires-Dist: langchain-core (>=0.3.52,<0.4.0)
-Requires-Dist: pydantic (>=2,<3)
-Project-URL: Repository, https://github.com/langchain-ai/langchain-google
-Project-URL: Source Code, https://github.com/langchain-ai/langchain-google/tree/main/libs/genai
-Description-Content-Type: text/markdown
-
-# langchain-google-genai
-
-This package contains the LangChain integrations for Gemini through their generative-ai SDK.
-
-## Installation
-
-```bash
-pip install -U langchain-google-genai
-```
-
-
-## Chat Models
-
-This package contains the `ChatGoogleGenerativeAI` class, which is the recommended way to interface with the Google Gemini series of models.
-
-To use, install the requirements, and configure your environment.
-
-```bash
-export GOOGLE_API_KEY=your-api-key
-```
-
-Then initialize
-
-```python
-from langchain_google_genai import ChatGoogleGenerativeAI
-
-llm = ChatGoogleGenerativeAI(model="gemini-pro")
-llm.invoke("Sing a ballad of LangChain.")
-```
-
-#### Multimodal inputs
-
-Gemini vision model supports image inputs when providing a single chat message. Example:
-
-```
-from langchain_core.messages import HumanMessage
-from langchain_google_genai import ChatGoogleGenerativeAI
-
-llm = ChatGoogleGenerativeAI(model="gemini-pro-vision")
-# example
-message = HumanMessage(
-    content=[
-        {
-            "type": "text",
-            "text": "What's in this image?",
-        },  # You can optionally provide text parts
-        {"type": "image_url", "image_url": "https://picsum.photos/seed/picsum/200/300"},
-    ]
-)
-llm.invoke([message])
-```
-
-The value of `image_url` can be any of the following:
-
-- A public image URL
-- An accessible gcs file (e.g., "gcs://path/to/file.png")
-- A base64 encoded image (e.g., `data:image/png;base64,abcd124`)
-
-#### Multimodal outputs
-
-Gemini 2.0 Flash Experimental model supports text output with inline images
-
-```
-from langchain_google_genai import ChatGoogleGenerativeAI
-
-llm = ChatGoogleGenerativeAI(model="models/gemini-2.0-flash-exp-image-generation")
-# example
-response = llm.invoke(
-    "Generate an image of a cat and say meow",
-    generation_config=dict(response_modalities=["TEXT", "IMAGE"]),
-)
-
-# Base64 encoded binary data of the image
-image_base64 = response.content[0].get("image_url").get("url").split(",")[-1]
-meow_str = response.content[1]
-```
-
-#### Multimodal Outputs in Chains
-
-```
-from langchain_core.runnables import RunnablePassthrough
-from langchain_core.prompts import ChatPromptTemplate
-
-from langchain_google_genai import ChatGoogleGenerativeAI, Modality
-
-llm = ChatGoogleGenerativeAI(
-    model="models/gemini-2.0-flash-exp-image-generation",
-    response_modalities=[Modality.TEXT, Modality.IMAGE],
-)
-
-prompt = ChatPromptTemplate(
-    [("human", "Generate an image of {animal} and tell me the sound of the animal")]
-)
-chain = {"animal": RunnablePassthrough()} | prompt | llm
-res = chain.invoke("cat")
-```
-
-#### Thinking support
-
-Gemini 2.5 Flash model supports reasoning through their thoughts
-
-```
-from langchain_google_genai import ChatGoogleGenerativeAI
-
-llm = ChatGoogleGenerativeAI(model="models/gemini-2.5-flash-preview-04-17", thinking_budget=1024)
-
-response = llm.invoke(
-    "How many O's are in Google? Please tell me how you double checked the result"
-)
-
-assert response.usage_metadata["output_token_details"]["reasoning"] > 0
-```
-
-## Embeddings
-
-This package also adds support for google's embeddings models.
-
-```
-from langchain_google_genai import GoogleGenerativeAIEmbeddings
-
-embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
-embeddings.embed_query("hello, world!")
-```
-
-## Semantic Retrieval
-
-Enables retrieval augmented generation (RAG) in your application.
-
-```
-# Create a new store for housing your documents.
-corpus_store = GoogleVectorStore.create_corpus(display_name="My Corpus")
-
-# Create a new document under the above corpus.
-document_store = GoogleVectorStore.create_document(
-    corpus_id=corpus_store.corpus_id, display_name="My Document"
-)
-
-# Upload some texts to the document.
-text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=0)
-for file in DirectoryLoader(path="data/").load():
-    documents = text_splitter.split_documents([file])
-    document_store.add_documents(documents)
-
-# Talk to your entire corpus with possibly many documents. 
-aqa = corpus_store.as_aqa()
-answer = aqa.invoke("What is the meaning of life?")
-
-# Read the response along with the attributed passages and answerability.
-print(response.answer)
-print(response.attributed_passages)
-print(response.answerable_probability)
-```
-