langchain-core 0.3.71__py3-none-any.whl → 0.4.0.dev0__py3-none-any.whl

langchain_core/__init__.py

@@ -4,7 +4,7 @@ The interfaces for core components like chat models, LLMs, vector stores, retrie
 and more are defined here. The universal invocation protocol (Runnables) along with
 a syntax for combining components (LangChain Expression Language) are also defined here.
 
-No third-party integrations are defined here. The dependencies are kept purposefully
+**No third-party integrations are defined here.** The dependencies are kept purposefully
 very lightweight.
 """
 

langchain_core/_api/beta_decorator.py

@@ -70,6 +70,7 @@ def beta(
             @beta
             def the_function_to_annotate():
                 pass
+
     """
 
     def beta(

langchain_core/_api/deprecation.py

@@ -136,6 +136,7 @@ def deprecated(
             @deprecated('1.4.0')
             def the_function_to_deprecate():
                 pass
+
     """
     _validate_deprecation_params(
         removal, alternative, alternative_import, pending=pending
@@ -549,6 +550,7 @@ def rename_parameter(
 
             @_api.rename_parameter("3.1", "bad_name", "good_name")
             def func(good_name): ...
+
     """
 
     def decorator(f: Callable[_P, _R]) -> Callable[_P, _R]:

langchain_core/beta/runnables/context.py

@@ -363,6 +363,7 @@ class Context:
             print(output["result"])  # Output: "hello"
             print(output["context"])  # Output: "What's your name?"
             print(output["input"])  # Output: "What's your name?
+
     """
 
     @staticmethod

langchain_core/callbacks/base.py

@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 
 from typing_extensions import Self
 
+from langchain_core.v1.messages import AIMessage, AIMessageChunk, MessageV1
+
 if TYPE_CHECKING:
     from collections.abc import Sequence
     from uuid import UUID
@@ -66,7 +68,9 @@ class LLMManagerMixin:
         self,
         token: str,
         *,
-        chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,
+        chunk: Optional[
+            Union[GenerationChunk, ChatGenerationChunk, AIMessageChunk]
+        ] = None,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
@@ -75,8 +79,8 @@ class LLMManagerMixin:
 
         Args:
             token (str): The new token.
-            chunk (GenerationChunk | ChatGenerationChunk): The new generated chunk,
-              containing content and other information.
+            chunk (GenerationChunk | ChatGenerationChunk | AIMessageChunk): The new
+              generated chunk, containing content and other information.
             run_id (UUID): The run ID. This is the ID of the current run.
             parent_run_id (UUID): The parent run ID. This is the ID of the parent run.
             kwargs (Any): Additional keyword arguments.
@@ -84,7 +88,7 @@ class LLMManagerMixin:
 
     def on_llm_end(
         self,
-        response: LLMResult,
+        response: Union[LLMResult, AIMessage],
         *,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
@@ -93,7 +97,7 @@ class LLMManagerMixin:
         """Run when LLM ends running.
 
         Args:
-            response (LLMResult): The response which was generated.
+            response (LLMResult | AIMessage): The response which was generated.
             run_id (UUID): The run ID. This is the ID of the current run.
             parent_run_id (UUID): The parent run ID. This is the ID of the parent run.
             kwargs (Any): Additional keyword arguments.
@@ -261,7 +265,7 @@ class CallbackManagerMixin:
     def on_chat_model_start(
         self,
         serialized: dict[str, Any],
-        messages: list[list[BaseMessage]],
+        messages: Union[list[list[BaseMessage]], list[MessageV1]],
         *,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
@@ -439,6 +443,9 @@ class BaseCallbackHandler(
     run_inline: bool = False
     """Whether to run the callback inline."""
 
+    accepts_new_messages: bool = False
+    """Whether the callback accepts new message format."""
+
     @property
     def ignore_llm(self) -> bool:
         """Whether to ignore LLM callbacks."""
@@ -509,7 +516,7 @@ class AsyncCallbackHandler(BaseCallbackHandler):
     async def on_chat_model_start(
         self,
         serialized: dict[str, Any],
-        messages: list[list[BaseMessage]],
+        messages: Union[list[list[BaseMessage]], list[MessageV1]],
         *,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
@@ -540,7 +547,9 @@ class AsyncCallbackHandler(BaseCallbackHandler):
         self,
         token: str,
         *,
-        chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,
+        chunk: Optional[
+            Union[GenerationChunk, ChatGenerationChunk, AIMessageChunk]
+        ] = None,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
         tags: Optional[list[str]] = None,
@@ -550,8 +559,8 @@ class AsyncCallbackHandler(BaseCallbackHandler):
 
         Args:
             token (str): The new token.
-            chunk (GenerationChunk | ChatGenerationChunk): The new generated chunk,
-              containing content and other information.
+            chunk (GenerationChunk | ChatGenerationChunk | AIMessageChunk): The new
+              generated chunk, containing content and other information.
             run_id (UUID): The run ID. This is the ID of the current run.
             parent_run_id (UUID): The parent run ID. This is the ID of the parent run.
             tags (Optional[list[str]]): The tags.
@@ -560,7 +569,7 @@ class AsyncCallbackHandler(BaseCallbackHandler):
 
     async def on_llm_end(
         self,
-        response: LLMResult,
+        response: Union[LLMResult, AIMessage],
         *,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
@@ -570,7 +579,7 @@ class AsyncCallbackHandler(BaseCallbackHandler):
         """Run when LLM ends running.
 
         Args:
-            response (LLMResult): The response which was generated.
+            response (LLMResult | AIMessage): The response which was generated.
             run_id (UUID): The run ID. This is the ID of the current run.
             parent_run_id (UUID): The parent run ID. This is the ID of the parent run.
             tags (Optional[list[str]]): The tags.
@@ -594,8 +603,8 @@ class AsyncCallbackHandler(BaseCallbackHandler):
             parent_run_id: The parent run ID. This is the ID of the parent run.
             tags: The tags.
             kwargs (Any): Additional keyword arguments.
-                - response (LLMResult): The response which was generated before
-                    the error occurred.
+                - response (LLMResult | AIMessage): The response which was generated
+                    before the error occurred.
         """
 
     async def on_chain_start(

langchain_core/callbacks/file.py

@@ -53,6 +53,7 @@ class FileCallbackHandler(BaseCallbackHandler):
         When not used as a context manager, a deprecation warning will be issued
         on first use. The file will be opened immediately in ``__init__`` and closed
         in ``__del__`` or when ``close()`` is called explicitly.
+
     """
 
     def __init__(

langchain_core/callbacks/manager.py

@@ -37,8 +37,16 @@ from langchain_core.callbacks.base import (
 )
 from langchain_core.callbacks.stdout import StdOutCallbackHandler
 from langchain_core.messages import BaseMessage, get_buffer_string
+from langchain_core.messages.utils import convert_from_v1_message
+from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, LLMResult
 from langchain_core.tracers.schemas import Run
 from langchain_core.utils.env import env_var_is_set
+from langchain_core.v1.messages import (
+    AIMessage,
+    AIMessageChunk,
+    MessageV1,
+    MessageV1Types,
+)
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Coroutine, Generator, Sequence
@@ -47,7 +55,7 @@ if TYPE_CHECKING:
 
     from langchain_core.agents import AgentAction, AgentFinish
     from langchain_core.documents import Document
-    from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult
+    from langchain_core.outputs import GenerationChunk
     from langchain_core.runnables.config import RunnableConfig
 
 logger = logging.getLogger(__name__)
@@ -105,6 +113,7 @@ def trace_as_chain_group(
                 # Use the callback manager for the chain group
                 res = llm.invoke(llm_input, {"callbacks": manager})
                 manager.on_chain_end({"output": res})
+
     """  # noqa: E501
     from langchain_core.tracers.context import _get_trace_callbacks
 
@@ -186,6 +195,7 @@ async def atrace_as_chain_group(
                 # Use the async callback manager for the chain group
                 res = await llm.ainvoke(llm_input, {"callbacks": manager})
                 await manager.on_chain_end({"output": res})
+
     """  # noqa: E501
     from langchain_core.tracers.context import _get_trace_callbacks
 
@@ -241,6 +251,46 @@ def shielded(func: Func) -> Func:
     return cast("Func", wrapped)
 
 
+def _convert_llm_events(
+    event_name: str, args: tuple[Any, ...], kwargs: dict[str, Any]
+) -> tuple[tuple[Any, ...], dict[str, Any]]:
+    args_list = list(args)
+    if (
+        event_name == "on_chat_model_start"
+        and isinstance(args_list[1], list)
+        and args_list[1]
+        and isinstance(args_list[1][0], MessageV1Types)
+    ):
+        batch = [
+            convert_from_v1_message(item)
+            for item in args_list[1]
+            if isinstance(item, MessageV1Types)
+        ]
+        args_list[1] = [batch]
+    elif (
+        event_name == "on_llm_new_token"
+        and "chunk" in kwargs
+        and isinstance(kwargs["chunk"], MessageV1Types)
+    ):
+        chunk = kwargs["chunk"]
+        kwargs["chunk"] = ChatGenerationChunk(text=chunk.text, message=chunk)
+    elif event_name == "on_llm_end" and isinstance(args_list[0], MessageV1Types):
+        args_list[0] = LLMResult(
+            generations=[
+                [
+                    ChatGeneration(
+                        text=args_list[0].text,
+                        message=convert_from_v1_message(args_list[0]),
+                    )
+                ]
+            ]
+        )
+    else:
+        pass
+
+    return tuple(args_list), kwargs
+
+
 def handle_event(
     handlers: list[BaseCallbackHandler],
     event_name: str,
@@ -269,6 +319,8 @@ def handle_event(
                 if ignore_condition_name is None or not getattr(
                     handler, ignore_condition_name
                 ):
+                    if not handler.accepts_new_messages:
+                        args, kwargs = _convert_llm_events(event_name, args, kwargs)
                     event = getattr(handler, event_name)(*args, **kwargs)
                     if asyncio.iscoroutine(event):
                         coros.append(event)
@@ -363,6 +415,8 @@ async def _ahandle_event_for_handler(
 ) -> None:
     try:
         if ignore_condition_name is None or not getattr(handler, ignore_condition_name):
+            if not handler.accepts_new_messages:
+                args, kwargs = _convert_llm_events(event_name, args, kwargs)
             event = getattr(handler, event_name)
             if asyncio.iscoroutinefunction(event):
                 await event(*args, **kwargs)
@@ -672,7 +726,9 @@ class CallbackManagerForLLMRun(RunManager, LLMManagerMixin):
         self,
         token: str,
         *,
-        chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,
+        chunk: Optional[
+            Union[GenerationChunk, ChatGenerationChunk, AIMessageChunk]
+        ] = None,
         **kwargs: Any,
     ) -> None:
         """Run when LLM generates a new token.
@@ -697,11 +753,11 @@ class CallbackManagerForLLMRun(RunManager, LLMManagerMixin):
             **kwargs,
         )
 
-    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+    def on_llm_end(self, response: Union[LLMResult, AIMessage], **kwargs: Any) -> None:
         """Run when LLM ends running.
 
         Args:
-            response (LLMResult): The LLM result.
+            response (LLMResult | AIMessage): The LLM result.
             **kwargs (Any): Additional keyword arguments.
         """
         if not self.handlers:
@@ -727,8 +783,8 @@ class CallbackManagerForLLMRun(RunManager, LLMManagerMixin):
         Args:
             error (Exception or KeyboardInterrupt): The error.
             kwargs (Any): Additional keyword arguments.
-                - response (LLMResult): The response which was generated before
-                    the error occurred.
+                - response (LLMResult | AIMessage): The response which was generated
+                  before the error occurred.
         """
         if not self.handlers:
             return
@@ -768,7 +824,9 @@ class AsyncCallbackManagerForLLMRun(AsyncRunManager, LLMManagerMixin):
         self,
         token: str,
         *,
-        chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,
+        chunk: Optional[
+            Union[GenerationChunk, ChatGenerationChunk, AIMessageChunk]
+        ] = None,
         **kwargs: Any,
     ) -> None:
         """Run when LLM generates a new token.
@@ -794,11 +852,13 @@ class AsyncCallbackManagerForLLMRun(AsyncRunManager, LLMManagerMixin):
         )
 
     @shielded
-    async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+    async def on_llm_end(
+        self, response: Union[LLMResult, AIMessage], **kwargs: Any
+    ) -> None:
         """Run when LLM ends running.
 
         Args:
-            response (LLMResult): The LLM result.
+            response (LLMResult | AIMessage): The LLM result.
             **kwargs (Any): Additional keyword arguments.
         """
         if not self.handlers:
@@ -825,11 +885,8 @@ class AsyncCallbackManagerForLLMRun(AsyncRunManager, LLMManagerMixin):
         Args:
             error (Exception or KeyboardInterrupt): The error.
             kwargs (Any): Additional keyword arguments.
-                - response (LLMResult): The response which was generated before
-                    the error occurred.
-
-
-
+                - response (LLMResult | AIMessage): The response which was generated
+                  before the error occurred.
         """
         if not self.handlers:
             return
@@ -1352,7 +1409,7 @@ class CallbackManager(BaseCallbackManager):
     def on_chat_model_start(
         self,
         serialized: dict[str, Any],
-        messages: list[list[BaseMessage]],
+        messages: Union[list[list[BaseMessage]], list[MessageV1]],
         run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> list[CallbackManagerForLLMRun]:
@@ -1360,7 +1417,7 @@ class CallbackManager(BaseCallbackManager):
 
         Args:
             serialized (dict[str, Any]): The serialized LLM.
-            messages (list[list[BaseMessage]]): The list of messages.
+            messages (list[list[BaseMessage | MessageV1]]): The list of messages.
             run_id (UUID, optional): The ID of the run. Defaults to None.
             **kwargs (Any): Additional keyword arguments.
 
@@ -1368,6 +1425,32 @@ class CallbackManager(BaseCallbackManager):
             list[CallbackManagerForLLMRun]: A callback manager for each
                 list of messages as an LLM run.
         """
+        if messages and isinstance(messages[0], MessageV1Types):
+            run_id_ = run_id if run_id is not None else uuid.uuid4()
+            handle_event(
+                self.handlers,
+                "on_chat_model_start",
+                "ignore_chat_model",
+                serialized,
+                messages,
+                run_id=run_id_,
+                parent_run_id=self.parent_run_id,
+                tags=self.tags,
+                metadata=self.metadata,
+                **kwargs,
+            )
+            return [
+                CallbackManagerForLLMRun(
+                    run_id=run_id_,
+                    handlers=self.handlers,
+                    inheritable_handlers=self.inheritable_handlers,
+                    parent_run_id=self.parent_run_id,
+                    tags=self.tags,
+                    inheritable_tags=self.inheritable_tags,
+                    metadata=self.metadata,
+                    inheritable_metadata=self.inheritable_metadata,
+                )
+            ]
         managers = []
         for message_list in messages:
             if run_id is not None:
@@ -1862,7 +1945,7 @@ class AsyncCallbackManager(BaseCallbackManager):
     async def on_chat_model_start(
         self,
         serialized: dict[str, Any],
-        messages: list[list[BaseMessage]],
+        messages: Union[list[list[BaseMessage]], list[MessageV1]],
         run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> list[AsyncCallbackManagerForLLMRun]:
@@ -1870,7 +1953,7 @@ class AsyncCallbackManager(BaseCallbackManager):
 
         Args:
             serialized (dict[str, Any]): The serialized LLM.
-            messages (list[list[BaseMessage]]): The list of messages.
+            messages (list[list[BaseMessage | MessageV1]]): The list of messages.
             run_id (UUID, optional): The ID of the run. Defaults to None.
             **kwargs (Any): Additional keyword arguments.
 
@@ -1879,10 +1962,51 @@ class AsyncCallbackManager(BaseCallbackManager):
                 async callback managers, one for each LLM Run
                 corresponding to each inner  message list.
         """
+        if messages and isinstance(messages[0], MessageV1Types):
+            run_id_ = run_id if run_id is not None else uuid.uuid4()
+            inline_tasks = []
+            non_inline_tasks = []
+            for handler in self.handlers:
+                task = ahandle_event(
+                    [handler],
+                    "on_chat_model_start",
+                    "ignore_chat_model",
+                    serialized,
+                    messages,
+                    run_id=run_id_,
+                    parent_run_id=self.parent_run_id,
+                    tags=self.tags,
+                    metadata=self.metadata,
+                    **kwargs,
+                )
+                if handler.run_inline:
+                    inline_tasks.append(task)
+                else:
+                    non_inline_tasks.append(task)
+            managers = [
+                AsyncCallbackManagerForLLMRun(
+                    run_id=run_id_,
+                    handlers=self.handlers,
+                    inheritable_handlers=self.inheritable_handlers,
+                    parent_run_id=self.parent_run_id,
+                    tags=self.tags,
+                    inheritable_tags=self.inheritable_tags,
+                    metadata=self.metadata,
+                    inheritable_metadata=self.inheritable_metadata,
+                )
+            ]
+            # Run inline tasks sequentially
+            for task in inline_tasks:
+                await task
+
+            # Run non-inline tasks concurrently
+            if non_inline_tasks:
+                await asyncio.gather(*non_inline_tasks)
+
+            return managers
         inline_tasks = []
         non_inline_tasks = []
         managers = []
-
         for message_list in messages:
             if run_id is not None:
                 run_id_ = run_id
@@ -2575,6 +2699,7 @@ async def adispatch_custom_event(
         behalf.
 
     .. versionadded:: 0.2.15
+
     """
     from langchain_core.runnables.config import (
         ensure_config,
@@ -2645,6 +2770,7 @@ def dispatch_custom_event(
             foo_.invoke({"a": "1"}, {"callbacks": [CustomCallbackManager()]})
 
     .. versionadded:: 0.2.15
+
     """
     from langchain_core.runnables.config import (
         ensure_config,

langchain_core/callbacks/streaming_stdout.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import sys
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Union
 
 from typing_extensions import override
 
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
     from langchain_core.agents import AgentAction, AgentFinish
     from langchain_core.messages import BaseMessage
     from langchain_core.outputs import LLMResult
+    from langchain_core.v1.messages import AIMessage, MessageV1
 
 
 class StreamingStdOutCallbackHandler(BaseCallbackHandler):
@@ -32,7 +33,7 @@ class StreamingStdOutCallbackHandler(BaseCallbackHandler):
     def on_chat_model_start(
         self,
         serialized: dict[str, Any],
-        messages: list[list[BaseMessage]],
+        messages: Union[list[list[BaseMessage]], list[MessageV1]],
         **kwargs: Any,
     ) -> None:
         """Run when LLM starts running.
@@ -54,7 +55,7 @@ class StreamingStdOutCallbackHandler(BaseCallbackHandler):
         sys.stdout.write(token)
         sys.stdout.flush()
 
-    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+    def on_llm_end(self, response: Union[LLMResult, AIMessage], **kwargs: Any) -> None:
         """Run when LLM ends running.
 
         Args:

langchain_core/callbacks/usage.py

@@ -4,14 +4,16 @@ import threading
 from collections.abc import Generator
 from contextlib import contextmanager
 from contextvars import ContextVar
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 from typing_extensions import override
 
 from langchain_core.callbacks import BaseCallbackHandler
 from langchain_core.messages import AIMessage
 from langchain_core.messages.ai import UsageMetadata, add_usage
+from langchain_core.messages.utils import convert_from_v1_message
 from langchain_core.outputs import ChatGeneration, LLMResult
+from langchain_core.v1.messages import AIMessage as AIMessageV1
 
 
 class UsageMetadataCallbackHandler(BaseCallbackHandler):
@@ -44,6 +46,7 @@ class UsageMetadataCallbackHandler(BaseCallbackHandler):
               'input_token_details': {'cache_read': 0, 'cache_creation': 0}}}
 
     .. versionadded:: 0.3.49
+
     """
 
     def __init__(self) -> None:
@@ -57,9 +60,17 @@ class UsageMetadataCallbackHandler(BaseCallbackHandler):
         return str(self.usage_metadata)
 
     @override
-    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+    def on_llm_end(
+        self, response: Union[LLMResult, AIMessageV1], **kwargs: Any
+    ) -> None:
         """Collect token usage."""
         # Check for usage_metadata (langchain-core >= 0.2.2)
+        if isinstance(response, AIMessageV1):
+            response = LLMResult(
+                generations=[
+                    [ChatGeneration(message=convert_from_v1_message(response))]
+                ]
+            )
         try:
             generation = response.generations[0][0]
         except IndexError:
@@ -98,7 +109,7 @@ def get_usage_metadata_callback(
 
     Args:
         name (str): The name of the context variable. Defaults to
-            ``"usage_metadata_callback"``.
+            ``'usage_metadata_callback'``.
 
     Example:
         .. code-block:: python
@@ -127,6 +138,7 @@ def get_usage_metadata_callback(
               'input_token_details': {'cache_read': 0, 'cache_creation': 0}}}
 
     .. versionadded:: 0.3.49
+
     """
     from langchain_core.tracers.context import register_configure_hook
 

langchain_core/chat_history.py

@@ -91,6 +91,7 @@ class BaseChatMessageHistory(ABC):
                 def clear(self):
                     with open(os.path.join(storage_path, session_id), "w") as f:
                         f.write("[]")
+
     """
 
     messages: list[BaseMessage]

langchain_core/document_loaders/langsmith.py

@@ -36,6 +36,7 @@ class LangSmithLoader(BaseLoader):
             # -> [Document("...", metadata={"inputs": {...}, "outputs": {...}, ...}), ...]
 
     .. versionadded:: 0.2.34
+
     """  # noqa: E501
 
     def __init__(
@@ -61,7 +62,7 @@ class LangSmithLoader(BaseLoader):
         Args:
             dataset_id: The ID of the dataset to filter by. Defaults to None.
             dataset_name: The name of the dataset to filter by. Defaults to None.
-            content_key: The inputs key to set as Document page content. ``"."`` characters
+            content_key: The inputs key to set as Document page content. ``'.'`` characters
                 are interpreted as nested keys. E.g. ``content_key="first.second"`` will
                 result in
                 ``Document(page_content=format_content(example.inputs["first"]["second"]))``

langchain_core/documents/base.py

@@ -102,6 +102,7 @@ class Blob(BaseMedia):
             # Read the blob as a byte stream
             with blob.as_bytes_io() as f:
                 print(f.read())
+
     """
 
     data: Union[bytes, str, None] = None
@@ -265,6 +266,7 @@ class Document(BaseMedia):
                 page_content="Hello, world!",
                 metadata={"source": "https://example.com"}
             )
+
     """
 
     page_content: str

langchain_core/embeddings/fake.py

@@ -46,6 +46,7 @@ class FakeEmbeddings(Embeddings, BaseModel):
 
             2
             [-0.5670477847544458, -0.31403828652395727, -0.5840547508955257]
+
     """
 
     size: int
@@ -103,6 +104,7 @@ class DeterministicFakeEmbedding(Embeddings, BaseModel):
 
             2
             [-0.5670477847544458, -0.31403828652395727, -0.5840547508955257]
+
     """
 
     size: int

langchain_core/indexing/api.py

@@ -444,6 +444,9 @@ def index(
     scoped_full_cleanup_source_ids: set[str] = set()
 
     for doc_batch in _batch(batch_size, doc_iterator):
+        # Track original batch size before deduplication
+        original_batch_size = len(doc_batch)
+
         hashed_docs = list(
             _deduplicate_in_order(
                 [
@@ -452,6 +455,8 @@ def index(
                 ]
             )
         )
+        # Count documents removed by within-batch deduplication
+        num_skipped += original_batch_size - len(hashed_docs)
 
         source_ids: Sequence[Optional[str]] = [
             source_id_assigner(hashed_doc) for hashed_doc in hashed_docs
@@ -784,6 +789,9 @@ async def aindex(
     scoped_full_cleanup_source_ids: set[str] = set()
 
     async for doc_batch in _abatch(batch_size, async_doc_iterator):
+        # Track original batch size before deduplication
+        original_batch_size = len(doc_batch)
+
         hashed_docs = list(
             _deduplicate_in_order(
                 [
@@ -792,6 +800,8 @@ async def aindex(
                 ]
             )
         )
+        # Count documents removed by within-batch deduplication
+        num_skipped += original_batch_size - len(hashed_docs)
 
         source_ids: Sequence[Optional[str]] = [
             source_id_assigner(doc) for doc in hashed_docs