langchain-core 0.3.74__py3-none-any.whl → 0.3.76__py3-none-any.whl

langchain_core/indexing/api.py

@@ -185,6 +185,9 @@ def _get_document_with_hash(
             When changing the key encoder, you must change the
             index as well to avoid duplicated documents in the cache.
 
+    Raises:
+        ValueError: If the metadata cannot be serialized using json.
+
     Returns:
         Document with a unique identifier based on the hash of the content and metadata.
     """
@@ -291,21 +294,21 @@ def index(
     documents were deleted, which documents should be skipped.
 
     For the time being, documents are indexed using their hashes, and users
-     are not able to specify the uid of the document.
+    are not able to specify the uid of the document.
 
     Important:
-       * In full mode, the loader should be returning
-         the entire dataset, and not just a subset of the dataset.
-         Otherwise, the auto_cleanup will remove documents that it is not
-         supposed to.
-       * In incremental mode, if documents associated with a particular
-         source id appear across different batches, the indexing API
-         will do some redundant work. This will still result in the
-         correct end state of the index, but will unfortunately not be
-         100% efficient. For example, if a given document is split into 15
-         chunks, and we index them using a batch size of 5, we'll have 3 batches
-         all with the same source id. In general, to avoid doing too much
-         redundant work select as big a batch size as possible.
+        * In full mode, the loader should be returning
+          the entire dataset, and not just a subset of the dataset.
+          Otherwise, the auto_cleanup will remove documents that it is not
+          supposed to.
+        * In incremental mode, if documents associated with a particular
+          source id appear across different batches, the indexing API
+          will do some redundant work. This will still result in the
+          correct end state of the index, but will unfortunately not be
+          100% efficient. For example, if a given document is split into 15
+          chunks, and we index them using a batch size of 5, we'll have 3 batches
+          all with the same source id. In general, to avoid doing too much
+          redundant work select as big a batch size as possible.
         * The `scoped_full` mode is suitable if determining an appropriate batch size
           is challenging or if your data loader cannot return the entire dataset at
           once. This mode keeps track of source IDs in memory, which should be fine
@@ -315,23 +318,22 @@ def index(
     Args:
         docs_source: Data loader or iterable of documents to index.
         record_manager: Timestamped set to keep track of which documents were
-                         updated.
+            updated.
         vector_store: VectorStore or DocumentIndex to index the documents into.
         batch_size: Batch size to use when indexing. Default is 100.
         cleanup: How to handle clean up of documents. Default is None.
+
             - incremental: Cleans up all documents that haven't been updated AND
-                           that are associated with source ids that were seen
-                           during indexing.
-                           Clean up is done continuously during indexing helping
-                           to minimize the probability of users seeing duplicated
-                           content.
+              that are associated with source ids that were seen during indexing.
+              Clean up is done continuously during indexing helping to minimize the
+              probability of users seeing duplicated content.
             - full: Delete all documents that have not been returned by the loader
-                    during this run of indexing.
-                    Clean up runs after all documents have been indexed.
-                    This means that users may see duplicated content during indexing.
+              during this run of indexing.
+              Clean up runs after all documents have been indexed.
+              This means that users may see duplicated content during indexing.
             - scoped_full: Similar to Full, but only deletes all documents
-                           that haven't been updated AND that are associated with
-                           source ids that were seen during indexing.
+              that haven't been updated AND that are associated with
+              source ids that were seen during indexing.
             - None: Do not delete any documents.
         source_id_key: Optional key that helps identify the original source
             of the document. Default is None.
@@ -358,10 +360,9 @@ def index(
             When changing the key encoder, you must change the
             index as well to avoid duplicated documents in the cache.
         upsert_kwargs: Additional keyword arguments to pass to the add_documents
-                       method of the VectorStore or the upsert method of the
-                       DocumentIndex. For example, you can use this to
-                       specify a custom vector_field:
-                       upsert_kwargs={"vector_field": "embedding"}
+            method of the VectorStore or the upsert method of the DocumentIndex.
+            For example, you can use this to specify a custom vector_field:
+            upsert_kwargs={"vector_field": "embedding"}
             .. versionadded:: 0.3.10
 
     Returns:
@@ -374,6 +375,9 @@ def index(
         ValueError: If vectorstore does not have
             "delete" and "add_documents" required methods.
         ValueError: If source_id_key is not None, but is not a string or callable.
+        TypeError: If ``vectorstore`` is not a VectorStore or a DocumentIndex.
+        AssertionError: If ``source_id`` is None when cleanup mode is incremental.
+            (should be unreachable code).
 
     .. version_modified:: 0.3.25
 
@@ -656,22 +660,22 @@ async def aindex(
     Args:
         docs_source: Data loader or iterable of documents to index.
         record_manager: Timestamped set to keep track of which documents were
-                         updated.
+            updated.
         vector_store: VectorStore or DocumentIndex to index the documents into.
         batch_size: Batch size to use when indexing. Default is 100.
         cleanup: How to handle clean up of documents. Default is None.
+
             - incremental: Cleans up all documents that haven't been updated AND
-                           that are associated with source ids that were seen
-                           during indexing.
-                           Clean up is done continuously during indexing helping
-                           to minimize the probability of users seeing duplicated
-                           content.
-            - full: Delete all documents that haven to been returned by the loader.
-                    Clean up runs after all documents have been indexed.
-                    This means that users may see duplicated content during indexing.
+              that are associated with source ids that were seen during indexing.
+              Clean up is done continuously during indexing helping to minimize the
+              probability of users seeing duplicated content.
+            - full: Delete all documents that have not been returned by the loader
+              during this run of indexing.
+              Clean up runs after all documents have been indexed.
+              This means that users may see duplicated content during indexing.
             - scoped_full: Similar to Full, but only deletes all documents
-                           that haven't been updated AND that are associated with
-                           source ids that were seen during indexing.
+              that haven't been updated AND that are associated with
+              source ids that were seen during indexing.
             - None: Do not delete any documents.
         source_id_key: Optional key that helps identify the original source
             of the document. Default is None.
@@ -680,6 +684,12 @@ async def aindex(
         force_update: Force update documents even if they are present in the
             record manager. Useful if you are re-indexing with updated embeddings.
             Default is False.
+        key_encoder: Hashing algorithm to use for hashing the document content and
+            metadata. Default is "sha1".
+            Other options include "blake2b", "sha256", and "sha512".
+
+            .. versionadded:: 0.3.66
+
         key_encoder: Hashing algorithm to use for hashing the document.
             If not provided, a default encoder using SHA-1 will be used.
             SHA-1 is not collision-resistant, and a motivated attacker
@@ -691,11 +701,10 @@ async def aindex(
 
             When changing the key encoder, you must change the
             index as well to avoid duplicated documents in the cache.
-        upsert_kwargs: Additional keyword arguments to pass to the aadd_documents
-                       method of the VectorStore or the aupsert method of the
-                       DocumentIndex. For example, you can use this to
-                       specify a custom vector_field:
-                       upsert_kwargs={"vector_field": "embedding"}
+        upsert_kwargs: Additional keyword arguments to pass to the add_documents
+            method of the VectorStore or the upsert method of the DocumentIndex.
+            For example, you can use this to specify a custom vector_field:
+            upsert_kwargs={"vector_field": "embedding"}
             .. versionadded:: 0.3.10
 
     Returns:
@@ -708,6 +717,9 @@ async def aindex(
         ValueError: If vectorstore does not have
             "adelete" and "aadd_documents" required methods.
         ValueError: If source_id_key is not None, but is not a string or callable.
+        TypeError: If ``vector_store`` is not a VectorStore or DocumentIndex.
+        AssertionError: If ``source_id_key`` is None when cleanup mode is
+            incremental or ``scoped_full`` (should be unreachable).
 
     .. version_modified:: 0.3.25
 

langchain_core/indexing/base.py

@@ -7,6 +7,8 @@ import time
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, Optional, TypedDict
 
+from typing_extensions import override
+
 from langchain_core._api import beta
 from langchain_core.retrievers import BaseRetriever
 from langchain_core.runnables import run_in_executor
@@ -254,14 +256,14 @@ class InMemoryRecordManager(RecordManager):
         """In-memory schema creation is simply ensuring the structure is initialized."""
 
     async def acreate_schema(self) -> None:
-        """Async in-memory schema creation is simply ensuring the structure is initialized."""  # noqa: E501
+        """In-memory schema creation is simply ensuring the structure is initialized."""
 
+    @override
     def get_time(self) -> float:
-        """Get the current server time as a high resolution timestamp!"""
         return time.time()
 
+    @override
     async def aget_time(self) -> float:
-        """Async get the current server time as a high resolution timestamp!"""
         return self.get_time()
 
     def update(
@@ -322,11 +324,6 @@ class InMemoryRecordManager(RecordManager):
                 raise an error.
                 This is meant to help prevent time-drift issues since
                 time may not be monotonically increasing!
-
-        Raises:
-            ValueError: If the length of keys doesn't match the length of group
-                ids.
-            ValueError: If time_at_least is in the future.
         """
         self.update(keys, group_ids=group_ids, time_at_least=time_at_least)
 
@@ -488,8 +485,8 @@ class DeleteResponse(TypedDict, total=False):
     failed: Sequence[str]
     """The IDs that failed to be deleted.
 
-    Please note that deleting an ID that
-    does not exist is **NOT** considered a failure.
+    .. warning::
+        Deleting an ID that does not exist is **NOT** considered a failure.
     """
 
     num_failed: int

langchain_core/indexing/in_memory.py

@@ -32,7 +32,17 @@ class InMemoryDocumentIndex(DocumentIndex):
 
     @override
     def upsert(self, items: Sequence[Document], /, **kwargs: Any) -> UpsertResponse:
-        """Upsert items into the index."""
+        """Upsert documents into the index.
+
+        Args:
+            items: Sequence of documents to add to the index.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            A response object that contains the list of IDs that were
+            successfully added or updated in the index and the list of IDs that
+            failed to be added or updated.
+        """
         ok_ids = []
 
         for item in items:
@@ -51,7 +61,18 @@ class InMemoryDocumentIndex(DocumentIndex):
 
     @override
     def delete(self, ids: Optional[list[str]] = None, **kwargs: Any) -> DeleteResponse:
-        """Delete by ID."""
+        """Delete by IDs.
+
+        Args:
+            ids: List of ids to delete.
+
+        Raises:
+            ValueError: If ids is None.
+
+        Returns:
+            A response object that contains the list of IDs that were successfully
+            deleted and the list of IDs that failed to be deleted.
+        """
         if ids is None:
             msg = "IDs must be provided for deletion"
             raise ValueError(msg)
@@ -69,7 +90,6 @@ class InMemoryDocumentIndex(DocumentIndex):
 
     @override
     def get(self, ids: Sequence[str], /, **kwargs: Any) -> list[Document]:
-        """Get by ids."""
         return [self.store[id_] for id_ in ids if id_ in self.store]
 
     @override

langchain_core/language_models/__init__.py

@@ -26,7 +26,8 @@ https://python.langchain.com/docs/how_to/custom_chat_model/
 **LLMs**
 
 Language models that takes a string as input and returns a string.
-These are traditionally older models (newer models generally are Chat Models, see below).
+These are traditionally older models (newer models generally are Chat Models,
+see below).
 
 Although the underlying models are string in, string out, the LangChain wrappers
 also allow these models to take messages as input. This gives them the same interface
@@ -39,7 +40,7 @@ Please see the following guide for more information on how to implement a custom
 https://python.langchain.com/docs/how_to/custom_llm/
 
 
-"""  # noqa: E501
+"""
 
 from typing import TYPE_CHECKING
 

langchain_core/language_models/base.py

@@ -22,19 +22,31 @@ from typing_extensions import TypeAlias, TypedDict, override
 from langchain_core._api import deprecated
 from langchain_core.caches import BaseCache
 from langchain_core.callbacks import Callbacks
+from langchain_core.globals import get_verbose
 from langchain_core.messages import (
     AnyMessage,
     BaseMessage,
     MessageLikeRepresentation,
     get_buffer_string,
 )
-from langchain_core.prompt_values import PromptValue
+from langchain_core.prompt_values import (
+    ChatPromptValueConcrete,
+    PromptValue,
+    StringPromptValue,
+)
 from langchain_core.runnables import Runnable, RunnableSerializable
 from langchain_core.utils import get_pydantic_field_names
 
 if TYPE_CHECKING:
     from langchain_core.outputs import LLMResult
 
+try:
+    from transformers import GPT2TokenizerFast  # type: ignore[import-not-found]
+
+    _HAS_TRANSFORMERS = True
+except ImportError:
+    _HAS_TRANSFORMERS = False
+
 
 class LangSmithParams(TypedDict, total=False):
     """LangSmith parameters for tracing."""
@@ -57,18 +69,22 @@ class LangSmithParams(TypedDict, total=False):
 def get_tokenizer() -> Any:
     """Get a GPT-2 tokenizer instance.
 
-    This function is cached to avoid re-loading the tokenizer
-    every time it is called.
+    This function is cached to avoid re-loading the tokenizer every time it is called.
+
+    Raises:
+        ImportError: If the transformers package is not installed.
+
+    Returns:
+        The GPT-2 tokenizer instance.
+
     """
-    try:
-        from transformers import GPT2TokenizerFast  # type: ignore[import-not-found]
-    except ImportError as e:
+    if not _HAS_TRANSFORMERS:
         msg = (
             "Could not import transformers python package. "
             "This is needed in order to calculate get_token_ids. "
             "Please install it with `pip install transformers`."
         )
-        raise ImportError(msg) from e
+        raise ImportError(msg)
     # create a GPT-2 tokenizer instance
     return GPT2TokenizerFast.from_pretrained("gpt2")
 
@@ -89,8 +105,6 @@ LanguageModelOutputVar = TypeVar("LanguageModelOutputVar", BaseMessage, str)
 
 
 def _get_verbosity() -> bool:
-    from langchain_core.globals import get_verbose
-
     return get_verbose()
 
 
@@ -99,7 +113,8 @@ class BaseLanguageModel(
 ):
     """Abstract base class for interfacing with language models.
 
-    All language model wrappers inherited from BaseLanguageModel.
+    All language model wrappers inherited from ``BaseLanguageModel``.
+
     """
 
     cache: Union[BaseCache, bool, None] = Field(default=None, exclude=True)
@@ -108,9 +123,10 @@ class BaseLanguageModel(
     * If true, will use the global cache.
     * If false, will not use a cache
     * If None, will use the global cache if it's set, otherwise no cache.
-    * If instance of BaseCache, will use the provided cache.
+    * If instance of ``BaseCache``, will use the provided cache.
 
     Caching is not currently supported for streaming methods of models.
+
     """
     verbose: bool = Field(default_factory=_get_verbosity, exclude=True, repr=False)
     """Whether to print out response text."""
@@ -140,6 +156,7 @@ class BaseLanguageModel(
 
         Returns:
             The verbosity setting to use.
+
         """
         if verbose is None:
             return _get_verbosity()
@@ -149,11 +166,6 @@ class BaseLanguageModel(
     @override
     def InputType(self) -> TypeAlias:
         """Get the input type for this runnable."""
-        from langchain_core.prompt_values import (
-            ChatPromptValueConcrete,
-            StringPromptValue,
-        )
-
         # This is a version of LanguageModelInput which replaces the abstract
         # base class BaseMessage with a union of its subclasses, which makes
         # for a much better schema.
@@ -177,10 +189,11 @@ class BaseLanguageModel(
         API.
 
         Use this method when you want to:
-            1. take advantage of batched calls,
-            2. need more output from the model than just the top generated value,
-            3. are building chains that are agnostic to the underlying language model
-                type (e.g., pure text completion models vs chat models).
+
+        1. Take advantage of batched calls,
+        2. Need more output from the model than just the top generated value,
+        3. Are building chains that are agnostic to the underlying language model
+           type (e.g., pure text completion models vs chat models).
 
         Args:
             prompts: List of PromptValues. A PromptValue is an object that can be
@@ -195,7 +208,8 @@ class BaseLanguageModel(
 
         Returns:
             An LLMResult, which contains a list of candidate Generations for each input
-                prompt and additional model provider-specific output.
+            prompt and additional model provider-specific output.
+
         """
 
     @abstractmethod
@@ -212,10 +226,11 @@ class BaseLanguageModel(
         API.
 
         Use this method when you want to:
-            1. take advantage of batched calls,
-            2. need more output from the model than just the top generated value,
-            3. are building chains that are agnostic to the underlying language model
-                type (e.g., pure text completion models vs chat models).
+
+        1. Take advantage of batched calls,
+        2. Need more output from the model than just the top generated value,
+        3. Are building chains that are agnostic to the underlying language model
+           type (e.g., pure text completion models vs chat models).
 
         Args:
             prompts: List of PromptValues. A PromptValue is an object that can be
@@ -229,8 +244,9 @@ class BaseLanguageModel(
                 to the model provider API call.
 
         Returns:
-            An LLMResult, which contains a list of candidate Generations for each input
-                prompt and additional model provider-specific output.
+            An ``LLMResult``, which contains a list of candidate Generations for each
+            input prompt and additional model provider-specific output.
+
         """
 
     def with_structured_output(
@@ -248,8 +264,8 @@ class BaseLanguageModel(
     ) -> str:
         """Pass a single string input to the model and return a string.
 
-         Use this method when passing in raw text. If you want to pass in specific
-            types of chat messages, use predict_messages.
+        Use this method when passing in raw text. If you want to pass in specific types
+        of chat messages, use predict_messages.
 
         Args:
             text: String input to pass to the model.
@@ -260,6 +276,7 @@ class BaseLanguageModel(
 
         Returns:
             Top model prediction as a string.
+
         """
 
     @deprecated("0.1.7", alternative="invoke", removal="1.0")
@@ -274,7 +291,7 @@ class BaseLanguageModel(
         """Pass a message sequence to the model and return a message.
 
         Use this method when passing in chat messages. If you want to pass in raw text,
-            use predict.
+        use predict.
 
         Args:
             messages: A sequence of chat messages corresponding to a single model input.
@@ -285,6 +302,7 @@ class BaseLanguageModel(
 
         Returns:
             Top model prediction as a message.
+
         """
 
     @deprecated("0.1.7", alternative="ainvoke", removal="1.0")
@@ -295,7 +313,7 @@ class BaseLanguageModel(
         """Asynchronously pass a string to the model and return a string.
 
         Use this method when calling pure text generation models and only the top
-            candidate generation is needed.
+        candidate generation is needed.
 
         Args:
             text: String input to pass to the model.
@@ -306,6 +324,7 @@ class BaseLanguageModel(
 
         Returns:
             Top model prediction as a string.
+
         """
 
     @deprecated("0.1.7", alternative="ainvoke", removal="1.0")
@@ -319,8 +338,8 @@ class BaseLanguageModel(
     ) -> BaseMessage:
         """Asynchronously pass messages to the model and return a message.
 
-        Use this method when calling chat models and only the top
-            candidate generation is needed.
+        Use this method when calling chat models and only the top candidate generation
+        is needed.
 
         Args:
             messages: A sequence of chat messages corresponding to a single model input.
@@ -331,6 +350,7 @@ class BaseLanguageModel(
 
         Returns:
             Top model prediction as a message.
+
         """
 
     @property
@@ -346,7 +366,8 @@ class BaseLanguageModel(
 
         Returns:
             A list of ids corresponding to the tokens in the text, in order they occur
-                in the text.
+            in the text.
+
         """
         if self.custom_get_token_ids is not None:
             return self.custom_get_token_ids(text)
@@ -362,6 +383,7 @@ class BaseLanguageModel(
 
         Returns:
             The integer number of tokens in the text.
+
         """
         return len(self.get_token_ids(text))
 
@@ -374,16 +396,18 @@ class BaseLanguageModel(
 
         Useful for checking if an input fits in a model's context window.
 
-        **Note**: the base implementation of get_num_tokens_from_messages ignores
-        tool schemas.
+        .. note::
+            The base implementation of ``get_num_tokens_from_messages`` ignores tool
+            schemas.
 
         Args:
             messages: The message inputs to tokenize.
-            tools: If provided, sequence of dict, BaseModel, function, or BaseTools
-                to be converted to tool schemas.
+            tools: If provided, sequence of dict, ``BaseModel``, function, or
+                ``BaseTools`` to be converted to tool schemas.
 
         Returns:
             The sum of the number of tokens across the messages.
+
         """
         if tools is not None:
             warnings.warn(
@@ -396,6 +420,7 @@ class BaseLanguageModel(
     def _all_required_field_names(cls) -> set:
         """DEPRECATED: Kept for backwards compatibility.
 
-        Use get_pydantic_field_names.
+        Use ``get_pydantic_field_names``.
+
         """
         return get_pydantic_field_names(cls)