langchain-core 0.4.0.dev0__py3-none-any.whl → 1.0.0__py3-none-any.whl

langchain_core/indexing/api.py

@@ -6,16 +6,20 @@ import hashlib
 import json
 import uuid
 import warnings
-from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence
+from collections.abc import (
+    AsyncIterable,
+    AsyncIterator,
+    Callable,
+    Iterable,
+    Iterator,
+    Sequence,
+)
 from itertools import islice
 from typing import (
     Any,
-    Callable,
     Literal,
-    Optional,
     TypedDict,
     TypeVar,
-    Union,
     cast,
 )
 
@@ -56,7 +60,7 @@ def _warn_about_sha1() -> None:
             "that map to the same fingerprint. If this matters in your "
             "threat model, switch to a stronger algorithm such "
             "as 'blake2b', 'sha256', or 'sha512' by specifying "
-            " `key_encoder` parameter in the the `index` or `aindex` function. ",
+            " `key_encoder` parameter in the `index` or `aindex` function. ",
             category=UserWarning,
             stacklevel=2,
         )
@@ -107,8 +111,8 @@ async def _abatch(size: int, iterable: AsyncIterable[T]) -> AsyncIterator[list[T
 
 
 def _get_source_id_assigner(
-    source_id_key: Union[str, Callable[[Document], str], None],
-) -> Callable[[Document], Union[str, None]]:
+    source_id_key: str | Callable[[Document], str] | None,
+) -> Callable[[Document], str | None]:
     """Get the source id from the document."""
     if source_id_key is None:
         return lambda _doc: None
@@ -162,9 +166,8 @@ def _calculate_hash(
 def _get_document_with_hash(
     document: Document,
     *,
-    key_encoder: Union[
-        Callable[[Document], str], Literal["sha1", "sha256", "sha512", "blake2b"]
-    ],
+    key_encoder: Callable[[Document], str]
+    | Literal["sha1", "sha256", "sha512", "blake2b"],
 ) -> Document:
     """Calculate a hash of the document, and assign it to the uid.
 
@@ -185,6 +188,9 @@ def _get_document_with_hash(
             When changing the key encoder, you must change the
             index as well to avoid duplicated documents in the cache.
 
+    Raises:
+        ValueError: If the metadata cannot be serialized using json.
+
     Returns:
         Document with a unique identifier based on the hash of the content and metadata.
     """
@@ -230,7 +236,7 @@ class _HashedDocument:
 
 
 def _delete(
-    vector_store: Union[VectorStore, DocumentIndex],
+    vector_store: VectorStore | DocumentIndex,
     ids: list[str],
 ) -> None:
     if isinstance(vector_store, VectorStore):
@@ -268,19 +274,18 @@ class IndexingResult(TypedDict):
 
 
 def index(
-    docs_source: Union[BaseLoader, Iterable[Document]],
+    docs_source: BaseLoader | Iterable[Document],
     record_manager: RecordManager,
-    vector_store: Union[VectorStore, DocumentIndex],
+    vector_store: VectorStore | DocumentIndex,
     *,
     batch_size: int = 100,
-    cleanup: Optional[Literal["incremental", "full", "scoped_full"]] = None,
-    source_id_key: Union[str, Callable[[Document], str], None] = None,
+    cleanup: Literal["incremental", "full", "scoped_full"] | None = None,
+    source_id_key: str | Callable[[Document], str] | None = None,
     cleanup_batch_size: int = 1_000,
     force_update: bool = False,
-    key_encoder: Union[
-        Literal["sha1", "sha256", "sha512", "blake2b"], Callable[[Document], str]
-    ] = "sha1",
-    upsert_kwargs: Optional[dict[str, Any]] = None,
+    key_encoder: Literal["sha1", "sha256", "sha512", "blake2b"]
+    | Callable[[Document], str] = "sha1",
+    upsert_kwargs: dict[str, Any] | None = None,
 ) -> IndexingResult:
     """Index data from the loader into the vector store.
 
@@ -291,21 +296,25 @@ def index(
     documents were deleted, which documents should be skipped.
 
     For the time being, documents are indexed using their hashes, and users
-     are not able to specify the uid of the document.
-
-    Important:
-       * In full mode, the loader should be returning
-         the entire dataset, and not just a subset of the dataset.
-         Otherwise, the auto_cleanup will remove documents that it is not
-         supposed to.
-       * In incremental mode, if documents associated with a particular
-         source id appear across different batches, the indexing API
-         will do some redundant work. This will still result in the
-         correct end state of the index, but will unfortunately not be
-         100% efficient. For example, if a given document is split into 15
-         chunks, and we index them using a batch size of 5, we'll have 3 batches
-         all with the same source id. In general, to avoid doing too much
-         redundant work select as big a batch size as possible.
+    are not able to specify the uid of the document.
+
+    !!! warning "Behavior changed in 0.3.25"
+        Added `scoped_full` cleanup mode.
+
+    !!! warning
+
+        * In full mode, the loader should be returning
+          the entire dataset, and not just a subset of the dataset.
+          Otherwise, the auto_cleanup will remove documents that it is not
+          supposed to.
+        * In incremental mode, if documents associated with a particular
+          source id appear across different batches, the indexing API
+          will do some redundant work. This will still result in the
+          correct end state of the index, but will unfortunately not be
+          100% efficient. For example, if a given document is split into 15
+          chunks, and we index them using a batch size of 5, we'll have 3 batches
+          all with the same source id. In general, to avoid doing too much
+          redundant work select as big a batch size as possible.
         * The `scoped_full` mode is suitable if determining an appropriate batch size
           is challenging or if your data loader cannot return the entire dataset at
           once. This mode keeps track of source IDs in memory, which should be fine
@@ -315,36 +324,32 @@ def index(
     Args:
         docs_source: Data loader or iterable of documents to index.
         record_manager: Timestamped set to keep track of which documents were
-                         updated.
+            updated.
         vector_store: VectorStore or DocumentIndex to index the documents into.
-        batch_size: Batch size to use when indexing. Default is 100.
-        cleanup: How to handle clean up of documents. Default is None.
+        batch_size: Batch size to use when indexing.
+        cleanup: How to handle clean up of documents.
+
             - incremental: Cleans up all documents that haven't been updated AND
-                           that are associated with source ids that were seen
-                           during indexing.
-                           Clean up is done continuously during indexing helping
-                           to minimize the probability of users seeing duplicated
-                           content.
+              that are associated with source ids that were seen during indexing.
+              Clean up is done continuously during indexing helping to minimize the
+              probability of users seeing duplicated content.
             - full: Delete all documents that have not been returned by the loader
-                    during this run of indexing.
-                    Clean up runs after all documents have been indexed.
-                    This means that users may see duplicated content during indexing.
+              during this run of indexing.
+              Clean up runs after all documents have been indexed.
+              This means that users may see duplicated content during indexing.
             - scoped_full: Similar to Full, but only deletes all documents
-                           that haven't been updated AND that are associated with
-                           source ids that were seen during indexing.
+              that haven't been updated AND that are associated with
+              source ids that were seen during indexing.
             - None: Do not delete any documents.
         source_id_key: Optional key that helps identify the original source
-            of the document. Default is None.
+            of the document.
         cleanup_batch_size: Batch size to use when cleaning up documents.
-            Default is 1_000.
         force_update: Force update documents even if they are present in the
             record manager. Useful if you are re-indexing with updated embeddings.
-            Default is False.
         key_encoder: Hashing algorithm to use for hashing the document content and
-            metadata. Default is "sha1".
-            Other options include "blake2b", "sha256", and "sha512".
+            metadata. Options include "blake2b", "sha256", and "sha512".
 
-            .. versionadded:: 0.3.66
+            !!! version-added "Added in version 0.3.66"
 
         key_encoder: Hashing algorithm to use for hashing the document.
             If not provided, a default encoder using SHA-1 will be used.
@@ -358,11 +363,10 @@ def index(
             When changing the key encoder, you must change the
             index as well to avoid duplicated documents in the cache.
         upsert_kwargs: Additional keyword arguments to pass to the add_documents
-                       method of the VectorStore or the upsert method of the
-                       DocumentIndex. For example, you can use this to
-                       specify a custom vector_field:
-                       upsert_kwargs={"vector_field": "embedding"}
-            .. versionadded:: 0.3.10
+            method of the VectorStore or the upsert method of the DocumentIndex.
+            For example, you can use this to specify a custom vector_field:
+            upsert_kwargs={"vector_field": "embedding"}
+            !!! version-added "Added in version 0.3.10"
 
     Returns:
         Indexing result which contains information about how many documents
@@ -374,10 +378,9 @@ def index(
         ValueError: If vectorstore does not have
             "delete" and "add_documents" required methods.
         ValueError: If source_id_key is not None, but is not a string or callable.
-
-    .. version_modified:: 0.3.25
-
-        * Added `scoped_full` cleanup mode.
+        TypeError: If `vectorstore` is not a VectorStore or a DocumentIndex.
+        AssertionError: If `source_id` is None when cleanup mode is incremental.
+            (should be unreachable code).
     """
     # Behavior is deprecated, but we keep it for backwards compatibility.
     # # Warn only once per process.
@@ -458,13 +461,13 @@ def index(
         # Count documents removed by within-batch deduplication
         num_skipped += original_batch_size - len(hashed_docs)
 
-        source_ids: Sequence[Optional[str]] = [
+        source_ids: Sequence[str | None] = [
             source_id_assigner(hashed_doc) for hashed_doc in hashed_docs
         ]
 
         if cleanup in {"incremental", "scoped_full"}:
             # source ids are required.
-            for source_id, hashed_doc in zip(source_ids, hashed_docs):
+            for source_id, hashed_doc in zip(source_ids, hashed_docs, strict=False):
                 if source_id is None:
                     msg = (
                         f"Source ids are required when cleanup mode is "
@@ -488,7 +491,7 @@ def index(
         docs_to_index = []
         uids_to_refresh = []
         seen_docs: set[str] = set()
-        for hashed_doc, doc_exists in zip(hashed_docs, exists_batch):
+        for hashed_doc, doc_exists in zip(hashed_docs, exists_batch, strict=False):
             hashed_id = cast("str", hashed_doc.id)
             if doc_exists:
                 if force_update:
@@ -559,7 +562,7 @@ def index(
     if cleanup == "full" or (
         cleanup == "scoped_full" and scoped_full_cleanup_source_ids
     ):
-        delete_group_ids: Optional[Sequence[str]] = None
+        delete_group_ids: Sequence[str] | None = None
         if cleanup == "scoped_full":
             delete_group_ids = list(scoped_full_cleanup_source_ids)
         while uids_to_delete := record_manager.list_keys(
@@ -587,7 +590,7 @@ async def _to_async_iterator(iterator: Iterable[T]) -> AsyncIterator[T]:
 
 
 async def _adelete(
-    vector_store: Union[VectorStore, DocumentIndex],
+    vector_store: VectorStore | DocumentIndex,
     ids: list[str],
 ) -> None:
     if isinstance(vector_store, VectorStore):
@@ -609,19 +612,18 @@ async def _adelete(
 
 
 async def aindex(
-    docs_source: Union[BaseLoader, Iterable[Document], AsyncIterator[Document]],
+    docs_source: BaseLoader | Iterable[Document] | AsyncIterator[Document],
     record_manager: RecordManager,
-    vector_store: Union[VectorStore, DocumentIndex],
+    vector_store: VectorStore | DocumentIndex,
     *,
     batch_size: int = 100,
-    cleanup: Optional[Literal["incremental", "full", "scoped_full"]] = None,
-    source_id_key: Union[str, Callable[[Document], str], None] = None,
+    cleanup: Literal["incremental", "full", "scoped_full"] | None = None,
+    source_id_key: str | Callable[[Document], str] | None = None,
     cleanup_batch_size: int = 1_000,
     force_update: bool = False,
-    key_encoder: Union[
-        Literal["sha1", "sha256", "sha512", "blake2b"], Callable[[Document], str]
-    ] = "sha1",
-    upsert_kwargs: Optional[dict[str, Any]] = None,
+    key_encoder: Literal["sha1", "sha256", "sha512", "blake2b"]
+    | Callable[[Document], str] = "sha1",
+    upsert_kwargs: dict[str, Any] | None = None,
 ) -> IndexingResult:
     """Async index data from the loader into the vector store.
 
@@ -632,54 +634,61 @@ async def aindex(
     documents were deleted, which documents should be skipped.
 
     For the time being, documents are indexed using their hashes, and users
-     are not able to specify the uid of the document.
-
-    Important:
-       * In full mode, the loader should be returning
-         the entire dataset, and not just a subset of the dataset.
-         Otherwise, the auto_cleanup will remove documents that it is not
-         supposed to.
-       * In incremental mode, if documents associated with a particular
-         source id appear across different batches, the indexing API
-         will do some redundant work. This will still result in the
-         correct end state of the index, but will unfortunately not be
-         100% efficient. For example, if a given document is split into 15
-         chunks, and we index them using a batch size of 5, we'll have 3 batches
-         all with the same source id. In general, to avoid doing too much
-         redundant work select as big a batch size as possible.
-       * The `scoped_full` mode is suitable if determining an appropriate batch size
-         is challenging or if your data loader cannot return the entire dataset at
-         once. This mode keeps track of source IDs in memory, which should be fine
-         for most use cases. If your dataset is large (10M+ docs), you will likely
-         need to parallelize the indexing process regardless.
+    are not able to specify the uid of the document.
+
+    !!! warning "Behavior changed in 0.3.25"
+        Added `scoped_full` cleanup mode.
+
+    !!! warning
+
+        * In full mode, the loader should be returning
+          the entire dataset, and not just a subset of the dataset.
+          Otherwise, the auto_cleanup will remove documents that it is not
+          supposed to.
+        * In incremental mode, if documents associated with a particular
+          source id appear across different batches, the indexing API
+          will do some redundant work. This will still result in the
+          correct end state of the index, but will unfortunately not be
+          100% efficient. For example, if a given document is split into 15
+          chunks, and we index them using a batch size of 5, we'll have 3 batches
+          all with the same source id. In general, to avoid doing too much
+          redundant work select as big a batch size as possible.
+        * The `scoped_full` mode is suitable if determining an appropriate batch size
+          is challenging or if your data loader cannot return the entire dataset at
+          once. This mode keeps track of source IDs in memory, which should be fine
+          for most use cases. If your dataset is large (10M+ docs), you will likely
+          need to parallelize the indexing process regardless.
 
     Args:
         docs_source: Data loader or iterable of documents to index.
         record_manager: Timestamped set to keep track of which documents were
-                         updated.
+            updated.
         vector_store: VectorStore or DocumentIndex to index the documents into.
-        batch_size: Batch size to use when indexing. Default is 100.
-        cleanup: How to handle clean up of documents. Default is None.
+        batch_size: Batch size to use when indexing.
+        cleanup: How to handle clean up of documents.
+
             - incremental: Cleans up all documents that haven't been updated AND
-                           that are associated with source ids that were seen
-                           during indexing.
-                           Clean up is done continuously during indexing helping
-                           to minimize the probability of users seeing duplicated
-                           content.
-            - full: Delete all documents that haven to been returned by the loader.
-                    Clean up runs after all documents have been indexed.
-                    This means that users may see duplicated content during indexing.
+              that are associated with source ids that were seen during indexing.
+              Clean up is done continuously during indexing helping to minimize the
+              probability of users seeing duplicated content.
+            - full: Delete all documents that have not been returned by the loader
+              during this run of indexing.
+              Clean up runs after all documents have been indexed.
+              This means that users may see duplicated content during indexing.
             - scoped_full: Similar to Full, but only deletes all documents
-                           that haven't been updated AND that are associated with
-                           source ids that were seen during indexing.
+              that haven't been updated AND that are associated with
+              source ids that were seen during indexing.
             - None: Do not delete any documents.
         source_id_key: Optional key that helps identify the original source
-            of the document. Default is None.
+            of the document.
         cleanup_batch_size: Batch size to use when cleaning up documents.
-            Default is 1_000.
         force_update: Force update documents even if they are present in the
             record manager. Useful if you are re-indexing with updated embeddings.
-            Default is False.
+        key_encoder: Hashing algorithm to use for hashing the document content and
+            metadata. Options include "blake2b", "sha256", and "sha512".
+
+            !!! version-added "Added in version 0.3.66"
+
         key_encoder: Hashing algorithm to use for hashing the document.
             If not provided, a default encoder using SHA-1 will be used.
             SHA-1 is not collision-resistant, and a motivated attacker
@@ -691,12 +700,11 @@ async def aindex(
 
             When changing the key encoder, you must change the
             index as well to avoid duplicated documents in the cache.
-        upsert_kwargs: Additional keyword arguments to pass to the aadd_documents
-                       method of the VectorStore or the aupsert method of the
-                       DocumentIndex. For example, you can use this to
-                       specify a custom vector_field:
-                       upsert_kwargs={"vector_field": "embedding"}
-            .. versionadded:: 0.3.10
+        upsert_kwargs: Additional keyword arguments to pass to the add_documents
+            method of the VectorStore or the upsert method of the DocumentIndex.
+            For example, you can use this to specify a custom vector_field:
+            upsert_kwargs={"vector_field": "embedding"}
+            !!! version-added "Added in version 0.3.10"
 
     Returns:
         Indexing result which contains information about how many documents
@@ -708,10 +716,9 @@ async def aindex(
         ValueError: If vectorstore does not have
             "adelete" and "aadd_documents" required methods.
         ValueError: If source_id_key is not None, but is not a string or callable.
-
-    .. version_modified:: 0.3.25
-
-        * Added `scoped_full` cleanup mode.
+        TypeError: If `vector_store` is not a VectorStore or DocumentIndex.
+        AssertionError: If `source_id_key` is None when cleanup mode is
+            incremental or `scoped_full` (should be unreachable).
     """
     # Behavior is deprecated, but we keep it for backwards compatibility.
     # # Warn only once per process.
@@ -803,13 +810,13 @@ async def aindex(
         # Count documents removed by within-batch deduplication
         num_skipped += original_batch_size - len(hashed_docs)
 
-        source_ids: Sequence[Optional[str]] = [
+        source_ids: Sequence[str | None] = [
             source_id_assigner(doc) for doc in hashed_docs
         ]
 
         if cleanup in {"incremental", "scoped_full"}:
             # If the cleanup mode is incremental, source ids are required.
-            for source_id, hashed_doc in zip(source_ids, hashed_docs):
+            for source_id, hashed_doc in zip(source_ids, hashed_docs, strict=False):
                 if source_id is None:
                     msg = (
                         f"Source ids are required when cleanup mode is "
@@ -833,7 +840,7 @@ async def aindex(
         docs_to_index: list[Document] = []
         uids_to_refresh = []
         seen_docs: set[str] = set()
-        for hashed_doc, doc_exists in zip(hashed_docs, exists_batch):
+        for hashed_doc, doc_exists in zip(hashed_docs, exists_batch, strict=False):
             hashed_id = cast("str", hashed_doc.id)
             if doc_exists:
                 if force_update:
@@ -905,7 +912,7 @@ async def aindex(
     if cleanup == "full" or (
         cleanup == "scoped_full" and scoped_full_cleanup_source_ids
     ):
-        delete_group_ids: Optional[Sequence[str]] = None
+        delete_group_ids: Sequence[str] | None = None
         if cleanup == "scoped_full":
             delete_group_ids = list(scoped_full_cleanup_source_ids)
         while uids_to_delete := await record_manager.alist_keys(