langchain-core 1.0.0a5__py3-none-any.whl → 1.0.3__py3-none-any.whl

langchain_core/indexing/api.py

@@ -6,16 +6,20 @@ import hashlib
 import json
 import uuid
 import warnings
-from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence
+from collections.abc import (
+    AsyncIterable,
+    AsyncIterator,
+    Callable,
+    Iterable,
+    Iterator,
+    Sequence,
+)
 from itertools import islice
 from typing import (
     Any,
-    Callable,
     Literal,
-    Optional,
     TypedDict,
     TypeVar,
-    Union,
     cast,
 )
 
@@ -107,8 +111,8 @@ async def _abatch(size: int, iterable: AsyncIterable[T]) -> AsyncIterator[list[T
 
 
 def _get_source_id_assigner(
-    source_id_key: Union[str, Callable[[Document], str], None],
-) -> Callable[[Document], Union[str, None]]:
+    source_id_key: str | Callable[[Document], str] | None,
+) -> Callable[[Document], str | None]:
     """Get the source id from the document."""
     if source_id_key is None:
         return lambda _doc: None
@@ -162,9 +166,8 @@ def _calculate_hash(
 def _get_document_with_hash(
     document: Document,
     *,
-    key_encoder: Union[
-        Callable[[Document], str], Literal["sha1", "sha256", "sha512", "blake2b"]
-    ],
+    key_encoder: Callable[[Document], str]
+    | Literal["sha1", "sha256", "sha512", "blake2b"],
 ) -> Document:
     """Calculate a hash of the document, and assign it to the uid.
 
@@ -233,7 +236,7 @@ class _HashedDocument:
 
 
 def _delete(
-    vector_store: Union[VectorStore, DocumentIndex],
+    vector_store: VectorStore | DocumentIndex,
     ids: list[str],
 ) -> None:
     if isinstance(vector_store, VectorStore):
@@ -271,19 +274,18 @@ class IndexingResult(TypedDict):
 
 
 def index(
-    docs_source: Union[BaseLoader, Iterable[Document]],
+    docs_source: BaseLoader | Iterable[Document],
     record_manager: RecordManager,
-    vector_store: Union[VectorStore, DocumentIndex],
+    vector_store: VectorStore | DocumentIndex,
     *,
     batch_size: int = 100,
-    cleanup: Optional[Literal["incremental", "full", "scoped_full"]] = None,
-    source_id_key: Union[str, Callable[[Document], str], None] = None,
+    cleanup: Literal["incremental", "full", "scoped_full"] | None = None,
+    source_id_key: str | Callable[[Document], str] | None = None,
     cleanup_batch_size: int = 1_000,
     force_update: bool = False,
-    key_encoder: Union[
-        Literal["sha1", "sha256", "sha512", "blake2b"], Callable[[Document], str]
-    ] = "sha1",
-    upsert_kwargs: Optional[dict[str, Any]] = None,
+    key_encoder: Literal["sha1", "sha256", "sha512", "blake2b"]
+    | Callable[[Document], str] = "sha1",
+    upsert_kwargs: dict[str, Any] | None = None,
 ) -> IndexingResult:
     """Index data from the loader into the vector store.
 
@@ -296,61 +298,58 @@ def index(
     For the time being, documents are indexed using their hashes, and users
     are not able to specify the uid of the document.
 
-    .. versionchanged:: 0.3.25
-        Added ``scoped_full`` cleanup mode.
+    !!! warning "Behavior changed in 0.3.25"
+        Added `scoped_full` cleanup mode.
 
-    .. important::
+    !!! warning
 
         * In full mode, the loader should be returning
-          the entire dataset, and not just a subset of the dataset.
-          Otherwise, the auto_cleanup will remove documents that it is not
-          supposed to.
+            the entire dataset, and not just a subset of the dataset.
+            Otherwise, the auto_cleanup will remove documents that it is not
+            supposed to.
         * In incremental mode, if documents associated with a particular
-          source id appear across different batches, the indexing API
-          will do some redundant work. This will still result in the
-          correct end state of the index, but will unfortunately not be
-          100% efficient. For example, if a given document is split into 15
-          chunks, and we index them using a batch size of 5, we'll have 3 batches
-          all with the same source id. In general, to avoid doing too much
-          redundant work select as big a batch size as possible.
-        * The ``scoped_full`` mode is suitable if determining an appropriate batch size
-          is challenging or if your data loader cannot return the entire dataset at
-          once. This mode keeps track of source IDs in memory, which should be fine
-          for most use cases. If your dataset is large (10M+ docs), you will likely
-          need to parallelize the indexing process regardless.
+            source id appear across different batches, the indexing API
+            will do some redundant work. This will still result in the
+            correct end state of the index, but will unfortunately not be
+            100% efficient. For example, if a given document is split into 15
+            chunks, and we index them using a batch size of 5, we'll have 3 batches
+            all with the same source id. In general, to avoid doing too much
+            redundant work select as big a batch size as possible.
+        * The `scoped_full` mode is suitable if determining an appropriate batch size
+            is challenging or if your data loader cannot return the entire dataset at
+            once. This mode keeps track of source IDs in memory, which should be fine
+            for most use cases. If your dataset is large (10M+ docs), you will likely
+            need to parallelize the indexing process regardless.
 
     Args:
         docs_source: Data loader or iterable of documents to index.
         record_manager: Timestamped set to keep track of which documents were
             updated.
-        vector_store: VectorStore or DocumentIndex to index the documents into.
-        batch_size: Batch size to use when indexing. Default is 100.
-        cleanup: How to handle clean up of documents. Default is None.
+        vector_store: `VectorStore` or DocumentIndex to index the documents into.
+        batch_size: Batch size to use when indexing.
+        cleanup: How to handle clean up of documents.
 
             - incremental: Cleans up all documents that haven't been updated AND
-              that are associated with source ids that were seen during indexing.
-              Clean up is done continuously during indexing helping to minimize the
-              probability of users seeing duplicated content.
+                that are associated with source IDs that were seen during indexing.
+                Clean up is done continuously during indexing helping to minimize the
+                probability of users seeing duplicated content.
             - full: Delete all documents that have not been returned by the loader
-              during this run of indexing.
-              Clean up runs after all documents have been indexed.
-              This means that users may see duplicated content during indexing.
+                during this run of indexing.
+                Clean up runs after all documents have been indexed.
+                This means that users may see duplicated content during indexing.
             - scoped_full: Similar to Full, but only deletes all documents
-              that haven't been updated AND that are associated with
-              source ids that were seen during indexing.
+                that haven't been updated AND that are associated with
+                source IDs that were seen during indexing.
             - None: Do not delete any documents.
         source_id_key: Optional key that helps identify the original source
-            of the document. Default is None.
+            of the document.
         cleanup_batch_size: Batch size to use when cleaning up documents.
-            Default is 1_000.
         force_update: Force update documents even if they are present in the
             record manager. Useful if you are re-indexing with updated embeddings.
-            Default is False.
         key_encoder: Hashing algorithm to use for hashing the document content and
-            metadata. Default is "sha1".
-            Other options include "blake2b", "sha256", and "sha512".
+            metadata. Options include "blake2b", "sha256", and "sha512".
 
-            .. versionadded:: 0.3.66
+            !!! version-added "Added in version 0.3.66"
 
         key_encoder: Hashing algorithm to use for hashing the document.
             If not provided, a default encoder using SHA-1 will be used.
@@ -364,10 +363,10 @@ def index(
             When changing the key encoder, you must change the
             index as well to avoid duplicated documents in the cache.
         upsert_kwargs: Additional keyword arguments to pass to the add_documents
-            method of the VectorStore or the upsert method of the DocumentIndex.
+            method of the `VectorStore` or the upsert method of the DocumentIndex.
             For example, you can use this to specify a custom vector_field:
             upsert_kwargs={"vector_field": "embedding"}
-            .. versionadded:: 0.3.10
+            !!! version-added "Added in version 0.3.10"
 
     Returns:
         Indexing result which contains information about how many documents
@@ -376,11 +375,11 @@ def index(
     Raises:
         ValueError: If cleanup mode is not one of 'incremental', 'full' or None
         ValueError: If cleanup mode is incremental and source_id_key is None.
-        ValueError: If vectorstore does not have
+        ValueError: If `VectorStore` does not have
             "delete" and "add_documents" required methods.
         ValueError: If source_id_key is not None, but is not a string or callable.
-        TypeError: If ``vectorstore`` is not a VectorStore or a DocumentIndex.
-        AssertionError: If ``source_id`` is None when cleanup mode is incremental.
+        TypeError: If `vectorstore` is not a `VectorStore` or a DocumentIndex.
+        AssertionError: If `source_id` is None when cleanup mode is incremental.
             (should be unreachable code).
     """
     # Behavior is deprecated, but we keep it for backwards compatibility.
@@ -416,7 +415,7 @@ def index(
                 raise ValueError(msg)
 
         if type(destination).delete == VectorStore.delete:
-            # Checking if the vectorstore has overridden the default delete method
+            # Checking if the VectorStore has overridden the default delete method
             # implementation which just raises a NotImplementedError
             msg = "Vectorstore has not implemented the delete method"
             raise ValueError(msg)
@@ -462,16 +461,16 @@ def index(
         # Count documents removed by within-batch deduplication
         num_skipped += original_batch_size - len(hashed_docs)
 
-        source_ids: Sequence[Optional[str]] = [
+        source_ids: Sequence[str | None] = [
             source_id_assigner(hashed_doc) for hashed_doc in hashed_docs
         ]
 
         if cleanup in {"incremental", "scoped_full"}:
-            # source ids are required.
-            for source_id, hashed_doc in zip(source_ids, hashed_docs):
+            # Source IDs are required.
+            for source_id, hashed_doc in zip(source_ids, hashed_docs, strict=False):
                 if source_id is None:
                     msg = (
-                        f"Source ids are required when cleanup mode is "
+                        f"Source IDs are required when cleanup mode is "
                         f"incremental or scoped_full. "
                         f"Document that starts with "
                         f"content: {hashed_doc.page_content[:100]} "
@@ -480,7 +479,7 @@ def index(
                     raise ValueError(msg)
                 if cleanup == "scoped_full":
                     scoped_full_cleanup_source_ids.add(source_id)
-            # source ids cannot be None after for loop above.
+            # Source IDs cannot be None after for loop above.
             source_ids = cast("Sequence[str]", source_ids)
 
         exists_batch = record_manager.exists(
@@ -492,7 +491,7 @@ def index(
         docs_to_index = []
         uids_to_refresh = []
         seen_docs: set[str] = set()
-        for hashed_doc, doc_exists in zip(hashed_docs, exists_batch):
+        for hashed_doc, doc_exists in zip(hashed_docs, exists_batch, strict=False):
             hashed_id = cast("str", hashed_doc.id)
             if doc_exists:
                 if force_update:
@@ -539,7 +538,7 @@ def index(
         # If source IDs are provided, we can do the deletion incrementally!
         if cleanup == "incremental":
             # Get the uids of the documents that were not returned by the loader.
-            # mypy isn't good enough to determine that source ids cannot be None
+            # mypy isn't good enough to determine that source IDs cannot be None
             # here due to a check that's happening above, so we check again.
             for source_id in source_ids:
                 if source_id is None:
@@ -563,7 +562,7 @@ def index(
     if cleanup == "full" or (
         cleanup == "scoped_full" and scoped_full_cleanup_source_ids
     ):
-        delete_group_ids: Optional[Sequence[str]] = None
+        delete_group_ids: Sequence[str] | None = None
         if cleanup == "scoped_full":
             delete_group_ids = list(scoped_full_cleanup_source_ids)
         while uids_to_delete := record_manager.list_keys(
@@ -591,7 +590,7 @@ async def _to_async_iterator(iterator: Iterable[T]) -> AsyncIterator[T]:
 
 
 async def _adelete(
-    vector_store: Union[VectorStore, DocumentIndex],
+    vector_store: VectorStore | DocumentIndex,
     ids: list[str],
 ) -> None:
     if isinstance(vector_store, VectorStore):
@@ -613,19 +612,18 @@ async def _adelete(
 
 
 async def aindex(
-    docs_source: Union[BaseLoader, Iterable[Document], AsyncIterator[Document]],
+    docs_source: BaseLoader | Iterable[Document] | AsyncIterator[Document],
     record_manager: RecordManager,
-    vector_store: Union[VectorStore, DocumentIndex],
+    vector_store: VectorStore | DocumentIndex,
     *,
     batch_size: int = 100,
-    cleanup: Optional[Literal["incremental", "full", "scoped_full"]] = None,
-    source_id_key: Union[str, Callable[[Document], str], None] = None,
+    cleanup: Literal["incremental", "full", "scoped_full"] | None = None,
+    source_id_key: str | Callable[[Document], str] | None = None,
     cleanup_batch_size: int = 1_000,
     force_update: bool = False,
-    key_encoder: Union[
-        Literal["sha1", "sha256", "sha512", "blake2b"], Callable[[Document], str]
-    ] = "sha1",
-    upsert_kwargs: Optional[dict[str, Any]] = None,
+    key_encoder: Literal["sha1", "sha256", "sha512", "blake2b"]
+    | Callable[[Document], str] = "sha1",
+    upsert_kwargs: dict[str, Any] | None = None,
 ) -> IndexingResult:
     """Async index data from the loader into the vector store.
 
@@ -638,61 +636,58 @@ async def aindex(
     For the time being, documents are indexed using their hashes, and users
     are not able to specify the uid of the document.
 
-    .. versionchanged:: 0.3.25
-        Added ``scoped_full`` cleanup mode.
+    !!! warning "Behavior changed in 0.3.25"
+        Added `scoped_full` cleanup mode.
 
-    .. important::
+    !!! warning
 
         * In full mode, the loader should be returning
-          the entire dataset, and not just a subset of the dataset.
-          Otherwise, the auto_cleanup will remove documents that it is not
-          supposed to.
+            the entire dataset, and not just a subset of the dataset.
+            Otherwise, the auto_cleanup will remove documents that it is not
+            supposed to.
         * In incremental mode, if documents associated with a particular
-          source id appear across different batches, the indexing API
-          will do some redundant work. This will still result in the
-          correct end state of the index, but will unfortunately not be
-          100% efficient. For example, if a given document is split into 15
-          chunks, and we index them using a batch size of 5, we'll have 3 batches
-          all with the same source id. In general, to avoid doing too much
-          redundant work select as big a batch size as possible.
-        * The ``scoped_full`` mode is suitable if determining an appropriate batch size
-          is challenging or if your data loader cannot return the entire dataset at
-          once. This mode keeps track of source IDs in memory, which should be fine
-          for most use cases. If your dataset is large (10M+ docs), you will likely
-          need to parallelize the indexing process regardless.
+            source id appear across different batches, the indexing API
+            will do some redundant work. This will still result in the
+            correct end state of the index, but will unfortunately not be
+            100% efficient. For example, if a given document is split into 15
+            chunks, and we index them using a batch size of 5, we'll have 3 batches
+            all with the same source id. In general, to avoid doing too much
+            redundant work select as big a batch size as possible.
+        * The `scoped_full` mode is suitable if determining an appropriate batch size
+            is challenging or if your data loader cannot return the entire dataset at
+            once. This mode keeps track of source IDs in memory, which should be fine
+            for most use cases. If your dataset is large (10M+ docs), you will likely
+            need to parallelize the indexing process regardless.
 
     Args:
         docs_source: Data loader or iterable of documents to index.
         record_manager: Timestamped set to keep track of which documents were
             updated.
-        vector_store: VectorStore or DocumentIndex to index the documents into.
-        batch_size: Batch size to use when indexing. Default is 100.
-        cleanup: How to handle clean up of documents. Default is None.
+        vector_store: `VectorStore` or DocumentIndex to index the documents into.
+        batch_size: Batch size to use when indexing.
+        cleanup: How to handle clean up of documents.
 
             - incremental: Cleans up all documents that haven't been updated AND
-              that are associated with source ids that were seen during indexing.
-              Clean up is done continuously during indexing helping to minimize the
-              probability of users seeing duplicated content.
+                that are associated with source IDs that were seen during indexing.
+                Clean up is done continuously during indexing helping to minimize the
+                probability of users seeing duplicated content.
             - full: Delete all documents that have not been returned by the loader
-              during this run of indexing.
-              Clean up runs after all documents have been indexed.
-              This means that users may see duplicated content during indexing.
+                during this run of indexing.
+                Clean up runs after all documents have been indexed.
+                This means that users may see duplicated content during indexing.
             - scoped_full: Similar to Full, but only deletes all documents
-              that haven't been updated AND that are associated with
-              source ids that were seen during indexing.
+                that haven't been updated AND that are associated with
+                source IDs that were seen during indexing.
             - None: Do not delete any documents.
         source_id_key: Optional key that helps identify the original source
-            of the document. Default is None.
+            of the document.
         cleanup_batch_size: Batch size to use when cleaning up documents.
-            Default is 1_000.
         force_update: Force update documents even if they are present in the
             record manager. Useful if you are re-indexing with updated embeddings.
-            Default is False.
         key_encoder: Hashing algorithm to use for hashing the document content and
-            metadata. Default is "sha1".
-            Other options include "blake2b", "sha256", and "sha512".
+            metadata. Options include "blake2b", "sha256", and "sha512".
 
-            .. versionadded:: 0.3.66
+            !!! version-added "Added in version 0.3.66"
 
         key_encoder: Hashing algorithm to use for hashing the document.
             If not provided, a default encoder using SHA-1 will be used.
@@ -706,10 +701,10 @@ async def aindex(
             When changing the key encoder, you must change the
             index as well to avoid duplicated documents in the cache.
         upsert_kwargs: Additional keyword arguments to pass to the add_documents
-            method of the VectorStore or the upsert method of the DocumentIndex.
+            method of the `VectorStore` or the upsert method of the DocumentIndex.
             For example, you can use this to specify a custom vector_field:
             upsert_kwargs={"vector_field": "embedding"}
-            .. versionadded:: 0.3.10
+            !!! version-added "Added in version 0.3.10"
 
     Returns:
         Indexing result which contains information about how many documents
@@ -718,12 +713,12 @@ async def aindex(
     Raises:
         ValueError: If cleanup mode is not one of 'incremental', 'full' or None
         ValueError: If cleanup mode is incremental and source_id_key is None.
-        ValueError: If vectorstore does not have
+        ValueError: If `VectorStore` does not have
             "adelete" and "aadd_documents" required methods.
         ValueError: If source_id_key is not None, but is not a string or callable.
-        TypeError: If ``vector_store`` is not a VectorStore or DocumentIndex.
-        AssertionError: If ``source_id_key`` is None when cleanup mode is
-            incremental or ``scoped_full`` (should be unreachable).
+        TypeError: If `vector_store` is not a `VectorStore` or DocumentIndex.
+        AssertionError: If `source_id_key` is None when cleanup mode is
+            incremental or `scoped_full` (should be unreachable).
     """
     # Behavior is deprecated, but we keep it for backwards compatibility.
     # # Warn only once per process.
@@ -762,7 +757,7 @@ async def aindex(
             type(destination).adelete == VectorStore.adelete
             and type(destination).delete == VectorStore.delete
         ):
-            # Checking if the vectorstore has overridden the default adelete or delete
+            # Checking if the VectorStore has overridden the default adelete or delete
             # methods implementation which just raises a NotImplementedError
             msg = "Vectorstore has not implemented the adelete or delete method"
             raise ValueError(msg)
@@ -815,16 +810,16 @@ async def aindex(
         # Count documents removed by within-batch deduplication
         num_skipped += original_batch_size - len(hashed_docs)
 
-        source_ids: Sequence[Optional[str]] = [
+        source_ids: Sequence[str | None] = [
             source_id_assigner(doc) for doc in hashed_docs
         ]
 
         if cleanup in {"incremental", "scoped_full"}:
-            # If the cleanup mode is incremental, source ids are required.
-            for source_id, hashed_doc in zip(source_ids, hashed_docs):
+            # If the cleanup mode is incremental, source IDs are required.
+            for source_id, hashed_doc in zip(source_ids, hashed_docs, strict=False):
                 if source_id is None:
                     msg = (
-                        f"Source ids are required when cleanup mode is "
+                        f"Source IDs are required when cleanup mode is "
                         f"incremental or scoped_full. "
                         f"Document that starts with "
                         f"content: {hashed_doc.page_content[:100]} "
@@ -833,7 +828,7 @@ async def aindex(
                     raise ValueError(msg)
                 if cleanup == "scoped_full":
                     scoped_full_cleanup_source_ids.add(source_id)
-            # source ids cannot be None after for loop above.
+            # Source IDs cannot be None after for loop above.
             source_ids = cast("Sequence[str]", source_ids)
 
         exists_batch = await record_manager.aexists(
@@ -845,7 +840,7 @@ async def aindex(
         docs_to_index: list[Document] = []
         uids_to_refresh = []
         seen_docs: set[str] = set()
-        for hashed_doc, doc_exists in zip(hashed_docs, exists_batch):
+        for hashed_doc, doc_exists in zip(hashed_docs, exists_batch, strict=False):
             hashed_id = cast("str", hashed_doc.id)
             if doc_exists:
                 if force_update:
@@ -893,7 +888,7 @@ async def aindex(
         if cleanup == "incremental":
             # Get the uids of the documents that were not returned by the loader.
 
-            # mypy isn't good enough to determine that source ids cannot be None
+            # mypy isn't good enough to determine that source IDs cannot be None
             # here due to a check that's happening above, so we check again.
             for source_id in source_ids:
                 if source_id is None:
@@ -917,7 +912,7 @@ async def aindex(
     if cleanup == "full" or (
         cleanup == "scoped_full" and scoped_full_cleanup_source_ids
     ):
-        delete_group_ids: Optional[Sequence[str]] = None
+        delete_group_ids: Sequence[str] | None = None
         if cleanup == "scoped_full":
             delete_group_ids = list(scoped_full_cleanup_source_ids)
         while uids_to_delete := await record_manager.alist_keys(