langchain 0.4.0.dev0__py3-none-any.whl → 1.0.0a2__py3-none-any.whl

langchain/chains/combine_documents/map_rerank.py

@@ -1,249 +0,0 @@
-"""Combining documents by mapping a chain over them first, then reranking results."""
-
-from __future__ import annotations
-
-from collections.abc import Sequence
-from typing import Any, Optional, Union, cast
-
-from langchain_core._api import deprecated
-from langchain_core.callbacks import Callbacks
-from langchain_core.documents import Document
-from langchain_core.runnables.config import RunnableConfig
-from langchain_core.utils.pydantic import create_model
-from pydantic import BaseModel, ConfigDict, model_validator
-from typing_extensions import Self, override
-
-from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
-from langchain.chains.llm import LLMChain
-from langchain.output_parsers.regex import RegexParser
-
-
-@deprecated(
-    since="0.3.1",
-    removal="1.0",
-    message=(
-        "This class is deprecated. Please see the migration guide here for "
-        "a recommended replacement: "
-        "https://python.langchain.com/docs/versions/migrating_chains/map_rerank_docs_chain/"
-    ),
-)
-class MapRerankDocumentsChain(BaseCombineDocumentsChain):
-    """Combining documents by mapping a chain over them, then reranking results.
-
-    This algorithm calls an LLMChain on each input document. The LLMChain is expected
-    to have an OutputParser that parses the result into both an answer (``answer_key``)
-    and a score (``rank_key``). The answer with the highest score is then returned.
-
-    Example:
-        .. code-block:: python
-
-            from langchain.chains import MapRerankDocumentsChain, LLMChain
-            from langchain_core.prompts import PromptTemplate
-            from langchain_community.llms import OpenAI
-            from langchain.output_parsers.regex import RegexParser
-
-            document_variable_name = "context"
-            llm = OpenAI()
-            # The prompt here should take as an input variable the
-            # `document_variable_name`
-            # The actual prompt will need to be a lot more complex, this is just
-            # an example.
-            prompt_template = (
-                "Use the following context to tell me the chemical formula "
-                "for water. Output both your answer and a score of how confident "
-                "you are. Context: {context}"
-            )
-            output_parser = RegexParser(
-                regex=r"(.*?)\nScore: (.*)",
-                output_keys=["answer", "score"],
-            )
-            prompt = PromptTemplate(
-                template=prompt_template,
-                input_variables=["context"],
-                output_parser=output_parser,
-            )
-            llm_chain = LLMChain(llm=llm, prompt=prompt)
-            chain = MapRerankDocumentsChain(
-                llm_chain=llm_chain,
-                document_variable_name=document_variable_name,
-                rank_key="score",
-                answer_key="answer",
-            )
-
-    """
-
-    llm_chain: LLMChain
-    """Chain to apply to each document individually."""
-    document_variable_name: str
-    """The variable name in the llm_chain to put the documents in.
-    If only one variable in the llm_chain, this need not be provided."""
-    rank_key: str
-    """Key in output of llm_chain to rank on."""
-    answer_key: str
-    """Key in output of llm_chain to return as answer."""
-    metadata_keys: Optional[list[str]] = None
-    """Additional metadata from the chosen document to return."""
-    return_intermediate_steps: bool = False
-    """Return intermediate steps.
-    Intermediate steps include the results of calling llm_chain on each document."""
-
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True,
-        extra="forbid",
-    )
-
-    @override
-    def get_output_schema(
-        self,
-        config: Optional[RunnableConfig] = None,
-    ) -> type[BaseModel]:
-        schema: dict[str, Any] = {
-            self.output_key: (str, None),
-        }
-        if self.return_intermediate_steps:
-            schema["intermediate_steps"] = (list[str], None)
-        if self.metadata_keys:
-            schema.update(dict.fromkeys(self.metadata_keys, (Any, None)))
-
-        return create_model("MapRerankOutput", **schema)
-
-    @property
-    def output_keys(self) -> list[str]:
-        """Expect input key.
-
-        :meta private:
-        """
-        _output_keys = super().output_keys
-        if self.return_intermediate_steps:
-            _output_keys = [*_output_keys, "intermediate_steps"]
-        if self.metadata_keys is not None:
-            _output_keys += self.metadata_keys
-        return _output_keys
-
-    @model_validator(mode="after")
-    def validate_llm_output(self) -> Self:
-        """Validate that the combine chain outputs a dictionary."""
-        output_parser = self.llm_chain.prompt.output_parser
-        if not isinstance(output_parser, RegexParser):
-            msg = (
-                "Output parser of llm_chain should be a RegexParser,"
-                f" got {output_parser}"
-            )
-            raise ValueError(msg)  # noqa: TRY004
-        output_keys = output_parser.output_keys
-        if self.rank_key not in output_keys:
-            msg = (
-                f"Got {self.rank_key} as key to rank on, but did not find "
-                f"it in the llm_chain output keys ({output_keys})"
-            )
-            raise ValueError(msg)
-        if self.answer_key not in output_keys:
-            msg = (
-                f"Got {self.answer_key} as key to return, but did not find "
-                f"it in the llm_chain output keys ({output_keys})"
-            )
-            raise ValueError(msg)
-        return self
-
-    @model_validator(mode="before")
-    @classmethod
-    def get_default_document_variable_name(cls, values: dict) -> Any:
-        """Get default document variable name, if not provided."""
-        if "llm_chain" not in values:
-            msg = "llm_chain must be provided"
-            raise ValueError(msg)
-
-        llm_chain_variables = values["llm_chain"].prompt.input_variables
-        if "document_variable_name" not in values:
-            if len(llm_chain_variables) == 1:
-                values["document_variable_name"] = llm_chain_variables[0]
-            else:
-                msg = (
-                    "document_variable_name must be provided if there are "
-                    "multiple llm_chain input_variables"
-                )
-                raise ValueError(msg)
-        elif values["document_variable_name"] not in llm_chain_variables:
-            msg = (
-                f"document_variable_name {values['document_variable_name']} was "
-                f"not found in llm_chain input_variables: {llm_chain_variables}"
-            )
-            raise ValueError(msg)
-        return values
-
-    def combine_docs(
-        self,
-        docs: list[Document],
-        callbacks: Callbacks = None,
-        **kwargs: Any,
-    ) -> tuple[str, dict]:
-        """Combine documents in a map rerank manner.
-
-        Combine by mapping first chain over all documents, then reranking the results.
-
-        Args:
-            docs: List of documents to combine
-            callbacks: Callbacks to be passed through
-            **kwargs: additional parameters to be passed to LLM calls (like other
-                input variables besides the documents)
-
-        Returns:
-            The first element returned is the single string output. The second
-            element returned is a dictionary of other keys to return.
-        """
-        results = self.llm_chain.apply_and_parse(
-            # FYI - this is parallelized and so it is fast.
-            [{self.document_variable_name: d.page_content, **kwargs} for d in docs],
-            callbacks=callbacks,
-        )
-        return self._process_results(docs, results)
-
-    async def acombine_docs(
-        self,
-        docs: list[Document],
-        callbacks: Callbacks = None,
-        **kwargs: Any,
-    ) -> tuple[str, dict]:
-        """Combine documents in a map rerank manner.
-
-        Combine by mapping first chain over all documents, then reranking the results.
-
-        Args:
-            docs: List of documents to combine
-            callbacks: Callbacks to be passed through
-            **kwargs: additional parameters to be passed to LLM calls (like other
-                input variables besides the documents)
-
-        Returns:
-            The first element returned is the single string output. The second
-            element returned is a dictionary of other keys to return.
-        """
-        results = await self.llm_chain.aapply_and_parse(
-            # FYI - this is parallelized and so it is fast.
-            [{self.document_variable_name: d.page_content, **kwargs} for d in docs],
-            callbacks=callbacks,
-        )
-        return self._process_results(docs, results)
-
-    def _process_results(
-        self,
-        docs: list[Document],
-        results: Sequence[Union[str, list[str], dict[str, str]]],
-    ) -> tuple[str, dict]:
-        typed_results = cast("list[dict]", results)
-        sorted_res = sorted(
-            zip(typed_results, docs),
-            key=lambda x: -int(x[0][self.rank_key]),
-        )
-        output, document = sorted_res[0]
-        extra_info = {}
-        if self.metadata_keys is not None:
-            for key in self.metadata_keys:
-                extra_info[key] = document.metadata[key]
-        if self.return_intermediate_steps:
-            extra_info["intermediate_steps"] = results
-        return output[self.answer_key], extra_info
-
-    @property
-    def _chain_type(self) -> str:
-        return "map_rerank_documents_chain"

langchain/chains/combine_documents/reduce.py

@@ -1,382 +0,0 @@
-"""Combine many documents together by recursively reducing them."""
-
-from __future__ import annotations
-
-from typing import Any, Callable, Optional, Protocol
-
-from langchain_core._api import deprecated
-from langchain_core.callbacks import Callbacks
-from langchain_core.documents import Document
-from pydantic import ConfigDict
-
-from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
-
-
-class CombineDocsProtocol(Protocol):
-    """Interface for the combine_docs method."""
-
-    def __call__(self, docs: list[Document], **kwargs: Any) -> str:
-        """Interface for the combine_docs method."""
-
-
-class AsyncCombineDocsProtocol(Protocol):
-    """Interface for the combine_docs method."""
-
-    async def __call__(self, docs: list[Document], **kwargs: Any) -> str:
-        """Async interface for the combine_docs method."""
-
-
-def split_list_of_docs(
-    docs: list[Document],
-    length_func: Callable,
-    token_max: int,
-    **kwargs: Any,
-) -> list[list[Document]]:
-    """Split Documents into subsets that each meet a cumulative length constraint.
-
-    Args:
-        docs: The full list of Documents.
-        length_func: Function for computing the cumulative length of a set of Documents.
-        token_max: The maximum cumulative length of any subset of Documents.
-        **kwargs: Arbitrary additional keyword params to pass to each call of the
-            length_func.
-
-    Returns:
-        A List[List[Document]].
-    """
-    new_result_doc_list = []
-    _sub_result_docs = []
-    for doc in docs:
-        _sub_result_docs.append(doc)
-        _num_tokens = length_func(_sub_result_docs, **kwargs)
-        if _num_tokens > token_max:
-            if len(_sub_result_docs) == 1:
-                msg = (
-                    "A single document was longer than the context length,"
-                    " we cannot handle this."
-                )
-                raise ValueError(msg)
-            new_result_doc_list.append(_sub_result_docs[:-1])
-            _sub_result_docs = _sub_result_docs[-1:]
-    new_result_doc_list.append(_sub_result_docs)
-    return new_result_doc_list
-
-
-def collapse_docs(
-    docs: list[Document],
-    combine_document_func: CombineDocsProtocol,
-    **kwargs: Any,
-) -> Document:
-    """Execute a collapse function on a set of documents and merge their metadatas.
-
-    Args:
-        docs: A list of Documents to combine.
-        combine_document_func: A function that takes in a list of Documents and
-            optionally addition keyword parameters and combines them into a single
-            string.
-        **kwargs: Arbitrary additional keyword params to pass to the
-            combine_document_func.
-
-    Returns:
-        A single Document with the output of combine_document_func for the page content
-            and the combined metadata's of all the input documents. All metadata values
-            are strings, and where there are overlapping keys across documents the
-            values are joined by ", ".
-    """
-    result = combine_document_func(docs, **kwargs)
-    combined_metadata = {k: str(v) for k, v in docs[0].metadata.items()}
-    for doc in docs[1:]:
-        for k, v in doc.metadata.items():
-            if k in combined_metadata:
-                combined_metadata[k] += f", {v}"
-            else:
-                combined_metadata[k] = str(v)
-    return Document(page_content=result, metadata=combined_metadata)
-
-
-async def acollapse_docs(
-    docs: list[Document],
-    combine_document_func: AsyncCombineDocsProtocol,
-    **kwargs: Any,
-) -> Document:
-    """Execute a collapse function on a set of documents and merge their metadatas.
-
-    Args:
-        docs: A list of Documents to combine.
-        combine_document_func: A function that takes in a list of Documents and
-            optionally addition keyword parameters and combines them into a single
-            string.
-        **kwargs: Arbitrary additional keyword params to pass to the
-            combine_document_func.
-
-    Returns:
-        A single Document with the output of combine_document_func for the page content
-            and the combined metadata's of all the input documents. All metadata values
-            are strings, and where there are overlapping keys across documents the
-            values are joined by ", ".
-    """
-    result = await combine_document_func(docs, **kwargs)
-    combined_metadata = {k: str(v) for k, v in docs[0].metadata.items()}
-    for doc in docs[1:]:
-        for k, v in doc.metadata.items():
-            if k in combined_metadata:
-                combined_metadata[k] += f", {v}"
-            else:
-                combined_metadata[k] = str(v)
-    return Document(page_content=result, metadata=combined_metadata)
-
-
-@deprecated(
-    since="0.3.1",
-    removal="1.0",
-    message=(
-        "This class is deprecated. Please see the migration guide here for "
-        "a recommended replacement: "
-        "https://python.langchain.com/docs/versions/migrating_chains/map_reduce_chain/"
-    ),
-)
-class ReduceDocumentsChain(BaseCombineDocumentsChain):
-    """Combine documents by recursively reducing them.
-
-    This involves
-
-    - combine_documents_chain
-
-    - collapse_documents_chain
-
-    `combine_documents_chain` is ALWAYS provided. This is final chain that is called.
-    We pass all previous results to this chain, and the output of this chain is
-    returned as a final result.
-
-    `collapse_documents_chain` is used if the documents passed in are too many to all
-    be passed to `combine_documents_chain` in one go. In this case,
-    `collapse_documents_chain` is called recursively on as big of groups of documents
-    as are allowed.
-
-    Example:
-        .. code-block:: python
-
-            from langchain.chains import (
-                StuffDocumentsChain, LLMChain, ReduceDocumentsChain
-            )
-            from langchain_core.prompts import PromptTemplate
-            from langchain_community.llms import OpenAI
-
-            # This controls how each document will be formatted. Specifically,
-            # it will be passed to `format_document` - see that function for more
-            # details.
-            document_prompt = PromptTemplate(
-                input_variables=["page_content"],
-                 template="{page_content}"
-            )
-            document_variable_name = "context"
-            llm = OpenAI()
-            # The prompt here should take as an input variable the
-            # `document_variable_name`
-            prompt = PromptTemplate.from_template(
-                "Summarize this content: {context}"
-            )
-            llm_chain = LLMChain(llm=llm, prompt=prompt)
-            combine_documents_chain = StuffDocumentsChain(
-                llm_chain=llm_chain,
-                document_prompt=document_prompt,
-                document_variable_name=document_variable_name
-            )
-            chain = ReduceDocumentsChain(
-                combine_documents_chain=combine_documents_chain,
-            )
-            # If we wanted to, we could also pass in collapse_documents_chain
-            # which is specifically aimed at collapsing documents BEFORE
-            # the final call.
-            prompt = PromptTemplate.from_template(
-                "Collapse this content: {context}"
-            )
-            llm_chain = LLMChain(llm=llm, prompt=prompt)
-            collapse_documents_chain = StuffDocumentsChain(
-                llm_chain=llm_chain,
-                document_prompt=document_prompt,
-                document_variable_name=document_variable_name
-            )
-            chain = ReduceDocumentsChain(
-                combine_documents_chain=combine_documents_chain,
-                collapse_documents_chain=collapse_documents_chain,
-            )
-
-    """
-
-    combine_documents_chain: BaseCombineDocumentsChain
-    """Final chain to call to combine documents.
-    This is typically a StuffDocumentsChain."""
-    collapse_documents_chain: Optional[BaseCombineDocumentsChain] = None
-    """Chain to use to collapse documents if needed until they can all fit.
-    If None, will use the combine_documents_chain.
-    This is typically a StuffDocumentsChain."""
-    token_max: int = 3000
-    """The maximum number of tokens to group documents into. For example, if
-    set to 3000 then documents will be grouped into chunks of no greater than
-    3000 tokens before trying to combine them into a smaller chunk."""
-    collapse_max_retries: Optional[int] = None
-    """The maximum number of retries to collapse documents to fit token_max.
-    If None, it will keep trying to collapse documents to fit token_max.
-    Otherwise, after it reaches the max number, it will throw an error"""
-
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True,
-        extra="forbid",
-    )
-
-    @property
-    def _collapse_chain(self) -> BaseCombineDocumentsChain:
-        if self.collapse_documents_chain is not None:
-            return self.collapse_documents_chain
-        return self.combine_documents_chain
-
-    def combine_docs(
-        self,
-        docs: list[Document],
-        token_max: Optional[int] = None,
-        callbacks: Callbacks = None,
-        **kwargs: Any,
-    ) -> tuple[str, dict]:
-        """Combine multiple documents recursively.
-
-        Args:
-            docs: List of documents to combine, assumed that each one is less than
-                `token_max`.
-            token_max: Recursively creates groups of documents less than this number
-                of tokens.
-            callbacks: Callbacks to be passed through
-            **kwargs: additional parameters to be passed to LLM calls (like other
-                input variables besides the documents)
-
-        Returns:
-            The first element returned is the single string output. The second
-            element returned is a dictionary of other keys to return.
-        """
-        result_docs, extra_return_dict = self._collapse(
-            docs,
-            token_max=token_max,
-            callbacks=callbacks,
-            **kwargs,
-        )
-        return self.combine_documents_chain.combine_docs(
-            docs=result_docs,
-            callbacks=callbacks,
-            **kwargs,
-        )
-
-    async def acombine_docs(
-        self,
-        docs: list[Document],
-        token_max: Optional[int] = None,
-        callbacks: Callbacks = None,
-        **kwargs: Any,
-    ) -> tuple[str, dict]:
-        """Async combine multiple documents recursively.
-
-        Args:
-            docs: List of documents to combine, assumed that each one is less than
-                `token_max`.
-            token_max: Recursively creates groups of documents less than this number
-                of tokens.
-            callbacks: Callbacks to be passed through
-            **kwargs: additional parameters to be passed to LLM calls (like other
-                input variables besides the documents)
-
-        Returns:
-            The first element returned is the single string output. The second
-            element returned is a dictionary of other keys to return.
-        """
-        result_docs, extra_return_dict = await self._acollapse(
-            docs,
-            token_max=token_max,
-            callbacks=callbacks,
-            **kwargs,
-        )
-        return await self.combine_documents_chain.acombine_docs(
-            docs=result_docs,
-            callbacks=callbacks,
-            **kwargs,
-        )
-
-    def _collapse(
-        self,
-        docs: list[Document],
-        token_max: Optional[int] = None,
-        callbacks: Callbacks = None,
-        **kwargs: Any,
-    ) -> tuple[list[Document], dict]:
-        result_docs = docs
-        length_func = self.combine_documents_chain.prompt_length
-        num_tokens = length_func(result_docs, **kwargs)
-
-        def _collapse_docs_func(docs: list[Document], **kwargs: Any) -> str:
-            return self._collapse_chain.run(
-                input_documents=docs,
-                callbacks=callbacks,
-                **kwargs,
-            )
-
-        _token_max = token_max or self.token_max
-        retries: int = 0
-        while num_tokens is not None and num_tokens > _token_max:
-            new_result_doc_list = split_list_of_docs(
-                result_docs,
-                length_func,
-                _token_max,
-                **kwargs,
-            )
-            result_docs = [
-                collapse_docs(docs_, _collapse_docs_func, **kwargs)
-                for docs_ in new_result_doc_list
-            ]
-            num_tokens = length_func(result_docs, **kwargs)
-            retries += 1
-            if self.collapse_max_retries and retries == self.collapse_max_retries:
-                msg = f"Exceed {self.collapse_max_retries} tries to \
-                        collapse document to {_token_max} tokens."
-                raise ValueError(msg)
-        return result_docs, {}
-
-    async def _acollapse(
-        self,
-        docs: list[Document],
-        token_max: Optional[int] = None,
-        callbacks: Callbacks = None,
-        **kwargs: Any,
-    ) -> tuple[list[Document], dict]:
-        result_docs = docs
-        length_func = self.combine_documents_chain.prompt_length
-        num_tokens = length_func(result_docs, **kwargs)
-
-        async def _collapse_docs_func(docs: list[Document], **kwargs: Any) -> str:
-            return await self._collapse_chain.arun(
-                input_documents=docs,
-                callbacks=callbacks,
-                **kwargs,
-            )
-
-        _token_max = token_max or self.token_max
-        retries: int = 0
-        while num_tokens is not None and num_tokens > _token_max:
-            new_result_doc_list = split_list_of_docs(
-                result_docs,
-                length_func,
-                _token_max,
-                **kwargs,
-            )
-            result_docs = [
-                await acollapse_docs(docs_, _collapse_docs_func, **kwargs)
-                for docs_ in new_result_doc_list
-            ]
-            num_tokens = length_func(result_docs, **kwargs)
-            retries += 1
-            if self.collapse_max_retries and retries == self.collapse_max_retries:
-                msg = f"Exceed {self.collapse_max_retries} tries to \
-                        collapse document to {_token_max} tokens."
-                raise ValueError(msg)
-        return result_docs, {}
-
-    @property
-    def _chain_type(self) -> str:
-        return "reduce_documents_chain"