langchain 0.4.0.dev0__py3-none-any.whl → 1.0.0__py3-none-any.whl

langchain/retrievers/document_compressors/cohere_rerank.py

@@ -1,126 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Sequence
-from copy import deepcopy
-from typing import Any, Optional, Union
-
-from langchain_core._api.deprecation import deprecated
-from langchain_core.callbacks import Callbacks
-from langchain_core.documents import BaseDocumentCompressor, Document
-from langchain_core.utils import get_from_dict_or_env
-from pydantic import ConfigDict, model_validator
-from typing_extensions import override
-
-
-@deprecated(
-    since="0.0.30",
-    removal="1.0",
-    alternative_import="langchain_cohere.CohereRerank",
-)
-class CohereRerank(BaseDocumentCompressor):
-    """Document compressor that uses `Cohere Rerank API`."""
-
-    client: Any = None
-    """Cohere client to use for compressing documents."""
-    top_n: Optional[int] = 3
-    """Number of documents to return."""
-    model: str = "rerank-english-v2.0"
-    """Model to use for reranking."""
-    cohere_api_key: Optional[str] = None
-    """Cohere API key. Must be specified directly or via environment variable
-        COHERE_API_KEY."""
-    user_agent: str = "langchain"
-    """Identifier for the application making the request."""
-
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True,
-        extra="forbid",
-    )
-
-    @model_validator(mode="before")
-    @classmethod
-    def validate_environment(cls, values: dict) -> Any:
-        """Validate that api key and python package exists in environment."""
-        if not values.get("client"):
-            try:
-                import cohere
-            except ImportError as e:
-                msg = (
-                    "Could not import cohere python package. "
-                    "Please install it with `pip install cohere`."
-                )
-                raise ImportError(msg) from e
-            cohere_api_key = get_from_dict_or_env(
-                values,
-                "cohere_api_key",
-                "COHERE_API_KEY",
-            )
-            client_name = values.get("user_agent", "langchain")
-            values["client"] = cohere.Client(cohere_api_key, client_name=client_name)
-        return values
-
-    def rerank(
-        self,
-        documents: Sequence[Union[str, Document, dict]],
-        query: str,
-        *,
-        model: Optional[str] = None,
-        top_n: Optional[int] = -1,
-        max_chunks_per_doc: Optional[int] = None,
-    ) -> list[dict[str, Any]]:
-        """Returns an ordered list of documents ordered by their relevance to the provided query.
-
-        Args:
-            query: The query to use for reranking.
-            documents: A sequence of documents to rerank.
-            model: The model to use for re-ranking. Default to self.model.
-            top_n : The number of results to return. If None returns all results.
-                Defaults to self.top_n.
-            max_chunks_per_doc : The maximum number of chunks derived from a document.
-        """  # noqa: E501
-        if len(documents) == 0:  # to avoid empty api call
-            return []
-        docs = [
-            doc.page_content if isinstance(doc, Document) else doc for doc in documents
-        ]
-        model = model or self.model
-        top_n = top_n if (top_n is None or top_n > 0) else self.top_n
-        results = self.client.rerank(
-            query=query,
-            documents=docs,
-            model=model,
-            top_n=top_n,
-            max_chunks_per_doc=max_chunks_per_doc,
-        )
-        if hasattr(results, "results"):
-            results = results.results
-        return [
-            {"index": res.index, "relevance_score": res.relevance_score}
-            for res in results
-        ]
-
-    @override
-    def compress_documents(
-        self,
-        documents: Sequence[Document],
-        query: str,
-        callbacks: Optional[Callbacks] = None,
-    ) -> Sequence[Document]:
-        """
-        Compress documents using Cohere's rerank API.
-
-        Args:
-            documents: A sequence of documents to compress.
-            query: The query to use for compressing the documents.
-            callbacks: Callbacks to run during the compression process.
-
-        Returns:
-            A sequence of compressed documents.
-        """
-        compressed = []
-        for res in self.rerank(documents, query):
-            doc = documents[res["index"]]
-            doc_copy = Document(doc.page_content, metadata=deepcopy(doc.metadata))
-            doc_copy.metadata["relevance_score"] = res["relevance_score"]
-            compressed.append(doc_copy)
-        return compressed

langchain/retrievers/document_compressors/cross_encoder.py

@@ -1,16 +0,0 @@
-from abc import ABC, abstractmethod
-
-
-class BaseCrossEncoder(ABC):
-    """Interface for cross encoder models."""
-
-    @abstractmethod
-    def score(self, text_pairs: list[tuple[str, str]]) -> list[float]:
-        """Score pairs' similarity.
-
-        Args:
-            text_pairs: List of pairs of texts.
-
-        Returns:
-            List of scores.
-        """

langchain/retrievers/document_compressors/cross_encoder_rerank.py

@@ -1,50 +0,0 @@
-from __future__ import annotations
-
-import operator
-from collections.abc import Sequence
-from typing import Optional
-
-from langchain_core.callbacks import Callbacks
-from langchain_core.documents import BaseDocumentCompressor, Document
-from pydantic import ConfigDict
-from typing_extensions import override
-
-from langchain.retrievers.document_compressors.cross_encoder import BaseCrossEncoder
-
-
-class CrossEncoderReranker(BaseDocumentCompressor):
-    """Document compressor that uses CrossEncoder for reranking."""
-
-    model: BaseCrossEncoder
-    """CrossEncoder model to use for scoring similarity
-      between the query and documents."""
-    top_n: int = 3
-    """Number of documents to return."""
-
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True,
-        extra="forbid",
-    )
-
-    @override
-    def compress_documents(
-        self,
-        documents: Sequence[Document],
-        query: str,
-        callbacks: Optional[Callbacks] = None,
-    ) -> Sequence[Document]:
-        """
-        Rerank documents using CrossEncoder.
-
-        Args:
-            documents: A sequence of documents to compress.
-            query: The query to use for compressing the documents.
-            callbacks: Callbacks to run during the compression process.
-
-        Returns:
-            A sequence of compressed documents.
-        """
-        scores = self.model.score([(query, doc.page_content) for doc in documents])
-        docs_with_scores = list(zip(documents, scores))
-        result = sorted(docs_with_scores, key=operator.itemgetter(1), reverse=True)
-        return [doc for doc, _ in result[: self.top_n]]

langchain/retrievers/document_compressors/embeddings_filter.py

@@ -1,140 +0,0 @@
-from collections.abc import Sequence
-from typing import Callable, Optional
-
-from langchain_core.callbacks import Callbacks
-from langchain_core.documents import BaseDocumentCompressor, Document
-from langchain_core.embeddings import Embeddings
-from langchain_core.utils import pre_init
-from pydantic import ConfigDict, Field
-from typing_extensions import override
-
-
-def _get_similarity_function() -> Callable:
-    try:
-        from langchain_community.utils.math import cosine_similarity
-    except ImportError as e:
-        msg = (
-            "To use please install langchain-community "
-            "with `pip install langchain-community`."
-        )
-        raise ImportError(msg) from e
-    return cosine_similarity
-
-
-class EmbeddingsFilter(BaseDocumentCompressor):
-    """Document compressor that uses embeddings to drop documents
-    unrelated to the query."""
-
-    embeddings: Embeddings
-    """Embeddings to use for embedding document contents and queries."""
-    similarity_fn: Callable = Field(default_factory=_get_similarity_function)
-    """Similarity function for comparing documents. Function expected to take as input
-    two matrices (List[List[float]]) and return a matrix of scores where higher values
-    indicate greater similarity."""
-    k: Optional[int] = 20
-    """The number of relevant documents to return. Can be set to None, in which case
-    `similarity_threshold` must be specified. Defaults to 20."""
-    similarity_threshold: Optional[float] = None
-    """Threshold for determining when two documents are similar enough
-    to be considered redundant. Defaults to None, must be specified if `k` is set
-    to None."""
-
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True,
-    )
-
-    @pre_init
-    def validate_params(cls, values: dict) -> dict:
-        """Validate similarity parameters."""
-        if values["k"] is None and values["similarity_threshold"] is None:
-            msg = "Must specify one of `k` or `similarity_threshold`."
-            raise ValueError(msg)
-        return values
-
-    @override
-    def compress_documents(
-        self,
-        documents: Sequence[Document],
-        query: str,
-        callbacks: Optional[Callbacks] = None,
-    ) -> Sequence[Document]:
-        """Filter documents based on similarity of their embeddings to the query."""
-        try:
-            from langchain_community.document_transformers.embeddings_redundant_filter import (  # noqa: E501
-                _get_embeddings_from_stateful_docs,
-                get_stateful_documents,
-            )
-        except ImportError as e:
-            msg = (
-                "To use please install langchain-community "
-                "with `pip install langchain-community`."
-            )
-            raise ImportError(msg) from e
-
-        try:
-            import numpy as np
-        except ImportError as e:
-            msg = "Could not import numpy, please install with `pip install numpy`."
-            raise ImportError(msg) from e
-        stateful_documents = get_stateful_documents(documents)
-        embedded_documents = _get_embeddings_from_stateful_docs(
-            self.embeddings,
-            stateful_documents,
-        )
-        embedded_query = self.embeddings.embed_query(query)
-        similarity = self.similarity_fn([embedded_query], embedded_documents)[0]
-        included_idxs: np.ndarray = np.arange(len(embedded_documents))
-        if self.k is not None:
-            included_idxs = np.argsort(similarity)[::-1][: self.k]
-        if self.similarity_threshold is not None:
-            similar_enough = np.where(
-                similarity[included_idxs] > self.similarity_threshold,
-            )
-            included_idxs = included_idxs[similar_enough]
-        for i in included_idxs:
-            stateful_documents[i].state["query_similarity_score"] = similarity[i]
-        return [stateful_documents[i] for i in included_idxs]
-
-    @override
-    async def acompress_documents(
-        self,
-        documents: Sequence[Document],
-        query: str,
-        callbacks: Optional[Callbacks] = None,
-    ) -> Sequence[Document]:
-        """Filter documents based on similarity of their embeddings to the query."""
-        try:
-            from langchain_community.document_transformers.embeddings_redundant_filter import (  # noqa: E501
-                _aget_embeddings_from_stateful_docs,
-                get_stateful_documents,
-            )
-        except ImportError as e:
-            msg = (
-                "To use please install langchain-community "
-                "with `pip install langchain-community`."
-            )
-            raise ImportError(msg) from e
-
-        try:
-            import numpy as np
-        except ImportError as e:
-            msg = "Could not import numpy, please install with `pip install numpy`."
-            raise ImportError(msg) from e
-        stateful_documents = get_stateful_documents(documents)
-        embedded_documents = await _aget_embeddings_from_stateful_docs(
-            self.embeddings,
-            stateful_documents,
-        )
-        embedded_query = await self.embeddings.aembed_query(query)
-        similarity = self.similarity_fn([embedded_query], embedded_documents)[0]
-        included_idxs: np.ndarray = np.arange(len(embedded_documents))
-        if self.k is not None:
-            included_idxs = np.argsort(similarity)[::-1][: self.k]
-        if self.similarity_threshold is not None:
-            similar_enough = np.where(
-                similarity[included_idxs] > self.similarity_threshold,
-            )
-            included_idxs = included_idxs[similar_enough]
-        for i in included_idxs:
-            stateful_documents[i].state["query_similarity_score"] = similarity[i]
-        return [stateful_documents[i] for i in included_idxs]

langchain/retrievers/document_compressors/flashrank_rerank.py

@@ -1,27 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain._api import create_importer
-
-if TYPE_CHECKING:
-    from langchain_community.document_compressors.flashrank_rerank import (
-        FlashrankRerank,
-    )
-
-# Create a way to dynamically look up deprecated imports.
-# Used to consolidate logic for raising deprecation warnings and
-# handling optional imports.
-DEPRECATED_LOOKUP = {
-    "FlashrankRerank": "langchain_community.document_compressors.flashrank_rerank",
-}
-
-_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)
-
-
-def __getattr__(name: str) -> Any:
-    """Look up attributes dynamically."""
-    return _import_attribute(name)
-
-
-__all__ = [
-    "FlashrankRerank",
-]

langchain/retrievers/document_compressors/listwise_rerank.py

@@ -1,145 +0,0 @@
-"""Filter that uses an LLM to rerank documents listwise and select top-k."""
-
-from collections.abc import Sequence
-from typing import Any, Optional
-
-from langchain_core.callbacks import Callbacks
-from langchain_core.documents import BaseDocumentCompressor, Document
-from langchain_core.language_models import BaseLanguageModel
-from langchain_core.prompts import BasePromptTemplate, ChatPromptTemplate
-from langchain_core.runnables import Runnable, RunnableLambda, RunnablePassthrough
-from pydantic import BaseModel, ConfigDict, Field
-
-_default_system_tmpl = """{context}
-
-Sort the Documents by their relevance to the Query."""
-_DEFAULT_PROMPT = ChatPromptTemplate.from_messages(
-    [("system", _default_system_tmpl), ("human", "{query}")],
-)
-
-
-def _get_prompt_input(input_: dict) -> dict[str, Any]:
-    """Return the compression chain input."""
-    documents = input_["documents"]
-    context = ""
-    for index, doc in enumerate(documents):
-        context += f"Document ID: {index}\n```{doc.page_content}```\n\n"
-    document_range = "empty list"
-    if len(documents) > 0:
-        document_range = f"Document ID: 0, ..., Document ID: {len(documents) - 1}"
-    context += f"Documents = [{document_range}]"
-    return {"query": input_["query"], "context": context}
-
-
-def _parse_ranking(results: dict) -> list[Document]:
-    ranking = results["ranking"]
-    docs = results["documents"]
-    return [docs[i] for i in ranking.ranked_document_ids]
-
-
-class LLMListwiseRerank(BaseDocumentCompressor):
-    """Document compressor that uses `Zero-Shot Listwise Document Reranking`.
-
-    Adapted from: https://arxiv.org/pdf/2305.02156.pdf
-
-    ``LLMListwiseRerank`` uses a language model to rerank a list of documents based on
-    their relevance to a query.
-
-    **NOTE**: requires that underlying model implement ``with_structured_output``.
-
-    Example usage:
-        .. code-block:: python
-
-            from langchain.retrievers.document_compressors.listwise_rerank import (
-                LLMListwiseRerank,
-            )
-            from langchain_core.documents import Document
-            from langchain_openai import ChatOpenAI
-
-            documents = [
-                Document("Sally is my friend from school"),
-                Document("Steve is my friend from home"),
-                Document("I didn't always like yogurt"),
-                Document("I wonder why it's called football"),
-                Document("Where's waldo"),
-            ]
-
-            reranker = LLMListwiseRerank.from_llm(
-                llm=ChatOpenAI(model="gpt-3.5-turbo"), top_n=3
-            )
-            compressed_docs = reranker.compress_documents(documents, "Who is steve")
-            assert len(compressed_docs) == 3
-            assert "Steve" in compressed_docs[0].page_content
-
-    """
-
-    reranker: Runnable[dict, list[Document]]
-    """LLM-based reranker to use for filtering documents. Expected to take in a dict
-        with 'documents: Sequence[Document]' and 'query: str' keys and output a
-        List[Document]."""
-
-    top_n: int = 3
-    """Number of documents to return."""
-
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True,
-    )
-
-    def compress_documents(
-        self,
-        documents: Sequence[Document],
-        query: str,
-        callbacks: Optional[Callbacks] = None,
-    ) -> Sequence[Document]:
-        """Filter down documents based on their relevance to the query."""
-        results = self.reranker.invoke(
-            {"documents": documents, "query": query},
-            config={"callbacks": callbacks},
-        )
-        return results[: self.top_n]
-
-    @classmethod
-    def from_llm(
-        cls,
-        llm: BaseLanguageModel,
-        *,
-        prompt: Optional[BasePromptTemplate] = None,
-        **kwargs: Any,
-    ) -> "LLMListwiseRerank":
-        """Create a LLMListwiseRerank document compressor from a language model.
-
-        Args:
-            llm: The language model to use for filtering. **Must implement
-                BaseLanguageModel.with_structured_output().**
-            prompt: The prompt to use for the filter.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            A LLMListwiseRerank document compressor that uses the given language model.
-        """
-
-        if llm.with_structured_output == BaseLanguageModel.with_structured_output:
-            msg = (
-                f"llm of type {type(llm)} does not implement `with_structured_output`."
-            )
-            raise ValueError(msg)
-
-        class RankDocuments(BaseModel):
-            """Rank the documents by their relevance to the user question.
-            Rank from most to least relevant."""
-
-            ranked_document_ids: list[int] = Field(
-                ...,
-                description=(
-                    "The integer IDs of the documents, sorted from most to least "
-                    "relevant to the user question."
-                ),
-            )
-
-        _prompt = prompt if prompt is not None else _DEFAULT_PROMPT
-        reranker = RunnablePassthrough.assign(
-            ranking=RunnableLambda(_get_prompt_input)
-            | _prompt
-            | llm.with_structured_output(RankDocuments),
-        ) | RunnableLambda(_parse_ranking)
-        return cls(reranker=reranker, **kwargs)

langchain/retrievers/elastic_search_bm25.py

@@ -1,23 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain._api import create_importer
-
-if TYPE_CHECKING:
-    from langchain_community.retrievers import ElasticSearchBM25Retriever
-
-# Create a way to dynamically look up deprecated imports.
-# Used to consolidate logic for raising deprecation warnings and
-# handling optional imports.
-DEPRECATED_LOOKUP = {"ElasticSearchBM25Retriever": "langchain_community.retrievers"}
-
-_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)
-
-
-def __getattr__(name: str) -> Any:
-    """Look up attributes dynamically."""
-    return _import_attribute(name)
-
-
-__all__ = [
-    "ElasticSearchBM25Retriever",
-]

langchain/retrievers/embedchain.py

@@ -1,23 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain._api import create_importer
-
-if TYPE_CHECKING:
-    from langchain_community.retrievers import EmbedchainRetriever
-
-# Create a way to dynamically look up deprecated imports.
-# Used to consolidate logic for raising deprecation warnings and
-# handling optional imports.
-DEPRECATED_LOOKUP = {"EmbedchainRetriever": "langchain_community.retrievers"}
-
-_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)
-
-
-def __getattr__(name: str) -> Any:
-    """Look up attributes dynamically."""
-    return _import_attribute(name)
-
-
-__all__ = [
-    "EmbedchainRetriever",
-]