langchain 0.4.0.dev0__py3-none-any.whl → 1.0.0__py3-none-any.whl

langchain/retrievers/milvus.py

@@ -1,28 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain._api import create_importer
-
-if TYPE_CHECKING:
-    from langchain_community.retrievers import MilvusRetriever
-    from langchain_community.retrievers.milvus import MilvusRetreiver
-
-# Create a way to dynamically look up deprecated imports.
-# Used to consolidate logic for raising deprecation warnings and
-# handling optional imports.
-DEPRECATED_LOOKUP = {
-    "MilvusRetriever": "langchain_community.retrievers",
-    "MilvusRetreiver": "langchain_community.retrievers.milvus",
-}
-
-_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)
-
-
-def __getattr__(name: str) -> Any:
-    """Look up attributes dynamically."""
-    return _import_attribute(name)
-
-
-__all__ = [
-    "MilvusRetreiver",
-    "MilvusRetriever",
-]

langchain/retrievers/multi_query.py

@@ -1,233 +0,0 @@
-import asyncio
-import logging
-from collections.abc import Sequence
-from typing import Optional
-
-from langchain_core.callbacks import (
-    AsyncCallbackManagerForRetrieverRun,
-    CallbackManagerForRetrieverRun,
-)
-from langchain_core.documents import Document
-from langchain_core.language_models import BaseLanguageModel
-from langchain_core.output_parsers import BaseOutputParser
-from langchain_core.prompts import BasePromptTemplate
-from langchain_core.prompts.prompt import PromptTemplate
-from langchain_core.retrievers import BaseRetriever
-from langchain_core.runnables import Runnable
-from typing_extensions import override
-
-from langchain.chains.llm import LLMChain
-
-logger = logging.getLogger(__name__)
-
-
-class LineListOutputParser(BaseOutputParser[list[str]]):
-    """Output parser for a list of lines."""
-
-    @override
-    def parse(self, text: str) -> list[str]:
-        lines = text.strip().split("\n")
-        return list(filter(None, lines))  # Remove empty lines
-
-
-# Default prompt
-DEFAULT_QUERY_PROMPT = PromptTemplate(
-    input_variables=["question"],
-    template="""You are an AI language model assistant. Your task is
-    to generate 3 different versions of the given user
-    question to retrieve relevant documents from a vector  database.
-    By generating multiple perspectives on the user question,
-    your goal is to help the user overcome some of the limitations
-    of distance-based similarity search. Provide these alternative
-    questions separated by newlines. Original question: {question}""",
-)
-
-
-def _unique_documents(documents: Sequence[Document]) -> list[Document]:
-    return [doc for i, doc in enumerate(documents) if doc not in documents[:i]]
-
-
-class MultiQueryRetriever(BaseRetriever):
-    """Given a query, use an LLM to write a set of queries.
-
-    Retrieve docs for each query. Return the unique union of all retrieved docs.
-    """
-
-    retriever: BaseRetriever
-    llm_chain: Runnable
-    verbose: bool = True
-    parser_key: str = "lines"
-    """DEPRECATED. parser_key is no longer used and should not be specified."""
-    include_original: bool = False
-    """Whether to include the original query in the list of generated queries."""
-
-    @classmethod
-    def from_llm(
-        cls,
-        retriever: BaseRetriever,
-        llm: BaseLanguageModel,
-        prompt: BasePromptTemplate = DEFAULT_QUERY_PROMPT,
-        parser_key: Optional[str] = None,  # noqa: ARG003
-        include_original: bool = False,  # noqa: FBT001,FBT002
-    ) -> "MultiQueryRetriever":
-        """Initialize from llm using default template.
-
-        Args:
-            retriever: retriever to query documents from
-            llm: llm for query generation using DEFAULT_QUERY_PROMPT
-            prompt: The prompt which aims to generate several different versions
-                of the given user query
-            include_original: Whether to include the original query in the list of
-                generated queries.
-
-        Returns:
-            MultiQueryRetriever
-        """
-        output_parser = LineListOutputParser()
-        llm_chain = prompt | llm | output_parser
-        return cls(
-            retriever=retriever,
-            llm_chain=llm_chain,
-            include_original=include_original,
-        )
-
-    async def _aget_relevant_documents(
-        self,
-        query: str,
-        *,
-        run_manager: AsyncCallbackManagerForRetrieverRun,
-    ) -> list[Document]:
-        """Get relevant documents given a user query.
-
-        Args:
-            query: user query
-
-        Returns:
-            Unique union of relevant documents from all generated queries
-        """
-        queries = await self.agenerate_queries(query, run_manager)
-        if self.include_original:
-            queries.append(query)
-        documents = await self.aretrieve_documents(queries, run_manager)
-        return self.unique_union(documents)
-
-    async def agenerate_queries(
-        self,
-        question: str,
-        run_manager: AsyncCallbackManagerForRetrieverRun,
-    ) -> list[str]:
-        """Generate queries based upon user input.
-
-        Args:
-            question: user query
-
-        Returns:
-            List of LLM generated queries that are similar to the user input
-        """
-        response = await self.llm_chain.ainvoke(
-            {"question": question},
-            config={"callbacks": run_manager.get_child()},
-        )
-        lines = response["text"] if isinstance(self.llm_chain, LLMChain) else response
-        if self.verbose:
-            logger.info("Generated queries: %s", lines)
-        return lines
-
-    async def aretrieve_documents(
-        self,
-        queries: list[str],
-        run_manager: AsyncCallbackManagerForRetrieverRun,
-    ) -> list[Document]:
-        """Run all LLM generated queries.
-
-        Args:
-            queries: query list
-
-        Returns:
-            List of retrieved Documents
-        """
-        document_lists = await asyncio.gather(
-            *(
-                self.retriever.ainvoke(
-                    query,
-                    config={"callbacks": run_manager.get_child()},
-                )
-                for query in queries
-            ),
-        )
-        return [doc for docs in document_lists for doc in docs]
-
-    def _get_relevant_documents(
-        self,
-        query: str,
-        *,
-        run_manager: CallbackManagerForRetrieverRun,
-    ) -> list[Document]:
-        """Get relevant documents given a user query.
-
-        Args:
-            query: user query
-
-        Returns:
-            Unique union of relevant documents from all generated queries
-        """
-        queries = self.generate_queries(query, run_manager)
-        if self.include_original:
-            queries.append(query)
-        documents = self.retrieve_documents(queries, run_manager)
-        return self.unique_union(documents)
-
-    def generate_queries(
-        self,
-        question: str,
-        run_manager: CallbackManagerForRetrieverRun,
-    ) -> list[str]:
-        """Generate queries based upon user input.
-
-        Args:
-            question: user query
-
-        Returns:
-            List of LLM generated queries that are similar to the user input
-        """
-        response = self.llm_chain.invoke(
-            {"question": question},
-            config={"callbacks": run_manager.get_child()},
-        )
-        lines = response["text"] if isinstance(self.llm_chain, LLMChain) else response
-        if self.verbose:
-            logger.info("Generated queries: %s", lines)
-        return lines
-
-    def retrieve_documents(
-        self,
-        queries: list[str],
-        run_manager: CallbackManagerForRetrieverRun,
-    ) -> list[Document]:
-        """Run all LLM generated queries.
-
-        Args:
-            queries: query list
-
-        Returns:
-            List of retrieved Documents
-        """
-        documents = []
-        for query in queries:
-            docs = self.retriever.invoke(
-                query,
-                config={"callbacks": run_manager.get_child()},
-            )
-            documents.extend(docs)
-        return documents
-
-    def unique_union(self, documents: list[Document]) -> list[Document]:
-        """Get unique Documents.
-
-        Args:
-            documents: List of retrieved Documents
-
-        Returns:
-            List of unique retrieved Documents
-        """
-        return _unique_documents(documents)

langchain/retrievers/multi_vector.py

@@ -1,135 +0,0 @@
-from enum import Enum
-from typing import Any, Optional
-
-from langchain_core.callbacks import (
-    AsyncCallbackManagerForRetrieverRun,
-    CallbackManagerForRetrieverRun,
-)
-from langchain_core.documents import Document
-from langchain_core.retrievers import BaseRetriever
-from langchain_core.stores import BaseStore, ByteStore
-from langchain_core.vectorstores import VectorStore
-from pydantic import Field, model_validator
-from typing_extensions import override
-
-from langchain.storage._lc_store import create_kv_docstore
-
-
-class SearchType(str, Enum):
-    """Enumerator of the types of search to perform."""
-
-    similarity = "similarity"
-    """Similarity search."""
-    similarity_score_threshold = "similarity_score_threshold"
-    """Similarity search with a score threshold."""
-    mmr = "mmr"
-    """Maximal Marginal Relevance reranking of similarity search."""
-
-
-class MultiVectorRetriever(BaseRetriever):
-    """Retrieve from a set of multiple embeddings for the same document."""
-
-    vectorstore: VectorStore
-    """The underlying vectorstore to use to store small chunks
-    and their embedding vectors"""
-    byte_store: Optional[ByteStore] = None
-    """The lower-level backing storage layer for the parent documents"""
-    docstore: BaseStore[str, Document]
-    """The storage interface for the parent documents"""
-    id_key: str = "doc_id"
-    search_kwargs: dict = Field(default_factory=dict)
-    """Keyword arguments to pass to the search function."""
-    search_type: SearchType = SearchType.similarity
-    """Type of search to perform (similarity / mmr)"""
-
-    @model_validator(mode="before")
-    @classmethod
-    def _shim_docstore(cls, values: dict) -> Any:
-        byte_store = values.get("byte_store")
-        docstore = values.get("docstore")
-        if byte_store is not None:
-            docstore = create_kv_docstore(byte_store)
-        elif docstore is None:
-            msg = "You must pass a `byte_store` parameter."
-            raise ValueError(msg)
-        values["docstore"] = docstore
-        return values
-
-    @override
-    def _get_relevant_documents(
-        self,
-        query: str,
-        *,
-        run_manager: CallbackManagerForRetrieverRun,
-    ) -> list[Document]:
-        """Get documents relevant to a query.
-        Args:
-            query: String to find relevant documents for
-            run_manager: The callbacks handler to use
-        Returns:
-            List of relevant documents
-        """
-        if self.search_type == SearchType.mmr:
-            sub_docs = self.vectorstore.max_marginal_relevance_search(
-                query,
-                **self.search_kwargs,
-            )
-        elif self.search_type == SearchType.similarity_score_threshold:
-            sub_docs_and_similarities = (
-                self.vectorstore.similarity_search_with_relevance_scores(
-                    query,
-                    **self.search_kwargs,
-                )
-            )
-            sub_docs = [sub_doc for sub_doc, _ in sub_docs_and_similarities]
-        else:
-            sub_docs = self.vectorstore.similarity_search(query, **self.search_kwargs)
-
-        # We do this to maintain the order of the ids that are returned
-        ids = []
-        for d in sub_docs:
-            if self.id_key in d.metadata and d.metadata[self.id_key] not in ids:
-                ids.append(d.metadata[self.id_key])
-        docs = self.docstore.mget(ids)
-        return [d for d in docs if d is not None]
-
-    @override
-    async def _aget_relevant_documents(
-        self,
-        query: str,
-        *,
-        run_manager: AsyncCallbackManagerForRetrieverRun,
-    ) -> list[Document]:
-        """Asynchronously get documents relevant to a query.
-        Args:
-            query: String to find relevant documents for
-            run_manager: The callbacks handler to use
-        Returns:
-            List of relevant documents
-        """
-        if self.search_type == SearchType.mmr:
-            sub_docs = await self.vectorstore.amax_marginal_relevance_search(
-                query,
-                **self.search_kwargs,
-            )
-        elif self.search_type == SearchType.similarity_score_threshold:
-            sub_docs_and_similarities = (
-                await self.vectorstore.asimilarity_search_with_relevance_scores(
-                    query,
-                    **self.search_kwargs,
-                )
-            )
-            sub_docs = [sub_doc for sub_doc, _ in sub_docs_and_similarities]
-        else:
-            sub_docs = await self.vectorstore.asimilarity_search(
-                query,
-                **self.search_kwargs,
-            )
-
-        # We do this to maintain the order of the ids that are returned
-        ids = []
-        for d in sub_docs:
-            if self.id_key in d.metadata and d.metadata[self.id_key] not in ids:
-                ids.append(d.metadata[self.id_key])
-        docs = await self.docstore.amget(ids)
-        return [d for d in docs if d is not None]

langchain/retrievers/outline.py

@@ -1,23 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain._api import create_importer
-
-if TYPE_CHECKING:
-    from langchain_community.retrievers import OutlineRetriever
-
-# Create a way to dynamically look up deprecated imports.
-# Used to consolidate logic for raising deprecation warnings and
-# handling optional imports.
-DEPRECATED_LOOKUP = {"OutlineRetriever": "langchain_community.retrievers"}
-
-_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)
-
-
-def __getattr__(name: str) -> Any:
-    """Look up attributes dynamically."""
-    return _import_attribute(name)
-
-
-__all__ = [
-    "OutlineRetriever",
-]

langchain/retrievers/parent_document_retriever.py

@@ -1,172 +0,0 @@
-import uuid
-from collections.abc import Sequence
-from typing import Any, Optional
-
-from langchain_core.documents import Document
-from langchain_text_splitters import TextSplitter
-
-from langchain.retrievers import MultiVectorRetriever
-
-
-class ParentDocumentRetriever(MultiVectorRetriever):
-    """Retrieve small chunks then retrieve their parent documents.
-
-    When splitting documents for retrieval, there are often conflicting desires:
-
-    1. You may want to have small documents, so that their embeddings can most
-        accurately reflect their meaning. If too long, then the embeddings can
-        lose meaning.
-    2. You want to have long enough documents that the context of each chunk is
-        retained.
-
-    The ParentDocumentRetriever strikes that balance by splitting and storing
-    small chunks of data. During retrieval, it first fetches the small chunks
-    but then looks up the parent ids for those chunks and returns those larger
-    documents.
-
-    Note that "parent document" refers to the document that a small chunk
-    originated from. This can either be the whole raw document OR a larger
-    chunk.
-
-    Examples:
-
-        .. code-block:: python
-
-            from langchain_chroma import Chroma
-            from langchain_community.embeddings import OpenAIEmbeddings
-            from langchain_text_splitters import RecursiveCharacterTextSplitter
-            from langchain.storage import InMemoryStore
-
-            # This text splitter is used to create the parent documents
-            parent_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, add_start_index=True)
-            # This text splitter is used to create the child documents
-            # It should create documents smaller than the parent
-            child_splitter = RecursiveCharacterTextSplitter(chunk_size=400, add_start_index=True)
-            # The vectorstore to use to index the child chunks
-            vectorstore = Chroma(embedding_function=OpenAIEmbeddings())
-            # The storage layer for the parent documents
-            store = InMemoryStore()
-
-            # Initialize the retriever
-            retriever = ParentDocumentRetriever(
-                vectorstore=vectorstore,
-                docstore=store,
-                child_splitter=child_splitter,
-                parent_splitter=parent_splitter,
-            )
-
-    """  # noqa: E501
-
-    child_splitter: TextSplitter
-    """The text splitter to use to create child documents."""
-
-    """The key to use to track the parent id. This will be stored in the
-    metadata of child documents."""
-    parent_splitter: Optional[TextSplitter] = None
-    """The text splitter to use to create parent documents.
-    If none, then the parent documents will be the raw documents passed in."""
-
-    child_metadata_fields: Optional[Sequence[str]] = None
-    """Metadata fields to leave in child documents. If None, leave all parent document
-        metadata.
-    """
-
-    def _split_docs_for_adding(
-        self,
-        documents: list[Document],
-        ids: Optional[list[str]] = None,
-        *,
-        add_to_docstore: bool = True,
-    ) -> tuple[list[Document], list[tuple[str, Document]]]:
-        if self.parent_splitter is not None:
-            documents = self.parent_splitter.split_documents(documents)
-        if ids is None:
-            doc_ids = [str(uuid.uuid4()) for _ in documents]
-            if not add_to_docstore:
-                msg = "If ids are not passed in, `add_to_docstore` MUST be True"
-                raise ValueError(msg)
-        else:
-            if len(documents) != len(ids):
-                msg = (
-                    "Got uneven list of documents and ids. "
-                    "If `ids` is provided, should be same length as `documents`."
-                )
-                raise ValueError(msg)
-            doc_ids = ids
-
-        docs = []
-        full_docs = []
-        for i, doc in enumerate(documents):
-            _id = doc_ids[i]
-            sub_docs = self.child_splitter.split_documents([doc])
-            if self.child_metadata_fields is not None:
-                for _doc in sub_docs:
-                    _doc.metadata = {
-                        k: _doc.metadata[k] for k in self.child_metadata_fields
-                    }
-            for _doc in sub_docs:
-                _doc.metadata[self.id_key] = _id
-            docs.extend(sub_docs)
-            full_docs.append((_id, doc))
-
-        return docs, full_docs
-
-    def add_documents(
-        self,
-        documents: list[Document],
-        ids: Optional[list[str]] = None,
-        add_to_docstore: bool = True,  # noqa: FBT001,FBT002
-        **kwargs: Any,
-    ) -> None:
-        """Adds documents to the docstore and vectorstores.
-
-        Args:
-            documents: List of documents to add
-            ids: Optional list of ids for documents. If provided should be the same
-                length as the list of documents. Can be provided if parent documents
-                are already in the document store and you don't want to re-add
-                to the docstore. If not provided, random UUIDs will be used as
-                ids.
-            add_to_docstore: Boolean of whether to add documents to docstore.
-                This can be false if and only if `ids` are provided. You may want
-                to set this to False if the documents are already in the docstore
-                and you don't want to re-add them.
-        """
-        docs, full_docs = self._split_docs_for_adding(
-            documents,
-            ids,
-            add_to_docstore=add_to_docstore,
-        )
-        self.vectorstore.add_documents(docs, **kwargs)
-        if add_to_docstore:
-            self.docstore.mset(full_docs)
-
-    async def aadd_documents(
-        self,
-        documents: list[Document],
-        ids: Optional[list[str]] = None,
-        add_to_docstore: bool = True,  # noqa: FBT001,FBT002
-        **kwargs: Any,
-    ) -> None:
-        """Adds documents to the docstore and vectorstores.
-
-        Args:
-            documents: List of documents to add
-            ids: Optional list of ids for documents. If provided should be the same
-                length as the list of documents. Can be provided if parent documents
-                are already in the document store and you don't want to re-add
-                to the docstore. If not provided, random UUIDs will be used as
-                ids.
-            add_to_docstore: Boolean of whether to add documents to docstore.
-                This can be false if and only if `ids` are provided. You may want
-                to set this to False if the documents are already in the docstore
-                and you don't want to re-add them.
-        """
-        docs, full_docs = self._split_docs_for_adding(
-            documents,
-            ids,
-            add_to_docstore=add_to_docstore,
-        )
-        await self.vectorstore.aadd_documents(docs, **kwargs)
-        if add_to_docstore:
-            await self.docstore.amset(full_docs)

langchain/retrievers/pinecone_hybrid_search.py

@@ -1,23 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain._api import create_importer
-
-if TYPE_CHECKING:
-    from langchain_community.retrievers import PineconeHybridSearchRetriever
-
-# Create a way to dynamically look up deprecated imports.
-# Used to consolidate logic for raising deprecation warnings and
-# handling optional imports.
-DEPRECATED_LOOKUP = {"PineconeHybridSearchRetriever": "langchain_community.retrievers"}
-
-_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)
-
-
-def __getattr__(name: str) -> Any:
-    """Look up attributes dynamically."""
-    return _import_attribute(name)
-
-
-__all__ = [
-    "PineconeHybridSearchRetriever",
-]

langchain/retrievers/pubmed.py

@@ -1,23 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain._api import create_importer
-
-if TYPE_CHECKING:
-    from langchain_community.retrievers import PubMedRetriever
-
-# Create a way to dynamically look up deprecated imports.
-# Used to consolidate logic for raising deprecation warnings and
-# handling optional imports.
-DEPRECATED_LOOKUP = {"PubMedRetriever": "langchain_community.retrievers"}
-
-_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)
-
-
-def __getattr__(name: str) -> Any:
-    """Look up attributes dynamically."""
-    return _import_attribute(name)
-
-
-__all__ = [
-    "PubMedRetriever",
-]

langchain/retrievers/pupmed.py

@@ -1,23 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain._api import create_importer
-
-if TYPE_CHECKING:
-    from langchain_community.retrievers import PubMedRetriever
-
-# Create a way to dynamically look up deprecated imports.
-# Used to consolidate logic for raising deprecation warnings and
-# handling optional imports.
-DEPRECATED_LOOKUP = {"PubMedRetriever": "langchain_community.retrievers"}
-
-_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)
-
-
-def __getattr__(name: str) -> Any:
-    """Look up attributes dynamically."""
-    return _import_attribute(name)
-
-
-__all__ = [
-    "PubMedRetriever",
-]