bisheng-langchain 0.2.3__py3-none-any.whl → 0.2.3.2__py3-none-any.whl

bisheng_langchain/agents/llm_functions_agent/base.py

@@ -82,7 +82,7 @@ def _parse_ai_message(message: BaseMessage) -> Union[AgentAction, AgentFinish]:
     if not isinstance(message, AIMessage):
         raise TypeError(f'Expected an AI message got {type(message)}')
 
-    function_call = message.additional_kwargs.get('function_call', {})
+    function_call = message.additional_kwargs.get('tool_calls', {})
 
     if function_call:
         function_name = function_call['name']

bisheng_langchain/chat_models/__init__.py

@@ -1,4 +1,4 @@
-from .host_llm import CustomLLMChat, HostBaichuanChat, HostChatGLM, HostLlama2Chat, HostQwenChat, HostYuanChat, HostYiChat
+from .host_llm import CustomLLMChat, HostBaichuanChat, HostChatGLM, HostLlama2Chat, HostQwenChat, HostQwen1_5Chat, HostYuanChat, HostYiChat
 from .minimax import ChatMinimaxAI
 from .proxy_llm import ProxyChatLLM
 from .qwen import ChatQWen
@@ -10,5 +10,5 @@ from .sensetime import SenseChat
 __all__ = [
     'ProxyChatLLM', 'ChatMinimaxAI', 'ChatWenxin', 'ChatZhipuAI', 'ChatXunfeiAI', 'HostChatGLM',
     'HostBaichuanChat', 'HostLlama2Chat', 'HostQwenChat', 'CustomLLMChat', 'ChatQWen', 'SenseChat',
-    'HostYuanChat', 'HostYiChat'
+    'HostYuanChat', 'HostYiChat', 'HostQwen1_5Chat'
 ]

bisheng_langchain/chat_models/host_llm.py

@@ -341,7 +341,7 @@ class BaseHostChatLLM(BaseChatModel):
                                 function_call['arguments'] += _function_call['arguments']
             message = _convert_dict_to_message({
                 'content': inner_completion,
-                'role': role,
+                'role': role if role is not None else 'assistant',
                 'function_call': function_call,
             })
             return ChatResult(generations=[ChatGeneration(message=message)])
@@ -508,6 +508,19 @@ class HostQwenChat(BaseHostChatLLM):
         return 'qwen_chat'
 
 
+class HostQwen1_5Chat(BaseHostChatLLM):
+    # Qwen-7B-Chat
+    model_name: str = Field('Qwen1.5-14B-Chat', alias='model')
+
+    temperature: float = 0
+    top_p: float = 1
+    max_tokens: int = 4096
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of chat model."""
+        return 'qwen1.5_chat'
+
 class HostLlama2Chat(BaseHostChatLLM):
     # Llama-2-7b-chat-hf, Llama-2-13b-chat-hf, Llama-2-70b-chat-hf
     model_name: str = Field('Llama-2-7b-chat-hf', alias='model')

bisheng_langchain/chat_models/interface/types.py

@@ -1,5 +1,4 @@
-# from typing import Union
-
+from typing import Union, List
 from pydantic import BaseModel
 
 
@@ -16,14 +15,14 @@ class Function(BaseModel):
 
 class ChatInput(BaseModel):
     model: str
-    messages: list[Message] = []
+    messages: List[Message] = []
     top_p: float = None
     temperature: float = None
     n: int = 1
     stream: bool = False
     stop: str = None
     max_tokens: int = 256
-    functions: list[Function] = []
+    functions: List[Function] = []
     function_call: str = None
 
 
@@ -46,7 +45,7 @@ class ChatOutput(BaseModel):
     object: str = None
     model: str = None
     created: int = None
-    choices: list[Choice] = []
+    choices: List[Choice] = []
     usage: Usage = None
 
 

bisheng_langchain/chat_models/qwen.py

@@ -105,7 +105,7 @@ class ChatQWen(BaseChatModel):
         .. code-block:: python
 
             from bisheng_langchain.chat_models import ChatQWen
-            chat_miniamaxai = ChatQWen(model_name="qwen-turbo")
+            chat_qwen = ChatQWen(model_name="qwen-turbo")
     """
 
     client: Optional[Any]  #: :meta private:
@@ -192,7 +192,11 @@ class ChatQWen(BaseChatModel):
             return self.client.post(url=url, json=inp).json()
 
         rsp_dict = _completion_with_retry(**kwargs)
-        if 'output' not in rsp_dict:
+        if 'code' in rsp_dict and rsp_dict['code'] == 'DataInspectionFailed':
+            output_res = {'choices': [{'finish_reason': 'stop', 'message': {'role': 'assistant', 'content': rsp_dict['message']}}]} 
+            usage_res = {'total_tokens': 2, 'output_tokens': 1, 'input_tokens': 1}
+            return output_res, usage_res
+        elif 'output' not in rsp_dict:
             logger.error(f'proxy_llm_error resp={rsp_dict}')
             message = rsp_dict['message']
             raise Exception(message)

bisheng_langchain/chat_models/zhipuai.py

@@ -226,8 +226,15 @@ class ChatZhipuAI(BaseChatModel):
     ) -> ChatResult:
         message_dicts, params = self._create_message_dicts(messages, stop)
         params = {**params, **kwargs}
-
         response = self.completion_with_retry(messages=message_dicts, **params)
+        if response['code'] == 1301:
+            response = {'code': 200, 
+                        'msg': '操作成功', 
+                        'data': {'request_id': '0', 'task_id': '0', 'task_status': 'SUCCESS', 
+                                 'choices': [{'role': 'assistant', 'content': '""'}], 
+                                 'usage': {'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0}}, 
+                        'success': True}
+
         return self._create_chat_result(response)
 
     async def _agenerate(

bisheng_langchain/embeddings/__init__.py

@@ -1,8 +1,11 @@
 from .host_embedding import (BGEZhEmbedding, CustomHostEmbedding, GTEEmbedding, HostEmbeddings,
-                             ME5Embedding)
+                             ME5Embedding, JINAEmbedding)
 from .wenxin import WenxinEmbeddings
+from .huggingfacemultilingual import HuggingFaceMultilingualEmbeddings
+from .huggingfacegte import HuggingFaceGteEmbeddings
 
 __all__ = [
     'WenxinEmbeddings', 'ME5Embedding', 'BGEZhEmbedding', 'GTEEmbedding',
-    'HostEmbeddings', 'CustomHostEmbedding'
+    'HostEmbeddings', 'CustomHostEmbedding', 'JINAEmbedding',
+    'HuggingFaceMultilingualEmbeddings', 'HuggingFaceGteEmbeddings'
 ]

bisheng_langchain/embeddings/host_embedding.py

@@ -158,6 +158,10 @@ class GTEEmbedding(HostEmbeddings):
     embedding_ctx_length: int = 512
 
 
+class JINAEmbedding(HostEmbeddings):
+    model: str = 'jina'
+    embedding_ctx_length: int = 512
+
 class CustomHostEmbedding(HostEmbeddings):
     model: str = Field('custom-embedding', alias='model')
     embedding_ctx_length: int = 512

bisheng_langchain/embeddings/huggingfacegte.py

@@ -0,0 +1,95 @@
+from typing import Any, Dict, List, Optional
+
+import requests
+from langchain_core.embeddings import Embeddings
+from langchain_core.pydantic_v1 import BaseModel, Extra, Field
+DEFAULT_Multilingual_MODEL = "thenlper/gte-large-zh"
+
+
+class HuggingFaceGteEmbeddings(BaseModel, Embeddings):
+    """HuggingFace BGE sentence_transformers embedding models.
+
+    To use, you should have the ``sentence_transformers`` python package installed.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_community.embeddings import HuggingFaceBgeEmbeddings
+
+            model_name = "BAAI/bge-large-en"
+            model_kwargs = {'device': 'cpu'}
+            encode_kwargs = {'normalize_embeddings': True}
+            hf = HuggingFaceBgeEmbeddings(
+                
+                model_name=model_name,
+                model_kwargs=model_kwargs,
+                encode_kwargs=encode_kwargs
+            )
+    """
+
+    client: Any  #: :meta private:
+    model_name: str = DEFAULT_Multilingual_MODEL
+    """Model name to use."""
+    cache_folder: Optional[str] = None
+    """Path to store models.
+    Can be also set by SENTENCE_TRANSFORMERS_HOME environment variable."""
+    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
+    """Keyword arguments to pass to the model."""
+    encode_kwargs: Dict[str, Any] = Field(default_factory=dict)
+    """Keyword arguments to pass when calling the `encode` method of the model."""
+    # query_instruction: str = DEFAULT_QUERY_BGE_INSTRUCTION_EN
+    """Instruction to use for embedding query."""
+
+    def __init__(self, **kwargs: Any):
+        """Initialize the sentence_transformer."""
+        super().__init__(**kwargs)
+        try:
+            import sentence_transformers
+
+        except ImportError as exc:
+            raise ImportError(
+                "Could not import sentence_transformers python package. "
+                "Please install it with `pip install sentence_transformers`."
+            ) from exc
+
+        self.client = sentence_transformers.SentenceTransformer(
+            self.model_name, cache_folder=self.cache_folder, **self.model_kwargs
+        )
+        # if "-zh" in self.model_name:
+        #     self.query_instruction = DEFAULT_QUERY_BGE_INSTRUCTION_ZH
+
+    # class Config:
+    #     """Configuration for this pydantic object."""
+
+    #     extra = Extra.forbid
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        """Compute doc embeddings using a HuggingFace transformer model.
+
+        Args:
+            texts: The list of texts to embed.
+
+        Returns:
+            List of embeddings, one for each text.
+        """
+        texts = [t.replace("\n", " ") for t in texts]
+        embeddings = self.client.encode(texts, **self.encode_kwargs)
+        return embeddings.tolist()
+
+    def embed_query(self, text: str) -> List[float]:
+        """Compute query embeddings using a HuggingFace transformer model.
+
+        Args:
+            text: The text to embed.
+
+        Returns:
+            Embeddings for the text.
+        """
+        text = text.replace("\n", " ")
+        embedding = self.client.encode(text, **self.encode_kwargs
+        )
+        return embedding.tolist()
+
+
+
+

bisheng_langchain/embeddings/huggingfacemultilingual.py

@@ -0,0 +1,104 @@
+from typing import Any, Dict, List, Optional
+
+import requests
+from langchain_core.embeddings import Embeddings
+from langchain_core.pydantic_v1 import BaseModel, Extra, Field
+DEFAULT_Multilingual_MODEL = "intfloat/multilingual-e5-large"
+
+
+class HuggingFaceMultilingualEmbeddings(BaseModel, Embeddings):
+    """HuggingFace BGE sentence_transformers embedding models.
+
+    To use, you should have the ``sentence_transformers`` python package installed.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_community.embeddings import HuggingFaceBgeEmbeddings
+
+            model_name = "BAAI/bge-large-en"
+            model_kwargs = {'device': 'cpu'}
+            encode_kwargs = {'normalize_embeddings': True}
+            hf = HuggingFaceBgeEmbeddings(
+                
+                model_name=model_name,
+                model_kwargs=model_kwargs,
+                encode_kwargs=encode_kwargs
+            )
+    """
+
+    client: Any  #: :meta private:
+    model_name: str = DEFAULT_Multilingual_MODEL
+    """Model name to use."""
+    cache_folder: Optional[str] = None
+    """Path to store models.
+    Can be also set by SENTENCE_TRANSFORMERS_HOME environment variable."""
+    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
+    """Keyword arguments to pass to the model."""
+    encode_kwargs: Dict[str, Any] = Field(default_factory=dict)
+    """Keyword arguments to pass when calling the `encode` method of the model."""
+    # query_instruction: str = DEFAULT_QUERY_BGE_INSTRUCTION_EN
+    """Instruction to use for embedding query."""
+
+    def __init__(self, **kwargs: Any):
+        """Initialize the sentence_transformer."""
+        super().__init__(**kwargs)
+        try:
+            import sentence_transformers
+
+        except ImportError as exc:
+            raise ImportError(
+                "Could not import sentence_transformers python package. "
+                "Please install it with `pip install sentence_transformers`."
+            ) from exc
+
+        self.client = sentence_transformers.SentenceTransformer(
+            self.model_name, cache_folder=self.cache_folder, **self.model_kwargs
+        )
+        # if "-zh" in self.model_name:
+        #     self.query_instruction = DEFAULT_QUERY_BGE_INSTRUCTION_ZH
+
+    # class Config:
+    #     """Configuration for this pydantic object."""
+
+    #     extra = Extra.forbid
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        """Compute doc embeddings using a HuggingFace transformer model.
+
+        Args:
+            texts: The list of texts to embed.
+
+        Returns:
+            List of embeddings, one for each text.
+        """
+        texts = [t.replace("\n", " ") for t in texts]
+        textschange = []
+        for t in texts:
+            textschange.append("passage: " + t)
+        # print(textschange)
+        
+        embeddings = self.client.encode(textschange, **self.encode_kwargs)
+        return embeddings.tolist()
+
+    def embed_query(self, text: str) -> List[float]:
+        """Compute query embeddings using a HuggingFace transformer model.
+
+        Args:
+            text: The text to embed.
+
+        Returns:
+            Embeddings for the text.
+        """
+        text = text.replace("\n", " ")
+        text = "query: " + text
+        # print(text)
+        embedding = self.client.encode(text, **self.encode_kwargs
+        )
+        return embedding.tolist()
+    
+    
+
+
+
+

bisheng_langchain/retrievers/__init__.py

@@ -1,5 +1,7 @@
 from bisheng_langchain.retrievers.mix_es_vector import MixEsVectorRetriever
+from bisheng_langchain.retrievers.ensemble import EnsembleRetriever
 
 __all__ = [
-    "MixEsVectorRetriever"
+    "MixEsVectorRetriever",
+    "EnsembleRetriever"
 ]

bisheng_langchain/retrievers/ensemble.py

@@ -0,0 +1,188 @@
+"""
+Ensemble retriever that ensemble the results of 
+multiple retrievers by using weighted  Reciprocal Rank Fusion
+"""
+
+from typing import Any, Dict, List
+
+from langchain_core.documents import Document
+from langchain_core.pydantic_v1 import root_validator
+from langchain_core.retrievers import BaseRetriever
+
+from langchain.callbacks.manager import (
+    AsyncCallbackManagerForRetrieverRun,
+    CallbackManagerForRetrieverRun,
+)
+
+
+class EnsembleRetriever(BaseRetriever):
+    """Retriever that ensembles the multiple retrievers.
+
+    It uses a rank fusion.
+
+    Args:
+        retrievers: A list of retrievers to ensemble.
+        weights: A list of weights corresponding to the retrievers. Defaults to equal
+            weighting for all retrievers.
+        c: A constant added to the rank, controlling the balance between the importance
+            of high-ranked items and the consideration given to lower-ranked items.
+            Default is 60.
+    """
+
+    retrievers: List[BaseRetriever]
+    weights: List[float]
+    c: int = 60
+
+    @root_validator(pre=True)
+    def set_weights(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if not values.get("weights"):
+            n_retrievers = len(values["retrievers"])
+            values["weights"] = [1 / n_retrievers] * n_retrievers
+        return values
+
+    def _get_relevant_documents(
+        self,
+        query: str,
+        *,
+        run_manager: CallbackManagerForRetrieverRun,
+        **kwagrs: Any,
+    ) -> List[Document]:
+        """
+        Get the relevant documents for a given query.
+
+        Args:
+            query: The query to search for.
+
+        Returns:
+            A list of reranked documents.
+        """
+
+        # Get fused result of the retrievers.
+        fused_documents = self.rank_fusion(query, run_manager, **kwagrs)
+
+        return fused_documents
+
+    async def _aget_relevant_documents(
+        self,
+        query: str,
+        *,
+        run_manager: AsyncCallbackManagerForRetrieverRun,
+    ) -> List[Document]:
+        """
+        Asynchronously get the relevant documents for a given query.
+
+        Args:
+            query: The query to search for.
+
+        Returns:
+            A list of reranked documents.
+        """
+
+        # Get fused result of the retrievers.
+        fused_documents = await self.arank_fusion(query, run_manager)
+
+        return fused_documents
+
+    def rank_fusion(
+        self,
+        query: str,
+        run_manager: CallbackManagerForRetrieverRun,
+        **kwagrs: Any,
+    ) -> List[Document]:
+        """
+        Retrieve the results of the retrievers and use rank_fusion_func to get
+        the final result.
+
+        Args:
+            query: The query to search for.
+
+        Returns:
+            A list of reranked documents.
+        """
+
+        # Get the results of all retrievers.
+        retriever_docs = [
+            retriever.get_relevant_documents(
+                query,
+                callbacks=run_manager.get_child(tag=f"retriever_{i+1}"),
+                **kwagrs,
+            )
+            for i, retriever in enumerate(self.retrievers)
+        ]
+
+        # apply rank fusion
+        fused_documents = self.weighted_reciprocal_rank(retriever_docs)
+
+        return fused_documents
+
+    async def arank_fusion(
+        self,
+        query: str,
+        run_manager: AsyncCallbackManagerForRetrieverRun,
+        **kwagrs: Any,
+    ) -> List[Document]:
+        """
+        Asynchronously retrieve the results of the retrievers
+        and use rank_fusion_func to get the final result.
+
+        Args:
+            query: The query to search for.
+
+        Returns:
+            A list of reranked documents.
+        """
+
+        # Get the results of all retrievers.
+        retriever_docs = [
+            await retriever.aget_relevant_documents(
+                query,
+                callbacks=run_manager.get_child(tag=f"retriever_{i+1}"),
+                **kwagrs,
+            )
+            for i, retriever in enumerate(self.retrievers)
+        ]
+
+        # apply rank fusion
+        fused_documents = self.weighted_reciprocal_rank(retriever_docs)
+
+        return fused_documents
+
+    def weighted_reciprocal_rank(self, doc_lists: List[List[Document]]) -> List[Document]:
+        """
+        Perform weighted Reciprocal Rank Fusion on multiple rank lists.
+        You can find more details about RRF here:
+        https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf
+
+        Args:
+            doc_lists: A list of rank lists, where each rank list contains unique items.
+
+        Returns:
+            list: The final aggregated list of items sorted by their weighted RRF
+                    scores in descending order.
+        """
+        if len(doc_lists) != len(self.weights):
+            raise ValueError("Number of rank lists must be equal to the number of weights.")
+
+        # Create a union of all unique documents in the input doc_lists
+        all_documents = set()
+        for doc_list in doc_lists:
+            for doc in doc_list:
+                all_documents.add(doc.page_content)
+
+        # Initialize the RRF score dictionary for each document
+        rrf_score_dic = {doc: 0.0 for doc in all_documents}
+
+        # Calculate RRF scores for each document
+        for doc_list, weight in zip(doc_lists, self.weights):
+            for rank, doc in enumerate(doc_list, start=1):
+                rrf_score = weight * (1 / (rank + self.c))
+                rrf_score_dic[doc.page_content] += rrf_score
+
+        # Sort documents by their RRF scores in descending order
+        sorted_documents = sorted(rrf_score_dic.keys(), key=lambda x: rrf_score_dic[x], reverse=True)
+
+        # Map the sorted page_content back to the original document objects
+        page_content_to_doc_map = {doc.page_content: doc for doc_list in doc_lists for doc in doc_list}
+        sorted_docs = [page_content_to_doc_map[page_content] for page_content in sorted_documents]
+
+        return sorted_docs

bisheng_langchain/vectorstores/elastic_keywords_search.py

@@ -120,11 +120,19 @@ class ElasticKeywordsSearch(VectorStore, ABC):
         self.llm_chain = llm_chain
         self.drop_old = drop_old
         _ssl_verify = ssl_verify or {}
+        self.elasticsearch_url = elasticsearch_url
+        self.ssl_verify = _ssl_verify
         try:
             self.client = elasticsearch.Elasticsearch(elasticsearch_url, **_ssl_verify)
         except ValueError as e:
             raise ValueError(f'Your elasticsearch client string is mis-formatted. Got error: {e} ')
 
+        if drop_old:
+            try:
+                self.client.indices.delete(index=index_name)
+            except elasticsearch.exceptions.NotFoundError:
+                pass
+        
     def add_texts(
         self,
         texts: Iterable[str],
@@ -250,6 +258,7 @@ class ElasticKeywordsSearch(VectorStore, ABC):
         refresh_indices: bool = True,
         llm: Optional[BaseLLM] = None,
         prompt: Optional[PromptTemplate] = DEFAULT_PROMPT,
+        drop_old: Optional[bool] = False,
         **kwargs: Any,
     ) -> ElasticKeywordsSearch:
         """Construct ElasticKeywordsSearch wrapper from raw documents.
@@ -279,13 +288,11 @@ class ElasticKeywordsSearch(VectorStore, ABC):
         index_name = index_name or uuid.uuid4().hex
         if llm:
             llm_chain = LLMChain(llm=llm, prompt=prompt)
-            vectorsearch = cls(elasticsearch_url, index_name, llm_chain=llm_chain, **kwargs)
+            vectorsearch = cls(elasticsearch_url, index_name, llm_chain=llm_chain, drop_old=drop_old, **kwargs)
         else:
-            vectorsearch = cls(elasticsearch_url, index_name, **kwargs)
-        vectorsearch.add_texts(texts,
-                               metadatas=metadatas,
-                               ids=ids,
-                               refresh_indices=refresh_indices)
+            vectorsearch = cls(elasticsearch_url, index_name, drop_old=drop_old, **kwargs)
+        vectorsearch.add_texts(texts, metadatas=metadatas, ids=ids, refresh_indices=refresh_indices)
+
         return vectorsearch
 
     def create_index(self, client: Any, index_name: str, mapping: Dict) -> None:

bisheng_langchain/vectorstores/milvus.py

@@ -8,8 +8,8 @@ from uuid import uuid4
 import numpy as np
 from langchain.docstore.document import Document
 from langchain.embeddings.base import Embeddings
-from langchain.vectorstores.milvus import Milvus as MilvusLangchain
 from langchain.vectorstores.utils import maximal_marginal_relevance
+from langchain_community.vectorstores.milvus import Milvus as MilvusLangchain
 
 logger = logging.getLogger(__name__)
 
@@ -185,6 +185,7 @@ class Milvus(MilvusLangchain):
         self.index_params = index_params
         self.search_params = search_params
         self.consistency_level = consistency_level
+        self.connection_args = connection_args
 
         # In order for a collection to be compatible, pk needs to be auto'id and int
         self._primary_field = primary_field
@@ -199,15 +200,23 @@ class Milvus(MilvusLangchain):
         # Create the connection to the server
         if connection_args is None:
             connection_args = DEFAULT_MILVUS_CONNECTION
+        # if 'timeout' not in connection_args:
+        # connection_args['timeout'] = 30
+
         self.alias = self._create_connection_alias(connection_args)
         self.col: Optional[Collection] = None
 
         # Grab the existing collection if it exists
-        if utility.has_collection(self.collection_name, using=self.alias):
-            self.col = Collection(
-                self.collection_name,
-                using=self.alias,
-            )
+        try:
+            if utility.has_collection(self.collection_name, using=self.alias):
+                self.col = Collection(
+                    self.collection_name,
+                    using=self.alias,
+                )
+        except Exception as e:
+            logger.error(f'milvus operating error={str(e)}')
+            self.close_connection(self.alias)
+            raise e
         # If need to drop old, drop it
         if drop_old and isinstance(self.col, Collection):
             self.col.drop()
@@ -216,6 +225,10 @@ class Milvus(MilvusLangchain):
         # Initialize the vector store
         self._init()
 
+    def close_connection(self, using):
+        from pymilvus import connections
+        connections.remove_connection(using)
+
     def _create_connection_alias(self, connection_args: dict) -> str:
         """Create the connection to the Milvus server."""
         from pymilvus import MilvusException, connections
@@ -430,6 +443,7 @@ class Milvus(MilvusLangchain):
         metadatas: Optional[List[dict]] = None,
         timeout: Optional[int] = None,
         batch_size: int = 1000,
+        no_embedding: bool = False,
         **kwargs: Any,
     ) -> List[str]:
         """Insert text data into Milvus.
@@ -460,15 +474,20 @@ class Milvus(MilvusLangchain):
         from pymilvus import Collection, MilvusException
 
         texts = list(texts)
+        if not no_embedding:
+            try:
+                embeddings = self.embedding_func.embed_documents(texts)
+            except NotImplementedError:
+                embeddings = [self.embedding_func.embed_query(x) for x in texts]
 
-        try:
-            embeddings = self.embedding_func.embed_documents(texts)
-        except NotImplementedError:
-            embeddings = [self.embedding_func.embed_query(x) for x in texts]
-
-        if len(embeddings) == 0:
-            logger.debug('Nothing to insert, skipping.')
-            return []
+            if len(embeddings) == 0:
+                logger.debug('Nothing to insert, skipping.')
+                return []
+        else:
+            embeddings = [[0.0]] * len(texts)
+            if len(embeddings) == 0:
+                logger.debug('Nothing to insert, skipping.')
+                return []
 
         # If the collection hasn't been initialized yet, perform all steps to do so
         if not isinstance(self.col, Collection):
@@ -842,6 +861,7 @@ class Milvus(MilvusLangchain):
         index_params: Optional[dict] = None,
         search_params: Optional[dict] = None,
         drop_old: bool = False,
+        no_embedding: bool = False,
         **kwargs: Any,
     ) -> Milvus:
         """Create a Milvus collection, indexes it with HNSW, and insert data.
@@ -877,7 +897,7 @@ class Milvus(MilvusLangchain):
             drop_old=drop_old,
             **kwargs,
         )
-        vector_db.add_texts(texts=texts, metadatas=metadatas)
+        vector_db.add_texts(texts=texts, metadatas=metadatas, no_embedding=no_embedding)
         return vector_db
 
     @staticmethod
@@ -888,3 +908,22 @@ class Milvus(MilvusLangchain):
 
     def _select_relevance_score_fn(self) -> Callable[[float], float]:
         return self._relevance_score_fn
+    
+    def query(self, expr: str, timeout: Optional[int] = None, **kwargs: Any) -> List[Document]:
+        output_fields = self.fields[:]
+        output_fields.remove(self._vector_field)
+        res = self.col.query(
+            expr=expr,
+            output_fields=output_fields,
+            timeout=timeout,
+            limit=1,
+            **kwargs,
+        )
+        # Organize results.
+        ret = []
+        for result in res:
+            meta = {x: result.get(x) for x in output_fields}
+            doc = Document(page_content=meta.pop(self._text_field), metadata=meta)
+            ret.append(doc)
+
+        return ret

{bisheng_langchain-0.2.3.dist-info → bisheng_langchain-0.2.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bisheng-langchain
-Version: 0.2.3
+Version: 0.2.3.2
 Summary: bisheng langchain modules
 Home-page: https://github.com/dataelement/bisheng
 Author: DataElem

{bisheng_langchain-0.2.3.dist-info → bisheng_langchain-0.2.3.2.dist-info}/RECORD

@@ -6,7 +6,7 @@ bisheng_langchain/agents/chatglm_functions_agent/base.py,sha256=tyytq0XIFXpfxDP0
 bisheng_langchain/agents/chatglm_functions_agent/output_parser.py,sha256=M7vDzQFqFUMmL250FHeNKXMwatkCdD0x1D0hyqGYRAA,3497
 bisheng_langchain/agents/chatglm_functions_agent/prompt.py,sha256=OiBTRUOhvhSyO2jO2ByUUiaCrkK_tIUH9pMWWKs-aF4,992
 bisheng_langchain/agents/llm_functions_agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bisheng_langchain/agents/llm_functions_agent/base.py,sha256=_bJNSYZJrY82Tcc0zwQLO4qyuTZbOkVDBA0hndGM12w,12311
+bisheng_langchain/agents/llm_functions_agent/base.py,sha256=4mzDOGheLGcP55xrGcYkLiH4kIII2IJjtYLAJAu41es,12308
 bisheng_langchain/autogen_role/__init__.py,sha256=MnTGbAOK770JM9l95Qcxu93s2gNAmhlil7K9HdFG81o,430
 bisheng_langchain/autogen_role/assistant.py,sha256=VGCoxJaRxRG6ZIJa2TsxcLZbMbF4KC8PRB76DOuznNU,4736
 bisheng_langchain/autogen_role/custom.py,sha256=8xxtAzNF_N1fysyChynVD19t659Qvtcyj_LNiOrE7ew,2499
@@ -26,19 +26,19 @@ bisheng_langchain/chains/retrieval/retrieval_chain.py,sha256=7VLJ-IPVjKfmAVgVET4
 bisheng_langchain/chains/router/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/chains/router/multi_rule.py,sha256=BiFryj3-7rOxfttD-MyOkKWLCSGB9LVYd2rjOsIfQC8,375
 bisheng_langchain/chains/router/rule_router.py,sha256=R2YRUnwn7s_7DbsSn27uPn4cIV0D-5iXEORXir0tNGM,1835
-bisheng_langchain/chat_models/__init__.py,sha256=7NuGJAUgeCF9yDqe7D3Yw69_5COlsExg811TSDErpps,599
-bisheng_langchain/chat_models/host_llm.py,sha256=zbrWUf9Vvc9_8dlVLPoY7Cm0NL7WhE9DSd5F7xYhY2A,22420
+bisheng_langchain/chat_models/__init__.py,sha256=4-HTLE_SXO4hmNJu6yQxiQKBt2IFca_ezllVBLmvbEE,635
+bisheng_langchain/chat_models/host_llm.py,sha256=HY2HtMwiW-0TeyVlDZ85Vr0ldRhr3M7ICMdrWqll1OM,22774
 bisheng_langchain/chat_models/minimax.py,sha256=JLs_f6vWD9beZYUtjD4FG28G8tZHrGUAWOwdLIuJomw,13901
 bisheng_langchain/chat_models/proxy_llm.py,sha256=wzVBZik9WC3-f7kyQ1eu3Ooibqpcocln08knf5lV1Nw,17082
-bisheng_langchain/chat_models/qwen.py,sha256=jGx_tW-LPxfegE6NvY6wID8ps2SsP813atjXnc04C-s,18841
+bisheng_langchain/chat_models/qwen.py,sha256=3_ncSsTJLaHH4FpWnfhU1ZJt0YlyhK4Utg_HSxepSiM,19172
 bisheng_langchain/chat_models/sensetime.py,sha256=fuQ5yYGO5F7o7iQ7us17MlL4TAWRRFCCpNN9bAF-ydc,17056
 bisheng_langchain/chat_models/wenxin.py,sha256=OBXmFWkUWZMu1lUz6hPAEawsbAcdgMWcm9WkJJLZyng,13671
 bisheng_langchain/chat_models/xunfeiai.py,sha256=Yz09-I8u6XhGVnT5mdel15Z3CCQZqApJkgnaxyiZNFk,14037
-bisheng_langchain/chat_models/zhipuai.py,sha256=KokWmDDwljsV2iFiRXZlylIaQRw4jDOq5aCnat53wnQ,14887
+bisheng_langchain/chat_models/zhipuai.py,sha256=MgN8pFInUB6q5agZSnAOipYxTIxAAGhh-Zq6NXs9Hxc,15342
 bisheng_langchain/chat_models/interface/__init__.py,sha256=KwcZMPSxFiXu6joXoZEgq6THxZeDXA8neZcOuLKBpUk,443
 bisheng_langchain/chat_models/interface/minimax.py,sha256=tF3S7ryFtYVXwh7jHHH9z1eY8nMCy0iLiFocsPSJ3pA,4423
 bisheng_langchain/chat_models/interface/openai.py,sha256=v4kxxglJoVMJ9kxaRDIJnWHBSvjl9vRhzQb5Fr-keg0,2081
-bisheng_langchain/chat_models/interface/types.py,sha256=SmbbDCI7hdWmxqRVEprW0NMPa4-F914Y8vYUhdWZ7Aw,1138
+bisheng_langchain/chat_models/interface/types.py,sha256=FZwQJPDnStQ3oJx5ubyGJlvrhnGCgqhhnZDYSLmqFOs,1141
 bisheng_langchain/chat_models/interface/utils.py,sha256=qww_uYsWDqK7cLuv-KzZmmlg9SZAHOi4R_6I6S4XLIk,65
 bisheng_langchain/chat_models/interface/wenxin.py,sha256=z_K1Nj78dDYYgiVIzc5sGkOiGr8OAoRwaKwmpWXssH0,4246
 bisheng_langchain/chat_models/interface/xunfei.py,sha256=DPHAZM_uHg0A8GnebgkRbLENhBW7bBtRHzKC0gFKZgc,7514
@@ -55,8 +55,10 @@ bisheng_langchain/document_loaders/parsers/ellm_client.py,sha256=B4Dea8xXXnGvB9j
 bisheng_langchain/document_loaders/parsers/image.py,sha256=7Vx4dD_WiSTojS4TMIJFxfE8nvze0kwNnwTd6f1cLds,938
 bisheng_langchain/document_loaders/parsers/ocr_client.py,sha256=rRh1coJYn24n7FaINBZH5yO6Edm9TRywY6UOXpcerVo,1612
 bisheng_langchain/document_loaders/parsers/test_image.py,sha256=EJHozq5oFfLBlLL5Lr6XFkrkvSttPpohprs9OjDzAKM,8685
-bisheng_langchain/embeddings/__init__.py,sha256=jaQ4F8e3yOvbwBhX-JJDmMHZrcDdIdtZLx752jbnb_E,309
-bisheng_langchain/embeddings/host_embedding.py,sha256=ZIHynOctSSSx2pPpEMw13vgPswJpUBMFsdO4ze0665o,6223
+bisheng_langchain/embeddings/__init__.py,sha256=_zLLb9cH4Ct4UpKQhtXr7V2IQ7LUnlCKkKTroTE_Enk,534
+bisheng_langchain/embeddings/host_embedding.py,sha256=CK_hZgOd3VJrkyh4Zyb3SrpkxlRRfy7ffanWfhkjIcE,6321
+bisheng_langchain/embeddings/huggingfacegte.py,sha256=RPfSXu7oMv6vgIjLqrPZ1Qz3K0yEuYn7VO0u7m7PzK8,3192
+bisheng_langchain/embeddings/huggingfacemultilingual.py,sha256=g7-yKJ-qIPUZQaRnGz312S-f3aJCGcdHemAR3znE-uo,3415
 bisheng_langchain/embeddings/wenxin.py,sha256=6zx53tSUguvny4gGe5CTmfwV-QtGqKmcT-Jlgf2xVUs,4737
 bisheng_langchain/embeddings/interface/__init__.py,sha256=GNY3tibpRxpAdAfSvQmXBKo0xKSLke_9y4clofi_WOE,98
 bisheng_langchain/embeddings/interface/types.py,sha256=VdurbtsnjCPdlOjPFcK2Mg6r9bJYYHb3tepvkk-y3nM,461
@@ -64,15 +66,16 @@ bisheng_langchain/embeddings/interface/wenxin.py,sha256=5d9gI4enmfkD80s0FHKiDt33
 bisheng_langchain/input_output/__init__.py,sha256=sW_GB7MlrHYsqY1Meb_LeimQqNsMz1gH-00Tqb2BUyM,153
 bisheng_langchain/input_output/input.py,sha256=I5YDmgbvvj1o2lO9wi8LE37wM0wP5jkhUREU32YrZMQ,1094
 bisheng_langchain/input_output/output.py,sha256=6U-az6-Cwz665C2YmcH3SYctWVjPFjmW8s70CA_qphk,11585
-bisheng_langchain/retrievers/__init__.py,sha256=TcyK31IMgFJcYaOCLd9O6qFzXt1VMbtLs-g4C6ml_3w,117
+bisheng_langchain/retrievers/__init__.py,sha256=XqBeNyPyNCJf-SzNBiFlkxtjrtHUFTTi5pe2yPyOKrA,210
+bisheng_langchain/retrievers/ensemble.py,sha256=nLsTKpJmaigrECCWzrvDUwhE-qs9Mg7gPRXfPo5qFMI,5942
 bisheng_langchain/retrievers/mix_es_vector.py,sha256=dSrrsuMPSgGiu181EOzACyIKiDXR0qNBQz_914USD3E,4465
 bisheng_langchain/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/utils/requests.py,sha256=20ooDlMDMkXig--rSyRqbnAlbGLscBvvkHzFk2AmyGM,8517
 bisheng_langchain/vectorstores/__init__.py,sha256=zCZgDe7LyQ0iDkfcm5UJ5NxwKQSRHnqrsjx700Fy11M,213
-bisheng_langchain/vectorstores/elastic_keywords_search.py,sha256=gt_uw_fSMcEZWxbiA3V0RyA-utLOZlUY-qxdwnsfZks,12664
-bisheng_langchain/vectorstores/milvus.py,sha256=44ZbDsIxdsbUnHOpEpCdrW5zvWnYvDdAVoDKjCFoyYI,34424
+bisheng_langchain/vectorstores/elastic_keywords_search.py,sha256=ACUzgeTwzVOVrm0EqBXF_VhzwrWZJbKYQgqNSW5VhbQ,12929
+bisheng_langchain/vectorstores/milvus.py,sha256=-gXIQzzmoPggLS2KEGUtp6kHg9peoawBlSVuOwQndHQ,35856
 bisheng_langchain/vectorstores/retriever.py,sha256=hj4nAAl352EV_ANnU2OHJn7omCH3nBK82ydo14KqMH4,4353
-bisheng_langchain-0.2.3.dist-info/METADATA,sha256=aS_6DGfFwxgfbvWFt84F7GZwRlm9PQ0b4gUfjmeiFPY,2297
-bisheng_langchain-0.2.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-bisheng_langchain-0.2.3.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
-bisheng_langchain-0.2.3.dist-info/RECORD,,
+bisheng_langchain-0.2.3.2.dist-info/METADATA,sha256=bBbyzXtoG8QPpc1m68dlwZBKBZffry7sCq_BY68h7fU,2299
+bisheng_langchain-0.2.3.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+bisheng_langchain-0.2.3.2.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
+bisheng_langchain-0.2.3.2.dist-info/RECORD,,

{bisheng_langchain-0.2.3.dist-info → bisheng_langchain-0.2.3.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any