bisheng-langchain 0.3.5.dev1__py3-none-any.whl → 0.3.7.dev1__py3-none-any.whl

bisheng_langchain/chains/qa_generation/base_v2.py

@@ -134,8 +134,6 @@ class TrainsetGenerator:
             chunk_size: int = 1024,
             seed: int = 42,
             prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
-            filter_lowquality_context: bool = False,
-            filter_lowquality_question: bool = False,
             answer_prompt: Optional[HumanMessagePromptTemplate] = ANSWER_FORMULATE,
     ) -> None:
         self.generator_llm = generator_llm
@@ -154,8 +152,6 @@ class TrainsetGenerator:
         self.threshold = 5.0
         self.rng = default_rng(seed)
         self.prompt = prompt
-        self.filter_lowquality_context = filter_lowquality_context
-        self.filter_lowquality_question = filter_lowquality_question
         if answer_prompt is None:
             answer_prompt = ANSWER_FORMULATE
         self.answer_prompt = answer_prompt
@@ -167,8 +163,6 @@ class TrainsetGenerator:
             chunk_size: int = 512,
             trainset_distribution: dict = DEFAULT_TRAIN_DISTRIBUTION,
             prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
-            filter_lowquality_context: bool = False,
-            filter_lowquality_question: bool = False,
             answer_prompt: Optional[PromptTemplate] = ANSWER_FORMULATE,
     ):
         generator_llm = llm
@@ -179,8 +173,6 @@ class TrainsetGenerator:
             chunk_size=chunk_size,
             trainset_distribution=trainset_distribution,
             prompt=prompt,
-            filter_lowquality_context=filter_lowquality_context,
-            filter_lowquality_question=filter_lowquality_question,
             answer_prompt=answer_prompt,
         )
 
@@ -324,17 +316,14 @@ class TrainsetGenerator:
             )
 
             text_chunk = " ".join([node.get_content() for node in nodes])
-            if self.filter_lowquality_context:
-                score = self._filter_context(text_chunk)
-                if not score:
-                    continue
+            score = self._filter_context(text_chunk)
+            if not score:
+                continue
             seed_question = self._seed_question(text_chunk)
 
             question = seed_question
-            if self.filter_lowquality_question:
-                is_valid_question = self._filter_question(question)
-            else:
-                is_valid_question = True
+            # is_valid_question = self._filter_question(question)
+            is_valid_question = True
             if is_valid_question:
                 context = [text_chunk] * len(question.split("\n"))
                 is_conv = len(context) > 1
@@ -372,8 +361,6 @@ class QAGenerationChainV2(Chain):
             llm: BaseLanguageModel,
             k: Optional[int] = None,
             chunk_size: int = 512,
-            filter_lowquality_context: bool = False,
-            filter_lowquality_question: bool = False,
             question_prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
             answer_prompt: Optional[HumanMessagePromptTemplate] = ANSWER_FORMULATE,
             **kwargs: Any,
@@ -390,14 +377,8 @@ class QAGenerationChainV2(Chain):
         Returns:
             a QAGenerationChain class
         """
-        generator = TrainsetGenerator.from_default(
-            llm, 
-            chunk_size=chunk_size, 
-            prompt=question_prompt, 
-            answer_prompt=answer_prompt,
-            filter_lowquality_context=filter_lowquality_context, 
-            filter_lowquality_question=filter_lowquality_question
-        )
+        generator = TrainsetGenerator.from_default(llm, chunk_size=chunk_size, prompt=question_prompt,
+                                                   answer_prompt=answer_prompt)
         return cls(documents=documents, generator=generator, k=k, **kwargs)
 
     @property
@@ -424,14 +405,14 @@ class QAGenerationChainV2(Chain):
         dataset = self.generator.generate(documents=self.documents, train_size=self.k)
         df = dataset.to_pandas()
         qa_pairs = df.to_dict("records")
-        qa = []
+        qa = ''
         for pair in qa_pairs:
-            qa.append({
-                "question": pair["question"],
-                "answer": pair["ground_truth"][0],
-                "context": pair["ground_truth_context"][0],
-            })
-        qa = f'```json\n{json.dumps(qa, ensure_ascii=False, indent=4)}\n```'
+            qa += json.dumps(
+                {
+                    "question": pair["question"],
+                    "answer": pair["ground_truth"][0],
+                    "context": pair["ground_truth_context"][0],
+                }, ensure_ascii=False)
         return {self.output_key: qa}
 
     async def _acall(

bisheng_langchain/memory/__init__.py

@@ -0,0 +1,3 @@
+from .redis import ConversationRedisMemory
+
+__all__ = ['ConversationRedisMemory']

bisheng_langchain/memory/redis.py

@@ -0,0 +1,104 @@
+import json
+from typing import Any, Dict, List, Optional
+
+import redis
+from langchain.memory.chat_memory import BaseChatMemory
+from langchain_core.messages import (AIMessage, BaseMessage, HumanMessage, get_buffer_string,
+                                     message_to_dict, messages_from_dict)
+from langchain_core.pydantic_v1 import root_validator
+from pydantic import Field
+
+
+class ConversationRedisMemory(BaseChatMemory):
+    """Using redis for storing conversation memory."""
+    redis_client: redis.Redis = Field(default=None, exclude=True)
+    human_prefix: str = 'Human'
+    ai_prefix: str = 'AI'
+    session_id: str = 'session'
+    memory_key: str = 'history'  #: :meta private:
+    redis_url: str
+    redis_prefix: str = 'redis_buffer_'
+    ttl: Optional[int] = None
+
+    @root_validator()
+    def validate_environment(cls, values: Dict) -> Dict:
+        redis_url = values.get('redis_url')
+        if not redis_url:
+            raise ValueError('Redis URL must be set')
+        pool = redis.ConnectionPool.from_url(redis_url, max_connections=1)
+        values['redis_client'] = redis.StrictRedis(connection_pool=pool)
+        return values
+
+    @property
+    def buffer(self) -> Any:
+        """String buffer of memory."""
+        return self.buffer_as_messages if self.return_messages else self.buffer_as_str
+
+    async def abuffer(self) -> Any:
+        """String buffer of memory."""
+        return (await self.abuffer_as_messages()
+                if self.return_messages else await self.abuffer_as_str())
+
+    def _buffer_as_str(self, messages: List[BaseMessage]) -> str:
+        return get_buffer_string(
+            messages,
+            human_prefix=self.human_prefix,
+            ai_prefix=self.ai_prefix,
+        )
+
+    @property
+    def buffer_as_str(self) -> str:
+        """Exposes the buffer as a string in case return_messages is True."""
+        messages = self.buffer_as_messages
+        return self._buffer_as_str(messages)
+
+        # return self._buffer_as_str(self.chat_memory.messages)
+
+    async def abuffer_as_str(self) -> str:
+        """Exposes the buffer as a string in case return_messages is True."""
+        # messages = await self.chat_memory.aget_messages()
+        messages = self.buffer_as_messages
+        return self._buffer_as_str(messages)
+
+    @property
+    def buffer_as_messages(self) -> List[BaseMessage]:
+        """Exposes the buffer as a list of messages in case return_messages is False."""
+        # return self.chat_memory.messages
+        redis_value = self.redis_client.lrange(self.redis_prefix + self.session_id, 0, -1)
+        items = [json.loads(m.decode('utf-8')) for m in redis_value[::-1]]
+        messages = messages_from_dict(items)
+        return messages
+
+    async def abuffer_as_messages(self) -> List[BaseMessage]:
+        """Exposes the buffer as a list of messages in case return_messages is False."""
+        self.buffer_as_messages
+
+    @property
+    def memory_variables(self) -> List[str]:
+        """Will always return list of memory variables.
+
+        :meta private:
+        """
+        return [self.memory_key]
+
+    def load_memory_variables(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        """Return history buffer."""
+        return {self.memory_key: self.buffer}
+
+    async def aload_memory_variables(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        """Return key-value pairs given the text input to the chain."""
+        buffer = await self.abuffer()
+        return {self.memory_key: buffer}
+
+    def save_context(self, inputs: Dict[str, Any], outputs: Dict[str, str]) -> None:
+        """Save context from this conversation to buffer."""
+        input_str, output_str = self._get_input_output(inputs, outputs)
+
+        input_message_str = json.dumps(message_to_dict(HumanMessage(content=input_str)),
+                                       ensure_ascii=False)
+        output_message_str = json.dumps(message_to_dict(AIMessage(content=output_str)),
+                                        ensure_ascii=False)
+        self.redis_client.lpush(self.redis_prefix + self.session_id, input_message_str)
+        self.redis_client.lpush(self.redis_prefix + self.session_id, output_message_str)
+        if self.ttl:
+            self.redis_client.expire(self.redis_prefix + self.session_id, self.ttl)

bisheng_langchain/rag/init_retrievers/keyword_retriever.py

@@ -16,7 +16,7 @@ from langchain.text_splitter import TextSplitter
 
 
 class KeywordRetriever(BaseRetriever):
-    keyword_store: VectorStore
+    keyword_store: ElasticKeywordsSearch
     text_splitter: TextSplitter
     search_type: str = 'similarity'
     search_kwargs: dict = Field(default_factory=dict)

bisheng_langchain/vectorstores/elastic_keywords_search.py

@@ -13,6 +13,7 @@ from langchain.llms.base import BaseLLM
 from langchain.prompts.prompt import PromptTemplate
 from langchain.utils import get_from_dict_or_env
 from langchain.vectorstores.base import VectorStore
+from loguru import logger
 
 if TYPE_CHECKING:
     from elasticsearch import Elasticsearch  # noqa: F401
@@ -326,6 +327,49 @@ class ElasticKeywordsSearch(VectorStore, ABC):
             response = client.search(index=index_name, body={'query': script_query, 'size': size})
         return response
 
-    def delete(self, **kwargs: Any) -> None:
+    def delete_index(self, **kwargs: Any) -> None:
         # TODO: Check if this can be done in bulk
         self.client.indices.delete(index=self.index_name)
+
+    def delete(
+        self,
+        ids: Optional[List[str]] = None,
+        refresh_indices: Optional[bool] = True,
+        **kwargs: Any,
+    ) -> Optional[bool]:
+        """Delete documents from the Elasticsearch index.
+
+        Args:
+            ids: List of ids of documents to delete.
+            refresh_indices: Whether to refresh the index
+                            after deleting documents. Defaults to True.
+        """
+        try:
+            from elasticsearch.helpers import BulkIndexError, bulk
+        except ImportError:
+            raise ImportError('Could not import elasticsearch python package. '
+                              'Please install it with `pip install elasticsearch`.')
+
+        body = []
+
+        if ids is None:
+            raise ValueError('ids must be provided.')
+
+        for _id in ids:
+            body.append({'_op_type': 'delete', '_index': self.index_name, '_id': _id})
+
+        if len(body) > 0:
+            try:
+                bulk(self.client, body, refresh=refresh_indices, ignore_status=404)
+                logger.debug(f'Deleted {len(body)} texts from index')
+
+                return True
+            except BulkIndexError as e:
+                logger.error(f'Error deleting texts: {e}')
+                firstError = e.errors[0].get('index', {}).get('error', {})
+                logger.error(f"First error reason: {firstError.get('reason')}")
+                raise e
+
+        else:
+            logger.debug('No texts to delete from index')
+            return False

{bisheng_langchain-0.3.5.dev1.dist-info → bisheng_langchain-0.3.7.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bisheng-langchain
-Version: 0.3.5.dev1
+Version: 0.3.7.dev1
 Summary: bisheng langchain modules
 Home-page: https://github.com/dataelement/bisheng
 Author: DataElem

{bisheng_langchain-0.3.5.dev1.dist-info → bisheng_langchain-0.3.7.dev1.dist-info}/RECORD

@@ -23,7 +23,7 @@ bisheng_langchain/chains/conversational_retrieval/__init__.py,sha256=47DEQpj8HBS
 bisheng_langchain/chains/conversational_retrieval/base.py,sha256=XiqBqov6No-wTVCou6qyMT5p2JQgoQI7OLQOYH8XUos,5313
 bisheng_langchain/chains/qa_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/chains/qa_generation/base.py,sha256=VYGmLDB0bnlDQ6T8ivLP55wwFbMo9HOzlPEDUuRx5fU,4148
-bisheng_langchain/chains/qa_generation/base_v2.py,sha256=2F2kGe3ermJraQu4oC-m8vm_ENBy_Zi4uHrJDcSOeJw,15460
+bisheng_langchain/chains/qa_generation/base_v2.py,sha256=ZtHEuNFwbE9txCGR3wx0oDAoj9V6bAxi3GXF8Z78cqQ,14580
 bisheng_langchain/chains/qa_generation/prompt.py,sha256=4eJk9aDUYDN1qaaYRPy9EobCIncnwS8BbQaDFzzePtM,1944
 bisheng_langchain/chains/qa_generation/prompt_v2.py,sha256=sQLanA_iOnLqrUIwzfTOTANt-1vJ44CM54HFDU8Jo1Q,8938
 bisheng_langchain/chains/question_answering/__init__.py,sha256=_gOZMc-SWprK6xc-Jj64jcr9nc-G4YkZbEYwfJNq_bY,8795
@@ -108,6 +108,8 @@ bisheng_langchain/gpts/tools/get_current_time/tool.py,sha256=3uvk7Yu07qhZy1sBrFM
 bisheng_langchain/input_output/__init__.py,sha256=sW_GB7MlrHYsqY1Meb_LeimQqNsMz1gH-00Tqb2BUyM,153
 bisheng_langchain/input_output/input.py,sha256=I5YDmgbvvj1o2lO9wi8LE37wM0wP5jkhUREU32YrZMQ,1094
 bisheng_langchain/input_output/output.py,sha256=6U-az6-Cwz665C2YmcH3SYctWVjPFjmW8s70CA_qphk,11585
+bisheng_langchain/memory/__init__.py,sha256=TNqe5l5BqUv4wh3_UH28fYPWQXGLBUYn6QJHsr7vanI,82
+bisheng_langchain/memory/redis.py,sha256=paz72ic5BfLXY6lj2cEbCxrTb8KVMnKMZmG9q7uh_9s,4291
 bisheng_langchain/rag/__init__.py,sha256=Rm_cDxOJINt0H4bOeUo3JctPxaI6xKKXZcS-R_wkoGs,198
 bisheng_langchain/rag/bisheng_rag_chain.py,sha256=2GMDUPJaW-D7tpOQ9qPt2vGZwmcXBS0UrcibO7J2S1g,5999
 bisheng_langchain/rag/bisheng_rag_pipeline.py,sha256=neoBK3TtuQ07_WeuJCzYlvtsDQNepUa_68NT8VCgytw,13749
@@ -126,7 +128,7 @@ bisheng_langchain/rag/config/baseline_s2b_mix.yaml,sha256=rkPfzU2-mvjRrZ0zMHaQsn
 bisheng_langchain/rag/config/baseline_v2.yaml,sha256=RP-DwIRIS_ZK8ixbXi2Z28rKqHD56pWmr2o2WWIwq3Y,2382
 bisheng_langchain/rag/init_retrievers/__init__.py,sha256=qpLLAuqZPtumTlJj17Ie5AbDDmiUiDxYefg_pumqu-c,218
 bisheng_langchain/rag/init_retrievers/baseline_vector_retriever.py,sha256=oRKZZpxlLQAtsubIcAXeXpf1a9h6Pt6uOtNTLeD2jps,2362
-bisheng_langchain/rag/init_retrievers/keyword_retriever.py,sha256=Da4Q5BrfN0GckJaeAgPYMlzQAp9ll7ZGGyvs7OdCQ5c,2513
+bisheng_langchain/rag/init_retrievers/keyword_retriever.py,sha256=NRT0fBx6HFR7j9IbRl_NBuqF7hnL-9v5GCqHpgnrfPQ,2523
 bisheng_langchain/rag/init_retrievers/mix_retriever.py,sha256=Whxq4kjNPLsxnHcVo60usdFFwLTCD-1jO38q08LXkVQ,4653
 bisheng_langchain/rag/init_retrievers/smaller_chunks_retriever.py,sha256=RQ7QLEOOhBrkw-EimXVJqIGa96D-KkNDik2h9hzg9fU,3805
 bisheng_langchain/rag/prompts/__init__.py,sha256=IUCq9gzqGQN_6IDk0D_F5t3mOUI_KbmSzYnnXoX4VKE,223
@@ -150,10 +152,10 @@ bisheng_langchain/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 bisheng_langchain/utils/azure_dalle_image_generator.py,sha256=96-_nO4hDSwyPE4rSYop5SgJ-U9CE2un4bTdW0E5RGU,6582
 bisheng_langchain/utils/requests.py,sha256=vWGKyNTxApVeaVdKxqACfIT1Q8wMy-jC3kUv2Ce9Mzc,8688
 bisheng_langchain/vectorstores/__init__.py,sha256=zCZgDe7LyQ0iDkfcm5UJ5NxwKQSRHnqrsjx700Fy11M,213
-bisheng_langchain/vectorstores/elastic_keywords_search.py,sha256=Pm1rS50GJ0HWbjBsFDgs28SVuVbjGSRPOor6yJlnE7w,13347
+bisheng_langchain/vectorstores/elastic_keywords_search.py,sha256=inZarhahRaesrvLqyeRCMQvHGAASY53opEVA0_o8S14,14901
 bisheng_langchain/vectorstores/milvus.py,sha256=xh7NokraKg_Xc9ofz0RVfJ_I36ftnprLJtV-1NfaeyQ,37162
 bisheng_langchain/vectorstores/retriever.py,sha256=hj4nAAl352EV_ANnU2OHJn7omCH3nBK82ydo14KqMH4,4353
-bisheng_langchain-0.3.5.dev1.dist-info/METADATA,sha256=Q20qBElwEheYunRPAoIvCRj8jH4RrXId03MA-SA6JnE,2476
-bisheng_langchain-0.3.5.dev1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-bisheng_langchain-0.3.5.dev1.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
-bisheng_langchain-0.3.5.dev1.dist-info/RECORD,,
+bisheng_langchain-0.3.7.dev1.dist-info/METADATA,sha256=PLzscscApd7BJDPDM4tQHtUUxaP9Rxjo-v1LYCLnTV0,2476
+bisheng_langchain-0.3.7.dev1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+bisheng_langchain-0.3.7.dev1.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
+bisheng_langchain-0.3.7.dev1.dist-info/RECORD,,