bisheng-langchain 0.3.7.dev1__py3-none-any.whl → 0.3.7.dev2__py3-none-any.whl

bisheng_langchain/chains/qa_generation/base_v2.py

@@ -134,6 +134,8 @@ class TrainsetGenerator:
             chunk_size: int = 1024,
             seed: int = 42,
             prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
+            filter_lowquality_context: bool = False,
+            filter_lowquality_question: bool = False,
             answer_prompt: Optional[HumanMessagePromptTemplate] = ANSWER_FORMULATE,
     ) -> None:
         self.generator_llm = generator_llm
@@ -152,6 +154,8 @@ class TrainsetGenerator:
         self.threshold = 5.0
         self.rng = default_rng(seed)
         self.prompt = prompt
+        self.filter_lowquality_context = filter_lowquality_context
+        self.filter_lowquality_question = filter_lowquality_question
         if answer_prompt is None:
             answer_prompt = ANSWER_FORMULATE
         self.answer_prompt = answer_prompt
@@ -163,6 +167,8 @@ class TrainsetGenerator:
             chunk_size: int = 512,
             trainset_distribution: dict = DEFAULT_TRAIN_DISTRIBUTION,
             prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
+            filter_lowquality_context: bool = False,
+            filter_lowquality_question: bool = False,
             answer_prompt: Optional[PromptTemplate] = ANSWER_FORMULATE,
     ):
         generator_llm = llm
@@ -173,6 +179,8 @@ class TrainsetGenerator:
             chunk_size=chunk_size,
             trainset_distribution=trainset_distribution,
             prompt=prompt,
+            filter_lowquality_context=filter_lowquality_context,
+            filter_lowquality_question=filter_lowquality_question,
             answer_prompt=answer_prompt,
         )
 
@@ -316,14 +324,17 @@ class TrainsetGenerator:
             )
 
             text_chunk = " ".join([node.get_content() for node in nodes])
-            score = self._filter_context(text_chunk)
-            if not score:
-                continue
+            if self.filter_lowquality_context:
+                score = self._filter_context(text_chunk)
+                if not score:
+                    continue
             seed_question = self._seed_question(text_chunk)
 
             question = seed_question
-            # is_valid_question = self._filter_question(question)
-            is_valid_question = True
+            if self.filter_lowquality_question:
+                is_valid_question = self._filter_question(question)
+            else:
+                is_valid_question = True
             if is_valid_question:
                 context = [text_chunk] * len(question.split("\n"))
                 is_conv = len(context) > 1
@@ -361,6 +372,8 @@ class QAGenerationChainV2(Chain):
             llm: BaseLanguageModel,
             k: Optional[int] = None,
             chunk_size: int = 512,
+            filter_lowquality_context: bool = False,
+            filter_lowquality_question: bool = False,
             question_prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
             answer_prompt: Optional[HumanMessagePromptTemplate] = ANSWER_FORMULATE,
             **kwargs: Any,
@@ -377,8 +390,14 @@ class QAGenerationChainV2(Chain):
         Returns:
             a QAGenerationChain class
         """
-        generator = TrainsetGenerator.from_default(llm, chunk_size=chunk_size, prompt=question_prompt,
-                                                   answer_prompt=answer_prompt)
+        generator = TrainsetGenerator.from_default(
+            llm, 
+            chunk_size=chunk_size, 
+            prompt=question_prompt, 
+            answer_prompt=answer_prompt,
+            filter_lowquality_context=filter_lowquality_context, 
+            filter_lowquality_question=filter_lowquality_question
+        )
         return cls(documents=documents, generator=generator, k=k, **kwargs)
 
     @property
@@ -405,14 +424,14 @@ class QAGenerationChainV2(Chain):
         dataset = self.generator.generate(documents=self.documents, train_size=self.k)
         df = dataset.to_pandas()
         qa_pairs = df.to_dict("records")
-        qa = ''
+        qa = []
         for pair in qa_pairs:
-            qa += json.dumps(
-                {
-                    "question": pair["question"],
-                    "answer": pair["ground_truth"][0],
-                    "context": pair["ground_truth_context"][0],
-                }, ensure_ascii=False)
+            qa.append({
+                "question": pair["question"],
+                "answer": pair["ground_truth"][0],
+                "context": pair["ground_truth_context"][0],
+            })
+        qa = f'```json\n{json.dumps(qa, ensure_ascii=False, indent=4)}\n```'
         return {self.output_key: qa}
 
     async def _acall(

{bisheng_langchain-0.3.7.dev1.dist-info → bisheng_langchain-0.3.7.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bisheng-langchain
-Version: 0.3.7.dev1
+Version: 0.3.7.dev2
 Summary: bisheng langchain modules
 Home-page: https://github.com/dataelement/bisheng
 Author: DataElem

{bisheng_langchain-0.3.7.dev1.dist-info → bisheng_langchain-0.3.7.dev2.dist-info}/RECORD

@@ -23,7 +23,7 @@ bisheng_langchain/chains/conversational_retrieval/__init__.py,sha256=47DEQpj8HBS
 bisheng_langchain/chains/conversational_retrieval/base.py,sha256=XiqBqov6No-wTVCou6qyMT5p2JQgoQI7OLQOYH8XUos,5313
 bisheng_langchain/chains/qa_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/chains/qa_generation/base.py,sha256=VYGmLDB0bnlDQ6T8ivLP55wwFbMo9HOzlPEDUuRx5fU,4148
-bisheng_langchain/chains/qa_generation/base_v2.py,sha256=ZtHEuNFwbE9txCGR3wx0oDAoj9V6bAxi3GXF8Z78cqQ,14580
+bisheng_langchain/chains/qa_generation/base_v2.py,sha256=2F2kGe3ermJraQu4oC-m8vm_ENBy_Zi4uHrJDcSOeJw,15460
 bisheng_langchain/chains/qa_generation/prompt.py,sha256=4eJk9aDUYDN1qaaYRPy9EobCIncnwS8BbQaDFzzePtM,1944
 bisheng_langchain/chains/qa_generation/prompt_v2.py,sha256=sQLanA_iOnLqrUIwzfTOTANt-1vJ44CM54HFDU8Jo1Q,8938
 bisheng_langchain/chains/question_answering/__init__.py,sha256=_gOZMc-SWprK6xc-Jj64jcr9nc-G4YkZbEYwfJNq_bY,8795
@@ -155,7 +155,7 @@ bisheng_langchain/vectorstores/__init__.py,sha256=zCZgDe7LyQ0iDkfcm5UJ5NxwKQSRHn
 bisheng_langchain/vectorstores/elastic_keywords_search.py,sha256=inZarhahRaesrvLqyeRCMQvHGAASY53opEVA0_o8S14,14901
 bisheng_langchain/vectorstores/milvus.py,sha256=xh7NokraKg_Xc9ofz0RVfJ_I36ftnprLJtV-1NfaeyQ,37162
 bisheng_langchain/vectorstores/retriever.py,sha256=hj4nAAl352EV_ANnU2OHJn7omCH3nBK82ydo14KqMH4,4353
-bisheng_langchain-0.3.7.dev1.dist-info/METADATA,sha256=PLzscscApd7BJDPDM4tQHtUUxaP9Rxjo-v1LYCLnTV0,2476
-bisheng_langchain-0.3.7.dev1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-bisheng_langchain-0.3.7.dev1.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
-bisheng_langchain-0.3.7.dev1.dist-info/RECORD,,
+bisheng_langchain-0.3.7.dev2.dist-info/METADATA,sha256=rPLG8c2G8ZAOn3mjAcIP4evhXJbe-CMeUQc9gtuIdCc,2476
+bisheng_langchain-0.3.7.dev2.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
+bisheng_langchain-0.3.7.dev2.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
+bisheng_langchain-0.3.7.dev2.dist-info/RECORD,,

{bisheng_langchain-0.3.7.dev1.dist-info → bisheng_langchain-0.3.7.dev2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: bdist_wheel (0.45.0)
 Root-Is-Purelib: true
 Tag: py3-none-any