bisheng-langchain 0.3.0rc0__py3-none-any.whl → 0.3.1__py3-none-any.whl

bisheng_langchain/rag/qa_corpus/qa_generator.py

@@ -0,0 +1,143 @@
+import os
+import random
+import json
+import copy
+import pandas as pd
+from loguru import logger
+from tqdm import tqdm
+from langchain.document_loaders import PyPDFLoader
+from langchain_core.prompts import PromptTemplate
+from bisheng_langchain.document_loaders import ElemUnstructuredLoader
+from ragas.trainset import TrainsetGenerator
+
+
+prompt_template = """Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+{context}
+
+Question: {question}
+Helpful Answer:"""
+PROMPT = PromptTemplate(
+    template=prompt_template, input_variables=["context", "question"]
+)
+
+
+class RagQAGenerator(object):
+
+    def __init__(self, 
+                 corpus_folder,
+                 qa_gen_folder,
+                 unstructured_api_url="https://bisheng.dataelem.com/api/v1/etl4llm/predict",
+                 model_name="gpt-4-0125-preview"):
+        self.unstructured_api_url = unstructured_api_url
+        self.corpus_folder = corpus_folder
+        self.qa_gen_folder = qa_gen_folder
+        self.model_name = model_name
+        if not os.path.exists(self.qa_gen_folder):
+            os.makedirs(self.qa_gen_folder)
+    
+    def generate(self):
+        for file_name in tqdm(os.listdir(self.corpus_folder)):
+            file_path = os.path.join(self.corpus_folder, file_name)
+            logger.info(f'{file_name} generate qa start ...')
+            # only consider pdf file
+            if file_name.endswith('.pdf'):
+                self.generate_qa_each_file(file_path)
+            else:
+                continue
+
+    def generate_qa_each_file(self, file_path, train_size=100):
+        file_name = os.path.basename(file_path)
+        loader = ElemUnstructuredLoader(file_name=file_name,
+                                        file_path=file_path,
+                                        unstructured_api_url=self.unstructured_api_url)
+        documents = loader.load()
+        for doc in documents:
+            doc.metadata = dict()
+        logger.info(f'documents: {len(documents)}')
+
+        trainsetgenerator = TrainsetGenerator.from_default(
+            openai_generator_llm=self.model_name,
+            openai_filter_llm=self.model_name)
+        trainset = trainsetgenerator.generate(documents, train_size=train_size)
+
+        save_path = os.path.join(self.qa_gen_folder, os.path.splitext(file_name)[0] + '_qa_gen.xlsx')
+        df = trainset.to_pandas()
+        df.to_excel(save_path, index=False)
+        return save_path
+    
+    def statistic_qa(self):
+        total_qa_num = 0
+        all_qa_info = dict()
+        for file_name in os.listdir(self.qa_gen_folder):
+            file_path = os.path.join(self.qa_gen_folder, file_name)
+            if file_name.endswith('.xlsx'):
+                df = pd.read_excel(file_path)
+                qa_info = df.to_dict('records')
+                logger.info(f'{file_name} qa num: {len(qa_info)}')
+                total_qa_num += len(qa_info)
+                all_qa_info[file_name] = qa_info
+        logger.info(f'total_file_num: {len(list(all_qa_info.keys()))}, total_qa_num: {total_qa_num}')
+        return all_qa_info
+
+    def format_qa_for_sft(self, min_context_num=3, max_context_num=7):
+        random.seed(123)
+        all_qa_info = self.statistic_qa()
+        train_samples = []
+        test_samples = []
+        for file_name in all_qa_info:
+            # each file qa
+            qa_info = all_qa_info[file_name]
+            if len(qa_info) == 0:
+                continue
+            contexts = []
+            for qa in qa_info:
+                ground_truth_context = str(eval(qa['ground_truth_context'])[0])
+                contexts.append(ground_truth_context)
+            
+            random.shuffle(qa_info)
+            for i, qa in enumerate(qa_info):
+                question = qa['question']
+                ground_truth_context = str(eval(qa['ground_truth_context'])[0])
+                ground_truth = str(eval(qa['ground_truth'])[0])
+                
+                # 加入其他干扰context
+                random_number = random.randint(
+                    min(min_context_num, len(contexts)), 
+                    min(max_context_num, len(contexts))
+                )
+                random_context = random.sample(contexts, random_number)
+                if ground_truth_context in random_context:
+                    random_context.remove(ground_truth_context)
+                # 将当前context随机插入到其他context中
+                insert_position = random.randint(0, len(random_context))
+                random_context.insert(insert_position, ground_truth_context)
+
+                random_context = '\n\n'.join(random_context)
+                prompt = PROMPT.format(context=random_context, question=question)
+                each_sample = {
+                    'instruction': '', 
+                    'input': prompt, 
+                    'output': ground_truth,
+                    'history': []
+                }
+                if i < 0.9 * len(qa_info):
+                    train_samples.append(each_sample)
+                else:
+                    test_samples.append(each_sample)
+             
+        logger.info(f'train_samples: {len(train_samples)} test_samples: {len(test_samples)}')
+        save_folder = os.path.dirname(self.qa_gen_folder)
+        with open(os.path.join(save_folder, f'train_samples_ganrao_chunk{max_context_num+1}.json'), 'w') as f:
+            json.dump(train_samples, f, indent=2, ensure_ascii=False)
+        with open(os.path.join(save_folder, f'test_samples_ganrao_chunk{max_context_num+1}.json'), 'w') as f:
+            json.dump(test_samples, f, indent=2, ensure_ascii=False)
+
+
+if __name__ == '__main__':
+    corpus_folder = '/home/public/rag_benchmark_v1.0/rag_benchmark_processed'
+    qa_gen_folder = '/home/public/rag_benchmark_v1.0/rag_qa_gen_filter'
+    generator = RagQAGenerator(corpus_folder=corpus_folder, qa_gen_folder=qa_gen_folder)
+    # generator.generate()
+    # generator.statistic_qa()
+    generator.format_qa_for_sft(min_context_num=5, max_context_num=11)

bisheng_langchain/rag/rerank/__init__.py

@@ -0,0 +1,5 @@
+from .rerank import CustomReranker
+
+__all__ = [
+    'CustomReranker',
+]

bisheng_langchain/rag/rerank/rerank.py

@@ -0,0 +1,48 @@
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+
+class CustomReranker:
+
+    def __init__(self, model_path, device_id='cuda:0', threshold=0.0):
+        self.device_id = device_id
+        self.threshold = threshold
+        self.tokenizer = AutoTokenizer.from_pretrained(model_path)
+        self.rank_model = AutoModelForSequenceClassification.from_pretrained(model_path).to(device_id)
+        self.rank_model.eval()
+
+    def match_score(self, chunk, query):
+        """
+        rerank模型计算query和chunk的相似度
+        """
+        pairs = [[query, chunk]]
+
+        with torch.no_grad():
+            inputs = self.tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=512).to(self.device_id)
+            scores = self.rank_model(**inputs, return_dict=True).logits.view(-1, ).float()
+            scores = torch.sigmoid(scores) 
+            scores = scores.cpu().numpy()
+            
+        return scores[0]
+
+    def sort_and_filter(self, query, all_chunks):
+        """
+        rerank模型对所有chunk进行排序
+        """
+        chunk_match_score = []
+        for index, chunk in enumerate(all_chunks):
+            chunk_text = chunk.page_content
+            chunk_match_score.append(self.match_score(chunk_text, query))
+
+        sorted_res = sorted(enumerate(chunk_match_score), key=lambda x: -x[1])
+        remain_chunks = [all_chunks[elem[0]] for elem in sorted_res if elem[1] >= self.threshold]
+        if not remain_chunks:
+            remain_chunks = [all_chunks[sorted_res[0][0]]]
+
+        # for index, chunk in enumerate(remain_chunks):
+        #     print('query:', query)
+        #     print('chunk_text:', chunk.page_content)
+        #     print('socre:', sorted_res[index][1])
+        #     print('***********')
+
+        return remain_chunks

bisheng_langchain/rag/rerank/rerank_benchmark.py

@@ -0,0 +1,139 @@
+import json
+import os
+import numpy as np
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained('/home/public/llm/bge-reranker-large')
+model = AutoModelForSequenceClassification.from_pretrained('/home/public/llm/bge-reranker-large').to('cuda:2')
+model.eval()
+
+
+def min_edit_distance(a, b):
+    dp = [[0 for i in range(len(b) + 1)] for j in range(len(a) + 1)]
+    for i in range(len(a) + 1):
+        dp[i][0] = i
+    for j in range(len(b) + 1):
+        dp[0][j] = j
+    for i in range(1, len(a) + 1):
+        for j in range(1, len(b) + 1):
+            if a[i - 1] == b[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1]
+            else:
+                dp[i][j] = min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + 1)
+    return dp[-1][-1]
+
+
+def is_matched(text0, text1, thrd=10):
+    text0.replace(" ", "").replace("\n", "")
+    text1.replace(" ", "").replace("\n", "")
+    dist = min_edit_distance(text0, text1)
+    if dist < thrd:
+        return True
+    return False
+
+
+def match_score(chunk, query):
+    """
+    rerank模型计算query和chunk的相似度
+    """
+    pairs = [[query, chunk]]
+
+    with torch.no_grad():
+        inputs = tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=512).to('cuda:2')
+        scores = model(**inputs, return_dict=True).logits.view(-1, ).float()
+        scores = torch.sigmoid(scores) 
+        scores = scores.cpu().numpy()
+        
+    return scores[0]
+
+
+def sort_filter_all_chunks_method1(d, th=0.0):
+    """
+    rerank模型对所有chunk进行排序
+    """
+    # answer关键词已提取好
+    query = d['question']
+    all_chunks = d['all_chunks']
+
+    chunk_match_score = []
+    for index, chunk in enumerate(all_chunks):
+        chunk_text = chunk['text']
+        chunk_match_score.append(match_score(chunk_text, query))
+
+    sorted_res = sorted(enumerate(chunk_match_score), key=lambda x: -x[1])
+    print('-----------')
+    print(sorted_res)
+    remain_chunks = [all_chunks[elem[0]] for elem in sorted_res if elem[1] >= th]
+    if not remain_chunks:
+        remain_chunks = [all_chunks[sorted_res[0][0]]]
+
+    # for index, chunk in enumerate(remain_chunks):
+    #     print('query:', query)
+    #     print('chunk_text:', chunk['text'])
+    #     print('socre:', sorted_res[index][1])
+    #     print('***********')
+
+    d['all_chunks'] = remain_chunks
+
+
+def calc_precision_recall(d):
+    """
+    计算分数
+    """
+    d_ves = d["all_chunks"][:10]
+    d_es = d["all_chunks"][10:]
+    all_chunks = []
+    for i in range(len(d_es)):
+        all_chunks.append(d_es[i])
+        all_chunks.append(d_ves[i])
+    all_chunks.extend(d_ves[i+1:])
+    d["all_chunks"] = all_chunks
+    
+    sort_filter_all_chunks_method1(d)
+    NCHUNK = len(d["chunks"])
+    NCHUNK_ALL = len(d["all_chunks"])
+    print('chunks:', NCHUNK, 'all_chunks:', NCHUNK_ALL)
+
+    scores = np.zeros((NCHUNK, NCHUNK_ALL))
+    for j in range(NCHUNK):
+        for i in range(NCHUNK):
+            if d["chunks"][j]["text"] == d["all_chunks"][i]['text']:
+                scores[j][i] = 1
+            elif abs(len(d["chunks"][j]["text"]) - len(d["all_chunks"][i]['text'])) > 10:
+                scores[j][i] = 0
+            elif is_matched(d["chunks"][j]["text"], d["all_chunks"][i]['text']):
+                scores[j][i] = 1
+
+    N_gt = NCHUNK
+    N_all = NCHUNK_ALL
+    if NCHUNK != 0:
+        N_right = sum(scores.max(axis=1))
+    else:
+        N_right = 0
+    recall = 0 if N_gt == 0 else N_right / N_gt
+    return recall, N_gt, N_right, N_all
+
+
+with open('zhaogushu_retriever_gt_convert_key.json', 'r') as f:
+    retriever_gt_list = json.load(f)
+mFieldRecall = 0
+total_N_gt, total_N_right, total_N_all = 0, 0, 0
+D_SCORES = {}
+nquestion = 0
+for d in retriever_gt_list:
+    recall, N_gt, N_right, N_all = calc_precision_recall(d)
+    mFieldRecall += recall
+    D_SCORES[d["question"]] = {'recall': recall}
+    total_N_gt += N_gt
+    total_N_right += N_right
+    total_N_all += N_all
+    nquestion += 1
+
+mFieldRecall = 0 if nquestion == 0 else mFieldRecall / nquestion
+mMethodRecall = total_N_right / total_N_gt
+
+print(f'mFieldRecall: {mFieldRecall * 100:.2f} %')
+print(f'mMethodRecall: {mMethodRecall * 100:.2f} %')
+print(f'total_N_right: {total_N_right}, total_N_gt: {total_N_gt}, total_N_all: {total_N_all}' )
+print(f'nquestion: {nquestion}, mean_N_right: {total_N_right / nquestion}, mean_N_gt: {total_N_gt / nquestion}, mean_N_all: {total_N_all / nquestion}')

bisheng_langchain/rag/run_qa_gen_web.py

@@ -0,0 +1,47 @@
+import os
+import tempfile
+import gradio as gr
+from bisheng_langchain.rag.qa_corpus.qa_generator import RagQAGenerator
+
+
+tmpdir = '/home/public/rag_benchmark_v1.0/tmp'
+if not os.path.exists(tmpdir):
+    os.makedirs(tmpdir)
+
+qa_gen_folder = '/home/public/rag_benchmark_v1.0/rag_qa_gen_demo'
+if not os.path.exists(qa_gen_folder):
+    os.makedirs(qa_gen_folder)
+model_name = "gpt-4-0125-preview"
+unstructured_api_url = "https://bisheng.dataelem.com/api/v1/etl4llm/predict"
+generator = RagQAGenerator(corpus_folder='',
+                           qa_gen_folder=qa_gen_folder, 
+                           unstructured_api_url=unstructured_api_url, 
+                           model_name=model_name)
+
+
+def qa_gen_run(intput_file, gen_qa_num):
+    gen_qa_num = int(gen_qa_num)
+    file_path = intput_file.name
+    output_file = generator.generate_qa_each_file(file_path, train_size=gen_qa_num)
+    return output_file
+
+
+with tempfile.TemporaryDirectory(dir=tmpdir) as tmpdir:
+    with gr.Blocks(css='#margin-top {margin-top: 15px} #center {text-align: center;} #description {text-align: center}') as demo:
+        with gr.Row(elem_id='center'):
+            gr.Markdown('# Bisheng qa auto generation Demo')
+
+        with gr.Row(elem_id = 'description'):
+            gr.Markdown("""Qa generation for anything.""")
+
+        with gr.Row():
+            intput_file = gr.components.File(label='FlowFile')
+            gen_qa_num = gr.Textbox(label='生成的问题数量', value=10, interactive=True, lines=2)
+
+        with gr.Row():
+            with gr.Column():
+                btn0 = gr.Button('Run Qa Gen')
+                out0 = gr.components.File(label='FlowFile')
+                btn0.click(fn=qa_gen_run, inputs=[intput_file, gen_qa_num], outputs=out0)
+                
+        demo.launch(server_name='192.168.106.20', server_port=9118, share=True)

bisheng_langchain/rag/run_rag_evaluate_web.py

@@ -0,0 +1,55 @@
+import os
+import gradio as gr
+import pandas as pd
+from pathlib import Path
+from gradio import components
+from bisheng_langchain.rag.scoring.ragas_score import RagScore
+
+
+save_folder = '/home/public/rag_benchmark_v1.0/rag_score_demo'
+if not os.path.exists(save_folder):
+    os.makedirs(save_folder)
+
+
+def rag_evaluate(excel_file, metric='answer_recall_bisheng', batch_size=5):
+    excel_path = excel_file.name
+    df = pd.read_excel(excel_path)
+    if '问题类型' not in df.columns:
+        df['问题类型'] = len(df['问题'].tolist()) * ['QA']
+    df.to_excel(excel_path, index=False)
+    params = {
+        'excel_path': excel_path,
+        'save_path': save_folder,
+        'question_column': '问题',
+        'query_type_column': '问题类型',
+        'gt_column': '人工标注',
+        'answer_column': '模型回答',
+        'metrics': [metric],
+        'batch_size': int(batch_size),
+    }
+    rag_score = RagScore(**params)
+    rag_score.score()
+
+    output_path = Path(save_folder) / f"{Path(excel_path).stem}_score.xlsx"
+    return str(output_path)
+
+
+if __name__ == '__main__':
+     title = """毕昇QA问答自动评估系统"""
+     with gr.Blocks() as demo:
+        gr.Markdown(title)
+        
+        with gr.Row():
+            with gr.Column(scale=2):
+                with gr.Row():
+                    eval_intput_file = gr.components.File(label='FlowFile')
+                    with gr.Column():
+                        metric_options = ["answer_recall_bisheng", "answer_correctness_bisheng"]
+                        metric = gr.components.Dropdown(label="评估方法", choices=metric_options, default=metric_options[0], interactive=True)
+                        # metric = gr.Textbox(label='评估方法', value='answer_recall_bisheng', interactive=True, lines=2)
+                        batch_size = gr.Textbox(label='批评估大小', value=5, interactive=True, lines=2)
+                btn0 = gr.Button('Run Evaluation')
+            eval_out_file = gr.components.File(label='FlowFile')
+
+        btn0.click(fn=rag_evaluate, inputs=[eval_intput_file, metric, batch_size], outputs=[eval_out_file])
+        demo.queue().launch(share=False, inbrowser=True, server_name="0.0.0.0", server_port=8218)

bisheng_langchain/rag/scoring/llama_index_score.py

@@ -0,0 +1,91 @@
+import os
+import shutil
+
+import httpx
+import nest_asyncio
+import pandas as pd
+
+nest_asyncio.apply()
+from collections import defaultdict
+
+from llama_index import ServiceContext
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.evaluation import CorrectnessEvaluator
+from llama_index.llms import OpenAI
+from tqdm import tqdm
+
+openai_api_key = os.environ.get('OPENAI_API_KEY', '')
+openai_proxy = os.environ.get('OPENAI_PROXY', '')
+
+
+def llama_index_answer_correctness(querys, responses, references):
+    embed = OpenAIEmbedding(api_key=openai_api_key, http_client=httpx.AsyncClient(proxies=openai_proxy))
+
+    model_name = "gpt-3.5-turbo-16k"
+    service_context = ServiceContext.from_defaults(
+        llm=OpenAI(model=model_name, api_key=openai_api_key, http_client=httpx.AsyncClient(proxies=openai_proxy)),
+        embed_model=embed,
+    )
+    evaluator = CorrectnessEvaluator(service_context=service_context)
+
+    correctness_scores = []
+    correctness_feedbacks = []
+    for i in tqdm(range(len(querys))):
+        result = evaluator.evaluate(
+            query=querys[i],
+            response=responses[i],
+            reference=references[i],
+        )
+        correctness = result.score
+        feedback = result.feedback
+        correctness_scores.append(correctness)
+        correctness_feedbacks.append(feedback)
+    return correctness_scores, correctness_feedbacks
+
+
+def rag_benchmark_scoring(excel_file):
+    if not os.path.exists(excel_file + '.bak'):
+        shutil.copy(excel_file, excel_file + '.bak')
+
+    df = pd.read_excel(excel_file)
+    df.dropna(subset=['问题', 'GT', 'rag_answer'], inplace=True)
+    all_questions_info = df.to_dict('records')
+
+    questions = []
+    ground_truths = []
+    answers = []
+    for question_info in all_questions_info:
+        question = question_info['问题']
+        gt = question_info['GT']
+        pred = question_info['rag_answer']
+
+        questions.append(question)
+        answers.append(pred)
+        ground_truths.append(gt)
+
+    correctness_scores, correctness_feedbacks = llama_index_answer_correctness(questions, answers, ground_truths)
+
+    score_map = {
+        'llama_index_correctness': correctness_scores,
+    }
+
+    for metric, scores in score_map.items():
+        df[metric] = df.index.map({i: score for i, score in enumerate(scores)})
+    df.to_excel(excel_file, index=False)
+
+    if '问题类型' in df.columns:
+        grouped_df = (
+            df.groupby('问题类型')
+            .agg({'问题': 'count', **{metric: 'mean' for metric in score_map}})
+            .rename(columns={'问题': '问题数量'})
+        )
+        total_question = grouped_df['问题数量'].sum()
+        grouped_df.loc['all', '问题数量'] = total_question
+        for metric in score_map:
+            grouped_df.loc['all', metric] = df[metric].sum() / total_question
+        return grouped_df
+
+
+if __name__ == '__main__':
+    excel_file = './data/benchmark_v1.0.xlsx'
+    print(rag_benchmark_scoring(excel_file))