bisheng-langchain 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

bisheng_langchain/document_loaders/elem_unstrcutured_loader.py

@@ -64,6 +64,10 @@ class ElemUnstructuredLoader(BasePDFLoader):
                  file_path: str,
                  unstructured_api_key: str = None,
                  unstructured_api_url: str = None,
+                 force_ocr: bool = False,
+                 enable_formular: bool = True,
+                 filter_page_header_footer: bool = False,
+                 ocr_sdk_url: str = None,
                  start: int = 0,
                  n: int = None,
                  verbose: bool = False,
@@ -71,6 +75,10 @@ class ElemUnstructuredLoader(BasePDFLoader):
         """Initialize with a file path."""
         self.unstructured_api_url = unstructured_api_url
         self.unstructured_api_key = unstructured_api_key
+        self.force_ocr = force_ocr
+        self.enable_formular = enable_formular
+        self.filter_page_header_footer = filter_page_header_footer
+        self.ocr_sdk_url = ocr_sdk_url,
         self.headers = {'Content-Type': 'application/json'}
         self.file_name = file_name
         self.start = start
@@ -84,9 +92,13 @@ class ElemUnstructuredLoader(BasePDFLoader):
         b64_data = base64.b64encode(open(self.file_path, 'rb').read()).decode()
         parameters = {'start': self.start, 'n': self.n}
         parameters.update(self.extra_kwargs)
+        # TODO: add filter_page_header_footer into payload when elt4llm is ready.
         payload = dict(filename=os.path.basename(self.file_name),
                        b64_data=[b64_data],
                        mode='partition',
+                       force_ocr=self.force_ocr,
+                       enable_formula=self.enable_formular,
+                       ocr_sdk_url=self.ocr_sdk_url,
                        parameters=parameters)
 
         resp = requests.post(self.unstructured_api_url, headers=self.headers, json=payload)

bisheng_langchain/gpts/tools/code_interpreter/tool.py

@@ -183,19 +183,20 @@ def upload_minio(
     import minio
 
     minio_client = minio.Minio(
-        endpoint=param.get('MINIO_ENDPOINT'),
-        access_key=param.get('MINIO_ACCESS_KEY'),
-        secret_key=param.get('MINIO_SECRET_KEY'),
-        secure=param.get('SCHEMA'),
-        cert_check=param.get('CERT_CHECK'),
+        endpoint=param.get('endpoint'),
+        access_key=param.get('access_key'),
+        secret_key=param.get('secret_key'),
+        secure=param.get('schema'),
+        cert_check=param.get('cert_check'),
     )
     minio_share = minio.Minio(
-        endpoint=param.get('MINIO_SHAREPOIN'),
-        access_key=param.get('MINIO_ACCESS_KEY'),
-        secret_key=param.get('MINIO_SECRET_KEY'),
-        secure=param.get('SCHEMA'),
-        cert_check=param.get('CERT_CHECK'),
+        endpoint=param.get('sharepoint'),
+        access_key=param.get('access_key'),
+        secret_key=param.get('secret_key'),
+        secure=param.get('schema'),
+        cert_check=param.get('cert_check'),
     )
+    bucket = param.get('tmp_bucket', 'tmp-dir')
     logger.debug(
         'upload_file obj={} bucket={} file_paht={}',
         object_name,

bisheng_langchain/rag/extract_info.py

@@ -1,5 +1,5 @@
 from bisheng_langchain.chat_models import ChatQWen
-from langchain.chains import LLMChain
+from langchain.chains.llm import LLMChain
 from langchain.prompts.chat import (
     ChatPromptTemplate,
     SystemMessagePromptTemplate,
@@ -15,22 +15,32 @@ human_template = """
 """
 
 messages = [
-        SystemMessagePromptTemplate.from_template(system_template),
-        HumanMessagePromptTemplate.from_template(human_template),
-    ]
+    SystemMessagePromptTemplate.from_template(system_template),
+    HumanMessagePromptTemplate.from_template(human_template),
+]
 title_extract_prompt = ChatPromptTemplate.from_messages(messages)
 
 
-def extract_title(llm, text, max_length=7000) -> str:
-    chain = LLMChain(llm=llm, prompt=title_extract_prompt)
+def extract_title(llm, text, max_length=7000, abstract_prompt: str = None) -> str:
+    """
+    此方法在bisheng_langchain模型的还有两处调用用，在不能提供abstract_propmpt的情况下
+    使用原来现有提示词.
+    """
+    if abstract_prompt:
+        updated_messages = [
+            SystemMessagePromptTemplate.from_template(abstract_prompt),
+            HumanMessagePromptTemplate.from_template(human_template),
+        ]
+        updated_title_extract_prompt = ChatPromptTemplate.from_messages(updated_messages)
+        chain = LLMChain(llm=llm, prompt=updated_title_extract_prompt)
+    else:
+        chain = LLMChain(llm=llm, prompt=title_extract_prompt)
     ans = chain.run(context=text[:max_length])
-    return ans 
+    return ans
 
 
-if __name__ == '__main__':
-    llm = ChatQWen(model_name='qwen1.5-72b-chat', 
-                   api_key='', 
-                   temperature=0.01)
+if __name__ == "__main__":
+    llm = ChatQWen(model_name="qwen1.5-72b-chat", api_key="", temperature=0.01)
     text = "江苏蔚蓝锂芯股份有限公司\n2021 年年度报告 \n2022 年 03 月\n\n 第一节 重要提示、目录和释义\n公司董事会、监事会及董事、监事、高级管理人员保证年度报告内容的真实、准确、完整，不存在虚假记载、误导性陈述或重大遗漏，并承担个别和连带的法律责任。\n公司负责人 CHEN KAI、主管会计工作负责人林文华及会计机构负责人(会计主管人员)张宗红声明：保证本年度报告中财务报告的真实、准确、完整。\n所有董事均已出席了审议本报告的董事会会议。"
     ans = extract_title(llm, text)
-    print(ans)
+    print(ans)

bisheng_langchain/text_splitter.py

@@ -106,6 +106,7 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
         self._separator_rule = separator_rule or ['after' for _ in range(4)]
         self.separator_rule = {one: self._separator_rule[index] for index, one in enumerate(separators)}
         self._is_separator_regex = is_separator_regex
+        self._chunk_overlap = kwargs.get('chunk_overlap', 0)
 
     def split_documents(self, documents: Iterable[Document]) -> List[Document]:
         texts, metadatas = [], []
@@ -167,7 +168,6 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
         documents = []
         for i, text in enumerate(texts):
             index = -1
-            # metadata = copy.deepcopy(_metadatas[i])
             indexes = metadatas[i].get('indexes', [])
             pages = metadatas[i].get('pages', [])
             types = metadatas[i].get('types', [])
@@ -215,7 +215,7 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
                 #         for elem in box_no_duplicates:
                 #             new_metadata['chunk_bboxes'].append(
                 #                 {'page': elem[0], 'bbox': new_metadata['bboxes'][elem[1]]})
-
                 new_doc = Document(page_content=chunk, metadata=new_metadata)
+                prev_document = new_doc
                 documents.append(new_doc)
         return documents

{bisheng_langchain-1.2.1.dist-info → bisheng_langchain-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bisheng-langchain
-Version: 1.2.1
+Version: 1.3.0
 Summary: bisheng langchain modules
 Home-page: https://github.com/dataelement/bisheng
 Author: DataElem

{bisheng_langchain-1.2.1.dist-info → bisheng_langchain-1.3.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 bisheng_langchain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bisheng_langchain/text_splitter.py,sha256=yYpzMa0c1tRZNGyuFFJsLvSeqberYHV2-BR28pQim8I,8794
+bisheng_langchain/text_splitter.py,sha256=HJFGo6g_JGxWCAat98VIlKAKfaIYaR6f-g7SJwvhcRM,8840
 bisheng_langchain/agents/__init__.py,sha256=ctsKj77fS8qlkhz_9sS_AhCjFvFNxEpJ9KBYVrApLRg,226
 bisheng_langchain/agents/chatglm_functions_agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/agents/chatglm_functions_agent/base.py,sha256=IxPf9_atUKy8UMgIDYrgmWBkrVfOLdyJSDfVcD_rsDg,13724
@@ -54,7 +54,7 @@ bisheng_langchain/document_loaders/custom_kv.py,sha256=-7h7QqGUFPhpNYAUZBDmkr_pD
 bisheng_langchain/document_loaders/elem_html.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/document_loaders/elem_image.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/document_loaders/elem_pdf.py,sha256=WpRIStBl1DUDa0NCd594gKU2NsgURRv5jnKSM71ZPI8,22273
-bisheng_langchain/document_loaders/elem_unstrcutured_loader.py,sha256=SA02ClxYb7DSfBdAA9bYT8oxMETTsp5GdHPkAH1Wkk8,8075
+bisheng_langchain/document_loaders/elem_unstrcutured_loader.py,sha256=AIKnApzovohheEJFMolE7J-tGY_ClUfiGBb-SwYuC60,8697
 bisheng_langchain/document_loaders/universal_kv.py,sha256=7z19Z_NwtILmtkbIURf4qMyEJGjlE-5CkhqF2KFGc7I,4134
 bisheng_langchain/document_loaders/parsers/__init__.py,sha256=OOM_FJkwaU-zNS58fASw0TH8FNT6VXKb0VrvisgdrII,171
 bisheng_langchain/document_loaders/parsers/ellm_client.py,sha256=Y_CRYwBr-gFArOirF1b76KyI5N8eVpsLeDiIsKtYkpU,1641
@@ -104,7 +104,7 @@ bisheng_langchain/gpts/tools/bing_search/tool.py,sha256=FlaeNEiOO52YjxpXu62efaMH
 bisheng_langchain/gpts/tools/calculator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/gpts/tools/calculator/tool.py,sha256=5FFL3YAYGQqC2L7zFP3LK6zApZ4GFI9MjUa5VdQ9nvY,695
 bisheng_langchain/gpts/tools/code_interpreter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bisheng_langchain/gpts/tools/code_interpreter/tool.py,sha256=WK8FX1wLS8896rResE8jSZ7L5SyjAuaGiw3SWRocdQM,11428
+bisheng_langchain/gpts/tools/code_interpreter/tool.py,sha256=oUhlJzn5sQzk-GTxyzq_i89Is2IYaO4hoIYcoo9N_e4,11441
 bisheng_langchain/gpts/tools/dalle_image_generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/gpts/tools/dalle_image_generator/tool.py,sha256=m1c_1pVJ3cM8Qrp0wDXEgEQ_9XJo_CyCmcwRayBsIYE,7492
 bisheng_langchain/gpts/tools/get_current_time/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -126,7 +126,7 @@ bisheng_langchain/rag/bisheng_rag_chain.py,sha256=75HCIkUAMGrD1E6BaSA9dhnug137PZ
 bisheng_langchain/rag/bisheng_rag_pipeline.py,sha256=neoBK3TtuQ07_WeuJCzYlvtsDQNepUa_68NT8VCgytw,13749
 bisheng_langchain/rag/bisheng_rag_pipeline_v2.py,sha256=iOoF7mbLp9qDGPsV0fEmgph_Ba8VnECYvCPebXk8xmo,16144
 bisheng_langchain/rag/bisheng_rag_tool.py,sha256=JAxsoASwaCaGHrFlAylYOZQZ9ZdnMcfYvAaZZKvIz0g,13676
-bisheng_langchain/rag/extract_info.py,sha256=jtZ4Bchjv4tOaayC2MnkV-lLu3vDA0Hsk_S-ATni34g,1695
+bisheng_langchain/rag/extract_info.py,sha256=QEFlAaC_9-87dl4BtTz9ciQaQhfE_XjWUnQrwnUfRiw,2215
 bisheng_langchain/rag/run_qa_gen_web.py,sha256=-fIvHNnD3lD0iNU5m0Me1GDwRjlcsB8tE5RnPtFRG2s,1840
 bisheng_langchain/rag/run_rag_evaluate_web.py,sha256=a9vMhq-ZhEiHHr43uKUzKtjdk280uAP_UHQW_eOaQMw,2224
 bisheng_langchain/rag/utils.py,sha256=ecl4sDR8iUrVCBRPAAT0hZOHkH50-TLS3567GLP1sRM,7122
@@ -166,7 +166,7 @@ bisheng_langchain/vectorstores/__init__.py,sha256=zCZgDe7LyQ0iDkfcm5UJ5NxwKQSRHn
 bisheng_langchain/vectorstores/elastic_keywords_search.py,sha256=BxvT9FUTju4AZPtQFTbYLmIIKKw8bqcact5Cav_5H2I,15357
 bisheng_langchain/vectorstores/milvus.py,sha256=jWq_lce-ihOz07D1kwj5ctPzElYexNCjJ-xSv-pK1CI,37172
 bisheng_langchain/vectorstores/retriever.py,sha256=fNtk8qSwBo2Qrlt1NpZVXaNATW2tBywkyS0q0NtN5MI,4326
-bisheng_langchain-1.2.1.dist-info/METADATA,sha256=DFifMhbexNo46m3h6BXMn9LwUnpndls5v_X51yyu7Yk,2435
-bisheng_langchain-1.2.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-bisheng_langchain-1.2.1.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
-bisheng_langchain-1.2.1.dist-info/RECORD,,
+bisheng_langchain-1.3.0.dist-info/METADATA,sha256=Osl1j2o-wDqBmnqjsqVEWW-sNmiNHr-mTmRpHYbFy_g,2435
+bisheng_langchain-1.3.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+bisheng_langchain-1.3.0.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
+bisheng_langchain-1.3.0.dist-info/RECORD,,