bisheng-langchain 0.2.2.2__py3-none-any.whl → 0.2.2.3__py3-none-any.whl

bisheng_langchain/chat_models/host_llm.py

@@ -273,7 +273,7 @@ class BaseHostChatLLM(BaseChatModel):
             try:
                 async with self.client.apost(url=self.host_base_url, json=kwargs) as response:
                     if response.status != 200:
-                        raise ValueError(f'Error: {response.status}')
+                        raise ValueError(f'Error: {response.status} contet: {response.text}')
                     async for txt in response.content.iter_any():
                         if b'\n' in txt:
                             for txt_ in txt.split(b'\n'):
@@ -313,7 +313,7 @@ class BaseHostChatLLM(BaseChatModel):
         """Generate chat completion with retry."""
         message_dicts, params = self._create_message_dicts(messages, stop)
         params = {**params, **kwargs}
-        if self.streaming:
+        if self.streaming and 'infer' not in self.host_base_url:
             inner_completion = ''
             role = 'assistant'
             params['stream'] = True

bisheng_langchain/document_loaders/elem_pdf.py

@@ -2,22 +2,15 @@
 """Loads PDF with semantic splilter."""
 import io
 import json
-import logging
 import os
 import re
-import tempfile
 import time
-from abc import ABC
 from collections import Counter
 from copy import deepcopy
-from pathlib import Path
-from typing import Any, Iterator, List, Mapping, Optional, Union
-from urllib.parse import urlparse
+from typing import List, Optional, Union
 
 import fitz
 import numpy as np
-import pypdfium2
-import requests
 from bisheng_langchain.document_loaders.parsers import LayoutParser
 from langchain.docstore.document import Document
 from langchain.document_loaders.blob_loaders import Blob
@@ -72,8 +65,7 @@ def order_by_tbyx(block_info, th=10):
     for i in range(len(res) - 1):
         for j in range(i, 0, -1):
             # restore the order using the
-            if (abs(res[j + 1][1] - res[j][1]) < th
-                    and (res[j + 1][0] < res[j][0])):
+            if (abs(res[j + 1][1] - res[j][1]) < th and (res[j + 1][0] < res[j][0])):
                 tmp = deepcopy(res[j])
                 res[j] = deepcopy(res[j + 1])
                 res[j + 1] = deepcopy(tmp)
@@ -207,8 +199,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
                  html_output_file: str = None,
                  verbose: bool = False) -> None:
         """Initialize with a file path."""
-        self.layout_parser = LayoutParser(api_key=layout_api_key,
-                                          api_base_url=layout_api_url)
+        self.layout_parser = LayoutParser(api_key=layout_api_key, api_base_url=layout_api_url)
         self.with_columns = with_columns
         self.is_join_table = is_join_table
         self.support_rotate = support_rotate
@@ -286,8 +277,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
         texts = []
         for b in blocks:
             if b[-1] != IMG_BLOCK_TYPE:
-                text = re.sub(RE_MULTISPACE_INCLUDING_NEWLINES, ' ', b[4]
-                              or '').strip()
+                text = re.sub(RE_MULTISPACE_INCLUDING_NEWLINES, ' ', b[4] or '').strip()
                 if text:
                     texts.append(text)
                     text_ploys.append(Rect(b[0], b[1], b[2], b[3]))
@@ -301,8 +291,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
         layout_info = json.loads(layout.page_content)
         for info in layout_info:
             bbs = info['bbox']
-            coords = ((bbs[0], bbs[1]), (bbs[2], bbs[3]), (bbs[4], bbs[5]),
-                      (bbs[6], bbs[7]))
+            coords = ((bbs[0], bbs[1]), (bbs[2], bbs[3]), (bbs[4], bbs[5]), (bbs[6], bbs[7]))
             semantic_polys.append(Polygon(coords))
             semantic_labels.append(info['category_id'])
 
@@ -350,8 +339,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
                 rs = text_rects[ind]
                 ord_ind = np.min(ori_orders)
                 mask[ind] = 1
-                new_block_info.append(
-                    (rect[0], rect[1], rect[2], rect[3], ts, rs, ord_ind))
+                new_block_info.append((rect[0], rect[1], rect[2], rect[3], ts, rs, ord_ind))
 
             elif np.all(mask[start:end] == 0):
                 rect = merge_rects(text_rects[start:end])
@@ -367,16 +355,14 @@ class PDFWithSemanticLoader(BasePDFLoader):
                     rs = rs[arg_ind]
 
                 mask[start:end] = 1
-                new_block_info.append(
-                    (rect[0], rect[1], rect[2], rect[3], ts, rs, ord_ind))
+                new_block_info.append((rect[0], rect[1], rect[2], rect[3], ts, rs, ord_ind))
 
         for i in range(texts_cnt):
             if mask[i] == 0:
                 b = blocks[i]
                 r = np.asarray([b[0], b[1], b[2], b[3]])
                 ord_ind = b[-2]
-                new_block_info.append(
-                    (b[0], b[1], b[2], b[3], [texts[i]], [r], ord_ind))
+                new_block_info.append((b[0], b[1], b[2], b[3], [texts[i]], [r], ord_ind))
 
         if self.with_columns:
             new_blocks = sorted(new_block_info, key=lambda x: x[-1])
@@ -430,8 +416,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
         for label, b in zip(texts_labels, new_blocks):
             if label in effective_class_inds:
                 text = join_lines(b[4], label == TABLE_ID)
-                filtered_blocks.append(
-                    (b[0], b[1], b[2], b[3], text, b[5], label))
+                filtered_blocks.append((b[0], b[1], b[2], b[3], text, b[5], label))
 
         # print('---filtered_blocks---')
         # for b in filtered_blocks:
@@ -539,8 +524,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
                 if (c0 > LINE_FULL_THRESHOLD and c1 < START_THRESHOLD
                         and c2 < SIMI_HEIGHT_THRESHOLD):
                     new_text = join_lines([b0[4], b1[4]])
-                    new_block = (b0[0], b0[1], b0[2], b0[3], new_text, b0[5],
-                                 b0[6])
+                    new_block = (b0[0], b0[1], b0[2], b0[3], new_text, b0[5], b0[6])
                     groups[i - 1][-1] = new_block
                     groups[i].pop(0)
 
@@ -549,8 +533,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
                 c0 = (r1_w - r0_w) / r1_h
                 if c0 < SIMI_WIDTH_THRESHOLD:
                     new_text = join_lines([b0[4], b1[4]], True)
-                    new_block = (b0[0], b0[1], b0[2], b0[3], new_text, b0[5],
-                                 b0[6])
+                    new_block = (b0[0], b0[1], b0[2], b0[3], new_text, b0[5], b0[6])
                     groups[i - 1][-1] = new_block
                     groups[i].pop(0)
 
@@ -559,10 +542,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
         return groups
 
     def save_to_html(self, groups, output_file):
-        styles = [
-            'style="background-color: #EBEBEB;"',
-            'style="background-color: #ABBAEA;"'
-        ]
+        styles = ['style="background-color: #EBEBEB;"', 'style="background-color: #ABBAEA;"']
         idx = 0
         table_style = 'style="border:1px solid black;"'
 
@@ -578,8 +558,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
                         rows = b[4].split('\n')
                         content = []
                         for r in rows:
-                            content.append(
-                                f'<tr><td {table_style}>{r}</td></tr>')
+                            content.append(f'<tr><td {table_style}>{r}</td></tr>')
                         elem_text = '\n'.join(content)
                         text = f'<table {table_style}>{elem_text}</table>'
                     else:
@@ -610,6 +589,7 @@ class PDFWithSemanticLoader(BasePDFLoader):
 
     def load(self) -> List[Document]:
         """Load given path as pages."""
+        import pypdfium2
         blob = Blob.from_path(self.file_path)
         start = self.start
         groups = []

{bisheng_langchain-0.2.2.2.dist-info → bisheng_langchain-0.2.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bisheng-langchain
-Version: 0.2.2.2
+Version: 0.2.2.3
 Summary: bisheng langchain modules
 Home-page: https://github.com/dataelement/bisheng
 Author: DataElem
@@ -26,7 +26,6 @@ Requires-Dist: bisheng-pyautogen
 Requires-Dist: jieba ==0.42.1
 Requires-Dist: pydantic ==1.10.13
 Requires-Dist: pymupdf ==1.23.8
-Requires-Dist: pypdfium2 ==4.25.0
 Requires-Dist: shapely ==2.0.2
 Requires-Dist: filetype ==1.2.0
 

{bisheng_langchain-0.2.2.2.dist-info → bisheng_langchain-0.2.2.3.dist-info}/RECORD

@@ -25,7 +25,7 @@ bisheng_langchain/chains/router/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 bisheng_langchain/chains/router/multi_rule.py,sha256=BiFryj3-7rOxfttD-MyOkKWLCSGB9LVYd2rjOsIfQC8,375
 bisheng_langchain/chains/router/rule_router.py,sha256=R2YRUnwn7s_7DbsSn27uPn4cIV0D-5iXEORXir0tNGM,1835
 bisheng_langchain/chat_models/__init__.py,sha256=A3_KoMRp96UqHwwYX4Mt60peVNjCsuUHAixqaV44BP4,492
-bisheng_langchain/chat_models/host_llm.py,sha256=9FyGwpqd8SPG1UC65NBS12a6_QoRAHUbZ8kH1kFehgo,21576
+bisheng_langchain/chat_models/host_llm.py,sha256=phIBLY7AoGPIIxl9saFV2XAcRvgaJ8XUJA06bqiN8Uw,21638
 bisheng_langchain/chat_models/minimax.py,sha256=JLs_f6vWD9beZYUtjD4FG28G8tZHrGUAWOwdLIuJomw,13901
 bisheng_langchain/chat_models/proxy_llm.py,sha256=wzVBZik9WC3-f7kyQ1eu3Ooibqpcocln08knf5lV1Nw,17082
 bisheng_langchain/chat_models/qwen.py,sha256=jGx_tW-LPxfegE6NvY6wID8ps2SsP813atjXnc04C-s,18841
@@ -44,7 +44,7 @@ bisheng_langchain/document_loaders/__init__.py,sha256=LuQ-zMYxde2FeiEcvVtjQqnHoz
 bisheng_langchain/document_loaders/custom_kv.py,sha256=sUKeK0e8-cCmKyj1FsR7SzBNWjo5zRwHWVS5tKVxuPs,6656
 bisheng_langchain/document_loaders/elem_html.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/document_loaders/elem_image.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bisheng_langchain/document_loaders/elem_pdf.py,sha256=64kUITkrTVJe9CH6IAVSdDVcn2Ekx2PM-jT0cdClXlo,22716
+bisheng_langchain/document_loaders/elem_pdf.py,sha256=K-TXILGNFLFjavhun_MFbUF4t2_WGA3Z-kbnr75lmW8,22243
 bisheng_langchain/document_loaders/elem_unstrcutured_loader.py,sha256=N2jCmizi9gIsAO38zWL1zOzfNhoo_XmNIX7fteSeR0Q,4883
 bisheng_langchain/document_loaders/universal_kv.py,sha256=dJF_GQGKBMUjB_kX9CSp7xZRhXgwVuGPbMIzJwPh-C0,4063
 bisheng_langchain/document_loaders/parsers/__init__.py,sha256=OOM_FJkwaU-zNS58fASw0TH8FNT6VXKb0VrvisgdrII,171
@@ -69,7 +69,7 @@ bisheng_langchain/vectorstores/__init__.py,sha256=zCZgDe7LyQ0iDkfcm5UJ5NxwKQSRHn
 bisheng_langchain/vectorstores/elastic_keywords_search.py,sha256=gt_uw_fSMcEZWxbiA3V0RyA-utLOZlUY-qxdwnsfZks,12664
 bisheng_langchain/vectorstores/milvus.py,sha256=44ZbDsIxdsbUnHOpEpCdrW5zvWnYvDdAVoDKjCFoyYI,34424
 bisheng_langchain/vectorstores/retriever.py,sha256=hj4nAAl352EV_ANnU2OHJn7omCH3nBK82ydo14KqMH4,4353
-bisheng_langchain-0.2.2.2.dist-info/METADATA,sha256=Htg1v32q74rGVPN8Z6uSXLq7IXBSjkcmJT2ak517iJc,2333
-bisheng_langchain-0.2.2.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-bisheng_langchain-0.2.2.2.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
-bisheng_langchain-0.2.2.2.dist-info/RECORD,,
+bisheng_langchain-0.2.2.3.dist-info/METADATA,sha256=9UgVujQ3Ep2eBWGmbeoe8r-dPNqOjPxXHa0_WYSz7pw,2299
+bisheng_langchain-0.2.2.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+bisheng_langchain-0.2.2.3.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
+bisheng_langchain-0.2.2.3.dist-info/RECORD,,