bisheng-langchain 0.3.4.dev3__py3-none-any.whl → 0.3.5.dev1__py3-none-any.whl

bisheng_langchain/document_loaders/elem_unstrcutured_loader.py

@@ -34,7 +34,10 @@ def merge_partitions(partitions):
             elif label == 'Table':
                 doc_content.append('\n\n' + text)
             else:
-                doc_content.append(text_elem_sep + text)
+                if last_label == 'Table':
+                    doc_content.append(text_elem_sep * 2 + text)
+                else:
+                    doc_content.append(text_elem_sep + text)
 
         last_label = label
         metadata['bboxes'].extend(list(map(lambda x: list(map(int, x)), extra_data['bboxes'])))
@@ -73,6 +76,7 @@ class ElemUnstructuredLoader(BasePDFLoader):
         self.start = start
         self.n = n
         self.extra_kwargs = kwargs
+        self.partitions = None
         super().__init__(file_path)
 
     def load(self) -> List[Document]:
@@ -93,18 +97,33 @@ class ElemUnstructuredLoader(BasePDFLoader):
         resp = resp.json()
         if 200 != resp.get('status_code'):
             logger.info(f'file partition {os.path.basename(self.file_name)} error resp={resp}')
+            raise Exception(f'file partition error {os.path.basename(self.file_name)} error resp={resp}')
         partitions = resp['partitions']
-        if not partitions:
-            logger.info(f'partition_error resp={resp}')
+        if partitions:
+            logger.info(f'content_from_partitions')
+            self.partitions = partitions
+            content, metadata = merge_partitions(partitions)
+        elif resp.get('text'):
+            logger.info(f'content_from_text')
+            content = resp['text']
+            metadata = {
+                "bboxes": [],
+                "pages": [],
+                "indexes": [],
+                "types": [],
+            }
+        else:
+            logger.warning(f'content_is_empty resp={resp}')
+            content = ''
+            metadata = {}
+
         logger.info(f'unstruct_return code={resp.get("status_code")}')
 
         if resp.get('b64_pdf'):
             with open(self.file_path, 'wb') as f:
                 f.write(base64.b64decode(resp['b64_pdf']))
 
-        content, metadata = merge_partitions(partitions)
         metadata['source'] = self.file_name
-
         doc = Document(page_content=content, metadata=metadata)
         return [doc]
 

bisheng_langchain/gpts/tools/api_tools/sina.py

@@ -145,7 +145,10 @@ class StockInfo(APIToolBase):
             if resp.status_code != 200:
                 logger.info('api_call_fail res={}', resp.text)
             k_data = resp.text
-            data_array = json.loads(kLinePattern.search(k_data).group(1))
+            k_data = kLinePattern.search(k_data)
+            if not k_data:
+                return '{}'
+            data_array = json.loads(k_data.group(1))
             for item in data_array:
                 if item.get('day') == date:
                     return json.dumps(item)
@@ -173,7 +176,10 @@ class StockInfo(APIToolBase):
             count = datetime.today() - date_obj
             url = self.url.format(stockName=stock_number, stock=stock, count=count.days)
             k_data = await self.async_client.aget(url)
-            data_array = json.loads(kLinePattern.search(k_data).group(1))
+            k_data = kLinePattern.search(k_data)
+            if not k_data:
+                return '{}'
+            data_array = json.loads(k_data.group(1))
             for item in data_array:
                 if item.get('day') == date:
                     return json.dumps(item)

bisheng_langchain/text_splitter.py

@@ -21,17 +21,22 @@ logger = logging.getLogger(__name__)
 
 
 def _split_text_with_regex(
-    text: str, separator: str, keep_separator: bool
+        text: str, separator: str, keep_separator: bool, separator_rule: str
 ) -> List[str]:
     # Now that we have the separator, split the text
     if separator:
         if keep_separator:
             # The parentheses in the pattern keep the delimiters in the result.
             _splits = re.split(f'({separator})', text)
-            splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)]
-            if len(_splits) % 2 == 0:
-                splits += _splits[-1:]
-            splits = [_splits[0]] + splits
+
+            if separator_rule == "before":
+                splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)]
+                if len(_splits) % 2 == 0:
+                    splits += _splits[-1:]
+                splits = [_splits[0]] + splits
+            else:
+                splits = [_splits[i-1] + _splits[i] for i in range(1, len(_splits), 2)]
+                splits = splits + [_splits[-1]]
         else:
             splits = re.split(separator, text)
     else:
@@ -82,11 +87,14 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
     """
     todo
     """
+
     def __init__(
-        self,
-        separators: Optional[List[str]] = None,
-        keep_separator: bool = True,
-        **kwargs: Any,
+            self,
+            separators: Optional[List[str]] = None,
+            separator_rule: Optional[List[str]] = None,
+            is_separator_regex: bool = False,
+            keep_separator: bool = True,
+            **kwargs: Any,
     ) -> None:
         """Create a new TextSplitter."""
         super().__init__(
@@ -95,7 +103,9 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
             **kwargs
         )
         self._separators = separators or ['\n\n', '\n', ' ', '']
-        self._is_separator_regex = False
+        self._separator_rule = separator_rule or ['after' for _ in range(4)]
+        self.separator_rule = {one: self._separator_rule[index] for index, one in enumerate(separators)}
+        self._is_separator_regex = is_separator_regex
 
     def split_documents(self, documents: Iterable[Document]) -> List[Document]:
         texts, metadatas = [], []
@@ -110,19 +120,21 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
         final_chunks = []
         # Get appropriate separator to use
         separator = separators[-1]
+        separator_rule = 'after'
         new_separators = []
         for i, _s in enumerate(separators):
             _separator = _s if self._is_separator_regex else re.escape(_s)
+            separator_rule = self.separator_rule[_s]
             if _s == '':
                 separator = _s
                 break
             if re.search(_separator, text):
                 separator = _s
-                new_separators = separators[i + 1 :]
+                new_separators = separators[i + 1:]
                 break
 
         _separator = separator if self._is_separator_regex else re.escape(separator)
-        splits = _split_text_with_regex(text, _separator, self._keep_separator)
+        splits = _split_text_with_regex(text, _separator, self._keep_separator, separator_rule)
 
         # Now go merging things, recursively splitting longer texts.
         _good_splits = []
@@ -149,60 +161,60 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
         return self._split_text(text, self._separators)
 
     def create_documents(
-        self, texts: List[str], metadatas: Optional[List[dict]] = None
+            self, texts: List[str], metadatas: Optional[List[dict]] = None
     ) -> List[Document]:
         """Create documents from a list of texts."""
         documents = []
         for i, text in enumerate(texts):
             index = -1
             # metadata = copy.deepcopy(_metadatas[i])
-            indexes = metadatas[i]['indexes']
-            pages = metadatas[i]['pages']
-            types = metadatas[i]['types']
-            bboxes = metadatas[i]['bboxes']
+            indexes = metadatas[i].get('indexes', [])
+            pages = metadatas[i].get('pages', [])
+            types = metadatas[i].get('types', [])
+            bboxes = metadatas[i].get('bboxes', [])
             searcher = IntervalSearch(indexes)
             split_texts = self.split_text(text)
             for chunk in split_texts:
                 new_metadata = copy.deepcopy(metadatas[i])
-                index = text.find(chunk, index + 1)
-                inter0 = [index, index + len(chunk) - 1]
-                norm_inter = searcher.find(inter0)
-                new_metadata['chunk_bboxes'] = []
-                for j in range(norm_inter[0], norm_inter[1] + 1):
-                    new_metadata['chunk_bboxes'].append(
-                        {'page': pages[j], 'bbox': bboxes[j]})
-
-                c = Counter([types[j] for j in norm_inter])
-                chunk_type = c.most_common(1)[0][0]
-                new_metadata['chunk_type'] = chunk_type
-                new_metadata['source'] = metadatas[i].get('source', '')
-
-
-            # for chunk in split_texts:
-            #     new_metadata = {}
-            #     new_metadata['chunk_type'] = metadata.get('chunk_type', 'paragraph')
-            #     new_metadata['bboxes'] = metadata.get('bboxes', [])
-            #     new_metadata['source'] = metadata.get('source', '')
-            #     # chunk's start index in text
-            #     index = text.find(chunk, index + 1)
-            #     new_metadata['start'] = metadata.get('start', 0) + index
-            #     new_metadata['end'] = metadata.get('start', 0) + index + len(chunk) - 1
-
-            #     if 'page' in metadata:
-            #         new_metadata['page'] = metadata['page'][new_metadata['start']:new_metadata['end']+1]
-            #     if 'token_to_bbox' in metadata:
-            #         new_metadata['token_to_bbox'] = metadata['token_to_bbox'][new_metadata['start']:new_metadata['end']+1]
-
-            #     if 'page' in new_metadata and 'token_to_bbox' in new_metadata:
-            #         box_no_duplicates = set()
-            #         for index in range(len(new_metadata['page'])):
-            #             box_no_duplicates.add(
-            #                 (new_metadata['page'][index], new_metadata['token_to_bbox'][index]))
-
-            #         new_metadata['chunk_bboxes'] = []
-            #         for elem in box_no_duplicates:
-            #             new_metadata['chunk_bboxes'].append(
-            #                 {'page': elem[0], 'bbox': new_metadata['bboxes'][elem[1]]})
+                if indexes and bboxes:
+                    index = text.find(chunk, index + 1)
+                    inter0 = [index, index + len(chunk) - 1]
+                    norm_inter = searcher.find(inter0)
+                    new_metadata['chunk_bboxes'] = []
+                    for j in range(norm_inter[0], norm_inter[1] + 1):
+                        new_metadata['chunk_bboxes'].append(
+                            {'page': pages[j], 'bbox': bboxes[j]})
+
+                    c = Counter([types[j] for j in norm_inter])
+                    chunk_type = c.most_common(1)[0][0]
+                    new_metadata['chunk_type'] = chunk_type
+                    new_metadata['source'] = metadatas[i].get('source', '')
+
+                # for chunk in split_texts:
+                #     new_metadata = {}
+                #     new_metadata['chunk_type'] = metadata.get('chunk_type', 'paragraph')
+                #     new_metadata['bboxes'] = metadata.get('bboxes', [])
+                #     new_metadata['source'] = metadata.get('source', '')
+                #     # chunk's start index in text
+                #     index = text.find(chunk, index + 1)
+                #     new_metadata['start'] = metadata.get('start', 0) + index
+                #     new_metadata['end'] = metadata.get('start', 0) + index + len(chunk) - 1
+
+                #     if 'page' in metadata:
+                #         new_metadata['page'] = metadata['page'][new_metadata['start']:new_metadata['end']+1]
+                #     if 'token_to_bbox' in metadata:
+                #         new_metadata['token_to_bbox'] = metadata['token_to_bbox'][new_metadata['start']:new_metadata['end']+1]
+
+                #     if 'page' in new_metadata and 'token_to_bbox' in new_metadata:
+                #         box_no_duplicates = set()
+                #         for index in range(len(new_metadata['page'])):
+                #             box_no_duplicates.add(
+                #                 (new_metadata['page'][index], new_metadata['token_to_bbox'][index]))
+
+                #         new_metadata['chunk_bboxes'] = []
+                #         for elem in box_no_duplicates:
+                #             new_metadata['chunk_bboxes'].append(
+                #                 {'page': elem[0], 'bbox': new_metadata['bboxes'][elem[1]]})
 
                 new_doc = Document(page_content=chunk, metadata=new_metadata)
                 documents.append(new_doc)

bisheng_langchain/vectorstores/milvus.py

@@ -10,6 +10,7 @@ from langchain.docstore.document import Document
 from langchain.embeddings.base import Embeddings
 from langchain.vectorstores.utils import maximal_marginal_relevance
 from langchain_community.vectorstores.milvus import Milvus as MilvusLangchain
+from pymilvus.exceptions import ConnectionNotExistException
 
 logger = logging.getLogger(__name__)
 
@@ -231,7 +232,7 @@ class Milvus(MilvusLangchain):
         from pymilvus import connections
         connections.remove_connection(using)
 
-    def _create_connection_alias(self, connection_args: dict) -> str:
+    def _create_connection_alias(self, connection_args: dict, personal_alias: str = None) -> str:
         """Create the connection to the Milvus server."""
         from pymilvus import MilvusException, connections
 
@@ -269,7 +270,10 @@ class Milvus(MilvusLangchain):
                     return con[0]
 
         # Generate a new connection if one doesn't exist
-        alias = uuid4().hex
+        if personal_alias:
+            alias = personal_alias
+        else:
+            alias = uuid4().hex
         try:
             connections.connect(alias=alias, **connection_args)
             logger.debug('Created new connection using: %s', alias)
@@ -522,7 +526,14 @@ class Milvus(MilvusLangchain):
             insert_list = [insert_dict[x][i:end] for x in self.fields if x in insert_dict]
             # Insert into the collection.
             try:
-                res: Collection
+                res = self.col.insert(insert_list, timeout=timeout, **kwargs)
+                pks.extend(res.primary_keys)
+            except ConnectionNotExistException as e:
+                logger.warning("retrying connection to milvus")
+                # reconnect to milvus
+                self._create_connection_alias(self.connection_args, self.alias)
+
+                # insert data
                 res = self.col.insert(insert_list, timeout=timeout, **kwargs)
                 pks.extend(res.primary_keys)
             except MilvusException as e:

{bisheng_langchain-0.3.4.dev3.dist-info → bisheng_langchain-0.3.5.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bisheng-langchain
-Version: 0.3.4.dev3
+Version: 0.3.5.dev1
 Summary: bisheng langchain modules
 Home-page: https://github.com/dataelement/bisheng
 Author: DataElem
@@ -30,7 +30,7 @@ Requires-Dist: shapely==2.0.2
 Requires-Dist: filetype==1.2.0
 Requires-Dist: langgraph==0.0.50
 Requires-Dist: openai==1.14.3
-Requires-Dist: langchain-openai==0.1.0
+Requires-Dist: langchain-openai==0.1.5
 Requires-Dist: llama-index==0.9.48
 Requires-Dist: bisheng-ragas==1.0.0
 

{bisheng_langchain-0.3.4.dev3.dist-info → bisheng_langchain-0.3.5.dev1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 bisheng_langchain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bisheng_langchain/text_splitter.py,sha256=8snY_Fojh-A1EEGXBSEqCh0N77KQc_dqsgeptAlf344,7934
+bisheng_langchain/text_splitter.py,sha256=yYpzMa0c1tRZNGyuFFJsLvSeqberYHV2-BR28pQim8I,8794
 bisheng_langchain/agents/__init__.py,sha256=ctsKj77fS8qlkhz_9sS_AhCjFvFNxEpJ9KBYVrApLRg,226
 bisheng_langchain/agents/chatglm_functions_agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/agents/chatglm_functions_agent/base.py,sha256=tyytq0XIFXpfxDP0s5QKeprKOunMqi1fHMfQ0-kOmDE,13674
@@ -54,7 +54,7 @@ bisheng_langchain/document_loaders/custom_kv.py,sha256=xWUPhcr1hjbdya4zgEHG4Fl0s
 bisheng_langchain/document_loaders/elem_html.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/document_loaders/elem_image.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/document_loaders/elem_pdf.py,sha256=K-TXILGNFLFjavhun_MFbUF4t2_WGA3Z-kbnr75lmW8,22243
-bisheng_langchain/document_loaders/elem_unstrcutured_loader.py,sha256=FtoyfmE85CwZuMvr52_bqcHQCgypKCWMGwLZrzgQYbY,7353
+bisheng_langchain/document_loaders/elem_unstrcutured_loader.py,sha256=JW87AhzCY_KS_YYszyxU3GgPjxP4vWOHDfifJEpP5CI,8055
 bisheng_langchain/document_loaders/universal_kv.py,sha256=ZdIgFIc2fH2kkvJNb7j2wi6FLS_PaaatVy6z_YNV2hw,4114
 bisheng_langchain/document_loaders/parsers/__init__.py,sha256=OOM_FJkwaU-zNS58fASw0TH8FNT6VXKb0VrvisgdrII,171
 bisheng_langchain/document_loaders/parsers/ellm_client.py,sha256=Y_CRYwBr-gFArOirF1b76KyI5N8eVpsLeDiIsKtYkpU,1641
@@ -93,7 +93,7 @@ bisheng_langchain/gpts/tools/api_tools/base.py,sha256=fWQSDIOVb4JZrtJ9ML9q2ycsAa
 bisheng_langchain/gpts/tools/api_tools/flow.py,sha256=ot2YAYgQGWgUpb2nCECAmpqHY6m0SgzwkupF9kDT3lU,2461
 bisheng_langchain/gpts/tools/api_tools/macro_data.py,sha256=FyG-qtl2ECS1CDKt6olN0eDTDM91d-UvDkMDBiVLgYQ,27429
 bisheng_langchain/gpts/tools/api_tools/openapi.py,sha256=CzKt9FRkgngBcWgabD4emPqAXkAgagkD-pMjG680MTE,3903
-bisheng_langchain/gpts/tools/api_tools/sina.py,sha256=GGA4ZYvNEpqBZ_l8MUYqgkI8xZe9XcGa9-KlHZVqr6I,9542
+bisheng_langchain/gpts/tools/api_tools/sina.py,sha256=4KpK7_HUUtjpdJ-K4LjPlb-occyAZcRtmmCWqJ2BotE,9708
 bisheng_langchain/gpts/tools/api_tools/tianyancha.py,sha256=abDAz-yAH1-2rKiSmZ6TgnrNUnpgAZpDY8oDiWfWapc,6684
 bisheng_langchain/gpts/tools/bing_search/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bisheng_langchain/gpts/tools/bing_search/tool.py,sha256=v_VlqcMplITA5go5qWA4qZ5p43E1-1s0bzmyY7H0hqY,1710
@@ -151,9 +151,9 @@ bisheng_langchain/utils/azure_dalle_image_generator.py,sha256=96-_nO4hDSwyPE4rSY
 bisheng_langchain/utils/requests.py,sha256=vWGKyNTxApVeaVdKxqACfIT1Q8wMy-jC3kUv2Ce9Mzc,8688
 bisheng_langchain/vectorstores/__init__.py,sha256=zCZgDe7LyQ0iDkfcm5UJ5NxwKQSRHnqrsjx700Fy11M,213
 bisheng_langchain/vectorstores/elastic_keywords_search.py,sha256=Pm1rS50GJ0HWbjBsFDgs28SVuVbjGSRPOor6yJlnE7w,13347
-bisheng_langchain/vectorstores/milvus.py,sha256=8HHbIxoSbLYDFlFJSfmjLOfqGpOSZd24iVYWSYz3TX0,36637
+bisheng_langchain/vectorstores/milvus.py,sha256=xh7NokraKg_Xc9ofz0RVfJ_I36ftnprLJtV-1NfaeyQ,37162
 bisheng_langchain/vectorstores/retriever.py,sha256=hj4nAAl352EV_ANnU2OHJn7omCH3nBK82ydo14KqMH4,4353
-bisheng_langchain-0.3.4.dev3.dist-info/METADATA,sha256=lPUBJcKemYQziXamiJz2lVpuYDlM0H-RBOUiH5sq_rc,2476
-bisheng_langchain-0.3.4.dev3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-bisheng_langchain-0.3.4.dev3.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
-bisheng_langchain-0.3.4.dev3.dist-info/RECORD,,
+bisheng_langchain-0.3.5.dev1.dist-info/METADATA,sha256=Q20qBElwEheYunRPAoIvCRj8jH4RrXId03MA-SA6JnE,2476
+bisheng_langchain-0.3.5.dev1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+bisheng_langchain-0.3.5.dev1.dist-info/top_level.txt,sha256=Z6pPNyCo4ihyr9iqGQbH8sJiC4dAUwA_mAyGRQB5_Fs,18
+bisheng_langchain-0.3.5.dev1.dist-info/RECORD,,