orbitkit 0.8.34__tar.gz → 0.8.36__tar.gz

{orbitkit-0.8.34/orbitkit.egg-info → orbitkit-0.8.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: orbitkit
-Version: 0.8.34
+Version: 0.8.36
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.36/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.36

{orbitkit-0.8.34 → orbitkit-0.8.36}/orbitkit/pdf_extractor/pdf_extractor_netmind_v3.py

@@ -17,6 +17,7 @@ import fitz  # PyMuPDF
 import os
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor, as_completed
+
 logger = logging.getLogger(__name__)
 
 
@@ -53,8 +54,8 @@ class PdfExtractorNetmindFileAnalysis:
                                       aws_access_key_id=self.aws_access_key_id,
                                       aws_secret_access_key=self.aws_secret_access_key)
         self.s3_resource = boto3.resource('s3',
-                                        aws_access_key_id=self.aws_access_key_id,
-                                        aws_secret_access_key=self.aws_secret_access_key)
+                                          aws_access_key_id=self.aws_access_key_id,
+                                          aws_secret_access_key=self.aws_secret_access_key)
 
     def upload_file_to_s3(self, local_key: str, remote_key: str):
         _remote_key = f'{self.bucket_tmp_group}{remote_key}'
@@ -146,16 +147,16 @@ class PdfExtractorNetmindExtract:
         s3_path_obj = s3_split_path(self.s3_path)
         presigned_url = self._generate_presigned_url(s3_path_obj)
         logger.warning("Get presigned_url successfully...")
-        json_response = self.get_netmind_response(presigned_url)
-        return self._save_json_to_s3(json_response, s3_path_obj)
+        api_response_time, json_response = self.get_netmind_response(presigned_url)
+        return api_response_time, self._save_json_to_s3(json_response, s3_path_obj)
 
     def _generate_presigned_url(self, s3_path_obj):
         return self.s3_client.generate_presigned_url('get_object',
-                                                      Params={
-                                                          'Bucket': s3_path_obj["bucket"],
-                                                          'Key': s3_path_obj["store_path"]
-                                                      },
-                                                      ExpiresIn=604800)
+                                                     Params={
+                                                         'Bucket': s3_path_obj["bucket"],
+                                                         'Key': s3_path_obj["store_path"]
+                                                     },
+                                                     ExpiresIn=604800)
 
     def get_netmind_response(self, presigned_url):
         start = time.time()
@@ -165,8 +166,9 @@ class PdfExtractorNetmindExtract:
         )
         # 状态检查
         response.raise_for_status()
-        logger.info(f"Extract text by using Netmind successfully: {time.time() - start}")
-        return response.json()
+        api_response_time = time.time() - start
+        logger.info(f"Extract text by using Netmind successfully: {api_response_time}")
+        return api_response_time, response.json()
 
     def _save_json_to_s3(self, json_data, s3_path_obj):
         json_key = f"{s3_path_obj['store_path']}.json"  # 生成 JSON 文件名
@@ -176,12 +178,12 @@ class PdfExtractorNetmindExtract:
             input_folder = os.path.join(tmp_dir, 'input')
             if not os.path.exists(input_folder):
                 os.makedirs(input_folder)
-            local_key = os.path.join(input_folder, local_name) # 临时文件路径
+            local_key = os.path.join(input_folder, local_name)  # 临时文件路径
             # 将 JSON 数据写入临时文件
             with open(local_key, 'w') as json_file:
                 json_file.write(json_content)
             # 上传到 S3
-            self.upload_file_to_s3(s3_path_obj['bucket'],local_key, json_key)
+            self.upload_file_to_s3(s3_path_obj['bucket'], local_key, json_key)
         return f"s3://{s3_path_obj['bucket']}/{json_key}"
 
     def upload_file_to_s3(self, bucket, local_key: str, remote_key: str):
@@ -189,13 +191,13 @@ class PdfExtractorNetmindExtract:
         logger.warning(f"File {local_key} Uploaded To s3://{bucket}/{remote_key}")
 
 
-
 class PdfExtractorNetmindMerge:
     def __init__(self,
                  source_s3_path: str = None,
                  temp_folder: Optional[str] = None,
                  s3_util: Optional[S3Util] = None,
                  txt_vector: str = 'txt-vector',
+                 is_page_number_discontinuity_exception_thrown: bool = False, #页码不连续异常抛出
                  slice_option: Optional[SplitPageOptions] = SplitPageOptions(),
                  **kwargs):
         self.aws_access_key_id = get_from_dict_or_env(kwargs, "aws_access_key_id", "AWS_ACCESS_KEY_ID")
@@ -204,6 +206,7 @@ class PdfExtractorNetmindMerge:
         self.txt_vector = txt_vector
         self.slice_option = slice_option
         self.source_s3_path = source_s3_path
+        self.is_page_number_discontinuity_exception_thrown = is_page_number_discontinuity_exception_thrown
         if not source_s3_path:
             raise Exception('not params source source_s3_path')
         if s3_util:
@@ -221,17 +224,14 @@ class PdfExtractorNetmindMerge:
         self._s3_resource = self.s3_util.get_s3_resource()
         self._s3_client = self.s3_util.get_s3_client()
 
-
     def upload_file_to_s3(self, bucket, local_key: str, remote_key: str):
         self._s3_resource.Object(bucket, remote_key).upload_file(local_key)
         logger.warning(f"File {local_key} Uploaded To s3://{bucket}/{remote_key}")
 
-
     def download_file_from_s3(self, bucket: str, remote_key: str, local_key: str):
         self._s3_resource.Bucket(bucket).download_file(remote_key, local_key)
         logger.info(f"File s3://{bucket}/{remote_key} downloaded to {local_key}")
 
-
     def megre_json(self, json_s3_path_list):
         with tempfile.TemporaryDirectory() as tmp_dir:
             # page 字典
@@ -246,7 +246,8 @@ class PdfExtractorNetmindMerge:
                 with open(local_path, 'r') as file:
                     _split_response_json = json.load(file)
                 file_item_name = _s3_path.split('/')[-1].replace('.json', '')
-                start_page = (int(file_item_name.split('_')[-1].split('.')[0]) - 1) * self.slice_option.split_page_number
+                start_page = (int(
+                    file_item_name.split('_')[-1].split('.')[0]) - 1) * self.slice_option.split_page_number
                 meta_data[start_page] = _split_response_json
             logger.warning("[JSON] Down json result successfully...")
             # 合并
@@ -267,10 +268,13 @@ class PdfExtractorNetmindMerge:
                 for i in range(1, len(all_pages)):
                     if all_pages[i] != all_pages[i - 1] + 1:
                         missing_pages = list(range(all_pages[i - 1] + 1, all_pages[i]))
-                        raise ValueError(
-                            f"页码不连续错误！在 {all_pages[i - 1]} 页之后直接出现了 {all_pages[i]} 页，"
-                            f"缺少页码: {missing_pages}"
-                        )
+                        if self.is_page_number_discontinuity_exception_thrown:
+                            raise ValueError(
+                                f"页码不连续错误！在 {all_pages[i - 1]} 页之后直接出现了 {all_pages[i]} 页，"
+                                f"缺少页码: {missing_pages}"
+                            )
+                        else:
+                            print(f"页码不连续错误！在 {all_pages[i - 1]} 页之后直接出现了 {all_pages[i]} 页，缺少页码: {missing_pages}")
             logger.info("[JSON] Merge json result successfully...")
             return sorted_file_arr
 
@@ -439,4 +443,4 @@ class PdfExtractorNetmindMerge:
 
         if block_raw["type"] == "image":
             f_block["image_detail"] = block_raw["image_detail"]
-        return f_block
+        return f_block

{orbitkit-0.8.34 → orbitkit-0.8.36/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: orbitkit
-Version: 0.8.34
+Version: 0.8.36
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.34/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.34