orbitkit 0.8.33__tar.gz → 0.8.35__tar.gz

{orbitkit-0.8.33/orbitkit.egg-info → orbitkit-0.8.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: orbitkit
-Version: 0.8.33
+Version: 0.8.35
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.35/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.35

{orbitkit-0.8.33 → orbitkit-0.8.35}/orbitkit/pdf_extractor/pdf_extractor_netmind_v3.py

@@ -13,11 +13,11 @@ from orbitkit.util import s3_split_path, S3Util, get_from_dict_or_env, ExtenCons
     get_content_type_4_filename
 from typing import Optional
 import urllib3
-from retry import retry
 import fitz  # PyMuPDF
 import os
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor, as_completed
+
 logger = logging.getLogger(__name__)
 
 
@@ -54,8 +54,8 @@ class PdfExtractorNetmindFileAnalysis:
                                       aws_access_key_id=self.aws_access_key_id,
                                       aws_secret_access_key=self.aws_secret_access_key)
         self.s3_resource = boto3.resource('s3',
-                                        aws_access_key_id=self.aws_access_key_id,
-                                        aws_secret_access_key=self.aws_secret_access_key)
+                                          aws_access_key_id=self.aws_access_key_id,
+                                          aws_secret_access_key=self.aws_secret_access_key)
 
     def upload_file_to_s3(self, local_key: str, remote_key: str):
         _remote_key = f'{self.bucket_tmp_group}{remote_key}'
@@ -147,18 +147,17 @@ class PdfExtractorNetmindExtract:
         s3_path_obj = s3_split_path(self.s3_path)
         presigned_url = self._generate_presigned_url(s3_path_obj)
         logger.warning("Get presigned_url successfully...")
-        json_response = self.get_netmind_response(presigned_url)
-        return self._save_json_to_s3(json_response, s3_path_obj)
+        api_response_time, json_response = self.get_netmind_response(presigned_url)
+        return api_response_time, self._save_json_to_s3(json_response, s3_path_obj)
 
     def _generate_presigned_url(self, s3_path_obj):
         return self.s3_client.generate_presigned_url('get_object',
-                                                      Params={
-                                                          'Bucket': s3_path_obj["bucket"],
-                                                          'Key': s3_path_obj["store_path"]
-                                                      },
-                                                      ExpiresIn=604800)
+                                                     Params={
+                                                         'Bucket': s3_path_obj["bucket"],
+                                                         'Key': s3_path_obj["store_path"]
+                                                     },
+                                                     ExpiresIn=604800)
 
-    @retry(tries=2, delay=4)
     def get_netmind_response(self, presigned_url):
         start = time.time()
         files = {"url": (None, presigned_url)}
@@ -167,8 +166,9 @@ class PdfExtractorNetmindExtract:
         )
         # 状态检查
         response.raise_for_status()
-        logger.info(f"Extract text by using Netmind successfully: {time.time() - start}")
-        return response.json()
+        api_response_time = time.time() - start
+        logger.info(f"Extract text by using Netmind successfully: {api_response_time}")
+        return api_response_time, response.json()
 
     def _save_json_to_s3(self, json_data, s3_path_obj):
         json_key = f"{s3_path_obj['store_path']}.json"  # 生成 JSON 文件名
@@ -178,12 +178,12 @@ class PdfExtractorNetmindExtract:
             input_folder = os.path.join(tmp_dir, 'input')
             if not os.path.exists(input_folder):
                 os.makedirs(input_folder)
-            local_key = os.path.join(input_folder, local_name) # 临时文件路径
+            local_key = os.path.join(input_folder, local_name)  # 临时文件路径
             # 将 JSON 数据写入临时文件
             with open(local_key, 'w') as json_file:
                 json_file.write(json_content)
             # 上传到 S3
-            self.upload_file_to_s3(s3_path_obj['bucket'],local_key, json_key)
+            self.upload_file_to_s3(s3_path_obj['bucket'], local_key, json_key)
         return f"s3://{s3_path_obj['bucket']}/{json_key}"
 
     def upload_file_to_s3(self, bucket, local_key: str, remote_key: str):
@@ -191,7 +191,6 @@ class PdfExtractorNetmindExtract:
         logger.warning(f"File {local_key} Uploaded To s3://{bucket}/{remote_key}")
 
 
-
 class PdfExtractorNetmindMerge:
     def __init__(self,
                  source_s3_path: str = None,
@@ -223,17 +222,14 @@ class PdfExtractorNetmindMerge:
         self._s3_resource = self.s3_util.get_s3_resource()
         self._s3_client = self.s3_util.get_s3_client()
 
-
     def upload_file_to_s3(self, bucket, local_key: str, remote_key: str):
         self._s3_resource.Object(bucket, remote_key).upload_file(local_key)
         logger.warning(f"File {local_key} Uploaded To s3://{bucket}/{remote_key}")
 
-
     def download_file_from_s3(self, bucket: str, remote_key: str, local_key: str):
         self._s3_resource.Bucket(bucket).download_file(remote_key, local_key)
         logger.info(f"File s3://{bucket}/{remote_key} downloaded to {local_key}")
 
-
     def megre_json(self, json_s3_path_list):
         with tempfile.TemporaryDirectory() as tmp_dir:
             # page 字典
@@ -248,7 +244,8 @@ class PdfExtractorNetmindMerge:
                 with open(local_path, 'r') as file:
                     _split_response_json = json.load(file)
                 file_item_name = _s3_path.split('/')[-1].replace('.json', '')
-                start_page = (int(file_item_name.split('_')[-1].split('.')[0]) - 1) * self.slice_option.split_page_number
+                start_page = (int(
+                    file_item_name.split('_')[-1].split('.')[0]) - 1) * self.slice_option.split_page_number
                 meta_data[start_page] = _split_response_json
             logger.warning("[JSON] Down json result successfully...")
             # 合并
@@ -441,4 +438,4 @@ class PdfExtractorNetmindMerge:
 
         if block_raw["type"] == "image":
             f_block["image_detail"] = block_raw["image_detail"]
-        return f_block
+        return f_block

{orbitkit-0.8.33 → orbitkit-0.8.35/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: orbitkit
-Version: 0.8.33
+Version: 0.8.35
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.33/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.33