orbitkit 0.8.74__tar.gz → 0.8.76__tar.gz

{orbitkit-0.8.74/orbitkit.egg-info → orbitkit-0.8.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.74
+Version: 0.8.76
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.76/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.76

{orbitkit-0.8.74 → orbitkit-0.8.76}/orbitkit/airflow_handler/data_preprocessing.py

@@ -121,16 +121,16 @@ class DocumentProcessor:
         allow_embedding_types = {'10002', '10085', '10076', '10122', '10311', '10178', '10075', '10090', '10050'}
         return target_stage if any(rid in allow_embedding_types for rid in report_type_ids) else "extract"
 
-    # @staticmethod
-    # def update_target_stage_by_reported_at(doc, target_stage):
-    #     date_str = doc.get('x_reported_at_utc_date', '1970-01-01')
-    #     now = datetime.datetime.now()
-    #     one_year_ago = now - relativedelta(years=1)
-    #     try:
-    #         reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
-    #     except ValueError:
-    #         reported_date = datetime.datetime(1970, 1, 1)
-    #     return "extract" if reported_date < one_year_ago else target_stage
+    @staticmethod
+    def update_target_stage_by_reported_at(doc, target_stage):
+        date_str = doc.get('x_reported_at_utc_date', '1970-01-01')
+        now = datetime.datetime.now()
+        one_year_ago = now - relativedelta(years=1)
+        try:
+            reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
+        except ValueError:
+            reported_date = datetime.datetime(1970, 1, 1)
+        return "extract" if reported_date < one_year_ago else target_stage
 
     @staticmethod
     def update_target_stage_by_perm_match(doc, target_stage):
@@ -201,7 +201,7 @@ class DocumentProcessor:
         # 判断 特殊条件下的数据不做embedding ('19999'类型和报告日期小于2023-01-01)
         if target_stage == 'embedding' and not custom_process_step:
             target_stage = cls.update_target_stage_by_report_type(doc, target_stage)
-            # target_stage = cls.update_target_stage_by_reported_at(doc, target_stage)
+            target_stage = cls.update_target_stage_by_reported_at(doc, target_stage)
             target_stage = cls.update_target_stage_by_perm_match(doc, target_stage)
             # 特殊情况下只需要做embedding 但是这个数据被条件限制为只做到提取时状态异常
             if start_stage == 'embedding' and target_stage == 'extract':

{orbitkit-0.8.74 → orbitkit-0.8.76}/orbitkit/pdf_extractor/pdf_extractor_minerU_v1.py

@@ -54,8 +54,8 @@ class MinerUApiBase:
 
     BASE_URL = "https://mineru.net/api/v4"
     MAX_FILE_SIZE_MB = 200  # 最大文件大小 MB
-    MAX_PAGES = 600  # 最大页数
-    SPLIT_PAGES = 500  # 拆分时每个分片的页数（留余量）
+    MAX_PAGES = 200  # 最大页数（MinerU 服务端硬限制）
+    SPLIT_PAGES = 150  # 拆分时每个分片的页数（留 10% 余量避免边界踩限）
     MAX_BATCH_FILES = 200  # 单次批量上传最大文件数
 
     def __init__(self, api_token: str, temp_dir: str = None):
@@ -543,6 +543,37 @@ class MinerUExtract:
                     block_seq = block_seq - 1
         return block_seq, pages_body
 
+    def level_two_chart(self, block_seq, pages_block, pages_body):
+        pages_chart_blocks = pages_block["blocks"]
+        for _pages_chart_blocks in pages_chart_blocks:
+            block_seq = block_seq + 1
+            _pages_chart_blocks_type = _pages_chart_blocks['type']
+            pages_chart_blocks_line = _pages_chart_blocks["lines"]
+            bbox = pages_block['bbox']
+            if _pages_chart_blocks_type == 'chart_body':
+                block_type = 'chart'
+            elif _pages_chart_blocks_type in ['chart_caption', 'chart_footnote']:
+                block_type = 'sentence'
+            else:
+                raise Exception(f'Chart 异常目标值 {_pages_chart_blocks_type}:')
+            text_location = self.get_location(bbox)
+            _block_arr, _image_detail_arr = self.get_com_lines(pages_chart_blocks_line)
+            if len(_block_arr) > 0 or len(_image_detail_arr) > 0:
+                _block_arr_str = '\n'.join(_block_arr)
+                self.blocks.append({
+                    "id": id_srv.get_random_short_id(),
+                    "page": self.page,
+                    "seq_no": block_seq,
+                    "sentence": _block_arr_str,
+                    "type": block_type,
+                    "image_detail": _image_detail_arr,
+                    "text_location": text_location
+                })
+            else:
+                if block_seq > 0:
+                    block_seq = block_seq - 1
+        return block_seq, pages_body
+
     def level_two_interline_equation(self, block_seq, pages_block, pages_body):
         block_seq = block_seq + 1
         bbox = pages_block['bbox']
@@ -602,7 +633,7 @@ class MinerUExtract:
                     _c_s.append(_l['content'])
                 elif _l['type'] in ['table']:
                     _c_s.append(_l['html'])
-                elif _l['type'] in ['image']:
+                elif _l['type'] in ['image', 'chart']:
                     pass
                 else:
                     raise Exception(f'类型匹配异常 意外值: {_l["type"]}')

{orbitkit-0.8.74 → orbitkit-0.8.76/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.74
+Version: 0.8.76
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.74/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.74