orbitkit 0.9.26__tar.gz → 0.9.28__tar.gz

{orbitkit-0.9.26/orbitkit.egg-info → orbitkit-0.9.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.9.26
+Version: 0.9.28
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.9.28/orbitkit/VERSION

@@ -0,0 +1 @@
+0.9.28

{orbitkit-0.9.26 → orbitkit-0.9.28}/orbitkit/airflow_handler/file_flow_exit_process.py

@@ -8,6 +8,9 @@ from orbitkit.constant.report_schema import SchemaKafkaIgnore
 
 logger = logging.getLogger(__name__)
 
+# 页数异常失败码（op_attachment_failure_helper：MINERU_PAGE_COUNT_INVALID）
+PAGE_COUNT_INVALID_CODE = '30009'
+
 
 class FlowUpdater:
 
@@ -40,6 +43,7 @@ class FlowUpdater:
 
         store_path_set = set()
         parent_id_store_path_map = {i['parent_id']: i['store_path'] for i in attachments_pdf}
+        parent_id_page_size_map = {i['parent_id']: i.get('page_size') for i in attachments_pdf}
         x_attachments_pdf = []
         for item in attachments:
             # The video branch will generate store_path_pre, so check for the existence of store_path_pre first.
@@ -53,14 +57,18 @@ class FlowUpdater:
             store_path_set.add(store_path)
             # new_store_path = store_path if store_path.lower().endswith('.pdf') else store_path + '.pdf'
             new_store_path = parent_id_store_path_map[parent_id]
-            x_attachments_pdf.append({
+            attachment_pdf = {
                 "store_path": new_store_path,
                 "store_path_txt": "",
                 "store_path_pre": store_path,
                 "file_name": item['file_name'],
                 "raw_is_pdf": 'true' if store_path.lower().endswith('.pdf') else 'false',
                 "file_hash": id_srv.get_fix_short_id(new_store_path).get("short_id")
-            })
+            }
+            # 页数取 op_attachment.page_size（解析成功时由解析服务写入），取不到置为空
+            page_size = parent_id_page_size_map.get(parent_id)
+            attachment_pdf["page_count"] = page_size if page_size is not None and page_size > 0 else None
+            x_attachments_pdf.append(attachment_pdf)
         return {
             'x_attachments_pdf': x_attachments_pdf,
             'x_status_list.status_convert.status': 'convert_done'
@@ -128,6 +136,12 @@ class FlowUpdater:
             'details': details
         }
 
+    @staticmethod
+    def _is_page_count_invalid(attachment_pdf):
+        """转化件页数（op_attachment.page_size）有值但不是正数。为空不算异常，回写时 page_count 置为空。"""
+        page_size = attachment_pdf.get('page_size')
+        return page_size is not None and page_size <= 0
+
     def update_mongo_data(self, report_id, data_source, update_params, kafka_ignore):
         if kafka_ignore:
             update_params.update({self.kafka_ignore.X_OTHERS_K_IGNORE_KEY: self.kafka_ignore.get_k_ignore_val()})
@@ -173,7 +187,8 @@ class FlowUpdater:
 
         db_doc = self._check_and_create_collection(data_source).find_one({'_id': report_id},
                                                                          {'_id': 1, 'x_attachments': 1,
-                                                                          'x_status_list': 1})
+                                                                          'x_status_list': 1,
+                                                                          'x_attachments_pdf': 1})
 
         if not db_doc:
             logger.warning(f"No document found with id {report_id}.")
@@ -184,10 +199,22 @@ class FlowUpdater:
             return
 
         db_store_path = {f"s3://{i['bucket']}/{i['store_path']}" for i in db_doc['x_attachments']}
+        steps = self.step_tuple[start_index:end_index + 1]
+
+        # 页数检查：提取成功但有转化件页数异常，整篇按提取失败处理（失败码 30009）。
+        # 改的是 exit_point 交过来的同一份记录：status 置为 failed 后，下面写 mongo 走现有提取失败分支，
+        # exit_point 随后把 op_meta 转 manual，并把标了 exit_failure_code 的附件记为失败（见 FileFlowHandleV2.exit_point）
+        if status == 'success' and 'extract' in steps:
+            invalid_attachments = [att for att in attachments_pdf if self._is_page_count_invalid(att)]
+            if invalid_attachments:
+                logger.error(f"Invalid page count: {report_id} {[att.get('store_path') for att in invalid_attachments]}")
+                for att in invalid_attachments:
+                    att['exit_failure_code'] = PAGE_COUNT_INVALID_CODE
+                status = op_meta_record['status'] = 'failed'
 
         update_params = {}
         step_status = True
-        for index, step in enumerate(self.step_tuple[start_index:end_index + 1], 1):
+        for index, step in enumerate(steps, 1):
             if step == end_stage and status == 'failed':
                 step_status = False
             logger.info(f' Processing step-{index} {step} - {"successfully" if step_status else "failed"}.')
@@ -200,6 +227,16 @@ class FlowUpdater:
             if parse_record is not None:
                 update_params['x_parse_record'] = parse_record
 
+        # 只重做提取（不重建 x_attachments_pdf）：把 mongo 已有的 x_attachments_pdf 按 store_path 填上 PG 页数后整体写回，
+        # 以 PG 为准（取不到置为空）；meta 写的标题、语言、摘要等字段原样带回
+        if status == 'success' and 'extract' in steps and 'x_attachments_pdf' not in update_params:
+            page_counts = {att['store_path']: att['page_size'] for att in attachments_pdf}
+            db_attachments_pdf = db_doc['x_attachments_pdf']  # 只重做提取时 mongo 必是 convert_done，x_attachments_pdf 必有
+            for att in db_attachments_pdf:
+                if att['store_path'] in page_counts:
+                    att['page_count'] = page_counts[att['store_path']]
+            update_params['x_attachments_pdf'] = db_attachments_pdf
+
         if update_params:
             # logger.info(json.dumps(update_params, ensure_ascii=False, indent=2))
             self.update_mongo_data(report_id, data_source, update_params, kafka_ignore)

{orbitkit-0.9.26 → orbitkit-0.9.28}/orbitkit/airflow_handler/file_handler_v2.py

@@ -202,6 +202,23 @@ class FileFlowHandleV2:
 
         postgres_coon.execute(op_meta_stmt, {'op_meta_id': op_meta_id})
 
+    def _update_exit_failed_attachments(self, postgres_coon, meta_data):
+        """回更时判定失败的附件（FlowUpdater 标了 exit_failure_code，如页数异常 30009），按提取失败写回 op_attachment。"""
+        for attachment in meta_data.get('x_attachments_pdf') or []:
+            failure_code = attachment.get('exit_failure_code')
+            if not failure_code:
+                continue
+            stmt = (
+                update(self.table_op_attachment)
+                .where(self.table_op_attachment.c.id == attachment['id'])
+                .values(
+                    status_extract='failed',
+                    failure_code=failure_code,
+                    updated_at=datetime.datetime.now(datetime.timezone.utc)
+                )
+            )
+            postgres_coon.execute(stmt)
+
     def _update_low_priority_processing_data(self, meta_id_set, params):
         meta_id_list = list(meta_id_set)
         if not meta_id_list:
@@ -457,6 +474,7 @@ class FileFlowHandleV2:
                         self._delete_and_backup_data_from_queue(session, item)
                     elif status == 'failed':
                         self._update_failed_record_to_manual(session, meta_id)
+                        self._update_exit_failed_attachments(session, item)
                     else:
                         logger.info(f"Data is still processing. Current status: {status}")
                     session.commit()

{orbitkit-0.9.26 → orbitkit-0.9.28}/orbitkit/pdf_transformer/mineru/result_transformer.py

@@ -598,7 +598,7 @@ def _jsonl_bytes(records: List[dict]) -> bytes:
 #   - para_blocks 然后 discarded_blocks
 #   - btype=page_number 直接跳过
 #   - 文本类（text/title/discarded/index/header/footer/aside_text/footnote/
-#     reference/ref_text/equation/page_footnote/caption/abstract/code）走
+#     reference/ref_text/phonetic/equation/page_footnote/caption/abstract/code）走
 #     get_com_lines 提文本，title/header → out_type=title，其它 → sentence
 #   - list 递归 blk['blocks']（如有）
 #   - table 提 HTML（span.type=table 取 span.html），text spans 拆为单独 block
@@ -612,7 +612,7 @@ def _jsonl_bytes(records: List[dict]) -> bytes:
 _TEXT_LIKE_TYPES = frozenset({
     "text", "discarded", "title", "index",
     "header", "footer", "aside_text",
-    "footnote", "reference", "ref_text",
+    "footnote", "reference", "ref_text", "phonetic",
     "equation", "page_footnote",
     "caption", "abstract", "code",
 })

{orbitkit-0.9.26 → orbitkit-0.9.28/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.9.26
+Version: 0.9.28
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.9.26/orbitkit/VERSION

@@ -1 +0,0 @@
-0.9.26