orbitkit 0.8.30__tar.gz → 0.8.55__tar.gz

{orbitkit-0.8.30/orbitkit.egg-info → orbitkit-0.8.55}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.30
+Version: 0.8.55
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao
@@ -19,13 +19,22 @@ Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: boto3>=1.16.0
-Requires-Dist: requests>=2.12.1
-Requires-Dist: prettytable>=3.0.0
-Requires-Dist: pytz>=2022.1
+Requires-Dist: boto3>=1.40.46
+Requires-Dist: aioboto3>=15.5.0
+Requires-Dist: aiofiles>=25.1.0
+Requires-Dist: requests>=2.32.5
+Requires-Dist: prettytable>=3.16.0
+Requires-Dist: pytz>=2025.2
 Requires-Dist: Deprecated
 Requires-Dist: func_timeout
 Dynamic: author
@@ -35,6 +44,7 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
+Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
 Dynamic: platform

orbitkit-0.8.55/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.55

{orbitkit-0.8.30 → orbitkit-0.8.55}/orbitkit/airflow_handler/data_preprocessing.py

@@ -1,13 +1,18 @@
+import os
 import datetime
 from collections import defaultdict
+from importlib.metadata import version
+import googletrans
 
 
 class DocumentProcessor:
+    if version("googletrans") < "4.0.2":
+        raise ImportError(f"googletrans >= 4.0.2 is required for async support. {version('googletrans')}")
     AUDIO_SUFFIXES = [".mp3", ".wav", ".aac", ".wma", ".m4a"]
     VIDEO_SUFFIXES = [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".3gp", ".hevc"]
     PDF_SUFFIXES = [".pdf"]
     DOC_SUFFIXES = [".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx"]
-    TXT_SUFFIXES = [".txt", ".html", ".htm"]
+    TXT_SUFFIXES = [".txt", ".html", ".htm", ".xhtml"]
     ALL_ALLOWED_SUFFIXES = set(AUDIO_SUFFIXES + VIDEO_SUFFIXES + PDF_SUFFIXES + DOC_SUFFIXES + TXT_SUFFIXES)
 
     DATA_PROCESS_STEPS = ['convert', 'extract', 'embedding', 'success']
@@ -16,6 +21,13 @@ class DocumentProcessor:
     def get_file_suffix(file_path):
         return f".{file_path.split('.')[-1]}".lower()
 
+    @staticmethod
+    async def translate_text(text, dest='en'):
+        """异步翻译函数 https://pypi.org/project/googletrans/"""
+        translator = googletrans.Translator()
+        result = await translator.translate(text, dest=dest)
+        return result.text
+
     @staticmethod
     def create_xbrl_template():
         return {
@@ -62,9 +74,10 @@ class DocumentProcessor:
     def xbrl_type_check(cls, doc):
         is_xbrl = doc.get('x_info_data', {}).get('is_xbrl') == 'true'
         x_attachments = doc.get('x_attachments', [])
+        convert_status = doc.get('x_status_list', {}).get('status_convert', {}).get('status')
         xhtml_count = sum(1 for att in x_attachments if att['store_path'].lower().endswith('.xhtml'))
 
-        if is_xbrl or xhtml_count > 0:
+        if is_xbrl or xhtml_count > 0 and convert_status != 'convert_done':
             template = cls.create_xbrl_template()
             template['_id'] = doc['_id']
             template['source_type'] = doc.get('x_report_source', {}).get('source_type', '')
@@ -79,12 +92,15 @@ class DocumentProcessor:
         return False, None
 
     @staticmethod
-    def get_start_stage_target_stage(doc):
+    def get_start_stage_target_stage(doc, custom_process_step_list):
         status_info = doc.get('x_status_list', {}).get('status_convert', {})
         status = status_info.get('status')
         status_txt = status_info.get('status_txt')
         x_spider_name = doc['x_spider_name']
 
+        if custom_process_step_list:
+            return custom_process_step_list[0], custom_process_step_list[1], x_spider_name
+
         if status != 'convert_done':
             return 'convert', 'embedding', x_spider_name
 
@@ -97,24 +113,54 @@ class DocumentProcessor:
         return 'success', 'success', x_spider_name
 
     @staticmethod
-    def update_target_stage_for_report_type(doc, target_stage):
+    def update_target_stage_by_report_type(doc, target_stage):
         report_type_ids = doc.get('x_orbit_data', {}).get('report_type_id_list', [])
         return "extract" if report_type_ids == ['19999'] else target_stage
 
     @staticmethod
-    def create_record(doc, start_stage):
+    def update_target_stage_by_reported_at(doc, target_stage):
+        date_str = doc.get('x_reported_at_utc_date', '1970-01-01')
+        try:
+            reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
+        except ValueError:
+            reported_date = datetime.datetime(1970, 1, 1)
+        return "extract" if reported_date < datetime.datetime(2023, 1, 1) else target_stage
+
+    @staticmethod
+    def update_target_stage_by_perm_match(doc, target_stage):
+        perm_match_status = doc['x_status_list']['status_perm']['status']
+        return target_stage if perm_match_status in {'perm_match_part', 'perm_match'} else "extract"
+
+    @classmethod
+    async def create_record(cls, doc, start_stage, important_level):
         attachments = doc.get('x_attachments', []) if start_stage == 'convert' else doc.get('x_attachments_pdf', [])
         s3_path_info = []
+        add_extends = {}
         for att in attachments:
             if len(att['file_name']) > 2000 or len(att['file_name'].encode('utf-8')) > 2000:
                 return False, None
+
+            if start_stage == 'convert' and not add_extends:
+                _, ext = os.path.splitext(att['store_path'])
+                if ext in cls.AUDIO_SUFFIXES or ext in cls.VIDEO_SUFFIXES:
+                    add_extends = {
+                        "original_title": doc['x_orbit_data']['report_title'],
+                        "title": await cls.translate_text(text=doc['x_orbit_data']['report_title']),
+                        "published": doc['x_reported_at_utc_date'],
+                        "tickers": [],
+                        "perm_id_list": doc['x_orbit_data']['perm_id_list'],
+                        "report_type_id_list_str": doc['x_orbit_data']['report_type_id_list']
+                    }
+
             s3_path_info.append({
                 'store_path': f"s3://{att['bucket']}/{att['store_path']}" if start_stage == 'convert' else att[
                     'store_path'],
                 'file_name': att['file_name']
             })
-
-        return True, {'id': doc['_id'], 's3_path_info': s3_path_info}
+        result_dict = {'id': doc['_id'], 's3_path_info': s3_path_info, 'important_level': important_level}
+        if add_extends:
+            result_dict['extends'] = add_extends
+        return True, result_dict
 
     @staticmethod
     def create_result_info(process_type, message, result_data):
@@ -125,7 +171,7 @@ class DocumentProcessor:
         }
 
     @classmethod
-    def process(cls, doc):
+    async def process(cls, doc, custom_process_step, important_level):
         report_id = doc['_id']
         # 筛选文件
         doc = cls.stock_us_filter_by_is_primary(doc)
@@ -137,19 +183,28 @@ class DocumentProcessor:
 
         is_xbrl, xbrl_data = cls.xbrl_type_check(doc)
         if is_xbrl:
-            return cls.create_result_info("xbrl", "XBRL format cannot be processed.", xbrl_data)
+            return cls.create_result_info("xbrl", "XBRL or Xhtml format cannot be processed.", xbrl_data)
+
+        start_stage, target_stage, x_spider_name = cls.get_start_stage_target_stage(doc, custom_process_step)
 
-        start_stage, target_stage, x_spider_name = cls.get_start_stage_target_stage(doc)
-        target_stage = cls.update_target_stage_for_report_type(doc, target_stage)
+        # 判断 特殊条件下的数据不做embedding ('19999'类型和报告日期小于2023-01-01)
+        if target_stage == 'embedding' and not custom_process_step:
+            target_stage = cls.update_target_stage_by_report_type(doc, target_stage)
+            target_stage = cls.update_target_stage_by_reported_at(doc, target_stage)
+            target_stage = cls.update_target_stage_by_perm_match(doc, target_stage)
+            # 特殊情况下只需要做embedding 但是这个数据被条件限制为只做到提取时状态异常
+            if start_stage == 'embedding' and target_stage == 'extract':
+                start_stage = 'success'
+                target_stage = 'success'
 
         if cls.DATA_PROCESS_STEPS.index(target_stage) < cls.DATA_PROCESS_STEPS.index(start_stage):
             return cls.create_result_info("step_error",
                                           "Invalid process sequence: 'start_stage' occurs before 'target_stage'.",
-                                          doc['_id'])
+                                          report_id)
 
-        file_name_check_status, record = cls.create_record(doc, start_stage)
+        file_name_check_status, record = await cls.create_record(doc, start_stage, important_level)
         if not file_name_check_status:
-            return cls.create_result_info("step_error", "Document file name too lang.", report_id)
+            return cls.create_result_info("error", "Document file name too lang.", report_id)
 
         return cls.create_result_info("file_flow", "Success", [start_stage, target_stage, x_spider_name, record])
 

orbitkit-0.8.55/orbitkit/airflow_handler/file_flow_entry_process.py

@@ -0,0 +1,274 @@
+import os
+from collections import Counter
+from datetime import datetime
+from typing import Optional
+import logging
+import pymongo
+import pytz
+import boto3
+from sqlalchemy import create_engine, Table, MetaData, select
+from sqlalchemy.orm import sessionmaker, scoped_session
+from contextlib import contextmanager
+
+from orbitkit.airflow_handler.file_handler_v2 import FileFlowHandleV2
+from orbitkit.airflow_handler.data_preprocessing import DocumentProcessor
+from orbitkit.orbit_type import OrbitTypeMatcher
+
+logger = logging.getLogger(__name__)
+
+
+class FilingOfficialProcessor:
+
+    def __init__(self, mongo_uri=None, postgres_uri=None, aws_access_key_id=None, aws_secret_access_key=None, pi2_postgres_uri=None, pi2_database_name=None, databases_fileflow=None):
+        mongo_uri = os.environ.get('MONGO_URI_MAIN_USER_APP') if not mongo_uri else mongo_uri
+        if not mongo_uri:
+            raise KeyError('mongo_uri not set.')
+
+        if not aws_secret_access_key or not aws_access_key_id:
+            raise KeyError('aws_access_key_id and aws_secret_access_key not set.')
+
+        self.mongo_client = pymongo.MongoClient(mongo_uri)
+        self.data_xbrl_convert_collection = self.mongo_client['filing_reports']['data_xbrl_convert']
+
+        postgres_uri = os.environ.get('PG_URI_AIRFLOW12_USER_NEWSFEEDSITE') if not postgres_uri else postgres_uri
+        if not postgres_uri:
+            raise KeyError('postgres_uri not set.')
+        databases_fileflow = databases_fileflow or "process_net"
+        self.file_handler = FileFlowHandleV2(postgres_uri=postgres_uri, database_name=databases_fileflow)
+        self.data_processor = DocumentProcessor()
+        self.max_batch_size = 10000
+        self.all_stat_count = {'all': 0, 'skip': 0, 'doc_error': 0, 'step_error': 0, 'xbrl': 0, 'file_flow': 0}
+
+        self.s3_client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
+        self.matcher = OrbitTypeMatcher(self.s3_client)
+        self.report_type_id_name_map = {i["lv3_id"]: i["lv3_name"] for i in self.matcher.get_full_type_list()}
+
+        self.pi2_postgres_uri = pi2_postgres_uri or os.environ['PG_URI_CX45_USER_GLAUUIADMIN']
+        if not self.pi2_postgres_uri:
+            raise KeyError('pie_postgres_uri not set.')
+        self.databases = pi2_database_name or 'newsfeedsite'
+        self.postgres_engine = create_engine(f"{self.pi2_postgres_uri}/{self.databases}", connect_args={"sslmode": "require"})
+        self.postgres_session = sessionmaker(bind=self.postgres_engine)
+        self.Session = scoped_session(self.postgres_session)
+        self.postgres_metadata = MetaData()
+
+        self.pi2_table = Table(
+            'primary_instrument_2_release', self.postgres_metadata,
+            autoload_with=self.postgres_engine, schema='security_master'
+        )
+
+        self.postgres_engine2 = create_engine(f"{postgres_uri}/{databases_fileflow}",
+                                             connect_args={"sslmode": "require"})
+        self.postgres_session2 = sessionmaker(bind=self.postgres_engine2)
+        self.Session2 = scoped_session(self.postgres_session2)
+
+        self.op_meta = Table(
+            'op_meta', self.postgres_metadata,
+            autoload_with=self.postgres_engine2, schema='public'
+        )
+
+    @contextmanager
+    def session_scope(self, use_session=None):
+        session = self.Session() if not use_session else use_session
+        try:
+            yield session
+            session.commit()
+        except Exception:
+            session.rollback()
+            raise
+        finally:
+            self.Session.remove()
+
+    def create_spider_name_source_type_map(self, collection, label):
+
+        def find_duplicates(keys):
+            return [k for k, v in Counter(keys).items() if v > 1]
+
+        map_dict = {}
+        pipeline = [{'$group': {'_id': "$x_spider_name"}}]
+
+        for document in collection.aggregate(pipeline):
+            map_dict[document['_id']] = label
+
+        all_keys = list(map_dict.keys())
+        duplicates = find_duplicates(all_keys)
+        if duplicates:
+            raise KeyError(f"Duplicate x_spider_name found: {duplicates}")
+
+        return map_dict
+
+    def send_xbrl_data_to_mongo(self, xbrl_data_list):
+        if not xbrl_data_list:
+            return
+        report_id_list = list(set([i['_id'] for i in xbrl_data_list]))
+        result = self.data_xbrl_convert_collection.find({'_id': {'$in': report_id_list}}, {'_id': 1}).batch_size(self.max_batch_size)
+        exists_id_list = [i['_id'] for i in result]
+        new_xbrl_data_list = [i for i in xbrl_data_list if i['_id'] not in exists_id_list]
+        if not new_xbrl_data_list:
+            return
+        self.data_xbrl_convert_collection.insert_many(new_xbrl_data_list)
+        logger.info(f'{len(new_xbrl_data_list)}-xbrl data inserted.')
+
+    def update_doc_status_to_convert(self, collection, report_id_list):
+        if len(report_id_list) == 0:
+            return
+        collection.update_many(
+            {"_id": {"$in": report_id_list}},
+            {
+                "$set": {
+                    "x_status_list.status_convert.status": "convert_failed",
+                    "x_status_list.status_convert.status_txt": "convert_txt_init",
+                    "x_status_list.status_convert.status_meta": "meta_init",
+                    "x_updated_date": datetime.now(tz=pytz.timezone('UTC')).strftime("%Y-%m-%dT%H:%M:%S%z"),
+                },
+                "$unset": {
+                    "x_attachments_pdf": ""
+                }
+            }
+        )
+        logger.info(f'Unable to convert {len(report_id_list)} document(s) due to unsupported file type.')
+
+    def update_extends_fields(self, perm_id_list, file_flow_info):
+        stmt = select(self.pi2_table.c.orbit_entity_id, self.pi2_table.c.ticker).where(self.pi2_table.c.orbit_entity_id.in_(perm_id_list))
+        orbit_entity_id_ticker_map = {}
+        with self.session_scope() as session:
+            result = session.execute(stmt)
+            for row in result:
+                if row.orbit_entity_id not in orbit_entity_id_ticker_map:
+                    orbit_entity_id_ticker_map[row.orbit_entity_id] = []
+
+                if row.ticker is not None:
+                    orbit_entity_id_ticker_map[row.orbit_entity_id].append(row.ticker)
+        for step_info, records in file_flow_info.items():
+            for record in records:
+                if 'extends' in record and record.get('extends') is not None:
+                    tickers = set()
+                    for i in record['extends']['perm_id_list']:
+                        tickers.update(orbit_entity_id_ticker_map.get(i, []))
+                    record['extends']['tickers'] = list(tickers)
+
+                    record['extends']['report_type_id_list_str'] = [self.report_type_id_name_map.get(i) for i in record['extends']['report_type_id_list_str']]
+
+        return file_flow_info
+
+    def send_task(self, file_flow_info, tags, is_important, priority, spider_name_source_type):
+        for step_str, records in file_flow_info.items():
+            steps = step_str.split('@__@')
+            start_stage = steps[0]
+            target_stage = steps[1]
+            x_spider_name = steps[2]
+
+            if start_stage == 'success' or target_stage == 'success':
+                self.all_stat_count['skip'] += len(records)
+                logger.info(
+                    f"{len(records)}--{start_stage}-{target_stage}-{x_spider_name} status: False, message: 'File has already completed the embedding stage.' ")
+                continue
+
+            if is_important:
+                logger.info(f"is_important: {is_important} - {x_spider_name}")
+                status, ids, message = self.file_handler.entry_point_urgent(records=records, start_stage=start_stage,
+                                                                            target_stage=target_stage,
+                                                                            tags=tags,
+                                                                            tag=x_spider_name,
+                                                                            priority=priority,
+                                                                            source_type=spider_name_source_type[
+                                                                                x_spider_name])
+            else:
+                status, ids, message = self.file_handler.entry_point(records=records, start_stage=start_stage,
+                                                                     target_stage=target_stage, tags=tags,tag=x_spider_name,
+                                                                     priority=priority,
+                                                                     source_type=spider_name_source_type[x_spider_name])
+            self.all_stat_count['file_flow'] += len(records)
+            logger.info(f"{len(records)}--{start_stage}-{target_stage}-{x_spider_name} status: {status}, message: {message}")
+
+    def op_meat_deduplicate_docs(self, docs, buffer_size=1000):
+        buffer = []
+
+        for doc in docs:
+            buffer.append(doc)
+
+            if len(buffer) >= buffer_size:
+                doc_ids = [d['_id'] for d in buffer]
+                with self.session_scope(use_session=self.Session2) as session:
+                    existing_ids = session.query(self.op_meta.c.id).filter(self.op_meta.c.id.in_(doc_ids)).all()
+                    existing_ids = {i[0] for i in existing_ids}
+                for buffered_doc in buffer:
+                    self.all_stat_count['all'] += 1
+                    if buffered_doc['_id'] not in existing_ids:
+                        yield buffered_doc
+
+                buffer.clear()
+
+        if buffer:
+            doc_ids = [d['_id'] for d in buffer]
+            with self.session_scope(use_session=self.Session2) as session:
+                existing_ids = session.query(self.op_meta.c.id).filter(self.op_meta.c.id.in_(doc_ids)).all()
+                existing_ids = {i[0] for i in existing_ids}
+            for buffered_doc in buffer:
+                self.all_stat_count['all'] += 1
+                if buffered_doc['_id'] not in existing_ids:
+                    yield buffered_doc
+
+            buffer.clear()
+
+    async def process_task_entry(self, source: str,
+                           query: dict, tags: list[str], priority: str,
+                           is_important: bool = False, custom_step: Optional[list[str]] = None, important_level = None, db_name: str = None):
+
+        if not important_level or not isinstance(important_level, int):
+            important_level = 0
+
+        if important_level == 0:
+            raise ValueError(f'important_level must be an integer (int) greater than 0. {important_level}')
+
+        allowed_steps = {"convert", "extract", "embedding"}
+        if custom_step is not None:
+            if not isinstance(custom_step, list):
+                raise ValueError("custom_step must be a list or None.")
+            if len(custom_step) > 2:
+                raise ValueError("custom_step can contain at most two elements.")
+            for step in custom_step:
+                if step not in allowed_steps:
+                    raise ValueError(f"Invalid step '{step}'. Allowed steps are: {allowed_steps}")
+
+        collection = self.mongo_client[db_name if db_name else "filing_reports"][source]
+        spider_name_source_type = self.create_spider_name_source_type_map(collection, source)
+
+        process_data = []
+        perm_id_set = set()
+        logger.info(f"load {source} data.")
+        docs = collection.find(query).batch_size(1000)
+        duplicate_docs = self.op_meat_deduplicate_docs(docs, buffer_size=self.max_batch_size) if not is_important else docs
+        for doc in duplicate_docs:
+            for orbit_entity_id in doc['x_orbit_data']['perm_id_list']:
+                perm_id_set.add(orbit_entity_id)
+            result_record = await self.data_processor.process(doc=doc, custom_process_step=custom_step, important_level=important_level)
+            process_data.append(result_record)
+            if len(process_data) >= self.max_batch_size:
+                file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
+                    process_data)
+                file_flow_info = self.update_extends_fields(list(perm_id_set), file_flow_info)
+                self.all_stat_count['doc_error'] += len(doc_error_list)
+                self.all_stat_count['step_error'] += len(except_id_list)
+                self.all_stat_count['xbrl'] += len(xbrl_data)
+                self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
+                self.send_xbrl_data_to_mongo(xbrl_data)
+                self.update_doc_status_to_convert(collection, doc_error_list)
+                process_data.clear()
+                perm_id_set.clear()
+
+        if process_data:
+            file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
+                process_data)
+            file_flow_info = self.update_extends_fields(list(perm_id_set), file_flow_info)
+            self.all_stat_count['doc_error'] += len(doc_error_list)
+            self.all_stat_count['step_error'] += len(except_id_list)
+            self.all_stat_count['xbrl'] += len(xbrl_data)
+            self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
+            self.send_xbrl_data_to_mongo(xbrl_data)
+            self.update_doc_status_to_convert(collection, doc_error_list)
+            process_data.clear()
+            perm_id_set.clear()
+
+        logger.info(f"finish processing {self.all_stat_count}. \n")
+        self.all_stat_count = {'all': 0, 'skip': 0, 'doc_error': 0, 'step_error': 0, 'xbrl': 0, 'file_flow': 0}

{orbitkit-0.8.30 → orbitkit-0.8.55}/orbitkit/airflow_handler/file_flow_exit_process.py

@@ -27,9 +27,7 @@ class FlowUpdater:
             setattr(self, collection_name, self.coon[data_source])
         return getattr(self, collection_name)
 
-    def _handle_convert(self, status, attachments, db_store_path_set):
-        if not attachments:
-            raise ValueError("No attachments provided.")
+    def _handle_convert(self, status, attachments, db_store_path_set, attachments_pdf):
         if not status:
             return {
                 'x_status_list.status_convert.status': 'convert_failed',
@@ -37,16 +35,24 @@ class FlowUpdater:
                 'x_status_list.status_convert.status_meta': 'meta_init'
             }
 
+        if not attachments or not attachments_pdf:
+            raise ValueError("Missing attachments: neither 'attachments' nor 'attachments_pdf' was provided.")
+
         store_path_set = set()
+        parent_id_store_path_map = {i['parent_id']: i['store_path'] for i in attachments_pdf}
         x_attachments_pdf = []
         for item in attachments:
-            store_path = item['store_path']
+            # The video branch will generate store_path_pre, so check for the existence of store_path_pre first.
+            # For other branches, the value will be null.
+            store_path = item['store_path_pre'] or item['store_path']
+            parent_id = item['id']
             if store_path not in db_store_path_set:
                 raise ValueError(f"store_path not found in db: {store_path}")
             if store_path in store_path_set:
                 continue
             store_path_set.add(store_path)
-            new_store_path = store_path if store_path.lower().endswith('.pdf') else store_path + '.pdf'
+            # new_store_path = store_path if store_path.lower().endswith('.pdf') else store_path + '.pdf'
+            new_store_path = parent_id_store_path_map[parent_id]
             x_attachments_pdf.append({
                 "store_path": new_store_path,
                 "store_path_txt": "",
@@ -76,11 +82,11 @@ class FlowUpdater:
             return {}
         return {'x_status_list.status_convert.status_txt': 'convert_txt_embedding'}
 
-    def _step_handle(self, step_stage, status, attachments, db_store_path):
+    def _step_handle(self, step_stage, status, attachments, db_store_path, attachments_pdf):
         method_name = f"_handle_{step_stage}"
         method = getattr(self, method_name, None)
         if method:
-            return method(status, attachments=attachments,
+            return method(status, attachments=attachments, attachments_pdf=attachments_pdf,
                           db_store_path_set=db_store_path) if step_stage == 'convert' else method(status)
         else:
             raise ValueError(f"Unknown step_stage: {step_stage}")
@@ -102,16 +108,16 @@ class FlowUpdater:
         current_stage = op_meta_record['current_stage']
         target_stage = op_meta_record['target_stage']
         attachments = op_meta_record['x_attachments']
+        attachments_pdf = op_meta_record['x_attachments_pdf']
         data_source = op_meta_record['data_source']
 
-        # 校验参数
         if not report_id or not status or not start_stage or not current_stage or not target_stage or (not attachments and start_stage == 'convert'):
             raise ValueError(f"Invalid op_meta_record: {op_meta_record}")
         if status == 'success' and target_stage != current_stage:
-            raise ValueError(f"Invalid current_stage: {current_stage}-{report_id}")
+            logger.error(f"Invalid current_stage: {current_stage}-{report_id}")
+            return
         attachments = [i for i in attachments if i['category'] == 'x_attachments']
 
-        # 确定结束阶段
         end_stage = target_stage if status == 'success' else current_stage if status == 'failed' else None
         if end_stage is None:
             logger.info(f"Invalid status: {status}.")
@@ -121,11 +127,9 @@ class FlowUpdater:
         if start_index > end_index:
             raise ValueError(f"start_stage cannot be after end_stage: {start_stage} -> {end_stage}.")
 
-        # 开始执行回更逻辑
         logger.info(
             f"😊 _id: {report_id}-{status}, start_step: {self.step_tuple[start_index]}, end_step: {self.step_tuple[end_index]}")
 
-        # 查询这个报告是否存在于当前数据源
         db_doc = self._check_and_create_collection(data_source).find_one({'_id': report_id},
                                                                          {'_id': 1, 'x_attachments': 1,
                                                                           'x_status_list': 1})
@@ -137,20 +141,17 @@ class FlowUpdater:
             logger.warning(f"{db_doc['_id']} statxus is not 'crawl_downloaded'")
             return
 
-        # 构建数据库中存在store_path 防止出现数据库中x_attachments与x_attachments_pdf数据不一致问题
         db_store_path = {f"s3://{i['bucket']}/{i['store_path']}" for i in db_doc['x_attachments']}
 
-        # 构建更新参数
         update_params = {}
         step_status = True
         for index, step in enumerate(self.step_tuple[start_index:end_index + 1], 1):
             if step == end_stage and status == 'failed':
                 step_status = False
             logger.info(f' Processing step-{index} {step} - {"successfully" if step_status else "failed"}.')
-            item = self._step_handle(step, step_status, attachments, db_store_path)
+            item = self._step_handle(step, step_status, attachments, db_store_path, attachments_pdf)
             update_params.update(item)
 
-        # 执行更新
         if update_params:
             # logger.info(json.dumps(update_params, ensure_ascii=False, indent=2))
             self.update_mongo_data(report_id, data_source, update_params, kafka_ignore)