orbitkit 0.8.19__tar.gz → 0.8.55__tar.gz

{orbitkit-0.8.19/orbitkit.egg-info → orbitkit-0.8.55}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.19
+Version: 0.8.55
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao
@@ -19,13 +19,22 @@ Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: boto3>=1.16.0
-Requires-Dist: requests>=2.12.1
-Requires-Dist: prettytable>=3.0.0
-Requires-Dist: pytz>=2022.1
+Requires-Dist: boto3>=1.40.46
+Requires-Dist: aioboto3>=15.5.0
+Requires-Dist: aiofiles>=25.1.0
+Requires-Dist: requests>=2.32.5
+Requires-Dist: prettytable>=3.16.0
+Requires-Dist: pytz>=2025.2
 Requires-Dist: Deprecated
 Requires-Dist: func_timeout
 Dynamic: author
@@ -35,6 +44,7 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
+Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
 Dynamic: platform

orbitkit-0.8.55/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.55

orbitkit-0.8.55/orbitkit/airflow_handler/data_preprocessing.py

@@ -0,0 +1,234 @@
+import os
+import datetime
+from collections import defaultdict
+from importlib.metadata import version
+import googletrans
+
+
+class DocumentProcessor:
+    if version("googletrans") < "4.0.2":
+        raise ImportError(f"googletrans >= 4.0.2 is required for async support. {version('googletrans')}")
+    AUDIO_SUFFIXES = [".mp3", ".wav", ".aac", ".wma", ".m4a"]
+    VIDEO_SUFFIXES = [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".3gp", ".hevc"]
+    PDF_SUFFIXES = [".pdf"]
+    DOC_SUFFIXES = [".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx"]
+    TXT_SUFFIXES = [".txt", ".html", ".htm", ".xhtml"]
+    ALL_ALLOWED_SUFFIXES = set(AUDIO_SUFFIXES + VIDEO_SUFFIXES + PDF_SUFFIXES + DOC_SUFFIXES + TXT_SUFFIXES)
+
+    DATA_PROCESS_STEPS = ['convert', 'extract', 'embedding', 'success']
+
+    @staticmethod
+    def get_file_suffix(file_path):
+        return f".{file_path.split('.')[-1]}".lower()
+
+    @staticmethod
+    async def translate_text(text, dest='en'):
+        """异步翻译函数 https://pypi.org/project/googletrans/"""
+        translator = googletrans.Translator()
+        result = await translator.translate(text, dest=dest)
+        return result.text
+
+    @staticmethod
+    def create_xbrl_template():
+        return {
+            "_id": "",
+            "source_type": "",
+            "x_attachments": [],
+            "status": "init",
+            "logs": [],
+            "metadata": {},
+            "x_created_date": datetime.datetime.now(),
+            "x_updated_date": datetime.datetime.now(),
+        }
+
+    @classmethod
+    def stock_us_filter_by_is_primary(cls, doc):
+        if not doc:
+            return None
+        if doc.get('x_spider_name') != 'stock_us':
+            return doc
+        doc['x_attachments'] = [att for att in doc.get('x_attachments', []) if 'is_primary' in att]
+        if len(doc['x_attachments']) == 0:
+            return None
+        return doc
+
+    @classmethod
+    def stock_indiabse_filter_by_prefix(cls, doc):
+        if not doc:
+            return None
+        if doc.get('x_spider_name') != 'stock_indiabse':
+            return doc
+        doc['x_attachments'] = [att for att in doc.get('x_attachments', []) if att['file_type'].lower() != '.xml']
+        if len(doc['x_attachments']) == 0:
+            return None
+        return doc
+
+    @classmethod
+    def file_type_filter(cls, doc):
+        if not doc:
+            return None
+        suffixes = {cls.get_file_suffix(att['store_path']) for att in doc.get('x_attachments', [])}
+        return doc if suffixes.issubset(cls.ALL_ALLOWED_SUFFIXES) else None
+
+    @classmethod
+    def xbrl_type_check(cls, doc):
+        is_xbrl = doc.get('x_info_data', {}).get('is_xbrl') == 'true'
+        x_attachments = doc.get('x_attachments', [])
+        convert_status = doc.get('x_status_list', {}).get('status_convert', {}).get('status')
+        xhtml_count = sum(1 for att in x_attachments if att['store_path'].lower().endswith('.xhtml'))
+
+        if is_xbrl or xhtml_count > 0 and convert_status != 'convert_done':
+            template = cls.create_xbrl_template()
+            template['_id'] = doc['_id']
+            template['source_type'] = doc.get('x_report_source', {}).get('source_type', '')
+            template['x_attachments'] = [{
+                "file_hash": att['file_hash'],
+                "store_path": f"s3://{att['bucket']}/{att['store_path']}",
+                "store_path_converted_pdf": "",
+                "store_path_converted_pdf_image": "",
+            } for att in x_attachments]
+            return True, template
+
+        return False, None
+
+    @staticmethod
+    def get_start_stage_target_stage(doc, custom_process_step_list):
+        status_info = doc.get('x_status_list', {}).get('status_convert', {})
+        status = status_info.get('status')
+        status_txt = status_info.get('status_txt')
+        x_spider_name = doc['x_spider_name']
+
+        if custom_process_step_list:
+            return custom_process_step_list[0], custom_process_step_list[1], x_spider_name
+
+        if status != 'convert_done':
+            return 'convert', 'embedding', x_spider_name
+
+        if status_txt not in ['convert_txt_done', 'convert_txt_embedding']:
+            return 'extract', 'embedding', x_spider_name
+
+        if status_txt == 'convert_txt_done':
+            return 'embedding', 'embedding', x_spider_name
+
+        return 'success', 'success', x_spider_name
+
+    @staticmethod
+    def update_target_stage_by_report_type(doc, target_stage):
+        report_type_ids = doc.get('x_orbit_data', {}).get('report_type_id_list', [])
+        return "extract" if report_type_ids == ['19999'] else target_stage
+
+    @staticmethod
+    def update_target_stage_by_reported_at(doc, target_stage):
+        date_str = doc.get('x_reported_at_utc_date', '1970-01-01')
+        try:
+            reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
+        except ValueError:
+            reported_date = datetime.datetime(1970, 1, 1)
+        return "extract" if reported_date < datetime.datetime(2023, 1, 1) else target_stage
+
+    @staticmethod
+    def update_target_stage_by_perm_match(doc, target_stage):
+        perm_match_status = doc['x_status_list']['status_perm']['status']
+        return target_stage if perm_match_status in {'perm_match_part', 'perm_match'} else "extract"
+
+    @classmethod
+    async def create_record(cls, doc, start_stage, important_level):
+        attachments = doc.get('x_attachments', []) if start_stage == 'convert' else doc.get('x_attachments_pdf', [])
+        s3_path_info = []
+        add_extends = {}
+        for att in attachments:
+            if len(att['file_name']) > 2000 or len(att['file_name'].encode('utf-8')) > 2000:
+                return False, None
+
+            if start_stage == 'convert' and not add_extends:
+                _, ext = os.path.splitext(att['store_path'])
+                if ext in cls.AUDIO_SUFFIXES or ext in cls.VIDEO_SUFFIXES:
+                    add_extends = {
+                        "original_title": doc['x_orbit_data']['report_title'],
+                        "title": await cls.translate_text(text=doc['x_orbit_data']['report_title']),
+                        "published": doc['x_reported_at_utc_date'],
+                        "tickers": [],
+                        "perm_id_list": doc['x_orbit_data']['perm_id_list'],
+                        "report_type_id_list_str": doc['x_orbit_data']['report_type_id_list']
+                    }
+
+            s3_path_info.append({
+                'store_path': f"s3://{att['bucket']}/{att['store_path']}" if start_stage == 'convert' else att[
+                    'store_path'],
+                'file_name': att['file_name']
+            })
+        result_dict = {'id': doc['_id'], 's3_path_info': s3_path_info, 'important_level': important_level}
+        if add_extends:
+            result_dict['extends'] = add_extends
+        return True, result_dict
+
+    @staticmethod
+    def create_result_info(process_type, message, result_data):
+        return {
+            'process_type': process_type,
+            'message': message,
+            'result_data': result_data
+        }
+
+    @classmethod
+    async def process(cls, doc, custom_process_step, important_level):
+        report_id = doc['_id']
+        # 筛选文件
+        doc = cls.stock_us_filter_by_is_primary(doc)
+        doc = cls.stock_indiabse_filter_by_prefix(doc)
+        # 校验文件类型必须在筛选文件类型之后
+        doc = cls.file_type_filter(doc)
+        if doc is None:
+            return cls.create_result_info("error", "Document file type is not allowed.", report_id)
+
+        is_xbrl, xbrl_data = cls.xbrl_type_check(doc)
+        if is_xbrl:
+            return cls.create_result_info("xbrl", "XBRL or Xhtml format cannot be processed.", xbrl_data)
+
+        start_stage, target_stage, x_spider_name = cls.get_start_stage_target_stage(doc, custom_process_step)
+
+        # 判断 特殊条件下的数据不做embedding ('19999'类型和报告日期小于2023-01-01)
+        if target_stage == 'embedding' and not custom_process_step:
+            target_stage = cls.update_target_stage_by_report_type(doc, target_stage)
+            target_stage = cls.update_target_stage_by_reported_at(doc, target_stage)
+            target_stage = cls.update_target_stage_by_perm_match(doc, target_stage)
+            # 特殊情况下只需要做embedding 但是这个数据被条件限制为只做到提取时状态异常
+            if start_stage == 'embedding' and target_stage == 'extract':
+                start_stage = 'success'
+                target_stage = 'success'
+
+        if cls.DATA_PROCESS_STEPS.index(target_stage) < cls.DATA_PROCESS_STEPS.index(start_stage):
+            return cls.create_result_info("step_error",
+                                          "Invalid process sequence: 'start_stage' occurs before 'target_stage'.",
+                                          report_id)
+
+        file_name_check_status, record = await cls.create_record(doc, start_stage, important_level)
+        if not file_name_check_status:
+            return cls.create_result_info("error", "Document file name too lang.", report_id)
+
+        return cls.create_result_info("file_flow", "Success", [start_stage, target_stage, x_spider_name, record])
+
+    @classmethod
+    def split_data_by_spider_name_and_step(cls, process_records):
+        file_flow_info = defaultdict(list)
+        xbrl_data_list = []
+        except_id_list = []
+        doc_error_list = []
+
+        for item in process_records:
+            process_type = item.get('process_type')
+            if process_type == 'xbrl':
+                xbrl_data_list.append(item['result_data'])
+            elif process_type == 'file_flow':
+                start_stage, target_stage, x_spider_name, record = item['result_data']
+                key = f"{start_stage}@__@{target_stage}@__@{x_spider_name}"
+                file_flow_info[key].append(record)
+            elif process_type == 'step_error':
+                except_id_list.append(item['result_data'])
+            elif process_type == 'error':
+                doc_error_list.append(item['result_data'])
+            else:
+                raise KeyError(
+                    f"Unknown process_type: {process_type}. Expected one of ['xbrl', 'file_flow', 'step_error', 'error'].")
+
+        return file_flow_info, xbrl_data_list, except_id_list, doc_error_list

orbitkit-0.8.55/orbitkit/airflow_handler/file_flow_entry_process.py

@@ -0,0 +1,274 @@
+import os
+from collections import Counter
+from datetime import datetime
+from typing import Optional
+import logging
+import pymongo
+import pytz
+import boto3
+from sqlalchemy import create_engine, Table, MetaData, select
+from sqlalchemy.orm import sessionmaker, scoped_session
+from contextlib import contextmanager
+
+from orbitkit.airflow_handler.file_handler_v2 import FileFlowHandleV2
+from orbitkit.airflow_handler.data_preprocessing import DocumentProcessor
+from orbitkit.orbit_type import OrbitTypeMatcher
+
+logger = logging.getLogger(__name__)
+
+
+class FilingOfficialProcessor:
+
+    def __init__(self, mongo_uri=None, postgres_uri=None, aws_access_key_id=None, aws_secret_access_key=None, pi2_postgres_uri=None, pi2_database_name=None, databases_fileflow=None):
+        mongo_uri = os.environ.get('MONGO_URI_MAIN_USER_APP') if not mongo_uri else mongo_uri
+        if not mongo_uri:
+            raise KeyError('mongo_uri not set.')
+
+        if not aws_secret_access_key or not aws_access_key_id:
+            raise KeyError('aws_access_key_id and aws_secret_access_key not set.')
+
+        self.mongo_client = pymongo.MongoClient(mongo_uri)
+        self.data_xbrl_convert_collection = self.mongo_client['filing_reports']['data_xbrl_convert']
+
+        postgres_uri = os.environ.get('PG_URI_AIRFLOW12_USER_NEWSFEEDSITE') if not postgres_uri else postgres_uri
+        if not postgres_uri:
+            raise KeyError('postgres_uri not set.')
+        databases_fileflow = databases_fileflow or "process_net"
+        self.file_handler = FileFlowHandleV2(postgres_uri=postgres_uri, database_name=databases_fileflow)
+        self.data_processor = DocumentProcessor()
+        self.max_batch_size = 10000
+        self.all_stat_count = {'all': 0, 'skip': 0, 'doc_error': 0, 'step_error': 0, 'xbrl': 0, 'file_flow': 0}
+
+        self.s3_client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
+        self.matcher = OrbitTypeMatcher(self.s3_client)
+        self.report_type_id_name_map = {i["lv3_id"]: i["lv3_name"] for i in self.matcher.get_full_type_list()}
+
+        self.pi2_postgres_uri = pi2_postgres_uri or os.environ['PG_URI_CX45_USER_GLAUUIADMIN']
+        if not self.pi2_postgres_uri:
+            raise KeyError('pie_postgres_uri not set.')
+        self.databases = pi2_database_name or 'newsfeedsite'
+        self.postgres_engine = create_engine(f"{self.pi2_postgres_uri}/{self.databases}", connect_args={"sslmode": "require"})
+        self.postgres_session = sessionmaker(bind=self.postgres_engine)
+        self.Session = scoped_session(self.postgres_session)
+        self.postgres_metadata = MetaData()
+
+        self.pi2_table = Table(
+            'primary_instrument_2_release', self.postgres_metadata,
+            autoload_with=self.postgres_engine, schema='security_master'
+        )
+
+        self.postgres_engine2 = create_engine(f"{postgres_uri}/{databases_fileflow}",
+                                             connect_args={"sslmode": "require"})
+        self.postgres_session2 = sessionmaker(bind=self.postgres_engine2)
+        self.Session2 = scoped_session(self.postgres_session2)
+
+        self.op_meta = Table(
+            'op_meta', self.postgres_metadata,
+            autoload_with=self.postgres_engine2, schema='public'
+        )
+
+    @contextmanager
+    def session_scope(self, use_session=None):
+        session = self.Session() if not use_session else use_session
+        try:
+            yield session
+            session.commit()
+        except Exception:
+            session.rollback()
+            raise
+        finally:
+            self.Session.remove()
+
+    def create_spider_name_source_type_map(self, collection, label):
+
+        def find_duplicates(keys):
+            return [k for k, v in Counter(keys).items() if v > 1]
+
+        map_dict = {}
+        pipeline = [{'$group': {'_id': "$x_spider_name"}}]
+
+        for document in collection.aggregate(pipeline):
+            map_dict[document['_id']] = label
+
+        all_keys = list(map_dict.keys())
+        duplicates = find_duplicates(all_keys)
+        if duplicates:
+            raise KeyError(f"Duplicate x_spider_name found: {duplicates}")
+
+        return map_dict
+
+    def send_xbrl_data_to_mongo(self, xbrl_data_list):
+        if not xbrl_data_list:
+            return
+        report_id_list = list(set([i['_id'] for i in xbrl_data_list]))
+        result = self.data_xbrl_convert_collection.find({'_id': {'$in': report_id_list}}, {'_id': 1}).batch_size(self.max_batch_size)
+        exists_id_list = [i['_id'] for i in result]
+        new_xbrl_data_list = [i for i in xbrl_data_list if i['_id'] not in exists_id_list]
+        if not new_xbrl_data_list:
+            return
+        self.data_xbrl_convert_collection.insert_many(new_xbrl_data_list)
+        logger.info(f'{len(new_xbrl_data_list)}-xbrl data inserted.')
+
+    def update_doc_status_to_convert(self, collection, report_id_list):
+        if len(report_id_list) == 0:
+            return
+        collection.update_many(
+            {"_id": {"$in": report_id_list}},
+            {
+                "$set": {
+                    "x_status_list.status_convert.status": "convert_failed",
+                    "x_status_list.status_convert.status_txt": "convert_txt_init",
+                    "x_status_list.status_convert.status_meta": "meta_init",
+                    "x_updated_date": datetime.now(tz=pytz.timezone('UTC')).strftime("%Y-%m-%dT%H:%M:%S%z"),
+                },
+                "$unset": {
+                    "x_attachments_pdf": ""
+                }
+            }
+        )
+        logger.info(f'Unable to convert {len(report_id_list)} document(s) due to unsupported file type.')
+
+    def update_extends_fields(self, perm_id_list, file_flow_info):
+        stmt = select(self.pi2_table.c.orbit_entity_id, self.pi2_table.c.ticker).where(self.pi2_table.c.orbit_entity_id.in_(perm_id_list))
+        orbit_entity_id_ticker_map = {}
+        with self.session_scope() as session:
+            result = session.execute(stmt)
+            for row in result:
+                if row.orbit_entity_id not in orbit_entity_id_ticker_map:
+                    orbit_entity_id_ticker_map[row.orbit_entity_id] = []
+
+                if row.ticker is not None:
+                    orbit_entity_id_ticker_map[row.orbit_entity_id].append(row.ticker)
+        for step_info, records in file_flow_info.items():
+            for record in records:
+                if 'extends' in record and record.get('extends') is not None:
+                    tickers = set()
+                    for i in record['extends']['perm_id_list']:
+                        tickers.update(orbit_entity_id_ticker_map.get(i, []))
+                    record['extends']['tickers'] = list(tickers)
+
+                    record['extends']['report_type_id_list_str'] = [self.report_type_id_name_map.get(i) for i in record['extends']['report_type_id_list_str']]
+
+        return file_flow_info
+
+    def send_task(self, file_flow_info, tags, is_important, priority, spider_name_source_type):
+        for step_str, records in file_flow_info.items():
+            steps = step_str.split('@__@')
+            start_stage = steps[0]
+            target_stage = steps[1]
+            x_spider_name = steps[2]
+
+            if start_stage == 'success' or target_stage == 'success':
+                self.all_stat_count['skip'] += len(records)
+                logger.info(
+                    f"{len(records)}--{start_stage}-{target_stage}-{x_spider_name} status: False, message: 'File has already completed the embedding stage.' ")
+                continue
+
+            if is_important:
+                logger.info(f"is_important: {is_important} - {x_spider_name}")
+                status, ids, message = self.file_handler.entry_point_urgent(records=records, start_stage=start_stage,
+                                                                            target_stage=target_stage,
+                                                                            tags=tags,
+                                                                            tag=x_spider_name,
+                                                                            priority=priority,
+                                                                            source_type=spider_name_source_type[
+                                                                                x_spider_name])
+            else:
+                status, ids, message = self.file_handler.entry_point(records=records, start_stage=start_stage,
+                                                                     target_stage=target_stage, tags=tags,tag=x_spider_name,
+                                                                     priority=priority,
+                                                                     source_type=spider_name_source_type[x_spider_name])
+            self.all_stat_count['file_flow'] += len(records)
+            logger.info(f"{len(records)}--{start_stage}-{target_stage}-{x_spider_name} status: {status}, message: {message}")
+
+    def op_meat_deduplicate_docs(self, docs, buffer_size=1000):
+        buffer = []
+
+        for doc in docs:
+            buffer.append(doc)
+
+            if len(buffer) >= buffer_size:
+                doc_ids = [d['_id'] for d in buffer]
+                with self.session_scope(use_session=self.Session2) as session:
+                    existing_ids = session.query(self.op_meta.c.id).filter(self.op_meta.c.id.in_(doc_ids)).all()
+                    existing_ids = {i[0] for i in existing_ids}
+                for buffered_doc in buffer:
+                    self.all_stat_count['all'] += 1
+                    if buffered_doc['_id'] not in existing_ids:
+                        yield buffered_doc
+
+                buffer.clear()
+
+        if buffer:
+            doc_ids = [d['_id'] for d in buffer]
+            with self.session_scope(use_session=self.Session2) as session:
+                existing_ids = session.query(self.op_meta.c.id).filter(self.op_meta.c.id.in_(doc_ids)).all()
+                existing_ids = {i[0] for i in existing_ids}
+            for buffered_doc in buffer:
+                self.all_stat_count['all'] += 1
+                if buffered_doc['_id'] not in existing_ids:
+                    yield buffered_doc
+
+            buffer.clear()
+
+    async def process_task_entry(self, source: str,
+                           query: dict, tags: list[str], priority: str,
+                           is_important: bool = False, custom_step: Optional[list[str]] = None, important_level = None, db_name: str = None):
+
+        if not important_level or not isinstance(important_level, int):
+            important_level = 0
+
+        if important_level == 0:
+            raise ValueError(f'important_level must be an integer (int) greater than 0. {important_level}')
+
+        allowed_steps = {"convert", "extract", "embedding"}
+        if custom_step is not None:
+            if not isinstance(custom_step, list):
+                raise ValueError("custom_step must be a list or None.")
+            if len(custom_step) > 2:
+                raise ValueError("custom_step can contain at most two elements.")
+            for step in custom_step:
+                if step not in allowed_steps:
+                    raise ValueError(f"Invalid step '{step}'. Allowed steps are: {allowed_steps}")
+
+        collection = self.mongo_client[db_name if db_name else "filing_reports"][source]
+        spider_name_source_type = self.create_spider_name_source_type_map(collection, source)
+
+        process_data = []
+        perm_id_set = set()
+        logger.info(f"load {source} data.")
+        docs = collection.find(query).batch_size(1000)
+        duplicate_docs = self.op_meat_deduplicate_docs(docs, buffer_size=self.max_batch_size) if not is_important else docs
+        for doc in duplicate_docs:
+            for orbit_entity_id in doc['x_orbit_data']['perm_id_list']:
+                perm_id_set.add(orbit_entity_id)
+            result_record = await self.data_processor.process(doc=doc, custom_process_step=custom_step, important_level=important_level)
+            process_data.append(result_record)
+            if len(process_data) >= self.max_batch_size:
+                file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
+                    process_data)
+                file_flow_info = self.update_extends_fields(list(perm_id_set), file_flow_info)
+                self.all_stat_count['doc_error'] += len(doc_error_list)
+                self.all_stat_count['step_error'] += len(except_id_list)
+                self.all_stat_count['xbrl'] += len(xbrl_data)
+                self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
+                self.send_xbrl_data_to_mongo(xbrl_data)
+                self.update_doc_status_to_convert(collection, doc_error_list)
+                process_data.clear()
+                perm_id_set.clear()
+
+        if process_data:
+            file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
+                process_data)
+            file_flow_info = self.update_extends_fields(list(perm_id_set), file_flow_info)
+            self.all_stat_count['doc_error'] += len(doc_error_list)
+            self.all_stat_count['step_error'] += len(except_id_list)
+            self.all_stat_count['xbrl'] += len(xbrl_data)
+            self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
+            self.send_xbrl_data_to_mongo(xbrl_data)
+            self.update_doc_status_to_convert(collection, doc_error_list)
+            process_data.clear()
+            perm_id_set.clear()
+
+        logger.info(f"finish processing {self.all_stat_count}. \n")
+        self.all_stat_count = {'all': 0, 'skip': 0, 'doc_error': 0, 'step_error': 0, 'xbrl': 0, 'file_flow': 0}