orbitkit 0.8.47__tar.gz → 0.8.49__tar.gz

{orbitkit-0.8.47/orbitkit.egg-info → orbitkit-0.8.49}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: orbitkit
-Version: 0.8.47
+Version: 0.8.49
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao
@@ -37,19 +37,6 @@ Requires-Dist: prettytable>=3.17.0
 Requires-Dist: pytz>=2025.2
 Requires-Dist: Deprecated
 Requires-Dist: func_timeout
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
-Dynamic: license-file
-Dynamic: maintainer
-Dynamic: maintainer-email
-Dynamic: platform
-Dynamic: requires-dist
-Dynamic: summary
 
 # orbitkit
 

orbitkit-0.8.49/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.49

{orbitkit-0.8.47 → orbitkit-0.8.49}/orbitkit/airflow_handler/data_preprocessing.py

@@ -12,7 +12,7 @@ class DocumentProcessor:
     VIDEO_SUFFIXES = [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".3gp", ".hevc"]
     PDF_SUFFIXES = [".pdf"]
     DOC_SUFFIXES = [".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx"]
-    TXT_SUFFIXES = [".txt", ".html", ".htm"]
+    TXT_SUFFIXES = [".txt", ".html", ".htm", ".xhtml"]
     ALL_ALLOWED_SUFFIXES = set(AUDIO_SUFFIXES + VIDEO_SUFFIXES + PDF_SUFFIXES + DOC_SUFFIXES + TXT_SUFFIXES)
 
     DATA_PROCESS_STEPS = ['convert', 'extract', 'embedding', 'success']
@@ -74,9 +74,10 @@ class DocumentProcessor:
     def xbrl_type_check(cls, doc):
         is_xbrl = doc.get('x_info_data', {}).get('is_xbrl') == 'true'
         x_attachments = doc.get('x_attachments', [])
+        convert_status = doc.get('x_status_list', {}).get('status_convert', {}).get('status')
         xhtml_count = sum(1 for att in x_attachments if att['store_path'].lower().endswith('.xhtml'))
 
-        if is_xbrl or xhtml_count > 0:
+        if is_xbrl or xhtml_count > 0 and convert_status != 'convert_done':
             template = cls.create_xbrl_template()
             template['_id'] = doc['_id']
             template['source_type'] = doc.get('x_report_source', {}).get('source_type', '')
@@ -123,7 +124,7 @@ class DocumentProcessor:
             reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
         except ValueError:
             reported_date = datetime.datetime(1970, 1, 1)
-        return "extract" if reported_date < datetime.datetime(2020, 1, 1) else target_stage
+        return "extract" if reported_date < datetime.datetime(2023, 1, 1) else target_stage
 
     @classmethod
     async def create_record(cls, doc, start_stage, important_level):
@@ -177,7 +178,7 @@ class DocumentProcessor:
 
         is_xbrl, xbrl_data = cls.xbrl_type_check(doc)
         if is_xbrl:
-            return cls.create_result_info("xbrl", "XBRL format cannot be processed.", xbrl_data)
+            return cls.create_result_info("xbrl", "XBRL or Xhtml format cannot be processed.", xbrl_data)
 
         start_stage, target_stage, x_spider_name = cls.get_start_stage_target_stage(doc, custom_process_step)
 
@@ -185,7 +186,7 @@ class DocumentProcessor:
         if target_stage == 'embedding' and not custom_process_step:
             target_stage = cls.update_target_stage_by_report_type(doc, target_stage)
             target_stage = cls.update_target_stage_by_reported_at(doc, target_stage)
-            # 特殊情况下只需要做提取但是这个数据被过滤不需要做embedding
+            # 特殊情况下只需要做embedding 但是这个数据被条件限制为只做到提取时状态异常
             if start_stage == 'embedding' and target_stage == 'extract':
                 start_stage = 'success'
                 target_stage = 'success'
@@ -193,11 +194,11 @@ class DocumentProcessor:
         if cls.DATA_PROCESS_STEPS.index(target_stage) < cls.DATA_PROCESS_STEPS.index(start_stage):
             return cls.create_result_info("step_error",
                                           "Invalid process sequence: 'start_stage' occurs before 'target_stage'.",
-                                          doc['_id'])
+                                          report_id)
 
         file_name_check_status, record = await cls.create_record(doc, start_stage, important_level)
         if not file_name_check_status:
-            return cls.create_result_info("step_error", "Document file name too lang.", report_id)
+            return cls.create_result_info("error", "Document file name too lang.", report_id)
 
         return cls.create_result_info("file_flow", "Success", [start_stage, target_stage, x_spider_name, record])
 

{orbitkit-0.8.47 → orbitkit-0.8.49}/orbitkit/airflow_handler/file_flow_entry_process.py

@@ -1,7 +1,7 @@
 import os
 from collections import Counter
 from datetime import datetime
-from typing import Literal, Optional
+from typing import Optional
 import logging
 import pymongo
 import pytz
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 class FilingOfficialProcessor:
 
-    def __init__(self, mongo_uri=None, postgres_uri=None, aws_access_key_id=None, aws_secret_access_key=None, pi2_postgres_uri=None, pi2_database_name=None):
+    def __init__(self, mongo_uri=None, postgres_uri=None, aws_access_key_id=None, aws_secret_access_key=None, pi2_postgres_uri=None, pi2_database_name=None, databases_fileflow=None):
         mongo_uri = os.environ.get('MONGO_URI_MAIN_USER_APP') if not mongo_uri else mongo_uri
         if not mongo_uri:
             raise KeyError('mongo_uri not set.')
@@ -29,23 +29,15 @@ class FilingOfficialProcessor:
 
         self.mongo_client = pymongo.MongoClient(mongo_uri)
         self.data_xbrl_convert_collection = self.mongo_client['filing_reports']['data_xbrl_convert']
-        self.filing_data_collection = self.mongo_client['filing_reports']['filing_data']
-        self.annotation_reports_view_rows_collection = self.mongo_client['filing_reports'][
-            'annotation_reports_view_rows']
-        self.source_map = {
-            'filing_data': (self.filing_data_collection, 'filing_data'),
-            'G7_demo': (self.filing_data_collection, 'G7_demo'),
-            'reports_view': [
-                (self.filing_data_collection, 'filing_data')
-            ]
-        }
+
         postgres_uri = os.environ.get('PG_URI_AIRFLOW12_USER_NEWSFEEDSITE') if not postgres_uri else postgres_uri
         if not postgres_uri:
             raise KeyError('postgres_uri not set.')
-        self.file_handler = FileFlowHandleV2(postgres_uri=postgres_uri)
+        databases_fileflow = databases_fileflow or "process_net"
+        self.file_handler = FileFlowHandleV2(postgres_uri=postgres_uri, database_name=databases_fileflow)
         self.data_processor = DocumentProcessor()
         self.max_batch_size = 10000
-        self.all_stat_count = {'all': 0, 'skip': 0, 'step_error': 0, 'xbrl': 0, 'file_flow': 0}
+        self.all_stat_count = {'all': 0, 'skip': 0, 'doc_error': 0, 'step_error': 0, 'xbrl': 0, 'file_flow': 0}
 
         self.s3_client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
         self.matcher = OrbitTypeMatcher(self.s3_client)
@@ -65,9 +57,19 @@ class FilingOfficialProcessor:
             autoload_with=self.postgres_engine, schema='security_master'
         )
 
+        self.postgres_engine2 = create_engine(f"{postgres_uri}/{databases_fileflow}",
+                                             connect_args={"sslmode": "require"})
+        self.postgres_session2 = sessionmaker(bind=self.postgres_engine2)
+        self.Session2 = scoped_session(self.postgres_session2)
+
+        self.op_meta = Table(
+            'op_meta', self.postgres_metadata,
+            autoload_with=self.postgres_engine2, schema='public'
+        )
+
     @contextmanager
-    def session_scope(self):
-        session = self.Session()
+    def session_scope(self, use_session=None):
+        session = self.Session() if not use_session else use_session
         try:
             yield session
             session.commit()
@@ -77,7 +79,7 @@ class FilingOfficialProcessor:
         finally:
             self.Session.remove()
 
-    def create_spider_name_source_type_map(self, collections):
+    def create_spider_name_source_type_map(self, collection, label):
 
         def find_duplicates(keys):
             return [k for k, v in Counter(keys).items() if v > 1]
@@ -85,9 +87,8 @@ class FilingOfficialProcessor:
         map_dict = {}
         pipeline = [{'$group': {'_id': "$x_spider_name"}}]
 
-        for collection, label in collections:
-            for document in collection.aggregate(pipeline):
-                map_dict[document['_id']] = label
+        for document in collection.aggregate(pipeline):
+            map_dict[document['_id']] = label
 
         all_keys = list(map_dict.keys())
         duplicates = find_duplicates(all_keys)
@@ -174,10 +175,39 @@ class FilingOfficialProcessor:
             self.all_stat_count['file_flow'] += len(records)
             logger.info(f"{len(records)}--{start_stage}-{target_stage}-{x_spider_name} status: {status}, message: {message}")
 
+    def op_meat_deduplicate_docs(self, docs, buffer_size=1000):
+        buffer = []
+
+        for doc in docs:
+            buffer.append(doc)
+
+            if len(buffer) >= buffer_size:
+                doc_ids = [d['_id'] for d in buffer]
+                with self.session_scope(use_session=self.Session2) as session:
+                    existing_ids = session.query(self.op_meta.c.id).filter(self.op_meta.c.id.in_(doc_ids)).all()
+                    existing_ids = {i[0] for i in existing_ids}
+                for buffered_doc in buffer:
+                    self.all_stat_count['all'] += 1
+                    if buffered_doc['_id'] not in existing_ids:
+                        yield buffered_doc
+
+                buffer.clear()
+
+        if buffer:
+            doc_ids = [d['_id'] for d in buffer]
+            with self.session_scope(use_session=self.Session2) as session:
+                existing_ids = session.query(self.op_meta.c.id).filter(self.op_meta.c.id.in_(doc_ids)).all()
+                existing_ids = {i[0] for i in existing_ids}
+            for buffered_doc in buffer:
+                self.all_stat_count['all'] += 1
+                if buffered_doc['_id'] not in existing_ids:
+                    yield buffered_doc
+
+            buffer.clear()
 
-    async def process_task_entry(self, source: Literal["filing_data", "reports_view", "G7_demo"],
+    async def process_task_entry(self, source: str,
                            query: dict, tags: list[str], priority: str,
-                           is_important: bool = False, custom_step: Optional[list[str]] = None, important_level = None):
+                           is_important: bool = False, custom_step: Optional[list[str]] = None, important_level = None, db_name: str = None):
 
         if not important_level or not isinstance(important_level, int):
             important_level = 0
@@ -195,43 +225,24 @@ class FilingOfficialProcessor:
                 if step not in allowed_steps:
                     raise ValueError(f"Invalid step '{step}'. Allowed steps are: {allowed_steps}")
 
-        if source == 'reports_view':
-            collections = self.source_map[source]
-        else:
-            collections = [self.source_map[source]]
-
-        spider_name_source_type = self.create_spider_name_source_type_map(collections)
+        collection = self.mongo_client[db_name if db_name else "filing_reports"][source]
+        spider_name_source_type = self.create_spider_name_source_type_map(collection, source)
 
         process_data = []
         perm_id_set = set()
-        for collection, label in collections:
-            logger.info(f"load {label} data.")
-            docs = collection.find(query).batch_size(1000)
-
-            for doc in docs:
-                self.all_stat_count['all'] += 1
-                for orbit_entity_id in doc['x_orbit_data']['perm_id_list']:
-                    perm_id_set.add(orbit_entity_id)
-                result_record = await self.data_processor.process(doc=doc, custom_process_step=custom_step, important_level=important_level)
-                process_data.append(result_record)
-                if len(process_data) >= self.max_batch_size:
-                    file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
-                        process_data)
-                    file_flow_info = self.update_extends_fields(list(perm_id_set), file_flow_info)
-                    self.all_stat_count['skip'] += len(doc_error_list)
-                    self.all_stat_count['step_error'] += len(except_id_list)
-                    self.all_stat_count['xbrl'] += len(xbrl_data)
-                    self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
-                    self.send_xbrl_data_to_mongo(xbrl_data)
-                    self.update_doc_status_to_convert(collection, doc_error_list)
-                    process_data.clear()
-                    perm_id_set.clear()
-
-            if process_data:
+        logger.info(f"load {source} data.")
+        docs = collection.find(query).batch_size(1000)
+        duplicate_docs = self.op_meat_deduplicate_docs(docs, buffer_size=self.max_batch_size) if not is_important else docs
+        for doc in duplicate_docs:
+            for orbit_entity_id in doc['x_orbit_data']['perm_id_list']:
+                perm_id_set.add(orbit_entity_id)
+            result_record = await self.data_processor.process(doc=doc, custom_process_step=custom_step, important_level=important_level)
+            process_data.append(result_record)
+            if len(process_data) >= self.max_batch_size:
                 file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
                     process_data)
                 file_flow_info = self.update_extends_fields(list(perm_id_set), file_flow_info)
-                self.all_stat_count['skip'] += len(doc_error_list)
+                self.all_stat_count['doc_error'] += len(doc_error_list)
                 self.all_stat_count['step_error'] += len(except_id_list)
                 self.all_stat_count['xbrl'] += len(xbrl_data)
                 self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
@@ -240,5 +251,18 @@ class FilingOfficialProcessor:
                 process_data.clear()
                 perm_id_set.clear()
 
+        if process_data:
+            file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
+                process_data)
+            file_flow_info = self.update_extends_fields(list(perm_id_set), file_flow_info)
+            self.all_stat_count['doc_error'] += len(doc_error_list)
+            self.all_stat_count['step_error'] += len(except_id_list)
+            self.all_stat_count['xbrl'] += len(xbrl_data)
+            self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
+            self.send_xbrl_data_to_mongo(xbrl_data)
+            self.update_doc_status_to_convert(collection, doc_error_list)
+            process_data.clear()
+            perm_id_set.clear()
+
         logger.info(f"finish processing {self.all_stat_count}. \n")
-        self.all_stat_count = {'all': 0, 'skip': 0, 'step_error': 0, 'xbrl': 0, 'file_flow': 0}
+        self.all_stat_count = {'all': 0, 'skip': 0, 'doc_error': 0, 'step_error': 0, 'xbrl': 0, 'file_flow': 0}

{orbitkit-0.8.47 → orbitkit-0.8.49}/orbitkit/airflow_handler/file_handler_v2.py

@@ -44,7 +44,7 @@ class FileFlowHandleV2:
             autoload_with=self.postgres_engine, schema='public'
         )
 
-        self.not_allow_file_type_list = not_allow_file_type_list or ['.xhtml']
+        self.not_allow_file_type_list = not_allow_file_type_list or []
 
     @contextmanager
     def session_scope(self):
@@ -94,20 +94,23 @@ class FileFlowHandleV2:
             result = session.execute(stmt)
             return [row[0] for row in result.fetchall()]
 
-    def _check_records(self, records: List[Dict[str, Any]], clean_exist_data: bool) -> Tuple[bool, List[str], str]:
+    def _check_records(self, records: List[Dict[str, Any]], clean_exist_data: bool) -> Tuple[bool, List[str], str, List[str]]:
         ids = []
+        invalidate_ids = []
         record_count = 0
         for record in records:
             record_count += 1
             is_valid, msg = self._validate_record(record)
             if not is_valid:
-                return False, [record.get("id", "unknown")], msg
+                invalidate_ids.append(record.get('id', 'unknown'))
+                logger.error(f"Validation failed for record {record.get('id', 'unknown')}: {msg}")
+                continue
             ids.append(record["id"])
 
         existing_ids = self._get_existing_ids(ids)
         if not clean_exist_data and len(existing_ids) == record_count:
-            return False, existing_ids, "No new data has been inserted."
-        return True, existing_ids, f"Validation complete. total: {len(records)}. {len(existing_ids)} records already exist."
+            return False, existing_ids, "No new data has been inserted.", invalidate_ids
+        return True, existing_ids, f"Validation complete. total: {len(records)}. {len(existing_ids)} records already exist.", invalidate_ids
 
     def _build_insert_data(
             self, record: Dict[str, Any], params: Dict[str, Any]
@@ -324,7 +327,7 @@ class FileFlowHandleV2:
         elif not isinstance(records, list):
             raise ValueError("records must be a dict or list of dicts.")
 
-        is_valid, existing_ids, msg = self._check_records(records, clean_exist_data)
+        is_valid, existing_ids, msg, invalidate_ids = self._check_records(records, clean_exist_data)
         if not is_valid:
             return False, existing_ids, msg
         logger.info(msg)
@@ -334,6 +337,8 @@ class FileFlowHandleV2:
         exist_data_ids = set()
         count = 0
         for record in records:
+            if record['id'] in invalidate_ids:
+                continue
             count += 1
             if record['id'] in existing_ids:
                 if clean_exist_data:

orbitkit-0.8.49/orbitkit/orbit_type/__init__.py

@@ -0,0 +1 @@
+from .orbit_type_simple import OrbitTypeMatcher, OrbitTypeMatcherAsync

{orbitkit-0.8.47 → orbitkit-0.8.49}/orbitkit/orbit_type/orbit_type_simple.py

@@ -2,9 +2,10 @@ import abc
 import json
 import logging
 import re
-from typing import List, Literal
+from typing import List, Literal, Optional
 from botocore.exceptions import ClientError
 import boto3
+import aioboto3
 
 logger = logging.getLogger(__name__)
 
@@ -459,6 +460,281 @@ class OrbitTypeMatcher:
             }
 
 
+class OrbitTypeMatcherAsync:
+    """
+    OrbitTypeMatcher 的异步版本
+    使用 aioboto3 进行异步 S3 操作，提升性能
+    """
+
+    def __init__(
+        self,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+        file_bucket: str = 'ot-cdn',
+        key_prefix: str = 'orbit-typing/typing-prod'
+    ):
+        """
+        初始化异步类型匹配器
+        
+        :param aws_access_key_id: AWS access key ID（可选，不提供则使用默认凭证链）
+        :param aws_secret_access_key: AWS secret access key（可选）
+        :param file_bucket: S3 bucket 名称
+        :param key_prefix: S3 key 前缀
+        """
+        # 创建 aioboto3 session
+        if aws_access_key_id and aws_secret_access_key:
+            self.session = aioboto3.Session(
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key
+            )
+        else:
+            # 使用默认凭证链（环境变量、AWS CLI 配置、IAM 角色等）
+            self.session = aioboto3.Session()
+        
+        self.file_bucket = file_bucket
+        self.key_prefix = key_prefix
+        self.matcher = None
+        self.json_file = None
+        self.version = None
+
+    @classmethod
+    async def create(
+        cls,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+        file_bucket: str = 'ot-cdn',
+        key_prefix: str = 'orbit-typing/typing-prod',
+        version: Optional[str] = None
+    ):
+        """
+        异步工厂方法，用于创建已初始化的 OrbitTypeMatcherAsync 实例
+        
+        :param aws_access_key_id: AWS access key ID（可选）
+        :param aws_secret_access_key: AWS secret access key（可选）
+        :param file_bucket: S3 bucket 名称
+        :param key_prefix: S3 key 前缀
+        :param version: 类型版本号（可选，不提供则使用最新版本）
+        :return: 已初始化的 OrbitTypeMatcherAsync 实例
+        
+        示例：
+            matcher = await OrbitTypeMatcherAsync.create(version='0.3.4')
+            result = matcher.match_type(title='Press Release')
+        """
+        instance = cls(
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            file_bucket=file_bucket,
+            key_prefix=key_prefix
+        )
+        
+        # 异步加载版本和配置文件
+        if not version:
+            version = await instance.get_newest_version()
+        
+        source_key = f'{key_prefix}/{version}.json'
+        json_file = await instance.read_s3_file(
+            bucket_name=file_bucket,
+            file_name=source_key
+        )
+        
+        if not json_file:
+            raise Exception(f'该 S3 文件不存在: {source_key}')
+        
+        instance.json_file = json_file
+        instance.matcher = L3RuleListMatcher(json_file)
+        instance.version = version
+        
+        return instance
+
+    async def read_s3_file(self, bucket_name: str, file_name: str):
+        """
+        异步读取 S3 中的 JSON 文件
+        
+        :param bucket_name: S3 bucket 名称
+        :param file_name: 文件路径
+        :return: JSON 数据或 None
+        """
+        try:
+            async with self.session.client('s3') as s3_client:
+                file_obj = await s3_client.get_object(Bucket=bucket_name, Key=file_name)
+                content = await file_obj['Body'].read()
+                lines = content.decode('utf-8')
+                json_data = json.loads(lines)
+                return json_data
+        except ClientError as e:
+            logger.error(f"读取 S3 文件失败: {bucket_name}/{file_name}, 错误: {e}")
+            return None
+
+    async def get_newest_version(self):
+        """
+        异步获取最新的类型版本号
+        
+        :return: 最新版本号字符串
+        """
+        async with self.session.client('s3') as s3_client:
+            response = await s3_client.list_objects_v2(
+                Bucket=self.file_bucket,
+                Prefix=self.key_prefix
+            )
+
+            # 遍历存储桶中的对象，获取文件名
+            file_names = []
+            for obj in response.get('Contents', []):
+                key = obj['Key']
+                if key.endswith('/'):  # 排除文件夹本身
+                    continue
+                file_names.append(key.split('/')[-1])  # 获取文件名部分
+
+            result = []
+            for key in file_names:
+                if 'tmp.json' in key:
+                    continue
+                if '.json' not in key:
+                    continue
+                result.append(key.replace('.json', ''))
+
+            def sort_vision_json_file(filename):
+                parts = filename.replace('.json', '').split('.')
+                return tuple(int(part) for part in parts)
+
+            sorted_versions = sorted(result, key=sort_vision_json_file, reverse=True)
+            return sorted_versions[0]
+
+    def get_full_type_list(self):
+        """
+        获取完整包含 1、2、3 级类型的列表
+        注意：此方法是同步的，因为它只进行内存操作
+        
+        :return: 完整类型列表
+        """
+        lv1_list = self.json_file['lv1_list']
+        lv1_dict = {}
+        for row in lv1_list:
+            lv1_id = row['lv1_id']
+            name = row['lv1_name']
+            lv1_dict[lv1_id] = name
+
+        lv2_list = self.json_file['lv2_list']
+        lv2_dict = {}
+        for row in lv2_list:
+            lv1_id = row['lv1_id']
+            lv2_id = row['lv2_id']
+            name = row['lv2_name']
+            lv2_dict[lv2_id] = {
+                'lv2_id': lv2_id,
+                'lv2_name': name,
+                'lv1_id': lv1_id,
+                'lv1_name': lv1_dict[lv1_id]
+            }
+
+        lv3_list = self.json_file['lv3_list']
+        lv3_data = []
+        for row in lv3_list:
+            lv2_id = row['level_2_id']
+            this_dict = {
+                'lv3_id': row['lv3_id'],
+                'lv3_name': row['lv3_name']
+            }
+            this_dict.update(lv2_dict[lv2_id])
+            lv3_data.append(this_dict)
+
+        sorted_list = sorted(
+            lv3_data,
+            key=lambda x: (int(x['lv3_id']), int(x['lv2_id']), int(x['lv1_id']))
+        )
+        return sorted_list
+
+    def match_type(self, match_flag: Literal['in_order', 'match_all'] = 'in_order', **match_kwargs):
+        """
+        用于匹配的函数，可以传入 n 个匹配参数
+        注意：此方法是同步的，因为匹配操作只涉及内存操作，不需要 I/O
+        
+        :param match_flag: 匹配模式
+            - 'in_order': 按顺序依次匹配，匹配到结果就停止
+            - 'match_all': 匹配全部的项，并将结果合并
+        :param match_kwargs: 匹配参数（如 title='...', url='...', description='...'）
+        :return: 匹配结果
+        
+        示例：
+            # 按顺序匹配
+            key, result = matcher.match_type(
+                match_flag='in_order',
+                title='Press Release',
+                description='Company news'
+            )
+            
+            # 匹配所有
+            result = matcher.match_type(
+                match_flag='match_all',
+                title='Press Release',
+                url='https://example.com/news/press-release'
+            )
+        """
+        if match_flag not in ['in_order', 'match_all']:
+            raise ValueError('match_flag 参数必须是 "in_order" 或 "match_all"')
+        if not match_kwargs:
+            raise ValueError('必须传入匹配关键词！')
+
+        default_result = [{'lv3_id': '19999', 'lv3_name': 'Miscellaneous'}]
+
+        def match_url(url: str):
+            split_url = url.replace('http://', '').replace('https://', '').split('/')
+            split_url = [x.strip().replace('%20', ' ') for x in split_url if len(x.strip()) > 4]
+            for part in reversed(split_url):
+                match_result = self.matcher.start_match_all(part)
+                if match_result:
+                    match_key = f'url#{part}'
+                    return match_key, match_result
+            return None, None
+
+        if match_flag == 'in_order':
+            for key, value in match_kwargs.items():
+                if key == 'url':
+                    url_key, match_result = match_url(value)
+                    if match_result:
+                        return url_key, match_result
+                else:
+                    match_result = self.matcher.start_match_all(value)
+                    if match_result:
+                        return key, match_result
+            return None, default_result
+
+        elif match_flag == 'match_all':
+            overall_results = []
+            match_detail = {}
+            for key, value in match_kwargs.items():
+                if key == 'url':
+                    url_key, match_result = match_url(value)
+                    if match_result:
+                        overall_results += match_result
+                        match_detail[url_key] = match_result
+                else:
+                    match_result = self.matcher.start_match_all(value)
+                    if match_result:
+                        overall_results += match_result
+                        match_detail[key] = match_result
+
+            if not overall_results:
+                overall_results = default_result
+
+            unique_dict = {d['lv3_id']: d for d in overall_results}
+            sorted_results = sorted(unique_dict.values(), key=lambda x: int(x['lv3_id']))
+
+            return {
+                'results': sorted_results,
+                'match_detail': match_detail
+            }
+
+
 if __name__ == '__main__':
+    # 同步版本示例
     matcher = OrbitTypeMatcher(version='0.2.1')
     matcher.match_type(title='asdf')
+    
+    # 异步版本示例
+    # import asyncio
+    # async def test_async():
+    #     matcher = await OrbitTypeMatcherAsync.create(version='0.3.4')
+    #     result = matcher.match_type(match_flag='in_order', title='Press Release')
+    #     print(result)
+    # asyncio.run(test_async())

{orbitkit-0.8.47 → orbitkit-0.8.49/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: orbitkit
-Version: 0.8.47
+Version: 0.8.49
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao
@@ -37,19 +37,6 @@ Requires-Dist: prettytable>=3.17.0
 Requires-Dist: pytz>=2025.2
 Requires-Dist: Deprecated
 Requires-Dist: func_timeout
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
-Dynamic: license-file
-Dynamic: maintainer
-Dynamic: maintainer-email
-Dynamic: platform
-Dynamic: requires-dist
-Dynamic: summary
 
 # orbitkit
 

orbitkit-0.8.47/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.47

orbitkit-0.8.47/orbitkit/orbit_type/__init__.py

@@ -1 +0,0 @@
-from .orbit_type_simple import OrbitTypeMatcher