orbitkit 0.8.78__tar.gz → 0.9.11__tar.gz

{orbitkit-0.8.78/orbitkit.egg-info → orbitkit-0.9.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.78
+Version: 0.9.11
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.9.11/orbitkit/VERSION

@@ -0,0 +1 @@
+0.9.11

{orbitkit-0.8.78 → orbitkit-0.9.11}/orbitkit/airflow_handler/data_preprocessing.py

@@ -160,6 +160,7 @@ class DocumentProcessor:
                         "title": title,
                         "published": doc['x_reported_at_utc_date'],
                         "tickers": [],
+                        "tickers_map_arr": [],
                         "perm_id_list": doc['x_orbit_data']['perm_id_list'],
                         "report_type_id_list_str": doc['x_orbit_data']['report_type_id_list']
                     }

orbitkit-0.9.11/orbitkit/airflow_handler/file_flow_entry.py

@@ -0,0 +1,46 @@
+"""file_flow 任务下发入口（独立运维工具，旁路）。
+
+通过 orbitkit.airflow_handler.FilingOfficialProcessor 把 MongoDB query 命中的 ID
+派发到 worker 队列。本入口与 .claude/skills/attachments-process-* 4-skill 体系并行，
+**不经 runtime registry**，仅作运维直跑使用。
+
+queue_type 选项：
+
+- ``MineruExtract``     : CPU 队列，默认，吞吐适中、成本低
+- ``MineruExtract-GPU`` : GPU 队列，吞吐高，需要 GPU 资源就绪
+
+切换 queue_type 时通常同步调整 priority（GPU 路径占用更高优先级）。
+
+与 4-skill 体系的关系见项目根 ``CLAUDE.md`` 的「attachments_process/（独立运维工具集，旁路）」段。
+"""
+import csv
+import datetime
+import logging
+import os
+import asyncio
+from orbitkit.airflow_handler.file_flow_entry_process import FilingOfficialProcessor
+from orbitkit.util.secret_manager import SecretManager
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+async def main_loop(max_process_count):
+    SecretManager.load_secret_2_envs(['prod/fileflow/data'])
+    processor = FilingOfficialProcessor(
+        mongo_uri=os.environ['MONGO_URI'],
+        postgres_uri=os.environ['POSTGRES_URI'],
+        aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
+        aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
+        pi2_postgres_uri=os.environ['PG_URI_CX45_USER_FILING']
+    )
+    query_dict = {
+        '_id': 'f_aXz7Y4n9qTlTRyuTc8sWsb',
+    }
+    await processor.process_task_entry(source='filing_data_staging', query=query_dict,
+                                            tags=['daily_increment_data'], allow_parse_over_limit=True,
+                                       priority='1', is_important=False, important_level=1, queue_type='MineruExtract-GPU')
+
+if __name__ == '__main__':
+    asyncio.run(main_loop(1))
+

{orbitkit-0.8.78 → orbitkit-0.9.11}/orbitkit/airflow_handler/file_flow_entry_process.py

@@ -29,6 +29,7 @@ class FilingOfficialProcessor:
 
         self.mongo_client = pymongo.MongoClient(mongo_uri)
         self.data_xbrl_convert_collection = self.mongo_client['filing_reports']['data_xbrl_convert']
+        self.sm_orbit_entity_id_cache_collection = self.mongo_client['stat_cache']['sm_orbit_entity_id_cache']
 
         postgres_uri = os.environ.get('PG_URI_AIRFLOW12_USER_NEWSFEEDSITE') if not postgres_uri else postgres_uri
         if not postgres_uri:
@@ -139,19 +140,33 @@ class FilingOfficialProcessor:
 
                 if row.ticker is not None:
                     orbit_entity_id_ticker_map[row.orbit_entity_id].append(row.ticker)
+        # 批量查询 organization_name / primary_ticker（走查 doc 的同一个 mongo 连接）
+        orbit_entity_id_org_map = {}
+        cache_cursor = self.sm_orbit_entity_id_cache_collection.find(
+            {'orbit_entity_id': {'$in': perm_id_list}},
+            {'orbit_entity_id': 1, 'organization_name': 1, 'primary_ticker': 1}
+        ).batch_size(self.max_batch_size)
+        for row in cache_cursor:
+            orbit_entity_id_org_map[row['orbit_entity_id']] = (row.get('organization_name'), row.get('primary_ticker'))
+
         for step_info, records in file_flow_info.items():
             for record in records:
                 if 'extends' in record and record.get('extends') is not None:
                     tickers = set()
+                    tickers_map_arr = []
                     for i in record['extends']['perm_id_list']:
                         tickers.update(orbit_entity_id_ticker_map.get(i, []))
+                        org_info = orbit_entity_id_org_map.get(i)
+                        if org_info and org_info[0] is not None:
+                            tickers_map_arr.append({org_info[0]: org_info[1]})
                     record['extends']['tickers'] = list(tickers)
+                    record['extends']['tickers_map_arr'] = tickers_map_arr
 
                     record['extends']['report_type_id_list_str'] = [self.report_type_id_name_map.get(i) for i in record['extends']['report_type_id_list_str']]
 
         return file_flow_info
 
-    def send_task(self, file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type):
+    def send_task(self, file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type, allow_parse_over_limit=False):
         for step_str, records in file_flow_info.items():
             steps = step_str.split('@__@')
             start_stage = steps[0]
@@ -173,13 +188,15 @@ class FilingOfficialProcessor:
                                                                             priority=priority,
                                                                             source_type=spider_name_source_type[
                                                                                 x_spider_name],
-                                                                            queue_type=queue_type)
+                                                                            queue_type=queue_type,
+                                                                            allow_parse_over_limit=allow_parse_over_limit)
             else:
                 status, ids, message = self.file_handler.entry_point(records=records, start_stage=start_stage,
                                                                      target_stage=target_stage, tags=tags,tag=x_spider_name,
                                                                      priority=priority,
                                                                      source_type=spider_name_source_type[x_spider_name],
-                                                                     queue_type=queue_type)
+                                                                     queue_type=queue_type,
+                                                                     allow_parse_over_limit=allow_parse_over_limit)
             self.all_stat_count['file_flow'] += len(records)
             logger.info(f"{len(records)}--{start_stage}-{target_stage}-{x_spider_name} status: {status}, message: {message}")
 
@@ -215,7 +232,7 @@ class FilingOfficialProcessor:
 
     async def process_task_entry(self, source: str,
                            query: dict, tags: list[str], priority: str,
-                           is_important: bool = False, custom_step: Optional[list[str]] = None, important_level = None, db_name: str = None, queue_type: str = 'Default'):
+                           is_important: bool = False, custom_step: Optional[list[str]] = None, important_level = None, db_name: str = None, queue_type: str = 'Default', allow_parse_over_limit: bool = False):
 
         if source == 'filing_data':
             raise ValueError(f"Cannot update documents from 'filing_data' source. ")
@@ -253,7 +270,7 @@ class FilingOfficialProcessor:
                 self.all_stat_count['doc_error'] += len(doc_error_list)
                 self.all_stat_count['step_error'] += len(except_id_list)
                 self.all_stat_count['xbrl'] += len(xbrl_data)
-                self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type)
+                self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type, allow_parse_over_limit)
                 self.send_xbrl_data_to_mongo(xbrl_data)
                 self.update_doc_status_to_convert(collection, doc_error_list)
                 process_data.clear()
@@ -266,7 +283,7 @@ class FilingOfficialProcessor:
             self.all_stat_count['doc_error'] += len(doc_error_list)
             self.all_stat_count['step_error'] += len(except_id_list)
             self.all_stat_count['xbrl'] += len(xbrl_data)
-            self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type)
+            self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type, allow_parse_over_limit)
             self.send_xbrl_data_to_mongo(xbrl_data)
             self.update_doc_status_to_convert(collection, doc_error_list)
             process_data.clear()

{orbitkit-0.8.78 → orbitkit-0.9.11}/orbitkit/airflow_handler/file_flow_exit_process.py

@@ -91,6 +91,43 @@ class FlowUpdater:
         else:
             raise ValueError(f"Unknown step_stage: {step_stage}")
 
+    @staticmethod
+    def _build_x_parse_record(attachments_pdf):
+        """整合同一报告下所有 x_attachments_pdf 行的 x_parse_record，回写 mongo。
+
+        逐附件无损保留到 details（附加 store_path）；summary 里放去重的 tools / tool_versions
+        及 is_mixed，方便报告级查询「用了什么工具/什么版本」。无有效记录返回 None。
+        """
+        details = []
+        tools = []
+        tool_versions = []
+        for att in attachments_pdf or []:
+            rec = att.get('x_parse_record')
+            if not rec:  # None / [] / {}
+                continue
+            detail = dict(rec)
+            detail['store_path'] = att.get('store_path')
+            details.append(detail)
+            name = rec.get('parsing_tool_name')
+            version = rec.get('parsing_tool_version')
+            if name:
+                tools.append(name)
+                if version:
+                    tool_versions.append(f"{name}@{version}")
+
+        if not details:
+            return None
+
+        tool_versions = sorted(set(tool_versions))
+        return {
+            'summary': {
+                'tools': sorted(set(tools)),
+                'tool_versions': tool_versions,
+                'is_mixed': len(tool_versions) > 1
+            },
+            'details': details
+        }
+
     def update_mongo_data(self, report_id, data_source, update_params, kafka_ignore):
         if kafka_ignore:
             update_params.update({self.kafka_ignore.X_OTHERS_K_IGNORE_KEY: self.kafka_ignore.get_k_ignore_val()})
@@ -157,6 +194,12 @@ class FlowUpdater:
             item = self._step_handle(step, step_status, attachments, db_store_path, attachments_pdf)
             update_params.update(item)
 
+        # 提取阶段完成即回写解析工具记录（不论解析成功与否）
+        if 'extract' in self.step_tuple[start_index:end_index + 1]:
+            parse_record = self._build_x_parse_record(attachments_pdf)
+            if parse_record is not None:
+                update_params['x_parse_record'] = parse_record
+
         if update_params:
             # logger.info(json.dumps(update_params, ensure_ascii=False, indent=2))
             self.update_mongo_data(report_id, data_source, update_params, kafka_ignore)

{orbitkit-0.8.78 → orbitkit-0.9.11}/orbitkit/airflow_handler/file_handler_v2.py

@@ -142,7 +142,8 @@ class FileFlowHandleV2:
             'tags': params['tags'],
             'tag': params['tag'],
             'important_level': record.get('important_level', 0),
-            'queue': params['queue']
+            'queue': params['queue'],
+            'allow_parse_over_limit': params.get('allow_parse_over_limit', False)
         }
 
         step = {
@@ -226,7 +227,8 @@ class FileFlowHandleV2:
             tag: str = None,
             priority: str = '1',
             source_type: Optional[str] = None,
-            queue_type: str = 'Default'
+            queue_type: str = 'Default',
+            allow_parse_over_limit: bool = False
     ) -> Tuple[bool, Any, str]:
         """
             普通任务接口
@@ -254,7 +256,8 @@ class FileFlowHandleV2:
         """
         return self._file_flow_entry_point_internal(
             records, start_stage, target_stage, tags, priority, source_type,tag,
-            urgent=False, clean_exist_data=False, queue_type=queue_type
+            urgent=False, clean_exist_data=False, queue_type=queue_type,
+            allow_parse_over_limit=allow_parse_over_limit
         )
 
     def entry_point_urgent(
@@ -266,7 +269,8 @@ class FileFlowHandleV2:
             tag: str = None,
             priority: str = '1',
             source_type: Optional[str] = None,
-            queue_type: str = 'Default'
+            queue_type: str = 'Default',
+            allow_parse_over_limit: bool = False
     ) -> Tuple[bool, Any, str]:
         """
             加急任务接口
@@ -294,7 +298,8 @@ class FileFlowHandleV2:
         """
         return self._file_flow_entry_point_internal(
             records, start_stage, target_stage, tags, priority, source_type, tag,
-            urgent=True, clean_exist_data=True, queue_type=queue_type
+            urgent=True, clean_exist_data=True, queue_type=queue_type,
+            allow_parse_over_limit=allow_parse_over_limit
         )
 
     def _file_flow_entry_point_internal(
@@ -308,7 +313,8 @@ class FileFlowHandleV2:
             tag: str,
             urgent: bool,
             clean_exist_data: bool = False,
-            queue_type: str = 'Default'
+            queue_type: str = 'Default',
+            allow_parse_over_limit: bool = False
     ) -> Tuple[bool, Any, str]:
         """核心处理逻辑"""
         params = {
@@ -320,7 +326,8 @@ class FileFlowHandleV2:
             'source_type': source_type,
             'tag': tag,
             'urgent': urgent,
-            'queue': queue_type
+            'queue': queue_type,
+            'allow_parse_over_limit': allow_parse_over_limit
         }
 
         is_valid, msg = self._validate_params(params)

orbitkit-0.9.11/orbitkit/cdn_uploader/__init__.py

@@ -0,0 +1,16 @@
+"""cdn_uploader：经自建 CDN 网关做对象下载 / 上传（基于 requests，不用 boto3）。
+
+用法::
+
+    from orbitkit.cdn_uploader import CdnUploader
+
+    with CdnUploader("https://oassets.orbitfin.ai", upload_key=KEY) as cdn:
+        cdn.upload_files("filing-reports", "txt-vector/<key>", result.to_files())
+"""
+
+from .uploader import DEFAULT_CONTENT_TYPE, CdnUploader
+
+__all__ = [
+    "CdnUploader",
+    "DEFAULT_CONTENT_TYPE",
+]

orbitkit-0.9.11/orbitkit/cdn_uploader/uploader.py

@@ -0,0 +1,151 @@
+"""自建 CDN 网关客户端（省 S3 出站费），基于 ``requests``，不依赖 boto3。
+
+设了 CDN 网关（形如 ``https://oassets.orbitfin.ai``，路径 ``/{bucket}/{key}``）
+后，把对象的下载 / 上传改走网关而不是直连 S3。收益：S3→CDN 回源免费 +
+CDN 出站费率低 / 有免费额度 → 省 egress。
+
+用**独立的 requests.Session（trust_env=False，不读全局环境代理）**，是否走代理
+由 ``proxies`` 显式控制：
+    - 传了 ``proxies`` → CDN 请求经该代理出境（例如 xproxy 开启时跟着走 xproxy）；
+    - 不传 → 直连网关域名。
+
+接口（与生产网关约定一致）：
+    下载  GET  {base}/{bucket}/{key}
+          - 带 ``x-upload-key`` 头（网关不需要会忽略）
+          - 成功 2xx，body = 原始字节
+    上传  PUT  {base}/{bucket}/{key}
+          - ``x-upload-key``: 鉴权 key（**绝不硬编码**，由调用方传入）
+          - ``x-amz-content-sha256``: body 的 sha256 十六进制小写
+          - ``Content-Type``
+          - body = 文件内容
+          - 成功 2xx
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from typing import Dict, Mapping, Optional
+from urllib.parse import quote
+
+import requests
+
+log = logging.getLogger(__name__)
+
+DEFAULT_CONTENT_TYPE = "application/octet-stream"
+
+
+def _proxies_arg(proxies: Optional[Mapping[str, str]], proxy_url: Optional[str]):
+    if proxies:
+        return dict(proxies)
+    if proxy_url:
+        return {"http": proxy_url, "https": proxy_url}
+    return None
+
+
+class CdnUploader:
+    """经自建 CDN 网关做对象的下载 / 上传（同步）。
+
+    Args:
+        base_url: 网关根地址，如 ``https://oassets.orbitfin.ai``。
+        upload_key: 网关鉴权 key（``x-upload-key``）；下载时也一并带上（网关
+            不需要会忽略）。**绝不硬编码**，由调用方从环境 / secret 传入。
+        timeout_seconds: 单次请求超时。
+        proxy_url: 简写：出境代理 URL（同时用于 http/https）。
+        proxies: requests 风格的 ``{"http": ..., "https": ...}``；给了则优先于
+            ``proxy_url``。两者都不给 = 直连。
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        upload_key: Optional[str] = None,
+        *,
+        timeout_seconds: int = 300,
+        proxy_url: Optional[str] = None,
+        proxies: Optional[Mapping[str, str]] = None,
+    ) -> None:
+        if not base_url:
+            raise ValueError("CdnUploader requires a non-empty base_url")
+        self._base = base_url.rstrip("/")
+        self._upload_key = upload_key
+        self._timeout = float(timeout_seconds)
+        self._proxies = _proxies_arg(proxies, proxy_url)
+        self._session = requests.Session()
+        # trust_env=False：不读全局 HTTP_PROXY/HTTPS_PROXY / netrc；代理只在
+        # 显式给了 proxies/proxy_url 时对这个 session 生效。
+        self._session.trust_env = False
+
+    def _url(self, bucket: str, key: str) -> str:
+        # key 里的 '/' 是路径分隔符要保留；其它特殊字符做 URL 编码。
+        safe_key = quote(key.lstrip("/"), safe="/")
+        return f"{self._base}/{bucket}/{safe_key}"
+
+    def download(self, bucket: str, key: str) -> bytes:
+        headers: Dict[str, str] = {}
+        if self._upload_key:
+            headers["x-upload-key"] = self._upload_key
+        resp = self._session.get(
+            self._url(bucket, key),
+            headers=headers,
+            proxies=self._proxies,
+            timeout=self._timeout,
+        )
+        resp.raise_for_status()
+        return resp.content
+
+    def upload(
+        self,
+        bucket: str,
+        key: str,
+        body: bytes,
+        content_type: str = DEFAULT_CONTENT_TYPE,
+    ) -> None:
+        headers = {
+            "x-amz-content-sha256": hashlib.sha256(body).hexdigest(),
+            "Content-Type": content_type,
+        }
+        if self._upload_key:
+            headers["x-upload-key"] = self._upload_key
+        resp = self._session.put(
+            self._url(bucket, key),
+            headers=headers,
+            data=body,
+            proxies=self._proxies,
+            timeout=self._timeout,
+        )
+        resp.raise_for_status()
+
+    def upload_files(
+        self,
+        bucket: str,
+        key_prefix: str,
+        files: Mapping[str, bytes],
+        *,
+        content_type_for: Optional[Mapping[str, str]] = None,
+    ) -> None:
+        """把 ``{相对路径: bytes}`` 批量上传到 ``{bucket}/{key_prefix}/{相对路径}``。
+
+        Args:
+            bucket: 目标 bucket。
+            key_prefix: key 前缀（会与相对路径以 ``/`` 拼接；空前缀直接用相对路径）。
+            files: 相对路径 → bytes，通常是 ``TransformResult.to_files()`` 的返回。
+            content_type_for: 可选，按相对路径覆盖 Content-Type；缺省用
+                ``application/octet-stream``。
+        """
+        prefix = key_prefix.strip("/")
+        for rel_path, body in files.items():
+            key = f"{prefix}/{rel_path}" if prefix else rel_path
+            ctype = DEFAULT_CONTENT_TYPE
+            if content_type_for and rel_path in content_type_for:
+                ctype = content_type_for[rel_path]
+            self.upload(bucket, key, body, ctype)
+
+    def close(self) -> None:
+        self._session.close()
+
+    def __enter__(self) -> "CdnUploader":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()

orbitkit-0.9.11/orbitkit/pdf_transformer/__init__.py

@@ -0,0 +1,16 @@
+"""pdf_transformer：各模型专属的 raw 结果 → 统一 blocks / pages 转换器集合。
+
+每个模型一个子包，产出同一套与模型无关的 blocks / pages 结构，交给
+:mod:`orbitkit.pdf_validator` 做减法过滤、:mod:`orbitkit.cdn_uploader` 上传：
+
+- :mod:`orbitkit.pdf_transformer.mineru` —— MinerU Router ZIP 转换器（现有）。
+- 后续：``datalab`` 等其它模型。
+
+用法::
+
+    from orbitkit.pdf_transformer.mineru import transform_zip, transform_chunk_zips
+"""
+
+from . import mineru
+
+__all__ = ["mineru"]

orbitkit-0.9.11/orbitkit/pdf_transformer/mineru/__init__.py

@@ -0,0 +1,41 @@
+"""MinerU 结果转换：raw ZIP → 结构化 blocks / pages（已旋转纠正 + 已 Y 翻转）+ 透传文件。
+
+本子包只负责 **模型专属的解析 + 几何归一**，产出与模型无关的 blocks / pages
+结构（:class:`TransformResult`）。**内容过滤**（减法）交给
+:mod:`orbitkit.pdf_validator`；**上传**交给 :mod:`orbitkit.cdn_uploader`。
+
+MinerU 是 :mod:`orbitkit.pdf_transformer` 下的其中一个模型实现（后续会有
+``datalab`` 等），各模型都产出同一套 blocks / pages 结构以复用 pdf_validator。
+
+典型用法::
+
+    from orbitkit.pdf_transformer.mineru import transform_zip
+    from orbitkit.pdf_validator import apply_block_filters, resolve_filters, parse_filter_names
+
+    result = transform_zip(zip_bytes, page_geometry=geom, bbox_rotate_correction=True)
+    result.blocks, result.pages = apply_block_filters(
+        result.blocks, result.pages,
+        resolve_filters(parse_filter_names("drop_non_text,drop_vlm_artifacts")),
+    )
+    files = result.to_files()   # Dict[相对路径, bytes]，可直接上传
+"""
+
+from .bbox_rotation import (
+    PageGeometry,
+    bbox_inverse_rotate,
+    correct_middle_bbox_rotation,
+)
+from .result_transformer import (
+    TransformResult,
+    transform_chunk_zips,
+    transform_zip,
+)
+
+__all__ = [
+    "TransformResult",
+    "transform_zip",
+    "transform_chunk_zips",
+    "PageGeometry",
+    "bbox_inverse_rotate",
+    "correct_middle_bbox_rotation",
+]