orbitkit 0.8.59__tar.gz → 0.8.61__tar.gz

{orbitkit-0.8.59/orbitkit.egg-info → orbitkit-0.8.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.59
+Version: 0.8.61
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.61/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.61

{orbitkit-0.8.59 → orbitkit-0.8.61}/orbitkit/audio_transcoder/netmind_extract_v1.py

@@ -104,7 +104,7 @@ def text_processing(netmind_data, lang, translate_model='gpt-4.1-mini'):
 def send_request_to_api(s3_remote_url, **kwargs):
     endpoint = kwargs['endpoint']
     token = kwargs['token']
-    lang = kwargs.get('lang', 'en')
+    lang = kwargs.get('lang', None)
     headers = {
         'Authorization': token,
     }
@@ -112,8 +112,9 @@ def send_request_to_api(s3_remote_url, **kwargs):
     files = {
         'model': (None, 'WhisperX'),
         'url': (None, s3_remote_url),
-        'language': (None, lang),
     }
+    if lang:
+        files['language'] = (None, lang)
 
     response = requests.post(endpoint, headers=headers, files=files)
     response.raise_for_status()
@@ -125,7 +126,7 @@ def send_request_to_api(s3_remote_url, **kwargs):
 def send_request_to_stream(file_steam, **kwargs):
     endpoint = kwargs['endpoint']
     token = kwargs['token']
-    lang = kwargs.get('lang', 'en')
+    lang = kwargs.get('lang', None)
     headers = {
         'Authorization': token,
     }
@@ -133,8 +134,9 @@ def send_request_to_stream(file_steam, **kwargs):
     files = {
         'model': (None, 'WhisperX'),
         'files': (None, file_steam),
-        'language': (None, lang),
     }
+    if lang:
+        files['language'] = (None, lang)
 
     response = requests.post(endpoint, headers=headers, files=files)
     response.raise_for_status()
@@ -143,7 +145,7 @@ def send_request_to_stream(file_steam, **kwargs):
 
 
 def request_wav_from_netmind(s3_client, s3_path=None, file_steam=None, **kwargs):
-    lang = kwargs.get('lang', 'en')
+    lang = kwargs.get('lang', None)
     folder = kwargs.get('folder', '')
     translate_model = kwargs.get('translate_model', 'gpt-4.1-mini')
     if s3_path:
@@ -166,7 +168,12 @@ def request_wav_from_netmind(s3_client, s3_path=None, file_steam=None, **kwargs)
     with open(json_netmind_wav_path, 'w', encoding='utf-8') as json_file:
         json.dump(data, json_file, ensure_ascii=False, indent=4)
 
-    net_process = text_processing(data, lang, translate_model=translate_model)
+    # lang 为 None 时跳过翻译，直接使用原始结果
+    if lang:
+        net_process = text_processing(data, lang, translate_model=translate_model)
+    else:
+        logger.info("lang=None, 跳过翻译，保持原语言")
+        net_process = data
 
     # 翻译接口处理
     json_netmind_lang_wav_path = os.path.join(folder, 'netmind_lang_wav.json')

orbitkit-0.8.61/orbitkit/pdf_extractor/mineru_demo.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+MinerU PDF 解析工具使用示例
+"""
+import os
+import tempfile
+import boto3
+from urllib.parse import urlparse
+from pdf_extractor_minerU_v1 import MinerUSingleTask
+
+
+def process_pdf(s3_path: str, api_token: str,
+                supported_versions: list = None, model_version: str = "vlm",
+                is_ocr: bool = True, enable_formula: bool = True,
+                enable_table: bool = True, timeout: int = 1800,
+                keep_all_files: bool = True) -> str:
+    """
+    处理 S3 上的 PDF 文件
+
+    :param s3_path: S3 路径 (如 s3://bucket/path/to/file.pdf)
+    :param api_token: MinerU API Token
+    :param supported_versions: 支持的版本列表，如 ["2.7.3"]，None 表示不检查
+    :param model_version: 模型版本 "pipeline" | "vlm" | "MinerU-HTML"
+    :param is_ocr: 是否启用 OCR
+    :param enable_formula: 是否启用公式识别
+    :param enable_table: 是否启用表格识别
+    :param timeout: 超时时间（秒）
+    :param keep_all_files: 是否保留所有文件
+    :return: S3 输出路径 (txt-vector/...)
+    """
+
+    # 版本检查回调
+    def check_version(version_info):
+        if supported_versions is None:
+            return
+        version = version_info.get("_version_name", "")
+        if version not in supported_versions:
+            raise Exception(
+                f"不支持的 MinerU API 版本: {version}\n"
+                f"当前支持: {supported_versions}\n"
+                f"请更新代码以支持新版本的 block 类型"
+            )
+        print(f"版本检查通过: {version}")
+
+    # 解析 S3 路径
+    parsed = urlparse(s3_path)
+    bucket = parsed.netloc
+    key = parsed.path.lstrip('/')
+    filename = os.path.basename(key)
+
+    # 初始化客户端
+    version_checker = check_version if supported_versions else None
+    client = MinerUSingleTask(api_token, version_checker=version_checker)
+    s3_client = boto3.client('s3')
+
+    print("=" * 60)
+    print("MinerU PDF 解析工具")
+    print("=" * 60)
+    print(f"输入: {s3_path}")
+
+    # 第零步：从 S3 下载 PDF 到本地
+    print("\n[步骤0] 从 S3 下载 PDF...")
+    with tempfile.TemporaryDirectory() as temp_dir:
+        local_pdf_path = os.path.join(temp_dir, filename)
+        s3_client.download_file(bucket, key, local_pdf_path)
+        print(f"下载完成: {local_pdf_path}")
+
+        # 第一步：上传文件
+        print("\n[步骤1] 上传文件到 MinerU...")
+        tasks = client.submit_task(
+            local_pdf_path,
+            model_version=model_version,
+            is_ocr=is_ocr,
+            enable_formula=enable_formula,
+            enable_table=enable_table
+        )
+
+        # 第二步：等待任务完成
+        print("\n[步骤2] 等待任务完成...")
+        results = client.wait_for_complete(tasks, timeout=timeout)
+
+        # 第三步：下载并处理结果（输出到 txt-vector/{原S3路径}）
+        print("\n[步骤3] 下载并处理结果...")
+        output_path = client.download_and_process(
+            results,
+            s3_path,  # 使用原 S3 路径生成输出目录
+            s3_client,
+            keep_all_files=keep_all_files
+        )
+
+    print("\n" + "=" * 60)
+    print(f"完成! 结果已上传到: {output_path}/")
+    print("=" * 60)
+
+    return output_path
+
+
+if __name__ == '__main__':
+    # ==================== 配置参数 ====================
+
+    # MinerU API Token（从 https://mineru.net 获取）
+    API_TOKEN = "你的API Token"
+
+    # PDF 文件的 S3 路径
+    S3_PATH = "s3://your-bucket/path/to/file.pdf"
+
+    # 支持的 API 版本（None 表示不检查版本）
+    SUPPORTED_VERSIONS = ["2.7.3"]
+
+    # 模型版本: "pipeline" | "vlm" | "MinerU-HTML"
+    MODEL_VERSION = "vlm"
+
+    # 其他选项
+    IS_OCR = True
+    ENABLE_FORMULA = True
+    ENABLE_TABLE = True
+    TIMEOUT = 1800
+    KEEP_ALL_FILES = True
+
+    # ==================== 执行 ====================
+
+    process_pdf(
+        s3_path=S3_PATH,
+        api_token=API_TOKEN,
+        supported_versions=SUPPORTED_VERSIONS,
+        model_version=MODEL_VERSION,
+        is_ocr=IS_OCR,
+        enable_formula=ENABLE_FORMULA,
+        enable_table=ENABLE_TABLE,
+        timeout=TIMEOUT,
+        keep_all_files=KEEP_ALL_FILES
+    )