openmd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openmd/__init__.py +3 -0
- openmd/__main__.py +5 -0
- openmd/adapters/__init__.py +1 -0
- openmd/adapters/base.py +95 -0
- openmd/adapters/bilibili/__init__.py +11 -0
- openmd/adapters/bilibili/adapter.py +104 -0
- openmd/adapters/bilibili/discovery.py +33 -0
- openmd/adapters/douyin/__init__.py +11 -0
- openmd/adapters/douyin/adapter.py +117 -0
- openmd/adapters/douyin/discovery.py +36 -0
- openmd/adapters/douyin/sidecar.py +77 -0
- openmd/adapters/generic_media/__init__.py +14 -0
- openmd/adapters/high_privacy.py +77 -0
- openmd/adapters/local_audio/__init__.py +14 -0
- openmd/adapters/local_audio/adapter.py +169 -0
- openmd/adapters/local_audio/asr.py +58 -0
- openmd/adapters/local_audio/backends/__init__.py +1 -0
- openmd/adapters/local_audio/backends/auto.py +177 -0
- openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
- openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
- openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
- openmd/adapters/opencli.py +770 -0
- openmd/adapters/pdf/__init__.py +11 -0
- openmd/adapters/pdf/adapter.py +153 -0
- openmd/adapters/pdf/backends/__init__.py +1 -0
- openmd/adapters/pdf/backends/markitdown.py +165 -0
- openmd/adapters/pdf/parser.py +56 -0
- openmd/adapters/qq_group/__init__.py +11 -0
- openmd/adapters/qq_group/adapter.py +289 -0
- openmd/adapters/qq_group/parsers/__init__.py +1 -0
- openmd/adapters/qq_group/parsers/html.py +60 -0
- openmd/adapters/qq_group/parsers/json.py +71 -0
- openmd/adapters/registry.py +149 -0
- openmd/adapters/screenshot/__init__.py +11 -0
- openmd/adapters/screenshot/adapter.py +154 -0
- openmd/adapters/screenshot/backends/__init__.py +1 -0
- openmd/adapters/screenshot/backends/paddle.py +15 -0
- openmd/adapters/screenshot/backends/rapidocr.py +176 -0
- openmd/adapters/screenshot/backends/tesseract.py +15 -0
- openmd/adapters/screenshot/ocr.py +84 -0
- openmd/adapters/wechat_group/__init__.py +11 -0
- openmd/adapters/wechat_group/adapter.py +160 -0
- openmd/adapters/wechat_group/wx_cli.py +268 -0
- openmd/adapters/wechat_moments/__init__.py +11 -0
- openmd/adapters/wechat_moments/adapter.py +167 -0
- openmd/adapters/wechat_moments/wx_cli.py +265 -0
- openmd/adapters/weixin/__init__.py +11 -0
- openmd/adapters/weixin/adapter.py +148 -0
- openmd/adapters/weixin/discovery.py +510 -0
- openmd/adapters/weixin/report.py +278 -0
- openmd/adapters/xiaohongshu.py +158 -0
- openmd/backends/__init__.py +1 -0
- openmd/backends/base.py +204 -0
- openmd/backends/bilibili.py +1480 -0
- openmd/backends/bilibili_wbi.py +277 -0
- openmd/backends/discourse.py +426 -0
- openmd/backends/douyin.py +868 -0
- openmd/backends/douyin_detail.py +124 -0
- openmd/backends/douyin_providers.py +602 -0
- openmd/backends/executor.py +546 -0
- openmd/backends/feishu.py +861 -0
- openmd/backends/generic_media.py +77 -0
- openmd/backends/interactive.py +60 -0
- openmd/backends/ocr.py +41 -0
- openmd/backends/opencli.py +82 -0
- openmd/backends/pdf.py +122 -0
- openmd/backends/registry.py +234 -0
- openmd/backends/screenshot.py +79 -0
- openmd/backends/video_metadata.py +167 -0
- openmd/backends/web_article.py +363 -0
- openmd/backends/wechat_group.py +35 -0
- openmd/backends/wechat_moments.py +35 -0
- openmd/backends/weixin.py +767 -0
- openmd/backends/x.py +455 -0
- openmd/backends/xiaoe.py +940 -0
- openmd/backends/xiaohongshu.py +911 -0
- openmd/backends/xiaohongshu_providers.py +876 -0
- openmd/backends/xiaoyuzhou.py +1017 -0
- openmd/backends/youtube.py +280 -0
- openmd/backends/youtube_discovery.py +1245 -0
- openmd/backends/youtube_transcript.py +213 -0
- openmd/browser_extension/background.js +231 -0
- openmd/browser_extension/icons/icon-128.png +0 -0
- openmd/browser_extension/icons/icon-16.png +0 -0
- openmd/browser_extension/icons/icon-32.png +0 -0
- openmd/browser_extension/icons/icon-48.png +0 -0
- openmd/browser_extension/icons/icon-512.png +0 -0
- openmd/browser_extension/idle.html +6 -0
- openmd/browser_extension/manifest.json +43 -0
- openmd/browser_extension/options.html +42 -0
- openmd/browser_extension/options.js +117 -0
- openmd/browser_extension/popup.css +71 -0
- openmd/browser_extension/popup.html +17 -0
- openmd/browser_extension/popup.js +59 -0
- openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
- openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
- openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
- openmd/browser_host.py +503 -0
- openmd/browser_setup.py +106 -0
- openmd/cli.py +2012 -0
- openmd/console.py +32 -0
- openmd/core/__init__.py +1 -0
- openmd/core/capabilities.py +162 -0
- openmd/core/content_validation.py +323 -0
- openmd/core/coverage.py +163 -0
- openmd/core/detector.py +247 -0
- openmd/core/doctor.py +104 -0
- openmd/core/execution_policy.py +164 -0
- openmd/core/filename.py +64 -0
- openmd/core/input_validation.py +91 -0
- openmd/core/local_config.py +96 -0
- openmd/core/manifest.py +156 -0
- openmd/core/models.py +168 -0
- openmd/core/opencli_output.py +34 -0
- openmd/core/operation_contract.py +60 -0
- openmd/core/output.py +194 -0
- openmd/core/package_contract.py +335 -0
- openmd/core/package_validator.py +489 -0
- openmd/core/package_writer.py +666 -0
- openmd/core/planner.py +345 -0
- openmd/core/preferences.py +61 -0
- openmd/core/readable.py +129 -0
- openmd/core/receipt.py +285 -0
- openmd/core/renderer.py +185 -0
- openmd/core/runlog.py +208 -0
- openmd/core/secrets.py +313 -0
- openmd/doctor_view.py +68 -0
- openmd/interactive.py +453 -0
- openmd/ocr/__init__.py +1 -0
- openmd/ocr/breakers.py +84 -0
- openmd/ocr/config.py +54 -0
- openmd/ocr/documents.py +522 -0
- openmd/ocr/endpoint.py +72 -0
- openmd/ocr/layout.py +92 -0
- openmd/ocr/layout_filter.py +118 -0
- openmd/ocr/layout_worker.py +89 -0
- openmd/ocr/native_math.py +220 -0
- openmd/ocr/office.py +305 -0
- openmd/ocr/ollama.py +200 -0
- openmd/ocr/otsl.py +51 -0
- openmd/ocr/ovis-prompt.txt +1 -0
- openmd/ocr/pipeline.py +384 -0
- openmd/ocr/settings.json +25 -0
- openmd/ocr/tiling.py +631 -0
- openmd/post_video.py +154 -0
- openmd/probe.py +397 -0
- openmd/routing/__init__.py +1 -0
- openmd/routing/cards/builtin.json +620 -0
- openmd/routing/cards/defuddle.json +70 -0
- openmd/routing/cards/x-tweet-fetcher.json +71 -0
- openmd/routing/cards/xyz-dl.json +60 -0
- openmd/routing/cards/yutto.json +68 -0
- openmd/routing/entry_shim.py +71 -0
- openmd/routing/lifecycle.py +459 -0
- openmd/routing/loader.py +138 -0
- openmd/routing/providers.py +158 -0
- openmd/routing/readers.py +108 -0
- openmd/routing/routes.json +504 -0
- openmd/routing/runner.py +192 -0
- openmd/routing/view.py +40 -0
- openmd/runtime/__init__.py +1 -0
- openmd/runtime/_windows_job.py +234 -0
- openmd/runtime/asr.py +217 -0
- openmd/runtime/asr_worker.py +119 -0
- openmd/runtime/gpu_lock.py +81 -0
- openmd/runtime/gpu_quarantine.py +85 -0
- openmd/runtime/interpreters.py +407 -0
- openmd/runtime/managed_tools.py +204 -0
- openmd/runtime/models.py +202 -0
- openmd/runtime/services.py +175 -0
- openmd/runtime/tempdirs.py +227 -0
- openmd_cli-0.1.0.dist-info/METADATA +168 -0
- openmd_cli-0.1.0.dist-info/RECORD +177 -0
- openmd_cli-0.1.0.dist-info/WHEEL +5 -0
- openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
openmd/routing/view.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Generate route views from the same declarations used by runtime selection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
|
|
8
|
+
from .loader import route_view
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def doctor_tools(*, probe_commands: bool = False):
|
|
12
|
+
from .providers import doctor_tools as build_report
|
|
13
|
+
|
|
14
|
+
return build_report(probe_commands=probe_commands)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def render_routes_markdown(*, source: str | None = None, effective: bool = False) -> str:
|
|
18
|
+
view = route_view(source, effective=effective)
|
|
19
|
+
lines = ["# Openmd routes", "", f"Generated from bundled cards and routes, version `{view['version']}`.", "", "| Source | Capability | Ordered backends |", "|---|---|---|"]
|
|
20
|
+
for route in view["routes"]:
|
|
21
|
+
lines.append(f"| {route['source']} | {route['capability']} | {' → '.join(route['chain'])} |")
|
|
22
|
+
lines += ["", "Tools run only with the gates declared in their cards. Media steps require local ASR before producing text.", "", "Local overlay applied: " + str(view["overlay_applied"]).lower() + ".", ""]
|
|
23
|
+
return "\n".join(lines)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def main(argv: list[str] | None = None) -> int:
|
|
27
|
+
parser = argparse.ArgumentParser()
|
|
28
|
+
parser.add_argument("--source")
|
|
29
|
+
parser.add_argument("--format", choices=("json", "table", "markdown"), default="table")
|
|
30
|
+
parser.add_argument("--bundled", action="store_true", help="Use bundled routes only when generating repository documentation.")
|
|
31
|
+
args = parser.parse_args(argv)
|
|
32
|
+
if args.format == "json":
|
|
33
|
+
print(json.dumps(route_view(args.source, effective=not args.bundled), ensure_ascii=False, indent=2))
|
|
34
|
+
else:
|
|
35
|
+
print(render_routes_markdown(source=args.source, effective=not args.bundled), end="")
|
|
36
|
+
return 0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
if __name__ == "__main__":
|
|
40
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Local capability runtimes; the controller retains its own dependencies."""
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Windows Job ownership, including atomic assignment before child execution.
|
|
2
|
+
|
|
3
|
+
Only imported on Windows. The small Popen creation hook uses STARTUPINFOEX's
|
|
4
|
+
JOB_LIST (Windows 10+) so even controller death between CreateProcess and
|
|
5
|
+
AssignProcessToJobObject cannot strand a suspended child. No global monkeypatch.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ctypes as C
|
|
11
|
+
from ctypes import wintypes as W
|
|
12
|
+
import os
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
kernel = C.WinDLL("kernel32", use_last_error=True)
|
|
19
|
+
SIZE = C.c_size_t
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class IO_COUNTERS(C.Structure):
|
|
23
|
+
_fields_ = [(name, C.c_ulonglong) for name in (
|
|
24
|
+
"ReadOperationCount", "WriteOperationCount", "OtherOperationCount",
|
|
25
|
+
"ReadTransferCount", "WriteTransferCount", "OtherTransferCount")]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class BASIC_LIMITS(C.Structure):
|
|
29
|
+
_fields_ = [("PerProcessUserTimeLimit", C.c_longlong),
|
|
30
|
+
("PerJobUserTimeLimit", C.c_longlong), ("LimitFlags", W.DWORD),
|
|
31
|
+
("MinimumWorkingSetSize", SIZE), ("MaximumWorkingSetSize", SIZE),
|
|
32
|
+
("ActiveProcessLimit", W.DWORD), ("Affinity", SIZE),
|
|
33
|
+
("PriorityClass", W.DWORD), ("SchedulingClass", W.DWORD)]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EXTENDED_LIMITS(C.Structure):
|
|
37
|
+
_fields_ = [("BasicLimitInformation", BASIC_LIMITS), ("IoInfo", IO_COUNTERS),
|
|
38
|
+
("ProcessMemoryLimit", SIZE), ("JobMemoryLimit", SIZE),
|
|
39
|
+
("PeakProcessMemoryUsed", SIZE), ("PeakJobMemoryUsed", SIZE)]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class BASIC_ACCOUNTING(C.Structure):
|
|
43
|
+
_fields_ = [(name, C.c_longlong) for name in ("TotalUserTime", "TotalKernelTime",
|
|
44
|
+
"ThisPeriodTotalUserTime", "ThisPeriodTotalKernelTime")] + [
|
|
45
|
+
(name, W.DWORD) for name in ("TotalPageFaultCount", "TotalProcesses",
|
|
46
|
+
"ActiveProcesses", "TotalTerminatedProcesses")]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class STARTUPINFO(C.Structure):
|
|
50
|
+
_fields_ = [("cb", W.DWORD), ("lpReserved", W.LPWSTR),
|
|
51
|
+
("lpDesktop", W.LPWSTR), ("lpTitle", W.LPWSTR),
|
|
52
|
+
*[(name, W.DWORD) for name in ("dwX", "dwY", "dwXSize", "dwYSize",
|
|
53
|
+
"dwXCountChars", "dwYCountChars", "dwFillAttribute", "dwFlags")],
|
|
54
|
+
("wShowWindow", W.WORD), ("cbReserved2", W.WORD),
|
|
55
|
+
("lpReserved2", C.c_void_p), ("hStdInput", W.HANDLE),
|
|
56
|
+
("hStdOutput", W.HANDLE), ("hStdError", W.HANDLE)]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class STARTUPINFOEX(C.Structure):
|
|
60
|
+
_fields_ = [("StartupInfo", STARTUPINFO), ("lpAttributeList", C.c_void_p)]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class PROCESS_INFORMATION(C.Structure):
|
|
64
|
+
_fields_ = [("hProcess", W.HANDLE), ("hThread", W.HANDLE),
|
|
65
|
+
("dwProcessId", W.DWORD), ("dwThreadId", W.DWORD)]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _api(name, result, *args):
|
|
69
|
+
function = getattr(kernel, name)
|
|
70
|
+
function.restype, function.argtypes = result, args
|
|
71
|
+
return function
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
CreateJobObject = _api("CreateJobObjectW", W.HANDLE, C.c_void_p, W.LPCWSTR)
|
|
75
|
+
SetInformationJobObject = _api("SetInformationJobObject", W.BOOL, W.HANDLE, C.c_int, C.c_void_p, W.DWORD)
|
|
76
|
+
AssignProcessToJobObject = _api("AssignProcessToJobObject", W.BOOL, W.HANDLE, W.HANDLE)
|
|
77
|
+
TerminateJobObject = _api("TerminateJobObject", W.BOOL, W.HANDLE, W.UINT)
|
|
78
|
+
QueryInformationJobObject = _api("QueryInformationJobObject", W.BOOL, W.HANDLE, C.c_int, C.c_void_p, W.DWORD, C.c_void_p)
|
|
79
|
+
WaitForSingleObject = _api("WaitForSingleObject", W.DWORD, W.HANDLE, W.DWORD)
|
|
80
|
+
CloseHandle = _api("CloseHandle", W.BOOL, W.HANDLE)
|
|
81
|
+
ResumeThread = _api("ResumeThread", W.DWORD, W.HANDLE)
|
|
82
|
+
TerminateProcess = _api("TerminateProcess", W.BOOL, W.HANDLE, W.UINT)
|
|
83
|
+
InitializeProcThreadAttributeList = _api("InitializeProcThreadAttributeList", W.BOOL, C.c_void_p, W.DWORD, W.DWORD, C.POINTER(SIZE))
|
|
84
|
+
UpdateProcThreadAttribute = _api("UpdateProcThreadAttribute", W.BOOL, C.c_void_p, W.DWORD, SIZE, C.c_void_p, SIZE, C.c_void_p, C.c_void_p)
|
|
85
|
+
DeleteProcThreadAttributeList = _api("DeleteProcThreadAttributeList", None, C.c_void_p)
|
|
86
|
+
CreateProcess = _api("CreateProcessW", W.BOOL, W.LPCWSTR, W.LPWSTR, C.c_void_p, C.c_void_p,
|
|
87
|
+
W.BOOL, W.DWORD, C.c_void_p, W.LPCWSTR,
|
|
88
|
+
C.POINTER(STARTUPINFOEX), C.POINTER(PROCESS_INFORMATION))
|
|
89
|
+
OpenThread = _api("OpenThread", W.HANDLE, W.DWORD, W.BOOL, W.DWORD)
|
|
90
|
+
CancelSynchronousIo = _api("CancelSynchronousIo", W.BOOL, W.HANDLE)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _checked(result):
|
|
94
|
+
if not result:
|
|
95
|
+
raise C.WinError(C.get_last_error())
|
|
96
|
+
return result
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class Job:
|
|
100
|
+
def __init__(self):
|
|
101
|
+
self.lock = threading.Lock()
|
|
102
|
+
self.handle = _checked(CreateJobObject(None, None))
|
|
103
|
+
try:
|
|
104
|
+
limits = EXTENDED_LIMITS()
|
|
105
|
+
limits.BasicLimitInformation.LimitFlags = 0x2000 # KILL_ON_JOB_CLOSE
|
|
106
|
+
_checked(SetInformationJobObject(self.handle, 9, C.byref(limits), C.sizeof(limits)))
|
|
107
|
+
except BaseException:
|
|
108
|
+
self.close()
|
|
109
|
+
raise
|
|
110
|
+
|
|
111
|
+
def close(self):
|
|
112
|
+
with self.lock:
|
|
113
|
+
if self.handle:
|
|
114
|
+
handle, self.handle = self.handle, None
|
|
115
|
+
try:
|
|
116
|
+
# Terminate explicitly; closing also protects controller crashes.
|
|
117
|
+
_checked(TerminateJobObject(handle, 1))
|
|
118
|
+
# Termination is asynchronous. Waiting on the Job object
|
|
119
|
+
# itself is the kernel synchronization point for the whole
|
|
120
|
+
# tree; ActiveProcesses can reach zero before the process
|
|
121
|
+
# handles and inherited file handles are fully released.
|
|
122
|
+
WaitForSingleObject(handle, 2000)
|
|
123
|
+
finally:
|
|
124
|
+
CloseHandle(handle)
|
|
125
|
+
|
|
126
|
+
def __del__(self):
|
|
127
|
+
if getattr(self, "handle", None):
|
|
128
|
+
CloseHandle(self.handle)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def cancel_thread_io(thread):
|
|
132
|
+
if thread is not None and thread.is_alive() and thread.native_id:
|
|
133
|
+
handle = OpenThread(0x0001, False, thread.native_id) # THREAD_TERMINATE
|
|
134
|
+
if handle:
|
|
135
|
+
try:
|
|
136
|
+
CancelSynchronousIo(handle)
|
|
137
|
+
finally:
|
|
138
|
+
CloseHandle(handle)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class JobPopen(subprocess.Popen):
|
|
142
|
+
"""Keep Popen's pipe/text/wait implementation; customize only OS creation.
|
|
143
|
+
|
|
144
|
+
Signature follows CPython's Windows Popen._execute_child; tested on 3.12.
|
|
145
|
+
Creation errors fail closed: never silently launch without Job ownership.
|
|
146
|
+
"""
|
|
147
|
+
|
|
148
|
+
def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds,
|
|
149
|
+
cwd, env, startupinfo, creationflags, shell,
|
|
150
|
+
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,
|
|
151
|
+
*unused, **unused_kwargs):
|
|
152
|
+
if pass_fds:
|
|
153
|
+
raise ValueError("pass_fds not supported on Windows")
|
|
154
|
+
if isinstance(args, (bytes, os.PathLike)):
|
|
155
|
+
if shell:
|
|
156
|
+
raise TypeError("bytes/path-like args is not allowed with shell")
|
|
157
|
+
args = subprocess.list2cmdline([args])
|
|
158
|
+
elif not isinstance(args, str):
|
|
159
|
+
args = subprocess.list2cmdline(args)
|
|
160
|
+
executable = os.fsdecode(executable) if executable is not None else None
|
|
161
|
+
startupinfo = startupinfo.copy() if startupinfo is not None else subprocess.STARTUPINFO()
|
|
162
|
+
if startupinfo.lpAttributeList and set(startupinfo.lpAttributeList) - {"handle_list"}:
|
|
163
|
+
raise ValueError("Unsupported Windows startup attribute")
|
|
164
|
+
native = STARTUPINFOEX()
|
|
165
|
+
native.StartupInfo.cb = C.sizeof(native)
|
|
166
|
+
native.StartupInfo.dwFlags = startupinfo.dwFlags
|
|
167
|
+
native.StartupInfo.wShowWindow = startupinfo.wShowWindow
|
|
168
|
+
handles = list((startupinfo.lpAttributeList or {}).get("handle_list", []))
|
|
169
|
+
if -1 not in (p2cread, c2pwrite, errwrite):
|
|
170
|
+
native.StartupInfo.dwFlags |= 0x100 # STARTF_USESTDHANDLES
|
|
171
|
+
native.StartupInfo.hStdInput = int(p2cread)
|
|
172
|
+
native.StartupInfo.hStdOutput = int(c2pwrite)
|
|
173
|
+
native.StartupInfo.hStdError = int(errwrite)
|
|
174
|
+
handles += [int(p2cread), int(c2pwrite), int(errwrite)]
|
|
175
|
+
handles = self._filter_handle_list(handles)
|
|
176
|
+
if shell:
|
|
177
|
+
native.StartupInfo.dwFlags |= 1
|
|
178
|
+
native.StartupInfo.wShowWindow = 0
|
|
179
|
+
executable = executable or os.environ.get("ComSpec") or os.path.join(os.environ.get("SystemRoot", ""), "System32", "cmd.exe")
|
|
180
|
+
if not os.path.isabs(executable):
|
|
181
|
+
raise FileNotFoundError("An absolute Windows shell path is required")
|
|
182
|
+
args = f'{executable} /c "{args}"'
|
|
183
|
+
cwd = os.fsdecode(cwd) if cwd is not None else None
|
|
184
|
+
environment = None
|
|
185
|
+
if env is not None:
|
|
186
|
+
entries = []
|
|
187
|
+
for key, value in sorted(env.items(), key=lambda pair: pair[0].upper()):
|
|
188
|
+
if "\0" in key or "\0" in value or "=" in key[1:] or not key:
|
|
189
|
+
raise ValueError("illegal environment variable")
|
|
190
|
+
entries.append(f"{key}={value}")
|
|
191
|
+
environment = C.create_unicode_buffer("\0".join(entries) + "\0\0")
|
|
192
|
+
size = SIZE()
|
|
193
|
+
InitializeProcThreadAttributeList(None, 2 if handles else 1, 0, C.byref(size))
|
|
194
|
+
attributes = C.create_string_buffer(size.value)
|
|
195
|
+
_checked(InitializeProcThreadAttributeList(attributes, 2 if handles else 1, 0, C.byref(size)))
|
|
196
|
+
native.lpAttributeList = C.cast(attributes, C.c_void_p)
|
|
197
|
+
info = PROCESS_INFORMATION()
|
|
198
|
+
try:
|
|
199
|
+
# JOB_LIST assigns at CreateProcess itself, closing the crash window.
|
|
200
|
+
jobs = (W.HANDLE * 1)(self._openmd_owner.handle)
|
|
201
|
+
_checked(UpdateProcThreadAttribute(attributes, 0, 0x2000D, jobs, C.sizeof(jobs), None, None))
|
|
202
|
+
if handles:
|
|
203
|
+
inherited = (W.HANDLE * len(handles))(*handles)
|
|
204
|
+
_checked(UpdateProcThreadAttribute(attributes, 0, 0x20002, inherited, C.sizeof(inherited), None, None))
|
|
205
|
+
sys.audit("subprocess.Popen", executable, args, cwd, env)
|
|
206
|
+
_checked(CreateProcess(executable, C.create_unicode_buffer(args), None, None,
|
|
207
|
+
bool(handles) or not close_fds,
|
|
208
|
+
creationflags | 0x4 | 0x80000 | 0x400, # SUSPENDED/EXTENDED/UNICODE
|
|
209
|
+
environment, cwd, C.byref(native), C.byref(info)))
|
|
210
|
+
self._child_created = True
|
|
211
|
+
self._handle = subprocess.Handle(info.hProcess)
|
|
212
|
+
self.pid = info.dwProcessId
|
|
213
|
+
# Already in this Job atomically; explicit assignment validates it
|
|
214
|
+
# before any child code can run or spawn a grandchild.
|
|
215
|
+
_checked(AssignProcessToJobObject(self._openmd_owner.handle, info.hProcess))
|
|
216
|
+
if ResumeThread(info.hThread) == 0xFFFFFFFF:
|
|
217
|
+
raise C.WinError(C.get_last_error())
|
|
218
|
+
except BaseException:
|
|
219
|
+
if info.hProcess:
|
|
220
|
+
TerminateProcess(info.hProcess, 1)
|
|
221
|
+
raise
|
|
222
|
+
finally:
|
|
223
|
+
if info.hThread:
|
|
224
|
+
CloseHandle(info.hThread)
|
|
225
|
+
DeleteProcThreadAttributeList(attributes)
|
|
226
|
+
self._close_pipe_fds(p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)
|
|
227
|
+
|
|
228
|
+
def __init__(self, *args, **kwargs):
|
|
229
|
+
self._openmd_owner = Job()
|
|
230
|
+
try:
|
|
231
|
+
super().__init__(*args, **kwargs)
|
|
232
|
+
except BaseException:
|
|
233
|
+
self._openmd_owner.close()
|
|
234
|
+
raise
|
openmd/runtime/asr.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""ASR factory and bounded worker protocol used by every controller entrypoint."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import hashlib
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import subprocess
|
|
10
|
+
import tempfile
|
|
11
|
+
from dataclasses import asdict, fields, replace
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from openmd.adapters.local_audio.asr import ASRBackend
|
|
16
|
+
from openmd.core.models import COMPONENT_STATUS_VALUES, FAILURE_CLASS_VALUES, SOURCE_LEVEL_VALUES, STATUS_VALUES, ExtractionComponent, TranscriptResult
|
|
17
|
+
from openmd.core.secrets import sanitize_error_summary
|
|
18
|
+
from .interpreters import owned_popen, AsrRuntime, display_path, kill_process_tree, process_options, remove_tree_with_retry, select_asr_runtime, terminate_quietly, wait_interruptibly, worker_command, worker_environment
|
|
19
|
+
|
|
20
|
+
from openmd.runtime.tempdirs import owned_mkdtemp
|
|
21
|
+
from openmd.runtime.interpreters import run_interruptibly
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
ASR_BACKENDS = ("asr.auto", "asr.faster_whisper", "asr.qwen3")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def local_backend(name: str) -> ASRBackend:
|
|
28
|
+
if name == "asr.auto":
|
|
29
|
+
from openmd.adapters.local_audio.backends.auto import AutoASRBackend
|
|
30
|
+
|
|
31
|
+
return AutoASRBackend()
|
|
32
|
+
if name == "asr.faster_whisper":
|
|
33
|
+
from openmd.adapters.local_audio.backends.faster_whisper import FasterWhisperASRBackend
|
|
34
|
+
|
|
35
|
+
return FasterWhisperASRBackend()
|
|
36
|
+
if name == "asr.qwen3":
|
|
37
|
+
from openmd.adapters.local_audio.backends.qwen3_asr import Qwen3ASRBackend
|
|
38
|
+
|
|
39
|
+
return Qwen3ASRBackend()
|
|
40
|
+
raise ValueError(f"Unknown local ASR backend: {name}")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def backend_execution_config(name: str) -> dict[str, Any]:
|
|
44
|
+
"""Constructing fingerprints never starts a GPU probe or imports a model."""
|
|
45
|
+
from openmd.core.local_config import load_local_config
|
|
46
|
+
from .models import checkout_root
|
|
47
|
+
|
|
48
|
+
config = local_backend(name).execution_config()
|
|
49
|
+
local = load_local_config()
|
|
50
|
+
explicit = os.getenv("OPENMD_ASR_PYTHON") or local.get("asr_python")
|
|
51
|
+
root = checkout_root()
|
|
52
|
+
candidates = [Path(explicit).expanduser()] if explicit else [Path(os.sys.executable)]
|
|
53
|
+
if not explicit and root:
|
|
54
|
+
candidates.append(root / "out/asr-bench/venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python"))
|
|
55
|
+
identities = []
|
|
56
|
+
for candidate in candidates:
|
|
57
|
+
if not candidate.is_absolute():
|
|
58
|
+
candidate = (root or Path.cwd()) / candidate
|
|
59
|
+
try:
|
|
60
|
+
stamp = candidate.stat().st_mtime_ns
|
|
61
|
+
except OSError:
|
|
62
|
+
stamp = None
|
|
63
|
+
identities.append({"path": str(candidate.resolve()), "mtime_ns": stamp})
|
|
64
|
+
# Hash locations/settings for resume identity without publishing machine paths
|
|
65
|
+
# or starting a GPU probe during argument parsing.
|
|
66
|
+
settings = {key: value for key, value in os.environ.items() if key.startswith(("OPENMD_FASTER_WHISPER_", "OPENMD_ASR_AUTO_", "OPENMD_QWEN3_")) or key in {"OPENMD_MODEL_ROOTS", "OPENMD_ASR_CHINESE_SCRIPT", "CUDA_VISIBLE_DEVICES", "HF_HOME", "HF_HUB_CACHE", "MODELSCOPE_CACHE"}}
|
|
67
|
+
identity = {"interpreters": identities, "model_roots": local.get("model_roots", []), "settings": settings, "backend": config}
|
|
68
|
+
config_id = hashlib.sha256(json.dumps(identity, sort_keys=True, ensure_ascii=True).encode()).hexdigest()
|
|
69
|
+
return {**config, "runtime_policy": "local_worker_v2", "runtime_configured": bool(explicit), "runtime_source": "explicit" if explicit else "checkout" if root else "current_interpreter", "runtime_config_id": config_id}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def get_asr_backend(name: str = "asr.auto") -> ASRBackend:
|
|
73
|
+
if name not in ASR_BACKENDS:
|
|
74
|
+
raise ValueError(f"Unknown local ASR backend: {name}")
|
|
75
|
+
# Resolve lazily: provider registration, health checks and execution fingerprints
|
|
76
|
+
# must not load torch or wait for a worker probe.
|
|
77
|
+
return WorkerASRBackend(name)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def failed_result(name: str, code: str, error: object) -> TranscriptResult:
|
|
81
|
+
return TranscriptResult(text="", status="failed", source_level="partial_extraction", backend=name, error_code=code, error_summary=sanitize_error_summary(error), source_notes=["Local ASR did not produce usable text; no cloud service or model download was used."], metadata={"profile": "media", "asr_backend": name})
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def serialize_result(result: TranscriptResult) -> dict[str, Any]:
|
|
85
|
+
return {"schema": "openmd.asr-result/v1", "result": asdict(result)}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def deserialize_result(payload: object) -> TranscriptResult:
|
|
89
|
+
if not isinstance(payload, dict) or payload.get("schema") != "openmd.asr-result/v1":
|
|
90
|
+
raise ValueError("invalid ASR worker schema")
|
|
91
|
+
result = payload.get("result")
|
|
92
|
+
if not isinstance(result, dict) or set(result) - {item.name for item in fields(TranscriptResult)}:
|
|
93
|
+
raise ValueError("invalid ASR result fields")
|
|
94
|
+
if not isinstance(result.get("text"), str) or result.get("status") not in STATUS_VALUES or result.get("source_level") not in SOURCE_LEVEL_VALUES or result.get("backend") not in ASR_BACKENDS:
|
|
95
|
+
raise ValueError("invalid ASR result status, source or backend")
|
|
96
|
+
for key in ("error_code", "error_summary", "title", "author", "published_at", "extraction_contract_version"):
|
|
97
|
+
if result.get(key) is not None and not isinstance(result[key], str):
|
|
98
|
+
raise ValueError(f"invalid ASR result {key}")
|
|
99
|
+
notes = result.get("source_notes", [])
|
|
100
|
+
metadata = result.get("metadata", {})
|
|
101
|
+
if not isinstance(notes, list) or not all(isinstance(note, str) for note in notes) or not isinstance(metadata, dict):
|
|
102
|
+
raise ValueError("invalid ASR notes or metadata")
|
|
103
|
+
components = result.get("components", [])
|
|
104
|
+
if not isinstance(components, list):
|
|
105
|
+
raise ValueError("invalid ASR components")
|
|
106
|
+
decoded: list[ExtractionComponent] = []
|
|
107
|
+
for component in components:
|
|
108
|
+
if not isinstance(component, dict) or set(component) - {item.name for item in fields(ExtractionComponent)}:
|
|
109
|
+
raise ValueError("invalid ASR component fields")
|
|
110
|
+
if component.get("status") not in COMPONENT_STATUS_VALUES or component.get("source_level") not in SOURCE_LEVEL_VALUES:
|
|
111
|
+
raise ValueError("invalid ASR component status")
|
|
112
|
+
if any(not isinstance(component.get(key), str) for key in ("id", "kind", "backend")):
|
|
113
|
+
raise ValueError("invalid ASR component identity")
|
|
114
|
+
if component.get("failure_class") is not None and component["failure_class"] not in FAILURE_CLASS_VALUES:
|
|
115
|
+
raise ValueError("invalid ASR component failure class")
|
|
116
|
+
for key in ("text_ref", "terminal_reason", "error_code"):
|
|
117
|
+
if component.get(key) is not None and not isinstance(component[key], str):
|
|
118
|
+
raise ValueError(f"invalid ASR component {key}")
|
|
119
|
+
warnings = component.get("warnings", [])
|
|
120
|
+
if not isinstance(warnings, list) or any(not isinstance(warning, str) for warning in warnings):
|
|
121
|
+
raise ValueError("invalid ASR component warnings")
|
|
122
|
+
decoded.append(ExtractionComponent(**component))
|
|
123
|
+
if result["status"] in {"verified", "partial"} and not result["text"].strip():
|
|
124
|
+
raise ValueError("successful ASR result has no text")
|
|
125
|
+
return TranscriptResult(**{**result, "source_notes": notes, "metadata": metadata, "components": decoded})
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _timeout_for_audio(path: Path) -> float:
|
|
129
|
+
try:
|
|
130
|
+
done = run_interruptibly(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", str(path)], stdin=subprocess.DEVNULL, capture_output=True, timeout=10, check=True)
|
|
131
|
+
duration = float(done.stdout.strip())
|
|
132
|
+
return max(600.0, min(duration * 2, 12 * 60 * 60)) if math.isfinite(duration) and duration >= 0 else 600.0
|
|
133
|
+
except (OSError, ValueError, subprocess.SubprocessError):
|
|
134
|
+
return 600.0
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class WorkerASRBackend(ASRBackend):
|
|
138
|
+
def __init__(self, name: str, python: Path | None = None) -> None:
|
|
139
|
+
self.name = name
|
|
140
|
+
self.python = python
|
|
141
|
+
|
|
142
|
+
def execution_config(self) -> dict[str, Any]:
|
|
143
|
+
return backend_execution_config(self.name)
|
|
144
|
+
|
|
145
|
+
def is_available(self) -> bool:
|
|
146
|
+
return self._runtime().mode != "unavailable"
|
|
147
|
+
|
|
148
|
+
def _runtime(self) -> AsrRuntime:
|
|
149
|
+
if self.python is not None:
|
|
150
|
+
return AsrRuntime("worker", self.python, "Explicit worker instance")
|
|
151
|
+
return select_asr_runtime(backend=self.name)
|
|
152
|
+
|
|
153
|
+
def transcribe(self, path: Path) -> TranscriptResult:
|
|
154
|
+
from .gpu_quarantine import GPUQuarantined
|
|
155
|
+
|
|
156
|
+
try:
|
|
157
|
+
return self._transcribe(path)
|
|
158
|
+
except TimeoutError as exc:
|
|
159
|
+
return failed_result(self.name, "gpu_lock_timeout", exc)
|
|
160
|
+
except GPUQuarantined as exc:
|
|
161
|
+
return failed_result(self.name, exc.code, exc)
|
|
162
|
+
|
|
163
|
+
def _transcribe(self, path: Path) -> TranscriptResult:
|
|
164
|
+
path = path.expanduser().resolve()
|
|
165
|
+
if not path.is_file():
|
|
166
|
+
return failed_result(self.name, "asr_file_not_found", "Local audio file not found")
|
|
167
|
+
try:
|
|
168
|
+
runtime = self._runtime()
|
|
169
|
+
except (OSError, ValueError) as exc:
|
|
170
|
+
return failed_result(self.name, "asr_runtime_unavailable", exc)
|
|
171
|
+
if runtime.mode == "in_process":
|
|
172
|
+
from .gpu_lock import gpu_lock, release_torch_cache
|
|
173
|
+
|
|
174
|
+
with gpu_lock():
|
|
175
|
+
try:
|
|
176
|
+
result = local_backend(self.name).transcribe(path)
|
|
177
|
+
finally:
|
|
178
|
+
release_torch_cache()
|
|
179
|
+
return replace(result, source_notes=[*result.source_notes, runtime.reason], metadata={**result.metadata, "asr_runtime": {"mode": "in_process", "reason": runtime.reason}})
|
|
180
|
+
if runtime.mode == "unavailable" or runtime.python is None:
|
|
181
|
+
reports = runtime.probe.get("attempts", [runtime.probe])
|
|
182
|
+
missing = sorted({name for report in reports if isinstance(report, dict) for name, model in report.get("models", {}).items() if not model.get("available", False)})
|
|
183
|
+
if missing:
|
|
184
|
+
return failed_result(self.name, "asr_model_missing", runtime.reason + "; missing or incomplete local models: " + ", ".join(missing))
|
|
185
|
+
return failed_result(self.name, "asr_runtime_unavailable", runtime.reason)
|
|
186
|
+
workdir = owned_mkdtemp(prefix="openmd-asr-worker-")
|
|
187
|
+
try:
|
|
188
|
+
result_path = Path(workdir) / "result.json"
|
|
189
|
+
command = worker_command(runtime.python, "--backend", self.name, "--input", str(path), "--result", str(result_path))
|
|
190
|
+
try:
|
|
191
|
+
# Library chatter is bounded to a file and never becomes protocol data.
|
|
192
|
+
with (Path(workdir) / "worker.log").open("wb") as log:
|
|
193
|
+
process = owned_popen(command, stdin=subprocess.DEVNULL, stdout=log, stderr=log, env=worker_environment(), cwd=workdir, **process_options())
|
|
194
|
+
try:
|
|
195
|
+
wait_interruptibly(process, _timeout_for_audio(path))
|
|
196
|
+
except subprocess.TimeoutExpired:
|
|
197
|
+
terminate_quietly(process)
|
|
198
|
+
return failed_result(self.name, "asr_worker_timeout", "Local ASR worker exceeded its bounded time limit")
|
|
199
|
+
except BaseException:
|
|
200
|
+
# Ctrl+C must surface as cancellation, never as a worker failure.
|
|
201
|
+
terminate_quietly(process)
|
|
202
|
+
raise
|
|
203
|
+
if process.returncode:
|
|
204
|
+
with (Path(workdir) / "worker.log").open("rb") as log:
|
|
205
|
+
log.seek(max(0, log.seek(0, 2) - 2048))
|
|
206
|
+
detail = log.read().decode("utf-8", errors="replace")
|
|
207
|
+
return failed_result(self.name, "asr_worker_failed", detail or f"Worker exited with code {process.returncode}")
|
|
208
|
+
if not result_path.is_file() or result_path.stat().st_size > 64 * 1024 * 1024:
|
|
209
|
+
raise ValueError("worker result missing or exceeds size limit")
|
|
210
|
+
result = deserialize_result(json.loads(result_path.read_text(encoding="utf-8")))
|
|
211
|
+
return replace(result, metadata={**result.metadata, "asr_runtime": {"mode": "worker", "python": display_path(runtime.python)}})
|
|
212
|
+
except (OSError, ValueError, TypeError, subprocess.SubprocessError) as exc:
|
|
213
|
+
return failed_result(self.name, "asr_worker_failed", exc)
|
|
214
|
+
finally:
|
|
215
|
+
# A killed worker's grandchild can briefly keep its cwd open on Windows; cleanup is
|
|
216
|
+
# retried and best-effort so it can never replace the original exception.
|
|
217
|
+
remove_tree_with_retry(Path(workdir))
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Offline ASR worker. Only a result file carries transcript protocol data."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import contextlib
|
|
7
|
+
import importlib.metadata
|
|
8
|
+
import importlib.util
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import platform
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
from dataclasses import replace
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def build_probe() -> dict:
|
|
19
|
+
from .interpreters import REQUIRED_MODULES, display_path
|
|
20
|
+
from .models import configured_models
|
|
21
|
+
|
|
22
|
+
modules = {name: importlib.util.find_spec(name) is not None for name in REQUIRED_MODULES}
|
|
23
|
+
versions = {}
|
|
24
|
+
for name, distribution in (("torch", "torch"), ("ctranslate2", "ctranslate2"), ("faster_whisper", "faster-whisper"), ("qwen_asr", "qwen-asr"), ("funasr", "funasr")):
|
|
25
|
+
try:
|
|
26
|
+
versions[name] = importlib.metadata.version(distribution)
|
|
27
|
+
except importlib.metadata.PackageNotFoundError:
|
|
28
|
+
versions[name] = None
|
|
29
|
+
report = {"schema": "openmd.asr-probe/v1", "python": display_path(Path(sys.executable)), "version": platform.python_version(), "modules": modules, "versions": versions, "torch_cuda_build": None, "torch_cuda_available": False, "ctranslate2_cuda_devices": 0, "gpu_name": None, "models": {key: model.as_dict() for key, model in configured_models().items()}, "ready": False}
|
|
30
|
+
errors = []
|
|
31
|
+
try:
|
|
32
|
+
if modules["torch"]:
|
|
33
|
+
import torch
|
|
34
|
+
|
|
35
|
+
report["torch_cuda_build"] = torch.version.cuda
|
|
36
|
+
report["torch_cuda_available"] = torch.cuda.is_available()
|
|
37
|
+
if report["torch_cuda_available"]:
|
|
38
|
+
report["gpu_name"] = torch.cuda.get_device_name(0)
|
|
39
|
+
if modules["faster_whisper"]:
|
|
40
|
+
from openmd.adapters.local_audio.backends.faster_whisper import _prepare_windows_dll_paths
|
|
41
|
+
|
|
42
|
+
_prepare_windows_dll_paths()
|
|
43
|
+
if modules["ctranslate2"]:
|
|
44
|
+
import ctranslate2
|
|
45
|
+
|
|
46
|
+
report["ctranslate2_cuda_devices"] = ctranslate2.get_cuda_device_count()
|
|
47
|
+
except Exception as exc:
|
|
48
|
+
from openmd.core.secrets import sanitize_error_summary
|
|
49
|
+
|
|
50
|
+
errors.append(sanitize_error_summary(exc))
|
|
51
|
+
report["ready"] = all(modules.values()) and report["torch_cuda_available"] and report["ctranslate2_cuda_devices"] > 0 and all(model["available"] for model in report["models"].values())
|
|
52
|
+
report["backend_readiness"] = {
|
|
53
|
+
"asr.faster_whisper": modules["faster_whisper"] and modules["ctranslate2"] and report["models"]["whisper-large-v3"]["available"],
|
|
54
|
+
"asr.qwen3": all(modules[key] for key in ("torch", "qwen_asr", "funasr", "soundfile")) and report["torch_cuda_available"] and all(report["models"][key]["available"] for key in ("qwen3-asr-1.7b", "fsmn-vad-zh")),
|
|
55
|
+
"asr.auto": report["ready"],
|
|
56
|
+
}
|
|
57
|
+
if os.getenv("OPENMD_ASR_AUTO_ZH_ENGINE") == "whisper":
|
|
58
|
+
report["backend_readiness"]["asr.auto"] = report["backend_readiness"]["asr.faster_whisper"] and report["models"]["whisper-large-v3-turbo"]["available"]
|
|
59
|
+
report["errors"] = errors
|
|
60
|
+
return report
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def main(argv: list[str] | None = None) -> int:
|
|
64
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
65
|
+
parser.add_argument("--probe", action="store_true")
|
|
66
|
+
parser.add_argument("--backend", choices=("asr.auto", "asr.faster_whisper", "asr.qwen3"))
|
|
67
|
+
parser.add_argument("--input", type=Path)
|
|
68
|
+
parser.add_argument("--result", type=Path)
|
|
69
|
+
args = parser.parse_args(argv)
|
|
70
|
+
os.environ.update({"OPENMD_ASR_WORKER": "1", "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "MODELSCOPE_OFFLINE": "1"})
|
|
71
|
+
if args.probe:
|
|
72
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
73
|
+
report = build_probe()
|
|
74
|
+
print(json.dumps(report, ensure_ascii=True, allow_nan=False))
|
|
75
|
+
return 0
|
|
76
|
+
if args.backend is None or args.input is None or args.result is None:
|
|
77
|
+
parser.error("--backend, --input and --result are required")
|
|
78
|
+
from .asr import serialize_result
|
|
79
|
+
from .gpu_quarantine import GPUQuarantined
|
|
80
|
+
|
|
81
|
+
# Share a structured failure with the controller; never initialize an ASR
|
|
82
|
+
# model when quarantine recovery fails inside the lease acquisition.
|
|
83
|
+
try:
|
|
84
|
+
return _transcribe_locked(args)
|
|
85
|
+
except GPUQuarantined as exc:
|
|
86
|
+
from .asr import failed_result
|
|
87
|
+
|
|
88
|
+
args.result.write_text(json.dumps(serialize_result(failed_result(args.backend, exc.code, exc))), encoding="utf-8")
|
|
89
|
+
return 0
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _transcribe_locked(args) -> int:
|
|
93
|
+
from .asr import local_backend, serialize_result
|
|
94
|
+
from .gpu_lock import gpu_lock, release_torch_cache
|
|
95
|
+
|
|
96
|
+
# The inference process owns the lease. If its controller crashes, an orphan
|
|
97
|
+
# worker keeps the lease until its inference exits, so a new OCR cannot race it.
|
|
98
|
+
with gpu_lock():
|
|
99
|
+
started = time.perf_counter()
|
|
100
|
+
try:
|
|
101
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
102
|
+
result = local_backend(args.backend).transcribe(args.input)
|
|
103
|
+
torch = sys.modules.get("torch")
|
|
104
|
+
execution = {"python_version": platform.python_version(), "torch_cuda_build": None, "torch_cuda_available": False,
|
|
105
|
+
"elapsed_seconds": round(time.perf_counter() - started, 3)}
|
|
106
|
+
if torch is not None:
|
|
107
|
+
execution.update({"torch_cuda_build": torch.version.cuda, "torch_cuda_available": torch.cuda.is_available()})
|
|
108
|
+
if execution["torch_cuda_available"]:
|
|
109
|
+
execution.update({"gpu_name": torch.cuda.get_device_name(0), "torch_peak_allocated_bytes": torch.cuda.max_memory_allocated(),
|
|
110
|
+
"torch_peak_reserved_bytes": torch.cuda.max_memory_reserved()})
|
|
111
|
+
finally:
|
|
112
|
+
release_torch_cache()
|
|
113
|
+
result = replace(result, metadata={**result.metadata, "asr_execution": execution})
|
|
114
|
+
args.result.write_text(json.dumps(serialize_result(result), ensure_ascii=False, allow_nan=False), encoding="utf-8")
|
|
115
|
+
return 0
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
if __name__ == "__main__":
|
|
119
|
+
raise SystemExit(main())
|