openmd-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. openmd/__init__.py +3 -0
  2. openmd/__main__.py +5 -0
  3. openmd/adapters/__init__.py +1 -0
  4. openmd/adapters/base.py +95 -0
  5. openmd/adapters/bilibili/__init__.py +11 -0
  6. openmd/adapters/bilibili/adapter.py +104 -0
  7. openmd/adapters/bilibili/discovery.py +33 -0
  8. openmd/adapters/douyin/__init__.py +11 -0
  9. openmd/adapters/douyin/adapter.py +117 -0
  10. openmd/adapters/douyin/discovery.py +36 -0
  11. openmd/adapters/douyin/sidecar.py +77 -0
  12. openmd/adapters/generic_media/__init__.py +14 -0
  13. openmd/adapters/high_privacy.py +77 -0
  14. openmd/adapters/local_audio/__init__.py +14 -0
  15. openmd/adapters/local_audio/adapter.py +169 -0
  16. openmd/adapters/local_audio/asr.py +58 -0
  17. openmd/adapters/local_audio/backends/__init__.py +1 -0
  18. openmd/adapters/local_audio/backends/auto.py +177 -0
  19. openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
  20. openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
  21. openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
  22. openmd/adapters/opencli.py +770 -0
  23. openmd/adapters/pdf/__init__.py +11 -0
  24. openmd/adapters/pdf/adapter.py +153 -0
  25. openmd/adapters/pdf/backends/__init__.py +1 -0
  26. openmd/adapters/pdf/backends/markitdown.py +165 -0
  27. openmd/adapters/pdf/parser.py +56 -0
  28. openmd/adapters/qq_group/__init__.py +11 -0
  29. openmd/adapters/qq_group/adapter.py +289 -0
  30. openmd/adapters/qq_group/parsers/__init__.py +1 -0
  31. openmd/adapters/qq_group/parsers/html.py +60 -0
  32. openmd/adapters/qq_group/parsers/json.py +71 -0
  33. openmd/adapters/registry.py +149 -0
  34. openmd/adapters/screenshot/__init__.py +11 -0
  35. openmd/adapters/screenshot/adapter.py +154 -0
  36. openmd/adapters/screenshot/backends/__init__.py +1 -0
  37. openmd/adapters/screenshot/backends/paddle.py +15 -0
  38. openmd/adapters/screenshot/backends/rapidocr.py +176 -0
  39. openmd/adapters/screenshot/backends/tesseract.py +15 -0
  40. openmd/adapters/screenshot/ocr.py +84 -0
  41. openmd/adapters/wechat_group/__init__.py +11 -0
  42. openmd/adapters/wechat_group/adapter.py +160 -0
  43. openmd/adapters/wechat_group/wx_cli.py +268 -0
  44. openmd/adapters/wechat_moments/__init__.py +11 -0
  45. openmd/adapters/wechat_moments/adapter.py +167 -0
  46. openmd/adapters/wechat_moments/wx_cli.py +265 -0
  47. openmd/adapters/weixin/__init__.py +11 -0
  48. openmd/adapters/weixin/adapter.py +148 -0
  49. openmd/adapters/weixin/discovery.py +510 -0
  50. openmd/adapters/weixin/report.py +278 -0
  51. openmd/adapters/xiaohongshu.py +158 -0
  52. openmd/backends/__init__.py +1 -0
  53. openmd/backends/base.py +204 -0
  54. openmd/backends/bilibili.py +1480 -0
  55. openmd/backends/bilibili_wbi.py +277 -0
  56. openmd/backends/discourse.py +426 -0
  57. openmd/backends/douyin.py +868 -0
  58. openmd/backends/douyin_detail.py +124 -0
  59. openmd/backends/douyin_providers.py +602 -0
  60. openmd/backends/executor.py +546 -0
  61. openmd/backends/feishu.py +861 -0
  62. openmd/backends/generic_media.py +77 -0
  63. openmd/backends/interactive.py +60 -0
  64. openmd/backends/ocr.py +41 -0
  65. openmd/backends/opencli.py +82 -0
  66. openmd/backends/pdf.py +122 -0
  67. openmd/backends/registry.py +234 -0
  68. openmd/backends/screenshot.py +79 -0
  69. openmd/backends/video_metadata.py +167 -0
  70. openmd/backends/web_article.py +363 -0
  71. openmd/backends/wechat_group.py +35 -0
  72. openmd/backends/wechat_moments.py +35 -0
  73. openmd/backends/weixin.py +767 -0
  74. openmd/backends/x.py +455 -0
  75. openmd/backends/xiaoe.py +940 -0
  76. openmd/backends/xiaohongshu.py +911 -0
  77. openmd/backends/xiaohongshu_providers.py +876 -0
  78. openmd/backends/xiaoyuzhou.py +1017 -0
  79. openmd/backends/youtube.py +280 -0
  80. openmd/backends/youtube_discovery.py +1245 -0
  81. openmd/backends/youtube_transcript.py +213 -0
  82. openmd/browser_extension/background.js +231 -0
  83. openmd/browser_extension/icons/icon-128.png +0 -0
  84. openmd/browser_extension/icons/icon-16.png +0 -0
  85. openmd/browser_extension/icons/icon-32.png +0 -0
  86. openmd/browser_extension/icons/icon-48.png +0 -0
  87. openmd/browser_extension/icons/icon-512.png +0 -0
  88. openmd/browser_extension/idle.html +6 -0
  89. openmd/browser_extension/manifest.json +43 -0
  90. openmd/browser_extension/options.html +42 -0
  91. openmd/browser_extension/options.js +117 -0
  92. openmd/browser_extension/popup.css +71 -0
  93. openmd/browser_extension/popup.html +17 -0
  94. openmd/browser_extension/popup.js +59 -0
  95. openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
  96. openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
  97. openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
  98. openmd/browser_host.py +503 -0
  99. openmd/browser_setup.py +106 -0
  100. openmd/cli.py +2012 -0
  101. openmd/console.py +32 -0
  102. openmd/core/__init__.py +1 -0
  103. openmd/core/capabilities.py +162 -0
  104. openmd/core/content_validation.py +323 -0
  105. openmd/core/coverage.py +163 -0
  106. openmd/core/detector.py +247 -0
  107. openmd/core/doctor.py +104 -0
  108. openmd/core/execution_policy.py +164 -0
  109. openmd/core/filename.py +64 -0
  110. openmd/core/input_validation.py +91 -0
  111. openmd/core/local_config.py +96 -0
  112. openmd/core/manifest.py +156 -0
  113. openmd/core/models.py +168 -0
  114. openmd/core/opencli_output.py +34 -0
  115. openmd/core/operation_contract.py +60 -0
  116. openmd/core/output.py +194 -0
  117. openmd/core/package_contract.py +335 -0
  118. openmd/core/package_validator.py +489 -0
  119. openmd/core/package_writer.py +666 -0
  120. openmd/core/planner.py +345 -0
  121. openmd/core/preferences.py +61 -0
  122. openmd/core/readable.py +129 -0
  123. openmd/core/receipt.py +285 -0
  124. openmd/core/renderer.py +185 -0
  125. openmd/core/runlog.py +208 -0
  126. openmd/core/secrets.py +313 -0
  127. openmd/doctor_view.py +68 -0
  128. openmd/interactive.py +453 -0
  129. openmd/ocr/__init__.py +1 -0
  130. openmd/ocr/breakers.py +84 -0
  131. openmd/ocr/config.py +54 -0
  132. openmd/ocr/documents.py +522 -0
  133. openmd/ocr/endpoint.py +72 -0
  134. openmd/ocr/layout.py +92 -0
  135. openmd/ocr/layout_filter.py +118 -0
  136. openmd/ocr/layout_worker.py +89 -0
  137. openmd/ocr/native_math.py +220 -0
  138. openmd/ocr/office.py +305 -0
  139. openmd/ocr/ollama.py +200 -0
  140. openmd/ocr/otsl.py +51 -0
  141. openmd/ocr/ovis-prompt.txt +1 -0
  142. openmd/ocr/pipeline.py +384 -0
  143. openmd/ocr/settings.json +25 -0
  144. openmd/ocr/tiling.py +631 -0
  145. openmd/post_video.py +154 -0
  146. openmd/probe.py +397 -0
  147. openmd/routing/__init__.py +1 -0
  148. openmd/routing/cards/builtin.json +620 -0
  149. openmd/routing/cards/defuddle.json +70 -0
  150. openmd/routing/cards/x-tweet-fetcher.json +71 -0
  151. openmd/routing/cards/xyz-dl.json +60 -0
  152. openmd/routing/cards/yutto.json +68 -0
  153. openmd/routing/entry_shim.py +71 -0
  154. openmd/routing/lifecycle.py +459 -0
  155. openmd/routing/loader.py +138 -0
  156. openmd/routing/providers.py +158 -0
  157. openmd/routing/readers.py +108 -0
  158. openmd/routing/routes.json +504 -0
  159. openmd/routing/runner.py +192 -0
  160. openmd/routing/view.py +40 -0
  161. openmd/runtime/__init__.py +1 -0
  162. openmd/runtime/_windows_job.py +234 -0
  163. openmd/runtime/asr.py +217 -0
  164. openmd/runtime/asr_worker.py +119 -0
  165. openmd/runtime/gpu_lock.py +81 -0
  166. openmd/runtime/gpu_quarantine.py +85 -0
  167. openmd/runtime/interpreters.py +407 -0
  168. openmd/runtime/managed_tools.py +204 -0
  169. openmd/runtime/models.py +202 -0
  170. openmd/runtime/services.py +175 -0
  171. openmd/runtime/tempdirs.py +227 -0
  172. openmd_cli-0.1.0.dist-info/METADATA +168 -0
  173. openmd_cli-0.1.0.dist-info/RECORD +177 -0
  174. openmd_cli-0.1.0.dist-info/WHEEL +5 -0
  175. openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
  176. openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
  177. openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
openmd/routing/view.py ADDED
@@ -0,0 +1,40 @@
1
+ """Generate route views from the same declarations used by runtime selection."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+
8
+ from .loader import route_view
9
+
10
+
11
+ def doctor_tools(*, probe_commands: bool = False):
12
+ from .providers import doctor_tools as build_report
13
+
14
+ return build_report(probe_commands=probe_commands)
15
+
16
+
17
+ def render_routes_markdown(*, source: str | None = None, effective: bool = False) -> str:
18
+ view = route_view(source, effective=effective)
19
+ lines = ["# Openmd routes", "", f"Generated from bundled cards and routes, version `{view['version']}`.", "", "| Source | Capability | Ordered backends |", "|---|---|---|"]
20
+ for route in view["routes"]:
21
+ lines.append(f"| {route['source']} | {route['capability']} | {' → '.join(route['chain'])} |")
22
+ lines += ["", "Tools run only with the gates declared in their cards. Media steps require local ASR before producing text.", "", "Local overlay applied: " + str(view["overlay_applied"]).lower() + ".", ""]
23
+ return "\n".join(lines)
24
+
25
+
26
+ def main(argv: list[str] | None = None) -> int:
27
+ parser = argparse.ArgumentParser()
28
+ parser.add_argument("--source")
29
+ parser.add_argument("--format", choices=("json", "table", "markdown"), default="table")
30
+ parser.add_argument("--bundled", action="store_true", help="Use bundled routes only when generating repository documentation.")
31
+ args = parser.parse_args(argv)
32
+ if args.format == "json":
33
+ print(json.dumps(route_view(args.source, effective=not args.bundled), ensure_ascii=False, indent=2))
34
+ else:
35
+ print(render_routes_markdown(source=args.source, effective=not args.bundled), end="")
36
+ return 0
37
+
38
+
39
+ if __name__ == "__main__":
40
+ raise SystemExit(main())
@@ -0,0 +1 @@
1
+ """Local capability runtimes; the controller retains its own dependencies."""
@@ -0,0 +1,234 @@
1
+ """Windows Job ownership, including atomic assignment before child execution.
2
+
3
+ Only imported on Windows. The small Popen creation hook uses STARTUPINFOEX's
4
+ JOB_LIST (Windows 10+) so even controller death between CreateProcess and
5
+ AssignProcessToJobObject cannot strand a suspended child. No global monkeypatch.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import ctypes as C
11
+ from ctypes import wintypes as W
12
+ import os
13
+ import subprocess
14
+ import sys
15
+ import threading
16
+
17
+
18
+ kernel = C.WinDLL("kernel32", use_last_error=True)
19
+ SIZE = C.c_size_t
20
+
21
+
22
+ class IO_COUNTERS(C.Structure):
23
+ _fields_ = [(name, C.c_ulonglong) for name in (
24
+ "ReadOperationCount", "WriteOperationCount", "OtherOperationCount",
25
+ "ReadTransferCount", "WriteTransferCount", "OtherTransferCount")]
26
+
27
+
28
+ class BASIC_LIMITS(C.Structure):
29
+ _fields_ = [("PerProcessUserTimeLimit", C.c_longlong),
30
+ ("PerJobUserTimeLimit", C.c_longlong), ("LimitFlags", W.DWORD),
31
+ ("MinimumWorkingSetSize", SIZE), ("MaximumWorkingSetSize", SIZE),
32
+ ("ActiveProcessLimit", W.DWORD), ("Affinity", SIZE),
33
+ ("PriorityClass", W.DWORD), ("SchedulingClass", W.DWORD)]
34
+
35
+
36
+ class EXTENDED_LIMITS(C.Structure):
37
+ _fields_ = [("BasicLimitInformation", BASIC_LIMITS), ("IoInfo", IO_COUNTERS),
38
+ ("ProcessMemoryLimit", SIZE), ("JobMemoryLimit", SIZE),
39
+ ("PeakProcessMemoryUsed", SIZE), ("PeakJobMemoryUsed", SIZE)]
40
+
41
+
42
+ class BASIC_ACCOUNTING(C.Structure):
43
+ _fields_ = [(name, C.c_longlong) for name in ("TotalUserTime", "TotalKernelTime",
44
+ "ThisPeriodTotalUserTime", "ThisPeriodTotalKernelTime")] + [
45
+ (name, W.DWORD) for name in ("TotalPageFaultCount", "TotalProcesses",
46
+ "ActiveProcesses", "TotalTerminatedProcesses")]
47
+
48
+
49
+ class STARTUPINFO(C.Structure):
50
+ _fields_ = [("cb", W.DWORD), ("lpReserved", W.LPWSTR),
51
+ ("lpDesktop", W.LPWSTR), ("lpTitle", W.LPWSTR),
52
+ *[(name, W.DWORD) for name in ("dwX", "dwY", "dwXSize", "dwYSize",
53
+ "dwXCountChars", "dwYCountChars", "dwFillAttribute", "dwFlags")],
54
+ ("wShowWindow", W.WORD), ("cbReserved2", W.WORD),
55
+ ("lpReserved2", C.c_void_p), ("hStdInput", W.HANDLE),
56
+ ("hStdOutput", W.HANDLE), ("hStdError", W.HANDLE)]
57
+
58
+
59
+ class STARTUPINFOEX(C.Structure):
60
+ _fields_ = [("StartupInfo", STARTUPINFO), ("lpAttributeList", C.c_void_p)]
61
+
62
+
63
+ class PROCESS_INFORMATION(C.Structure):
64
+ _fields_ = [("hProcess", W.HANDLE), ("hThread", W.HANDLE),
65
+ ("dwProcessId", W.DWORD), ("dwThreadId", W.DWORD)]
66
+
67
+
68
+ def _api(name, result, *args):
69
+ function = getattr(kernel, name)
70
+ function.restype, function.argtypes = result, args
71
+ return function
72
+
73
+
74
+ CreateJobObject = _api("CreateJobObjectW", W.HANDLE, C.c_void_p, W.LPCWSTR)
75
+ SetInformationJobObject = _api("SetInformationJobObject", W.BOOL, W.HANDLE, C.c_int, C.c_void_p, W.DWORD)
76
+ AssignProcessToJobObject = _api("AssignProcessToJobObject", W.BOOL, W.HANDLE, W.HANDLE)
77
+ TerminateJobObject = _api("TerminateJobObject", W.BOOL, W.HANDLE, W.UINT)
78
+ QueryInformationJobObject = _api("QueryInformationJobObject", W.BOOL, W.HANDLE, C.c_int, C.c_void_p, W.DWORD, C.c_void_p)
79
+ WaitForSingleObject = _api("WaitForSingleObject", W.DWORD, W.HANDLE, W.DWORD)
80
+ CloseHandle = _api("CloseHandle", W.BOOL, W.HANDLE)
81
+ ResumeThread = _api("ResumeThread", W.DWORD, W.HANDLE)
82
+ TerminateProcess = _api("TerminateProcess", W.BOOL, W.HANDLE, W.UINT)
83
+ InitializeProcThreadAttributeList = _api("InitializeProcThreadAttributeList", W.BOOL, C.c_void_p, W.DWORD, W.DWORD, C.POINTER(SIZE))
84
+ UpdateProcThreadAttribute = _api("UpdateProcThreadAttribute", W.BOOL, C.c_void_p, W.DWORD, SIZE, C.c_void_p, SIZE, C.c_void_p, C.c_void_p)
85
+ DeleteProcThreadAttributeList = _api("DeleteProcThreadAttributeList", None, C.c_void_p)
86
+ CreateProcess = _api("CreateProcessW", W.BOOL, W.LPCWSTR, W.LPWSTR, C.c_void_p, C.c_void_p,
87
+ W.BOOL, W.DWORD, C.c_void_p, W.LPCWSTR,
88
+ C.POINTER(STARTUPINFOEX), C.POINTER(PROCESS_INFORMATION))
89
+ OpenThread = _api("OpenThread", W.HANDLE, W.DWORD, W.BOOL, W.DWORD)
90
+ CancelSynchronousIo = _api("CancelSynchronousIo", W.BOOL, W.HANDLE)
91
+
92
+
93
+ def _checked(result):
94
+ if not result:
95
+ raise C.WinError(C.get_last_error())
96
+ return result
97
+
98
+
99
+ class Job:
100
+ def __init__(self):
101
+ self.lock = threading.Lock()
102
+ self.handle = _checked(CreateJobObject(None, None))
103
+ try:
104
+ limits = EXTENDED_LIMITS()
105
+ limits.BasicLimitInformation.LimitFlags = 0x2000 # KILL_ON_JOB_CLOSE
106
+ _checked(SetInformationJobObject(self.handle, 9, C.byref(limits), C.sizeof(limits)))
107
+ except BaseException:
108
+ self.close()
109
+ raise
110
+
111
+ def close(self):
112
+ with self.lock:
113
+ if self.handle:
114
+ handle, self.handle = self.handle, None
115
+ try:
116
+ # Terminate explicitly; closing also protects controller crashes.
117
+ _checked(TerminateJobObject(handle, 1))
118
+ # Termination is asynchronous. Waiting on the Job object
119
+ # itself is the kernel synchronization point for the whole
120
+ # tree; ActiveProcesses can reach zero before the process
121
+ # handles and inherited file handles are fully released.
122
+ WaitForSingleObject(handle, 2000)
123
+ finally:
124
+ CloseHandle(handle)
125
+
126
+ def __del__(self):
127
+ if getattr(self, "handle", None):
128
+ CloseHandle(self.handle)
129
+
130
+
131
+ def cancel_thread_io(thread):
132
+ if thread is not None and thread.is_alive() and thread.native_id:
133
+ handle = OpenThread(0x0001, False, thread.native_id) # THREAD_TERMINATE
134
+ if handle:
135
+ try:
136
+ CancelSynchronousIo(handle)
137
+ finally:
138
+ CloseHandle(handle)
139
+
140
+
141
+ class JobPopen(subprocess.Popen):
142
+ """Keep Popen's pipe/text/wait implementation; customize only OS creation.
143
+
144
+ Signature follows CPython's Windows Popen._execute_child; tested on 3.12.
145
+ Creation errors fail closed: never silently launch without Job ownership.
146
+ """
147
+
148
+ def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds,
149
+ cwd, env, startupinfo, creationflags, shell,
150
+ p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,
151
+ *unused, **unused_kwargs):
152
+ if pass_fds:
153
+ raise ValueError("pass_fds not supported on Windows")
154
+ if isinstance(args, (bytes, os.PathLike)):
155
+ if shell:
156
+ raise TypeError("bytes/path-like args is not allowed with shell")
157
+ args = subprocess.list2cmdline([args])
158
+ elif not isinstance(args, str):
159
+ args = subprocess.list2cmdline(args)
160
+ executable = os.fsdecode(executable) if executable is not None else None
161
+ startupinfo = startupinfo.copy() if startupinfo is not None else subprocess.STARTUPINFO()
162
+ if startupinfo.lpAttributeList and set(startupinfo.lpAttributeList) - {"handle_list"}:
163
+ raise ValueError("Unsupported Windows startup attribute")
164
+ native = STARTUPINFOEX()
165
+ native.StartupInfo.cb = C.sizeof(native)
166
+ native.StartupInfo.dwFlags = startupinfo.dwFlags
167
+ native.StartupInfo.wShowWindow = startupinfo.wShowWindow
168
+ handles = list((startupinfo.lpAttributeList or {}).get("handle_list", []))
169
+ if -1 not in (p2cread, c2pwrite, errwrite):
170
+ native.StartupInfo.dwFlags |= 0x100 # STARTF_USESTDHANDLES
171
+ native.StartupInfo.hStdInput = int(p2cread)
172
+ native.StartupInfo.hStdOutput = int(c2pwrite)
173
+ native.StartupInfo.hStdError = int(errwrite)
174
+ handles += [int(p2cread), int(c2pwrite), int(errwrite)]
175
+ handles = self._filter_handle_list(handles)
176
+ if shell:
177
+ native.StartupInfo.dwFlags |= 1
178
+ native.StartupInfo.wShowWindow = 0
179
+ executable = executable or os.environ.get("ComSpec") or os.path.join(os.environ.get("SystemRoot", ""), "System32", "cmd.exe")
180
+ if not os.path.isabs(executable):
181
+ raise FileNotFoundError("An absolute Windows shell path is required")
182
+ args = f'{executable} /c "{args}"'
183
+ cwd = os.fsdecode(cwd) if cwd is not None else None
184
+ environment = None
185
+ if env is not None:
186
+ entries = []
187
+ for key, value in sorted(env.items(), key=lambda pair: pair[0].upper()):
188
+ if "\0" in key or "\0" in value or "=" in key[1:] or not key:
189
+ raise ValueError("illegal environment variable")
190
+ entries.append(f"{key}={value}")
191
+ environment = C.create_unicode_buffer("\0".join(entries) + "\0\0")
192
+ size = SIZE()
193
+ InitializeProcThreadAttributeList(None, 2 if handles else 1, 0, C.byref(size))
194
+ attributes = C.create_string_buffer(size.value)
195
+ _checked(InitializeProcThreadAttributeList(attributes, 2 if handles else 1, 0, C.byref(size)))
196
+ native.lpAttributeList = C.cast(attributes, C.c_void_p)
197
+ info = PROCESS_INFORMATION()
198
+ try:
199
+ # JOB_LIST assigns at CreateProcess itself, closing the crash window.
200
+ jobs = (W.HANDLE * 1)(self._openmd_owner.handle)
201
+ _checked(UpdateProcThreadAttribute(attributes, 0, 0x2000D, jobs, C.sizeof(jobs), None, None))
202
+ if handles:
203
+ inherited = (W.HANDLE * len(handles))(*handles)
204
+ _checked(UpdateProcThreadAttribute(attributes, 0, 0x20002, inherited, C.sizeof(inherited), None, None))
205
+ sys.audit("subprocess.Popen", executable, args, cwd, env)
206
+ _checked(CreateProcess(executable, C.create_unicode_buffer(args), None, None,
207
+ bool(handles) or not close_fds,
208
+ creationflags | 0x4 | 0x80000 | 0x400, # SUSPENDED/EXTENDED/UNICODE
209
+ environment, cwd, C.byref(native), C.byref(info)))
210
+ self._child_created = True
211
+ self._handle = subprocess.Handle(info.hProcess)
212
+ self.pid = info.dwProcessId
213
+ # Already in this Job atomically; explicit assignment validates it
214
+ # before any child code can run or spawn a grandchild.
215
+ _checked(AssignProcessToJobObject(self._openmd_owner.handle, info.hProcess))
216
+ if ResumeThread(info.hThread) == 0xFFFFFFFF:
217
+ raise C.WinError(C.get_last_error())
218
+ except BaseException:
219
+ if info.hProcess:
220
+ TerminateProcess(info.hProcess, 1)
221
+ raise
222
+ finally:
223
+ if info.hThread:
224
+ CloseHandle(info.hThread)
225
+ DeleteProcThreadAttributeList(attributes)
226
+ self._close_pipe_fds(p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)
227
+
228
+ def __init__(self, *args, **kwargs):
229
+ self._openmd_owner = Job()
230
+ try:
231
+ super().__init__(*args, **kwargs)
232
+ except BaseException:
233
+ self._openmd_owner.close()
234
+ raise
openmd/runtime/asr.py ADDED
@@ -0,0 +1,217 @@
1
+ """ASR factory and bounded worker protocol used by every controller entrypoint."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import hashlib
7
+ import math
8
+ import os
9
+ import subprocess
10
+ import tempfile
11
+ from dataclasses import asdict, fields, replace
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from openmd.adapters.local_audio.asr import ASRBackend
16
+ from openmd.core.models import COMPONENT_STATUS_VALUES, FAILURE_CLASS_VALUES, SOURCE_LEVEL_VALUES, STATUS_VALUES, ExtractionComponent, TranscriptResult
17
+ from openmd.core.secrets import sanitize_error_summary
18
+ from .interpreters import owned_popen, AsrRuntime, display_path, kill_process_tree, process_options, remove_tree_with_retry, select_asr_runtime, terminate_quietly, wait_interruptibly, worker_command, worker_environment
19
+
20
+ from openmd.runtime.tempdirs import owned_mkdtemp
21
+ from openmd.runtime.interpreters import run_interruptibly
22
+
23
+
24
+ ASR_BACKENDS = ("asr.auto", "asr.faster_whisper", "asr.qwen3")
25
+
26
+
27
+ def local_backend(name: str) -> ASRBackend:
28
+ if name == "asr.auto":
29
+ from openmd.adapters.local_audio.backends.auto import AutoASRBackend
30
+
31
+ return AutoASRBackend()
32
+ if name == "asr.faster_whisper":
33
+ from openmd.adapters.local_audio.backends.faster_whisper import FasterWhisperASRBackend
34
+
35
+ return FasterWhisperASRBackend()
36
+ if name == "asr.qwen3":
37
+ from openmd.adapters.local_audio.backends.qwen3_asr import Qwen3ASRBackend
38
+
39
+ return Qwen3ASRBackend()
40
+ raise ValueError(f"Unknown local ASR backend: {name}")
41
+
42
+
43
+ def backend_execution_config(name: str) -> dict[str, Any]:
44
+ """Constructing fingerprints never starts a GPU probe or imports a model."""
45
+ from openmd.core.local_config import load_local_config
46
+ from .models import checkout_root
47
+
48
+ config = local_backend(name).execution_config()
49
+ local = load_local_config()
50
+ explicit = os.getenv("OPENMD_ASR_PYTHON") or local.get("asr_python")
51
+ root = checkout_root()
52
+ candidates = [Path(explicit).expanduser()] if explicit else [Path(os.sys.executable)]
53
+ if not explicit and root:
54
+ candidates.append(root / "out/asr-bench/venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python"))
55
+ identities = []
56
+ for candidate in candidates:
57
+ if not candidate.is_absolute():
58
+ candidate = (root or Path.cwd()) / candidate
59
+ try:
60
+ stamp = candidate.stat().st_mtime_ns
61
+ except OSError:
62
+ stamp = None
63
+ identities.append({"path": str(candidate.resolve()), "mtime_ns": stamp})
64
+ # Hash locations/settings for resume identity without publishing machine paths
65
+ # or starting a GPU probe during argument parsing.
66
+ settings = {key: value for key, value in os.environ.items() if key.startswith(("OPENMD_FASTER_WHISPER_", "OPENMD_ASR_AUTO_", "OPENMD_QWEN3_")) or key in {"OPENMD_MODEL_ROOTS", "OPENMD_ASR_CHINESE_SCRIPT", "CUDA_VISIBLE_DEVICES", "HF_HOME", "HF_HUB_CACHE", "MODELSCOPE_CACHE"}}
67
+ identity = {"interpreters": identities, "model_roots": local.get("model_roots", []), "settings": settings, "backend": config}
68
+ config_id = hashlib.sha256(json.dumps(identity, sort_keys=True, ensure_ascii=True).encode()).hexdigest()
69
+ return {**config, "runtime_policy": "local_worker_v2", "runtime_configured": bool(explicit), "runtime_source": "explicit" if explicit else "checkout" if root else "current_interpreter", "runtime_config_id": config_id}
70
+
71
+
72
+ def get_asr_backend(name: str = "asr.auto") -> ASRBackend:
73
+ if name not in ASR_BACKENDS:
74
+ raise ValueError(f"Unknown local ASR backend: {name}")
75
+ # Resolve lazily: provider registration, health checks and execution fingerprints
76
+ # must not load torch or wait for a worker probe.
77
+ return WorkerASRBackend(name)
78
+
79
+
80
+ def failed_result(name: str, code: str, error: object) -> TranscriptResult:
81
+ return TranscriptResult(text="", status="failed", source_level="partial_extraction", backend=name, error_code=code, error_summary=sanitize_error_summary(error), source_notes=["Local ASR did not produce usable text; no cloud service or model download was used."], metadata={"profile": "media", "asr_backend": name})
82
+
83
+
84
+ def serialize_result(result: TranscriptResult) -> dict[str, Any]:
85
+ return {"schema": "openmd.asr-result/v1", "result": asdict(result)}
86
+
87
+
88
+ def deserialize_result(payload: object) -> TranscriptResult:
89
+ if not isinstance(payload, dict) or payload.get("schema") != "openmd.asr-result/v1":
90
+ raise ValueError("invalid ASR worker schema")
91
+ result = payload.get("result")
92
+ if not isinstance(result, dict) or set(result) - {item.name for item in fields(TranscriptResult)}:
93
+ raise ValueError("invalid ASR result fields")
94
+ if not isinstance(result.get("text"), str) or result.get("status") not in STATUS_VALUES or result.get("source_level") not in SOURCE_LEVEL_VALUES or result.get("backend") not in ASR_BACKENDS:
95
+ raise ValueError("invalid ASR result status, source or backend")
96
+ for key in ("error_code", "error_summary", "title", "author", "published_at", "extraction_contract_version"):
97
+ if result.get(key) is not None and not isinstance(result[key], str):
98
+ raise ValueError(f"invalid ASR result {key}")
99
+ notes = result.get("source_notes", [])
100
+ metadata = result.get("metadata", {})
101
+ if not isinstance(notes, list) or not all(isinstance(note, str) for note in notes) or not isinstance(metadata, dict):
102
+ raise ValueError("invalid ASR notes or metadata")
103
+ components = result.get("components", [])
104
+ if not isinstance(components, list):
105
+ raise ValueError("invalid ASR components")
106
+ decoded: list[ExtractionComponent] = []
107
+ for component in components:
108
+ if not isinstance(component, dict) or set(component) - {item.name for item in fields(ExtractionComponent)}:
109
+ raise ValueError("invalid ASR component fields")
110
+ if component.get("status") not in COMPONENT_STATUS_VALUES or component.get("source_level") not in SOURCE_LEVEL_VALUES:
111
+ raise ValueError("invalid ASR component status")
112
+ if any(not isinstance(component.get(key), str) for key in ("id", "kind", "backend")):
113
+ raise ValueError("invalid ASR component identity")
114
+ if component.get("failure_class") is not None and component["failure_class"] not in FAILURE_CLASS_VALUES:
115
+ raise ValueError("invalid ASR component failure class")
116
+ for key in ("text_ref", "terminal_reason", "error_code"):
117
+ if component.get(key) is not None and not isinstance(component[key], str):
118
+ raise ValueError(f"invalid ASR component {key}")
119
+ warnings = component.get("warnings", [])
120
+ if not isinstance(warnings, list) or any(not isinstance(warning, str) for warning in warnings):
121
+ raise ValueError("invalid ASR component warnings")
122
+ decoded.append(ExtractionComponent(**component))
123
+ if result["status"] in {"verified", "partial"} and not result["text"].strip():
124
+ raise ValueError("successful ASR result has no text")
125
+ return TranscriptResult(**{**result, "source_notes": notes, "metadata": metadata, "components": decoded})
126
+
127
+
128
+ def _timeout_for_audio(path: Path) -> float:
129
+ try:
130
+ done = run_interruptibly(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", str(path)], stdin=subprocess.DEVNULL, capture_output=True, timeout=10, check=True)
131
+ duration = float(done.stdout.strip())
132
+ return max(600.0, min(duration * 2, 12 * 60 * 60)) if math.isfinite(duration) and duration >= 0 else 600.0
133
+ except (OSError, ValueError, subprocess.SubprocessError):
134
+ return 600.0
135
+
136
+
137
+ class WorkerASRBackend(ASRBackend):
138
+ def __init__(self, name: str, python: Path | None = None) -> None:
139
+ self.name = name
140
+ self.python = python
141
+
142
+ def execution_config(self) -> dict[str, Any]:
143
+ return backend_execution_config(self.name)
144
+
145
+ def is_available(self) -> bool:
146
+ return self._runtime().mode != "unavailable"
147
+
148
+ def _runtime(self) -> AsrRuntime:
149
+ if self.python is not None:
150
+ return AsrRuntime("worker", self.python, "Explicit worker instance")
151
+ return select_asr_runtime(backend=self.name)
152
+
153
+ def transcribe(self, path: Path) -> TranscriptResult:
154
+ from .gpu_quarantine import GPUQuarantined
155
+
156
+ try:
157
+ return self._transcribe(path)
158
+ except TimeoutError as exc:
159
+ return failed_result(self.name, "gpu_lock_timeout", exc)
160
+ except GPUQuarantined as exc:
161
+ return failed_result(self.name, exc.code, exc)
162
+
163
+ def _transcribe(self, path: Path) -> TranscriptResult:
164
+ path = path.expanduser().resolve()
165
+ if not path.is_file():
166
+ return failed_result(self.name, "asr_file_not_found", "Local audio file not found")
167
+ try:
168
+ runtime = self._runtime()
169
+ except (OSError, ValueError) as exc:
170
+ return failed_result(self.name, "asr_runtime_unavailable", exc)
171
+ if runtime.mode == "in_process":
172
+ from .gpu_lock import gpu_lock, release_torch_cache
173
+
174
+ with gpu_lock():
175
+ try:
176
+ result = local_backend(self.name).transcribe(path)
177
+ finally:
178
+ release_torch_cache()
179
+ return replace(result, source_notes=[*result.source_notes, runtime.reason], metadata={**result.metadata, "asr_runtime": {"mode": "in_process", "reason": runtime.reason}})
180
+ if runtime.mode == "unavailable" or runtime.python is None:
181
+ reports = runtime.probe.get("attempts", [runtime.probe])
182
+ missing = sorted({name for report in reports if isinstance(report, dict) for name, model in report.get("models", {}).items() if not model.get("available", False)})
183
+ if missing:
184
+ return failed_result(self.name, "asr_model_missing", runtime.reason + "; missing or incomplete local models: " + ", ".join(missing))
185
+ return failed_result(self.name, "asr_runtime_unavailable", runtime.reason)
186
+ workdir = owned_mkdtemp(prefix="openmd-asr-worker-")
187
+ try:
188
+ result_path = Path(workdir) / "result.json"
189
+ command = worker_command(runtime.python, "--backend", self.name, "--input", str(path), "--result", str(result_path))
190
+ try:
191
+ # Library chatter is bounded to a file and never becomes protocol data.
192
+ with (Path(workdir) / "worker.log").open("wb") as log:
193
+ process = owned_popen(command, stdin=subprocess.DEVNULL, stdout=log, stderr=log, env=worker_environment(), cwd=workdir, **process_options())
194
+ try:
195
+ wait_interruptibly(process, _timeout_for_audio(path))
196
+ except subprocess.TimeoutExpired:
197
+ terminate_quietly(process)
198
+ return failed_result(self.name, "asr_worker_timeout", "Local ASR worker exceeded its bounded time limit")
199
+ except BaseException:
200
+ # Ctrl+C must surface as cancellation, never as a worker failure.
201
+ terminate_quietly(process)
202
+ raise
203
+ if process.returncode:
204
+ with (Path(workdir) / "worker.log").open("rb") as log:
205
+ log.seek(max(0, log.seek(0, 2) - 2048))
206
+ detail = log.read().decode("utf-8", errors="replace")
207
+ return failed_result(self.name, "asr_worker_failed", detail or f"Worker exited with code {process.returncode}")
208
+ if not result_path.is_file() or result_path.stat().st_size > 64 * 1024 * 1024:
209
+ raise ValueError("worker result missing or exceeds size limit")
210
+ result = deserialize_result(json.loads(result_path.read_text(encoding="utf-8")))
211
+ return replace(result, metadata={**result.metadata, "asr_runtime": {"mode": "worker", "python": display_path(runtime.python)}})
212
+ except (OSError, ValueError, TypeError, subprocess.SubprocessError) as exc:
213
+ return failed_result(self.name, "asr_worker_failed", exc)
214
+ finally:
215
+ # A killed worker's grandchild can briefly keep its cwd open on Windows; cleanup is
216
+ # retried and best-effort so it can never replace the original exception.
217
+ remove_tree_with_retry(Path(workdir))
@@ -0,0 +1,119 @@
1
+ """Offline ASR worker. Only a result file carries transcript protocol data."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import contextlib
7
+ import importlib.metadata
8
+ import importlib.util
9
+ import json
10
+ import os
11
+ import platform
12
+ import sys
13
+ import time
14
+ from dataclasses import replace
15
+ from pathlib import Path
16
+
17
+
18
+ def build_probe() -> dict:
19
+ from .interpreters import REQUIRED_MODULES, display_path
20
+ from .models import configured_models
21
+
22
+ modules = {name: importlib.util.find_spec(name) is not None for name in REQUIRED_MODULES}
23
+ versions = {}
24
+ for name, distribution in (("torch", "torch"), ("ctranslate2", "ctranslate2"), ("faster_whisper", "faster-whisper"), ("qwen_asr", "qwen-asr"), ("funasr", "funasr")):
25
+ try:
26
+ versions[name] = importlib.metadata.version(distribution)
27
+ except importlib.metadata.PackageNotFoundError:
28
+ versions[name] = None
29
+ report = {"schema": "openmd.asr-probe/v1", "python": display_path(Path(sys.executable)), "version": platform.python_version(), "modules": modules, "versions": versions, "torch_cuda_build": None, "torch_cuda_available": False, "ctranslate2_cuda_devices": 0, "gpu_name": None, "models": {key: model.as_dict() for key, model in configured_models().items()}, "ready": False}
30
+ errors = []
31
+ try:
32
+ if modules["torch"]:
33
+ import torch
34
+
35
+ report["torch_cuda_build"] = torch.version.cuda
36
+ report["torch_cuda_available"] = torch.cuda.is_available()
37
+ if report["torch_cuda_available"]:
38
+ report["gpu_name"] = torch.cuda.get_device_name(0)
39
+ if modules["faster_whisper"]:
40
+ from openmd.adapters.local_audio.backends.faster_whisper import _prepare_windows_dll_paths
41
+
42
+ _prepare_windows_dll_paths()
43
+ if modules["ctranslate2"]:
44
+ import ctranslate2
45
+
46
+ report["ctranslate2_cuda_devices"] = ctranslate2.get_cuda_device_count()
47
+ except Exception as exc:
48
+ from openmd.core.secrets import sanitize_error_summary
49
+
50
+ errors.append(sanitize_error_summary(exc))
51
+ report["ready"] = all(modules.values()) and report["torch_cuda_available"] and report["ctranslate2_cuda_devices"] > 0 and all(model["available"] for model in report["models"].values())
52
+ report["backend_readiness"] = {
53
+ "asr.faster_whisper": modules["faster_whisper"] and modules["ctranslate2"] and report["models"]["whisper-large-v3"]["available"],
54
+ "asr.qwen3": all(modules[key] for key in ("torch", "qwen_asr", "funasr", "soundfile")) and report["torch_cuda_available"] and all(report["models"][key]["available"] for key in ("qwen3-asr-1.7b", "fsmn-vad-zh")),
55
+ "asr.auto": report["ready"],
56
+ }
57
+ if os.getenv("OPENMD_ASR_AUTO_ZH_ENGINE") == "whisper":
58
+ report["backend_readiness"]["asr.auto"] = report["backend_readiness"]["asr.faster_whisper"] and report["models"]["whisper-large-v3-turbo"]["available"]
59
+ report["errors"] = errors
60
+ return report
61
+
62
+
63
+ def main(argv: list[str] | None = None) -> int:
64
+ parser = argparse.ArgumentParser(description=__doc__)
65
+ parser.add_argument("--probe", action="store_true")
66
+ parser.add_argument("--backend", choices=("asr.auto", "asr.faster_whisper", "asr.qwen3"))
67
+ parser.add_argument("--input", type=Path)
68
+ parser.add_argument("--result", type=Path)
69
+ args = parser.parse_args(argv)
70
+ os.environ.update({"OPENMD_ASR_WORKER": "1", "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "MODELSCOPE_OFFLINE": "1"})
71
+ if args.probe:
72
+ with contextlib.redirect_stdout(sys.stderr):
73
+ report = build_probe()
74
+ print(json.dumps(report, ensure_ascii=True, allow_nan=False))
75
+ return 0
76
+ if args.backend is None or args.input is None or args.result is None:
77
+ parser.error("--backend, --input and --result are required")
78
+ from .asr import serialize_result
79
+ from .gpu_quarantine import GPUQuarantined
80
+
81
+ # Share a structured failure with the controller; never initialize an ASR
82
+ # model when quarantine recovery fails inside the lease acquisition.
83
+ try:
84
+ return _transcribe_locked(args)
85
+ except GPUQuarantined as exc:
86
+ from .asr import failed_result
87
+
88
+ args.result.write_text(json.dumps(serialize_result(failed_result(args.backend, exc.code, exc))), encoding="utf-8")
89
+ return 0
90
+
91
+
92
+ def _transcribe_locked(args) -> int:
93
+ from .asr import local_backend, serialize_result
94
+ from .gpu_lock import gpu_lock, release_torch_cache
95
+
96
+ # The inference process owns the lease. If its controller crashes, an orphan
97
+ # worker keeps the lease until its inference exits, so a new OCR cannot race it.
98
+ with gpu_lock():
99
+ started = time.perf_counter()
100
+ try:
101
+ with contextlib.redirect_stdout(sys.stderr):
102
+ result = local_backend(args.backend).transcribe(args.input)
103
+ torch = sys.modules.get("torch")
104
+ execution = {"python_version": platform.python_version(), "torch_cuda_build": None, "torch_cuda_available": False,
105
+ "elapsed_seconds": round(time.perf_counter() - started, 3)}
106
+ if torch is not None:
107
+ execution.update({"torch_cuda_build": torch.version.cuda, "torch_cuda_available": torch.cuda.is_available()})
108
+ if execution["torch_cuda_available"]:
109
+ execution.update({"gpu_name": torch.cuda.get_device_name(0), "torch_peak_allocated_bytes": torch.cuda.max_memory_allocated(),
110
+ "torch_peak_reserved_bytes": torch.cuda.max_memory_reserved()})
111
+ finally:
112
+ release_torch_cache()
113
+ result = replace(result, metadata={**result.metadata, "asr_execution": execution})
114
+ args.result.write_text(json.dumps(serialize_result(result), ensure_ascii=False, allow_nan=False), encoding="utf-8")
115
+ return 0
116
+
117
+
118
+ if __name__ == "__main__":
119
+ raise SystemExit(main())