openmd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openmd/__init__.py +3 -0
- openmd/__main__.py +5 -0
- openmd/adapters/__init__.py +1 -0
- openmd/adapters/base.py +95 -0
- openmd/adapters/bilibili/__init__.py +11 -0
- openmd/adapters/bilibili/adapter.py +104 -0
- openmd/adapters/bilibili/discovery.py +33 -0
- openmd/adapters/douyin/__init__.py +11 -0
- openmd/adapters/douyin/adapter.py +117 -0
- openmd/adapters/douyin/discovery.py +36 -0
- openmd/adapters/douyin/sidecar.py +77 -0
- openmd/adapters/generic_media/__init__.py +14 -0
- openmd/adapters/high_privacy.py +77 -0
- openmd/adapters/local_audio/__init__.py +14 -0
- openmd/adapters/local_audio/adapter.py +169 -0
- openmd/adapters/local_audio/asr.py +58 -0
- openmd/adapters/local_audio/backends/__init__.py +1 -0
- openmd/adapters/local_audio/backends/auto.py +177 -0
- openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
- openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
- openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
- openmd/adapters/opencli.py +770 -0
- openmd/adapters/pdf/__init__.py +11 -0
- openmd/adapters/pdf/adapter.py +153 -0
- openmd/adapters/pdf/backends/__init__.py +1 -0
- openmd/adapters/pdf/backends/markitdown.py +165 -0
- openmd/adapters/pdf/parser.py +56 -0
- openmd/adapters/qq_group/__init__.py +11 -0
- openmd/adapters/qq_group/adapter.py +289 -0
- openmd/adapters/qq_group/parsers/__init__.py +1 -0
- openmd/adapters/qq_group/parsers/html.py +60 -0
- openmd/adapters/qq_group/parsers/json.py +71 -0
- openmd/adapters/registry.py +149 -0
- openmd/adapters/screenshot/__init__.py +11 -0
- openmd/adapters/screenshot/adapter.py +154 -0
- openmd/adapters/screenshot/backends/__init__.py +1 -0
- openmd/adapters/screenshot/backends/paddle.py +15 -0
- openmd/adapters/screenshot/backends/rapidocr.py +176 -0
- openmd/adapters/screenshot/backends/tesseract.py +15 -0
- openmd/adapters/screenshot/ocr.py +84 -0
- openmd/adapters/wechat_group/__init__.py +11 -0
- openmd/adapters/wechat_group/adapter.py +160 -0
- openmd/adapters/wechat_group/wx_cli.py +268 -0
- openmd/adapters/wechat_moments/__init__.py +11 -0
- openmd/adapters/wechat_moments/adapter.py +167 -0
- openmd/adapters/wechat_moments/wx_cli.py +265 -0
- openmd/adapters/weixin/__init__.py +11 -0
- openmd/adapters/weixin/adapter.py +148 -0
- openmd/adapters/weixin/discovery.py +510 -0
- openmd/adapters/weixin/report.py +278 -0
- openmd/adapters/xiaohongshu.py +158 -0
- openmd/backends/__init__.py +1 -0
- openmd/backends/base.py +204 -0
- openmd/backends/bilibili.py +1480 -0
- openmd/backends/bilibili_wbi.py +277 -0
- openmd/backends/discourse.py +426 -0
- openmd/backends/douyin.py +868 -0
- openmd/backends/douyin_detail.py +124 -0
- openmd/backends/douyin_providers.py +602 -0
- openmd/backends/executor.py +546 -0
- openmd/backends/feishu.py +861 -0
- openmd/backends/generic_media.py +77 -0
- openmd/backends/interactive.py +60 -0
- openmd/backends/ocr.py +41 -0
- openmd/backends/opencli.py +82 -0
- openmd/backends/pdf.py +122 -0
- openmd/backends/registry.py +234 -0
- openmd/backends/screenshot.py +79 -0
- openmd/backends/video_metadata.py +167 -0
- openmd/backends/web_article.py +363 -0
- openmd/backends/wechat_group.py +35 -0
- openmd/backends/wechat_moments.py +35 -0
- openmd/backends/weixin.py +767 -0
- openmd/backends/x.py +455 -0
- openmd/backends/xiaoe.py +940 -0
- openmd/backends/xiaohongshu.py +911 -0
- openmd/backends/xiaohongshu_providers.py +876 -0
- openmd/backends/xiaoyuzhou.py +1017 -0
- openmd/backends/youtube.py +280 -0
- openmd/backends/youtube_discovery.py +1245 -0
- openmd/backends/youtube_transcript.py +213 -0
- openmd/browser_extension/background.js +231 -0
- openmd/browser_extension/icons/icon-128.png +0 -0
- openmd/browser_extension/icons/icon-16.png +0 -0
- openmd/browser_extension/icons/icon-32.png +0 -0
- openmd/browser_extension/icons/icon-48.png +0 -0
- openmd/browser_extension/icons/icon-512.png +0 -0
- openmd/browser_extension/idle.html +6 -0
- openmd/browser_extension/manifest.json +43 -0
- openmd/browser_extension/options.html +42 -0
- openmd/browser_extension/options.js +117 -0
- openmd/browser_extension/popup.css +71 -0
- openmd/browser_extension/popup.html +17 -0
- openmd/browser_extension/popup.js +59 -0
- openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
- openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
- openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
- openmd/browser_host.py +503 -0
- openmd/browser_setup.py +106 -0
- openmd/cli.py +2012 -0
- openmd/console.py +32 -0
- openmd/core/__init__.py +1 -0
- openmd/core/capabilities.py +162 -0
- openmd/core/content_validation.py +323 -0
- openmd/core/coverage.py +163 -0
- openmd/core/detector.py +247 -0
- openmd/core/doctor.py +104 -0
- openmd/core/execution_policy.py +164 -0
- openmd/core/filename.py +64 -0
- openmd/core/input_validation.py +91 -0
- openmd/core/local_config.py +96 -0
- openmd/core/manifest.py +156 -0
- openmd/core/models.py +168 -0
- openmd/core/opencli_output.py +34 -0
- openmd/core/operation_contract.py +60 -0
- openmd/core/output.py +194 -0
- openmd/core/package_contract.py +335 -0
- openmd/core/package_validator.py +489 -0
- openmd/core/package_writer.py +666 -0
- openmd/core/planner.py +345 -0
- openmd/core/preferences.py +61 -0
- openmd/core/readable.py +129 -0
- openmd/core/receipt.py +285 -0
- openmd/core/renderer.py +185 -0
- openmd/core/runlog.py +208 -0
- openmd/core/secrets.py +313 -0
- openmd/doctor_view.py +68 -0
- openmd/interactive.py +453 -0
- openmd/ocr/__init__.py +1 -0
- openmd/ocr/breakers.py +84 -0
- openmd/ocr/config.py +54 -0
- openmd/ocr/documents.py +522 -0
- openmd/ocr/endpoint.py +72 -0
- openmd/ocr/layout.py +92 -0
- openmd/ocr/layout_filter.py +118 -0
- openmd/ocr/layout_worker.py +89 -0
- openmd/ocr/native_math.py +220 -0
- openmd/ocr/office.py +305 -0
- openmd/ocr/ollama.py +200 -0
- openmd/ocr/otsl.py +51 -0
- openmd/ocr/ovis-prompt.txt +1 -0
- openmd/ocr/pipeline.py +384 -0
- openmd/ocr/settings.json +25 -0
- openmd/ocr/tiling.py +631 -0
- openmd/post_video.py +154 -0
- openmd/probe.py +397 -0
- openmd/routing/__init__.py +1 -0
- openmd/routing/cards/builtin.json +620 -0
- openmd/routing/cards/defuddle.json +70 -0
- openmd/routing/cards/x-tweet-fetcher.json +71 -0
- openmd/routing/cards/xyz-dl.json +60 -0
- openmd/routing/cards/yutto.json +68 -0
- openmd/routing/entry_shim.py +71 -0
- openmd/routing/lifecycle.py +459 -0
- openmd/routing/loader.py +138 -0
- openmd/routing/providers.py +158 -0
- openmd/routing/readers.py +108 -0
- openmd/routing/routes.json +504 -0
- openmd/routing/runner.py +192 -0
- openmd/routing/view.py +40 -0
- openmd/runtime/__init__.py +1 -0
- openmd/runtime/_windows_job.py +234 -0
- openmd/runtime/asr.py +217 -0
- openmd/runtime/asr_worker.py +119 -0
- openmd/runtime/gpu_lock.py +81 -0
- openmd/runtime/gpu_quarantine.py +85 -0
- openmd/runtime/interpreters.py +407 -0
- openmd/runtime/managed_tools.py +204 -0
- openmd/runtime/models.py +202 -0
- openmd/runtime/services.py +175 -0
- openmd/runtime/tempdirs.py +227 -0
- openmd_cli-0.1.0.dist-info/METADATA +168 -0
- openmd_cli-0.1.0.dist-info/RECORD +177 -0
- openmd_cli-0.1.0.dist-info/WHEEL +5 -0
- openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Tools Openmd installs and runs for the user, so one `pip install openmd-cli` is enough.
|
|
2
|
+
|
|
3
|
+
opencli (Apache-2.0, npm `@jackwener/opencli`) powers the browser-session
|
|
4
|
+
routes (Bilibili subtitles, Weixin, X, Douyin ...). Node.js comes from the
|
|
5
|
+
`nodejs-wheel-binaries` dependency; opencli itself is installed once, on first
|
|
6
|
+
need, into ``<data>/tools/opencli`` at a pinned version.
|
|
7
|
+
|
|
8
|
+
A user's own `opencli` on PATH always wins and is never modified.
|
|
9
|
+
Set ``OPENMD_AUTO_INSTALL_TOOLS=0`` to disable the automatic install, and
|
|
10
|
+
``OPENMD_NPM_REGISTRY`` to use a registry mirror.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import importlib.util
|
|
16
|
+
import os
|
|
17
|
+
import shutil
|
|
18
|
+
import stat
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
import tempfile
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Callable
|
|
24
|
+
|
|
25
|
+
from openmd.core.preferences import data_directory
|
|
26
|
+
from openmd.runtime.interpreters import run_interruptibly
|
|
27
|
+
|
|
28
|
+
OPENCLI_PACKAGE = "@jackwener/opencli"
|
|
29
|
+
OPENCLI_VERSION = "2.0.3"
|
|
30
|
+
OPENCLI_BRIDGE_STORE_URL = "https://chromewebstore.google.com/detail/opencli/ildkmabpimmkaediidaifkhjpohdnifk"
|
|
31
|
+
_host_package_warning_shown = False
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def node_dir() -> Path | None:
|
|
35
|
+
"""Directory holding the bundled `node` executable, if the wheel is installed."""
|
|
36
|
+
spec = importlib.util.find_spec("nodejs_wheel")
|
|
37
|
+
if spec is None or not spec.origin:
|
|
38
|
+
return None
|
|
39
|
+
root = Path(spec.origin).parent
|
|
40
|
+
folder = root if os.name == "nt" else root / "bin"
|
|
41
|
+
return folder if (folder / ("node.exe" if os.name == "nt" else "node")).is_file() else None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _npm_cli(node: Path) -> Path | None:
|
|
45
|
+
root = node if os.name == "nt" else node.parent
|
|
46
|
+
candidate = root / "lib" / "node_modules" / "npm" / "bin" / "npm-cli.js"
|
|
47
|
+
return candidate if candidate.is_file() else None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def managed_root() -> Path:
|
|
51
|
+
return data_directory() / "tools" / "opencli"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def managed_bin() -> Path:
|
|
55
|
+
return managed_root() / "node_modules" / ".bin"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def managed_package_root() -> Path:
|
|
59
|
+
"""Package root used by plugins when the managed host is active."""
|
|
60
|
+
return managed_root() / "node_modules" / "@jackwener" / "opencli"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _managed_ready() -> bool:
|
|
64
|
+
return (managed_bin() / ("opencli.cmd" if os.name == "nt" else "opencli")).is_file()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _prepend_path(*folders: Path) -> None:
|
|
68
|
+
current = os.environ.get("PATH", "").split(os.pathsep)
|
|
69
|
+
additions = [str(f) for f in folders if str(f) not in current]
|
|
70
|
+
if additions:
|
|
71
|
+
os.environ["PATH"] = os.pathsep.join([*additions, *current])
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _is_junction(path: Path) -> bool:
|
|
75
|
+
# Path.is_symlink() does not identify junctions on Python 3.10/3.11.
|
|
76
|
+
try:
|
|
77
|
+
return getattr(path.lstat(), "st_reparse_tag", None) == getattr(stat, "IO_REPARSE_TAG_MOUNT_POINT", 0xA0000003)
|
|
78
|
+
except FileNotFoundError:
|
|
79
|
+
return False
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _create_junction(source: Path, target: Path) -> None:
|
|
83
|
+
if os.name != "nt":
|
|
84
|
+
raise OSError("Directory junctions require Windows")
|
|
85
|
+
import _winapi
|
|
86
|
+
|
|
87
|
+
_winapi.CreateJunction(str(source.resolve()), str(target))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _prepare_bundled_node_host_package() -> None:
|
|
91
|
+
"""Expose the managed host at the bundled Node resolution root.
|
|
92
|
+
|
|
93
|
+
Some user plugins were compiled by older OpenCLI versions that resolve
|
|
94
|
+
``@jackwener/opencli`` relative to ``process.execPath``. The managed
|
|
95
|
+
install keeps Node and npm under one private runtime directory, so provide
|
|
96
|
+
the same package shape there without touching ``~/.opencli`` or a user's
|
|
97
|
+
plugin directory.
|
|
98
|
+
"""
|
|
99
|
+
global _host_package_warning_shown
|
|
100
|
+
node = node_dir()
|
|
101
|
+
source = managed_package_root()
|
|
102
|
+
if node is None or not source.is_dir():
|
|
103
|
+
return
|
|
104
|
+
target = node / "node_modules" / "@jackwener" / "opencli"
|
|
105
|
+
try:
|
|
106
|
+
junction = _is_junction(target)
|
|
107
|
+
if junction or target.is_symlink():
|
|
108
|
+
try:
|
|
109
|
+
if target.resolve() == source.resolve():
|
|
110
|
+
return
|
|
111
|
+
except OSError:
|
|
112
|
+
pass
|
|
113
|
+
# rmdir removes only the junction, never the linked package tree.
|
|
114
|
+
if junction:
|
|
115
|
+
target.rmdir()
|
|
116
|
+
else:
|
|
117
|
+
target.unlink(missing_ok=True)
|
|
118
|
+
elif target.exists():
|
|
119
|
+
# Never replace a real package supplied by the Node runtime.
|
|
120
|
+
return
|
|
121
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
122
|
+
try:
|
|
123
|
+
target.symlink_to(source, target_is_directory=True)
|
|
124
|
+
except OSError:
|
|
125
|
+
_create_junction(source, target)
|
|
126
|
+
except OSError:
|
|
127
|
+
if not _host_package_warning_shown:
|
|
128
|
+
print("Openmd: browser plugin compatibility link could not be prepared; check runtime directory permissions and retry.", file=sys.stderr)
|
|
129
|
+
_host_package_warning_shown = True
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def activate() -> str:
|
|
133
|
+
"""Make a usable opencli visible to this process and its children.
|
|
134
|
+
|
|
135
|
+
Returns "user" (own install on PATH), "managed" (Openmd's copy) or "missing".
|
|
136
|
+
Browser reads default to a background (minimized) window.
|
|
137
|
+
"""
|
|
138
|
+
show = os.environ.get("OPENMD_SHOW_BROWSER", "").strip().lower() in {"1", "true", "yes", "on"}
|
|
139
|
+
os.environ.setdefault("OPENCLI_WINDOW", "foreground" if show else "background")
|
|
140
|
+
if _managed_ready():
|
|
141
|
+
if shutil.which("opencli") and Path(shutil.which("opencli")).parent != managed_bin():
|
|
142
|
+
return "user"
|
|
143
|
+
node = node_dir()
|
|
144
|
+
_prepend_path(*([node] if node else []), managed_bin())
|
|
145
|
+
_prepare_bundled_node_host_package()
|
|
146
|
+
return "managed"
|
|
147
|
+
return "user" if shutil.which("opencli") else "missing"
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def auto_install_enabled() -> bool:
|
|
151
|
+
return os.environ.get("OPENMD_AUTO_INSTALL_TOOLS", "1").strip().lower() not in {"0", "false", "no", "off"}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def ensure_opencli(say: Callable[[str], None] | None = None) -> str:
|
|
155
|
+
"""Install the pinned opencli once if no opencli is available.
|
|
156
|
+
|
|
157
|
+
Returns the activate() state after the attempt; never raises for a failed
|
|
158
|
+
install (the route then reports opencli as missing with a hint).
|
|
159
|
+
"""
|
|
160
|
+
state = activate()
|
|
161
|
+
if state != "missing" or not auto_install_enabled():
|
|
162
|
+
return state
|
|
163
|
+
node = node_dir()
|
|
164
|
+
npm = _npm_cli(node) if node else None
|
|
165
|
+
if node is None or npm is None:
|
|
166
|
+
return "missing"
|
|
167
|
+
say = say or (lambda message: print(message, file=sys.stderr))
|
|
168
|
+
say("首次使用这类链接,正在准备浏览器读取组件(约 20MB,只需一次)…")
|
|
169
|
+
root = managed_root()
|
|
170
|
+
root.parent.mkdir(parents=True, exist_ok=True)
|
|
171
|
+
staging = Path(tempfile.mkdtemp(prefix="opencli-", dir=root.parent))
|
|
172
|
+
command = [str(node / ("node.exe" if os.name == "nt" else "node")), str(npm), "install", "--prefix", str(staging),
|
|
173
|
+
f"{OPENCLI_PACKAGE}@{OPENCLI_VERSION}", "--no-audit", "--no-fund", "--loglevel=error", "--save-exact"]
|
|
174
|
+
registry = os.environ.get("OPENMD_NPM_REGISTRY")
|
|
175
|
+
if registry:
|
|
176
|
+
command += ["--registry", registry]
|
|
177
|
+
env = {**os.environ, "PATH": os.pathsep.join([str(node), os.environ.get("PATH", "")])}
|
|
178
|
+
try:
|
|
179
|
+
completed = run_interruptibly(command, env=env, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=900, check=False)
|
|
180
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
181
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
182
|
+
say(f"浏览器读取组件准备失败:{type(exc).__name__};检查网络,或设置 OPENMD_NPM_REGISTRY 使用 npm 镜像后重试。")
|
|
183
|
+
return "missing"
|
|
184
|
+
if completed.returncode != 0:
|
|
185
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
186
|
+
tail = (completed.stderr or completed.stdout or "").strip().splitlines()[-1:] or [""]
|
|
187
|
+
say(f"浏览器读取组件准备失败({tail[0][:120]});检查网络,或设置 OPENMD_NPM_REGISTRY 使用 npm 镜像后重试。")
|
|
188
|
+
return "missing"
|
|
189
|
+
shutil.rmtree(root, ignore_errors=True)
|
|
190
|
+
staging.rename(root)
|
|
191
|
+
say("浏览器读取组件已就绪。")
|
|
192
|
+
return activate()
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def status() -> dict[str, object]:
|
|
196
|
+
"""Doctor view: where opencli comes from, without probing the network."""
|
|
197
|
+
state = activate()
|
|
198
|
+
return {
|
|
199
|
+
"opencli": state,
|
|
200
|
+
"opencli_version_pinned": OPENCLI_VERSION,
|
|
201
|
+
"managed_path": str(managed_root()),
|
|
202
|
+
"bundled_node": node_dir() is not None,
|
|
203
|
+
"auto_install": auto_install_enabled(),
|
|
204
|
+
}
|
openmd/runtime/models.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Resolve ASR models from existing local files without calling a model hub."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Iterator
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class ModelSpec:
|
|
15
|
+
logical_id: str
|
|
16
|
+
repo: str
|
|
17
|
+
family: str
|
|
18
|
+
required: tuple[str, ...]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class ResolvedModel:
|
|
23
|
+
logical_id: str
|
|
24
|
+
path: Path | None
|
|
25
|
+
origin: str
|
|
26
|
+
missing: tuple[str, ...] = ()
|
|
27
|
+
searched: tuple[str, ...] = ()
|
|
28
|
+
|
|
29
|
+
def as_dict(self) -> dict[str, object]:
|
|
30
|
+
return {
|
|
31
|
+
"logical_id": self.logical_id,
|
|
32
|
+
"available": self.path is not None,
|
|
33
|
+
"origin": self.origin,
|
|
34
|
+
"missing": list(self.missing),
|
|
35
|
+
"searched": list(self.searched),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
MODEL_SPECS = (
|
|
40
|
+
ModelSpec("whisper-large-v3", "Systran/faster-whisper-large-v3", "whisper", ("model.bin", "config.json", "tokenizer.json")),
|
|
41
|
+
ModelSpec("whisper-large-v3-turbo", "mobiuslabsgmbh/faster-whisper-large-v3-turbo", "whisper", ("model.bin", "config.json", "tokenizer.json")),
|
|
42
|
+
ModelSpec("qwen3-asr-1.7b", "Qwen/Qwen3-ASR-1.7B", "qwen3", ("config.json", "tokenizer_config.json", "preprocessor_config.json")),
|
|
43
|
+
ModelSpec("fsmn-vad-zh", "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch", "vad", ("model.pt", "config.yaml", "am.mvn")),
|
|
44
|
+
)
|
|
45
|
+
_SPECS = {spec.logical_id: spec for spec in MODEL_SPECS}
|
|
46
|
+
_ALIASES = {spec.repo: spec.logical_id for spec in MODEL_SPECS}
|
|
47
|
+
_ALIASES.update({"large-v3": "whisper-large-v3", "large": "whisper-large-v3", "large-v3-turbo": "whisper-large-v3-turbo", "turbo": "whisper-large-v3-turbo"})
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ModelNotFoundError(FileNotFoundError):
|
|
51
|
+
def __init__(self, resolution: ResolvedModel) -> None:
|
|
52
|
+
self.resolution = resolution
|
|
53
|
+
super().__init__(
|
|
54
|
+
f"Local ASR model {resolution.logical_id!r} is missing or incomplete "
|
|
55
|
+
f"({', '.join(resolution.missing)}). Searched: {', '.join(resolution.searched) or 'explicit path'}. "
|
|
56
|
+
"Set OPENMD_MODEL_ROOTS or the model's OPENMD_*_MODEL variable to an installed local directory; no download was attempted."
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def checkout_root() -> Path | None:
|
|
61
|
+
root = Path(__file__).resolve().parents[3]
|
|
62
|
+
return root if (root / "pyproject.toml").is_file() and (root / "src" / "openmd").is_dir() else None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _spec_for(value: str, family: str | None) -> ModelSpec:
|
|
66
|
+
logical = _ALIASES.get(value, value)
|
|
67
|
+
if logical in _SPECS:
|
|
68
|
+
return _SPECS[logical]
|
|
69
|
+
explicit = Path(value).expanduser()
|
|
70
|
+
if explicit.is_dir() or explicit.is_absolute() or value.startswith((".", "~")) or "\\" in value:
|
|
71
|
+
known_family = family or "whisper"
|
|
72
|
+
template = _SPECS[{"whisper": "whisper-large-v3", "qwen3": "qwen3-asr-1.7b", "vad": "fsmn-vad-zh"}[known_family]]
|
|
73
|
+
return ModelSpec("local-" + explicit.name, value, known_family, template.required)
|
|
74
|
+
if family == "whisper":
|
|
75
|
+
# Preserve explicit legacy faster-whisper sizes while staying offline.
|
|
76
|
+
repo = value if "/" in value else f"Systran/faster-whisper-{value}"
|
|
77
|
+
return ModelSpec(value, repo, "whisper", ("model.bin", "config.json", "tokenizer.json"))
|
|
78
|
+
fallback = _SPECS["qwen3-asr-1.7b" if family == "qwen3" else "fsmn-vad-zh"]
|
|
79
|
+
return ModelSpec(value, value, family or fallback.family, fallback.required)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def model_label(value: str) -> str:
|
|
83
|
+
"""Keep logical model names useful while omitting a machine's full path."""
|
|
84
|
+
path = Path(value).expanduser()
|
|
85
|
+
return "local-" + path.name if path.is_dir() or path.is_absolute() or value.startswith((".", "~")) or "\\" in value else value
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _missing(path: Path, spec: ModelSpec) -> tuple[str, ...]:
|
|
89
|
+
if not path.is_dir():
|
|
90
|
+
return ("directory",)
|
|
91
|
+
missing = [name for name in spec.required if not (path / name).is_file() or (path / name).stat().st_size == 0]
|
|
92
|
+
if spec.family == "qwen3":
|
|
93
|
+
weights = list(path.glob("*.safetensors"))
|
|
94
|
+
if not weights or any(weight.stat().st_size == 0 for weight in weights):
|
|
95
|
+
missing.append("*.safetensors")
|
|
96
|
+
index = path / "model.safetensors.index.json"
|
|
97
|
+
if index.is_file():
|
|
98
|
+
try:
|
|
99
|
+
weight_map = json.loads(index.read_text(encoding="utf-8"))["weight_map"]
|
|
100
|
+
if not isinstance(weight_map, dict) or not weight_map:
|
|
101
|
+
raise ValueError("empty weight map")
|
|
102
|
+
for name in sorted(set(weight_map.values())):
|
|
103
|
+
candidate = (path / name).resolve()
|
|
104
|
+
if not candidate.is_relative_to(path.resolve()) or not candidate.is_file() or candidate.stat().st_size == 0:
|
|
105
|
+
missing.append("weight shard")
|
|
106
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
107
|
+
missing.append("valid model.safetensors.index.json")
|
|
108
|
+
if not (path / "tokenizer.json").is_file() and not all((path / name).is_file() for name in ("vocab.json", "merges.txt")):
|
|
109
|
+
missing.append("tokenizer.json or vocab.json + merges.txt")
|
|
110
|
+
return tuple(dict.fromkeys(missing))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _snapshot_paths(root: Path) -> Iterator[Path]:
|
|
114
|
+
if not root.is_dir():
|
|
115
|
+
return
|
|
116
|
+
yield root
|
|
117
|
+
refs = root / "refs" / "main"
|
|
118
|
+
if refs.is_file():
|
|
119
|
+
revision = refs.read_text(encoding="utf-8").strip()
|
|
120
|
+
snapshot = (root / "snapshots" / revision).resolve()
|
|
121
|
+
if snapshot.is_relative_to(root.resolve()):
|
|
122
|
+
yield snapshot
|
|
123
|
+
yield from sorted((root / "snapshots").glob("*"), reverse=True)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _local_config() -> dict:
|
|
127
|
+
from openmd.core.local_config import load_local_config
|
|
128
|
+
|
|
129
|
+
return load_local_config()
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def resolve_model(value: str, *, family: str | None = None) -> ResolvedModel:
|
|
133
|
+
"""Return a validated directory or an explicit missing result; never download."""
|
|
134
|
+
spec = _spec_for(value, family)
|
|
135
|
+
explicit = Path(value).expanduser()
|
|
136
|
+
if explicit.is_dir() or explicit.is_absolute() or value.startswith((".", "~")) or "\\" in value:
|
|
137
|
+
missing = _missing(explicit, spec)
|
|
138
|
+
return ResolvedModel(spec.logical_id, explicit.resolve() if not missing else None, "explicit_path", missing, ("explicit_path",))
|
|
139
|
+
|
|
140
|
+
candidates: list[tuple[Path, str]] = []
|
|
141
|
+
repo_parts = spec.repo.split("/")
|
|
142
|
+
compact = spec.repo.replace("/", "--")
|
|
143
|
+
configured = _local_config().get("model_roots", [])
|
|
144
|
+
roots = [*(Path(part).expanduser() for part in os.getenv("OPENMD_MODEL_ROOTS", "").split(os.pathsep) if part), *(Path(part).expanduser() for part in configured)]
|
|
145
|
+
for root in roots:
|
|
146
|
+
candidates.extend((path, "configured_root") for path in (root / spec.logical_id, root / compact, root / "models" / compact, root / ("models--" + compact), root.joinpath(*repo_parts), root / "models" / Path(*repo_parts)))
|
|
147
|
+
checkout = checkout_root()
|
|
148
|
+
if checkout:
|
|
149
|
+
candidates.append((checkout / "out" / "asr-bench" / "models" / "models" / compact, "checkout_model_cache"))
|
|
150
|
+
hf_roots = [Path(os.environ["HF_HUB_CACHE"]).expanduser()] if os.getenv("HF_HUB_CACHE") else []
|
|
151
|
+
if os.getenv("HF_HOME"):
|
|
152
|
+
hf_roots.append(Path(os.environ["HF_HOME"]).expanduser() / "hub")
|
|
153
|
+
hf_roots.append(Path.home() / ".cache" / "huggingface" / "hub")
|
|
154
|
+
candidates.extend((root / ("models--" + compact), "huggingface_cache") for root in hf_roots)
|
|
155
|
+
ms_roots = [Path(os.environ["MODELSCOPE_CACHE"]).expanduser()] if os.getenv("MODELSCOPE_CACHE") else []
|
|
156
|
+
ms_roots.append(Path.home() / ".cache" / "modelscope" / "hub")
|
|
157
|
+
for root in ms_roots:
|
|
158
|
+
candidates.extend((path, "modelscope_cache") for path in (root.joinpath(*repo_parts), root / "models" / Path(*repo_parts), root / "models" / compact))
|
|
159
|
+
|
|
160
|
+
searched: list[str] = []
|
|
161
|
+
incomplete: tuple[str, ...] = ()
|
|
162
|
+
for root, origin in candidates:
|
|
163
|
+
if origin not in searched:
|
|
164
|
+
searched.append(origin)
|
|
165
|
+
for candidate in _snapshot_paths(root):
|
|
166
|
+
missing = _missing(candidate, spec)
|
|
167
|
+
if not missing:
|
|
168
|
+
return ResolvedModel(spec.logical_id, candidate.resolve(), origin, (), tuple(searched))
|
|
169
|
+
if candidate.is_dir() and missing != ("directory",):
|
|
170
|
+
incomplete = missing
|
|
171
|
+
return ResolvedModel(spec.logical_id, None, "missing", incomplete or spec.required, tuple(searched))
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def require_model(value: str, *, family: str) -> Path:
|
|
175
|
+
resolved = resolve_model(value, family=family)
|
|
176
|
+
if resolved.path is None:
|
|
177
|
+
raise ModelNotFoundError(resolved)
|
|
178
|
+
return resolved.path
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def configured_models() -> dict[str, ResolvedModel]:
|
|
182
|
+
return {
|
|
183
|
+
"whisper-large-v3": resolve_model(os.getenv("OPENMD_FASTER_WHISPER_MODEL") or "large-v3", family="whisper"),
|
|
184
|
+
"whisper-large-v3-turbo": resolve_model(os.getenv("OPENMD_ASR_AUTO_EN_MODEL") or "large-v3-turbo", family="whisper"),
|
|
185
|
+
"qwen3-asr-1.7b": resolve_model(os.getenv("OPENMD_QWEN3_MODEL") or "Qwen/Qwen3-ASR-1.7B", family="qwen3"),
|
|
186
|
+
"fsmn-vad-zh": resolve_model(os.getenv("OPENMD_QWEN3_VAD_MODEL") or "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch", family="vad"),
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@contextmanager
|
|
191
|
+
def offline_model_loading() -> Iterator[None]:
|
|
192
|
+
keys = ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "MODELSCOPE_OFFLINE")
|
|
193
|
+
previous = {key: os.environ.get(key) for key in keys}
|
|
194
|
+
os.environ.update({key: "1" for key in keys})
|
|
195
|
+
try:
|
|
196
|
+
yield
|
|
197
|
+
finally:
|
|
198
|
+
for key, value in previous.items():
|
|
199
|
+
if value is None:
|
|
200
|
+
os.environ.pop(key, None)
|
|
201
|
+
else:
|
|
202
|
+
os.environ[key] = value
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""The sole persistent-process exception: the installed OpenCLI browser daemon.
|
|
2
|
+
|
|
3
|
+
Actual commands, status checks and all other workers retain launch-time ownership.
|
|
4
|
+
No Ollama lifecycle operations, model downloads or general detached launcher exist.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
import shlex
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
from openmd.core.opencli_output import load_opencli_json
|
|
18
|
+
|
|
19
|
+
_START_LOCK = threading.Lock()
|
|
20
|
+
_DAEMONS: list[subprocess.Popen] = []
|
|
21
|
+
_CONTROL_COMMANDS = {"daemon", "--help", "-h", "--version", "-V", "help", "list", "completion"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _opencli_launch(command, env=None) -> tuple[list[str], list[str]] | None:
|
|
25
|
+
"""Recognize npm shims and native Node entries, without executing a shell."""
|
|
26
|
+
if isinstance(command, str):
|
|
27
|
+
try:
|
|
28
|
+
argv = [part.strip('"') for part in shlex.split(command, posix=False)]
|
|
29
|
+
except ValueError:
|
|
30
|
+
return None
|
|
31
|
+
elif isinstance(command, (list, tuple)):
|
|
32
|
+
argv = [os.fsdecode(part) for part in command]
|
|
33
|
+
else:
|
|
34
|
+
return None
|
|
35
|
+
if not argv:
|
|
36
|
+
return None
|
|
37
|
+
first = Path(argv[0])
|
|
38
|
+
if first.name.lower() in {"opencli", "opencli.cmd", "opencli.ps1", "opencli.exe", "opencli.bat"}:
|
|
39
|
+
resolved = shutil.which(argv[0], path=None if env is None else env.get("PATH"))
|
|
40
|
+
shim = Path(resolved or argv[0]).resolve()
|
|
41
|
+
# Unix npm installs symlink the command directly to the package entry.
|
|
42
|
+
if shim.as_posix().lower().endswith("/@jackwener/opencli/dist/src/main.js"):
|
|
43
|
+
entry = shim
|
|
44
|
+
elif shim.parent.name == ".bin": # local install, e.g. Openmd's managed copy
|
|
45
|
+
entry = shim.parent.parent / "@jackwener/opencli/dist/src/main.js"
|
|
46
|
+
else:
|
|
47
|
+
entry = shim.parent / "node_modules/@jackwener/opencli/dist/src/main.js"
|
|
48
|
+
node = shim.parent / ("node.exe" if os.name == "nt" else "node")
|
|
49
|
+
binary = str(node) if node.is_file() else shutil.which("node", path=None if env is None else env.get("PATH"))
|
|
50
|
+
tail = argv[1:]
|
|
51
|
+
elif first.name.lower() in {"node", "node.exe"} and len(argv) > 1:
|
|
52
|
+
entry = Path(argv[1]).resolve()
|
|
53
|
+
if not entry.as_posix().lower().endswith("/@jackwener/opencli/dist/src/main.js"):
|
|
54
|
+
return None
|
|
55
|
+
binary, tail = argv[0], argv[2:]
|
|
56
|
+
else:
|
|
57
|
+
return None
|
|
58
|
+
# Status/stop/restart must never start the daemon as a side effect. The
|
|
59
|
+
# recursive owned status command below consequently bypasses this helper.
|
|
60
|
+
while tail and tail[0].startswith("--profile="):
|
|
61
|
+
tail = tail[1:]
|
|
62
|
+
if tail and tail[0] == "--profile":
|
|
63
|
+
tail = tail[2:]
|
|
64
|
+
if not tail or tail[0] in _CONTROL_COMMANDS:
|
|
65
|
+
return None
|
|
66
|
+
daemon = entry.with_name("daemon.js")
|
|
67
|
+
if not binary or not entry.is_file() or not daemon.is_file():
|
|
68
|
+
return None
|
|
69
|
+
return [str(binary), str(entry)], [str(binary), str(daemon)]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _daemon_running(launcher, *, env=None, cwd=None) -> bool:
|
|
73
|
+
from .interpreters import run_interruptibly
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
result = run_interruptibly([*launcher, "daemon", "status"], env=env, cwd=cwd,
|
|
77
|
+
stdin=subprocess.DEVNULL, capture_output=True,
|
|
78
|
+
text=True, encoding="utf-8", errors="replace", timeout=10,
|
|
79
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
80
|
+
payload = load_opencli_json(result.stdout)
|
|
81
|
+
if result.returncode or not isinstance(payload, dict) or not isinstance(payload.get("running"), bool):
|
|
82
|
+
raise ValueError("invalid status")
|
|
83
|
+
return payload["running"]
|
|
84
|
+
except (ValueError, TypeError, subprocess.TimeoutExpired) as exc:
|
|
85
|
+
# Do not blindly launch a second service when the status is unknown.
|
|
86
|
+
raise OSError("OpenCLI daemon status unavailable; run opencli daemon status.") from exc
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _start_opencli_daemon_detached(daemon_argv, *, env=None):
|
|
90
|
+
"""Only called with the two-item installed Node/daemon.js launch specification.
|
|
91
|
+
|
|
92
|
+
This is deliberately outside owned_popen: a command's kill-on-close Job must
|
|
93
|
+
never acquire this intentional persistent service. No Job flag is weakened.
|
|
94
|
+
"""
|
|
95
|
+
binary, script = daemon_argv
|
|
96
|
+
if Path(binary).name.lower() not in {"node", "node.exe"} or not Path(script).as_posix().lower().endswith("/@jackwener/opencli/dist/src/daemon.js"):
|
|
97
|
+
raise ValueError("Only the installed OpenCLI daemon may be detached")
|
|
98
|
+
options = {"creationflags": subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP} if os.name == "nt" else {"start_new_session": True}
|
|
99
|
+
process = subprocess.Popen([binary, script], env=env, cwd=str(Path(script).parent),
|
|
100
|
+
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
|
101
|
+
stderr=subprocess.DEVNULL, close_fds=True, **options)
|
|
102
|
+
_DAEMONS[:] = [child for child in _DAEMONS if child.poll() is None]
|
|
103
|
+
_DAEMONS.append(process)
|
|
104
|
+
return process
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
OPENMD_BRIDGE_PROFILE = "openmd"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _bridge_profiles(timeout: float = 1.0) -> list[str]:
|
|
111
|
+
"""Connected Browser Bridge profile ids from the local daemon (never proxied)."""
|
|
112
|
+
import json
|
|
113
|
+
import urllib.request
|
|
114
|
+
|
|
115
|
+
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
116
|
+
request = urllib.request.Request("http://127.0.0.1:19825/status", headers={"X-OpenCLI": "1"})
|
|
117
|
+
try:
|
|
118
|
+
with opener.open(request, timeout=timeout) as response: # noqa: S310 - loopback only
|
|
119
|
+
payload = json.loads(response.read().decode("utf-8"))
|
|
120
|
+
except (OSError, ValueError):
|
|
121
|
+
return []
|
|
122
|
+
return [str(p.get("contextId")) for p in payload.get("profiles") or [] if isinstance(p, dict) and p.get("contextId")]
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def bridge_profile_env(command, env=None):
|
|
126
|
+
"""Select Openmd's built-in bridge when it is connected.
|
|
127
|
+
|
|
128
|
+
With both the standalone OpenCLI extension and Openmd's built-in bridge
|
|
129
|
+
connected, opencli refuses to guess (profile_required). Prefer ours, keep a
|
|
130
|
+
user's explicit OPENCLI_PROFILE, and leave the single-bridge case untouched.
|
|
131
|
+
Returns a new env mapping, or None when nothing needs to change.
|
|
132
|
+
"""
|
|
133
|
+
base = os.environ if env is None else env
|
|
134
|
+
if base.get("OPENCLI_PROFILE") or _opencli_launch(command, env) is None:
|
|
135
|
+
return None
|
|
136
|
+
deadline = time.monotonic() + 2
|
|
137
|
+
profiles = _bridge_profiles()
|
|
138
|
+
while not profiles and time.monotonic() < deadline: # bridges reconnect to a fresh daemon
|
|
139
|
+
time.sleep(.2)
|
|
140
|
+
profiles = _bridge_profiles()
|
|
141
|
+
if OPENMD_BRIDGE_PROFILE in profiles and len(profiles) > 1:
|
|
142
|
+
return {**base, "OPENCLI_PROFILE": OPENMD_BRIDGE_PROFILE}
|
|
143
|
+
return None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def ensure_opencli_daemon(command, *, env=None, cwd=None) -> None:
|
|
147
|
+
launch = _opencli_launch(command, env)
|
|
148
|
+
if launch is None:
|
|
149
|
+
return
|
|
150
|
+
launcher, daemon_argv = launch
|
|
151
|
+
# Lock waits stay in short Python slices, like the owned process waiter.
|
|
152
|
+
while not _START_LOCK.acquire(timeout=.1):
|
|
153
|
+
pass
|
|
154
|
+
# Pre-start is only an optimization (keep the daemon alive across owned
|
|
155
|
+
# commands). Unknown status or a slow/failed start must not block the actual
|
|
156
|
+
# command: it then runs exactly as before and reports its own errors.
|
|
157
|
+
try:
|
|
158
|
+
try:
|
|
159
|
+
if _daemon_running(launcher, env=env, cwd=cwd):
|
|
160
|
+
return
|
|
161
|
+
except OSError:
|
|
162
|
+
return # status unknown: never launch a second service blindly
|
|
163
|
+
process = _start_opencli_daemon_detached(daemon_argv, env=env)
|
|
164
|
+
deadline = time.monotonic() + 5
|
|
165
|
+
while time.monotonic() < deadline:
|
|
166
|
+
try:
|
|
167
|
+
if _daemon_running(launcher, env=env, cwd=cwd):
|
|
168
|
+
return
|
|
169
|
+
except OSError:
|
|
170
|
+
pass
|
|
171
|
+
if process.poll() is not None:
|
|
172
|
+
return
|
|
173
|
+
time.sleep(.1)
|
|
174
|
+
finally:
|
|
175
|
+
_START_LOCK.release()
|