openmd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openmd/__init__.py +3 -0
- openmd/__main__.py +5 -0
- openmd/adapters/__init__.py +1 -0
- openmd/adapters/base.py +95 -0
- openmd/adapters/bilibili/__init__.py +11 -0
- openmd/adapters/bilibili/adapter.py +104 -0
- openmd/adapters/bilibili/discovery.py +33 -0
- openmd/adapters/douyin/__init__.py +11 -0
- openmd/adapters/douyin/adapter.py +117 -0
- openmd/adapters/douyin/discovery.py +36 -0
- openmd/adapters/douyin/sidecar.py +77 -0
- openmd/adapters/generic_media/__init__.py +14 -0
- openmd/adapters/high_privacy.py +77 -0
- openmd/adapters/local_audio/__init__.py +14 -0
- openmd/adapters/local_audio/adapter.py +169 -0
- openmd/adapters/local_audio/asr.py +58 -0
- openmd/adapters/local_audio/backends/__init__.py +1 -0
- openmd/adapters/local_audio/backends/auto.py +177 -0
- openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
- openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
- openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
- openmd/adapters/opencli.py +770 -0
- openmd/adapters/pdf/__init__.py +11 -0
- openmd/adapters/pdf/adapter.py +153 -0
- openmd/adapters/pdf/backends/__init__.py +1 -0
- openmd/adapters/pdf/backends/markitdown.py +165 -0
- openmd/adapters/pdf/parser.py +56 -0
- openmd/adapters/qq_group/__init__.py +11 -0
- openmd/adapters/qq_group/adapter.py +289 -0
- openmd/adapters/qq_group/parsers/__init__.py +1 -0
- openmd/adapters/qq_group/parsers/html.py +60 -0
- openmd/adapters/qq_group/parsers/json.py +71 -0
- openmd/adapters/registry.py +149 -0
- openmd/adapters/screenshot/__init__.py +11 -0
- openmd/adapters/screenshot/adapter.py +154 -0
- openmd/adapters/screenshot/backends/__init__.py +1 -0
- openmd/adapters/screenshot/backends/paddle.py +15 -0
- openmd/adapters/screenshot/backends/rapidocr.py +176 -0
- openmd/adapters/screenshot/backends/tesseract.py +15 -0
- openmd/adapters/screenshot/ocr.py +84 -0
- openmd/adapters/wechat_group/__init__.py +11 -0
- openmd/adapters/wechat_group/adapter.py +160 -0
- openmd/adapters/wechat_group/wx_cli.py +268 -0
- openmd/adapters/wechat_moments/__init__.py +11 -0
- openmd/adapters/wechat_moments/adapter.py +167 -0
- openmd/adapters/wechat_moments/wx_cli.py +265 -0
- openmd/adapters/weixin/__init__.py +11 -0
- openmd/adapters/weixin/adapter.py +148 -0
- openmd/adapters/weixin/discovery.py +510 -0
- openmd/adapters/weixin/report.py +278 -0
- openmd/adapters/xiaohongshu.py +158 -0
- openmd/backends/__init__.py +1 -0
- openmd/backends/base.py +204 -0
- openmd/backends/bilibili.py +1480 -0
- openmd/backends/bilibili_wbi.py +277 -0
- openmd/backends/discourse.py +426 -0
- openmd/backends/douyin.py +868 -0
- openmd/backends/douyin_detail.py +124 -0
- openmd/backends/douyin_providers.py +602 -0
- openmd/backends/executor.py +546 -0
- openmd/backends/feishu.py +861 -0
- openmd/backends/generic_media.py +77 -0
- openmd/backends/interactive.py +60 -0
- openmd/backends/ocr.py +41 -0
- openmd/backends/opencli.py +82 -0
- openmd/backends/pdf.py +122 -0
- openmd/backends/registry.py +234 -0
- openmd/backends/screenshot.py +79 -0
- openmd/backends/video_metadata.py +167 -0
- openmd/backends/web_article.py +363 -0
- openmd/backends/wechat_group.py +35 -0
- openmd/backends/wechat_moments.py +35 -0
- openmd/backends/weixin.py +767 -0
- openmd/backends/x.py +455 -0
- openmd/backends/xiaoe.py +940 -0
- openmd/backends/xiaohongshu.py +911 -0
- openmd/backends/xiaohongshu_providers.py +876 -0
- openmd/backends/xiaoyuzhou.py +1017 -0
- openmd/backends/youtube.py +280 -0
- openmd/backends/youtube_discovery.py +1245 -0
- openmd/backends/youtube_transcript.py +213 -0
- openmd/browser_extension/background.js +231 -0
- openmd/browser_extension/icons/icon-128.png +0 -0
- openmd/browser_extension/icons/icon-16.png +0 -0
- openmd/browser_extension/icons/icon-32.png +0 -0
- openmd/browser_extension/icons/icon-48.png +0 -0
- openmd/browser_extension/icons/icon-512.png +0 -0
- openmd/browser_extension/idle.html +6 -0
- openmd/browser_extension/manifest.json +43 -0
- openmd/browser_extension/options.html +42 -0
- openmd/browser_extension/options.js +117 -0
- openmd/browser_extension/popup.css +71 -0
- openmd/browser_extension/popup.html +17 -0
- openmd/browser_extension/popup.js +59 -0
- openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
- openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
- openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
- openmd/browser_host.py +503 -0
- openmd/browser_setup.py +106 -0
- openmd/cli.py +2012 -0
- openmd/console.py +32 -0
- openmd/core/__init__.py +1 -0
- openmd/core/capabilities.py +162 -0
- openmd/core/content_validation.py +323 -0
- openmd/core/coverage.py +163 -0
- openmd/core/detector.py +247 -0
- openmd/core/doctor.py +104 -0
- openmd/core/execution_policy.py +164 -0
- openmd/core/filename.py +64 -0
- openmd/core/input_validation.py +91 -0
- openmd/core/local_config.py +96 -0
- openmd/core/manifest.py +156 -0
- openmd/core/models.py +168 -0
- openmd/core/opencli_output.py +34 -0
- openmd/core/operation_contract.py +60 -0
- openmd/core/output.py +194 -0
- openmd/core/package_contract.py +335 -0
- openmd/core/package_validator.py +489 -0
- openmd/core/package_writer.py +666 -0
- openmd/core/planner.py +345 -0
- openmd/core/preferences.py +61 -0
- openmd/core/readable.py +129 -0
- openmd/core/receipt.py +285 -0
- openmd/core/renderer.py +185 -0
- openmd/core/runlog.py +208 -0
- openmd/core/secrets.py +313 -0
- openmd/doctor_view.py +68 -0
- openmd/interactive.py +453 -0
- openmd/ocr/__init__.py +1 -0
- openmd/ocr/breakers.py +84 -0
- openmd/ocr/config.py +54 -0
- openmd/ocr/documents.py +522 -0
- openmd/ocr/endpoint.py +72 -0
- openmd/ocr/layout.py +92 -0
- openmd/ocr/layout_filter.py +118 -0
- openmd/ocr/layout_worker.py +89 -0
- openmd/ocr/native_math.py +220 -0
- openmd/ocr/office.py +305 -0
- openmd/ocr/ollama.py +200 -0
- openmd/ocr/otsl.py +51 -0
- openmd/ocr/ovis-prompt.txt +1 -0
- openmd/ocr/pipeline.py +384 -0
- openmd/ocr/settings.json +25 -0
- openmd/ocr/tiling.py +631 -0
- openmd/post_video.py +154 -0
- openmd/probe.py +397 -0
- openmd/routing/__init__.py +1 -0
- openmd/routing/cards/builtin.json +620 -0
- openmd/routing/cards/defuddle.json +70 -0
- openmd/routing/cards/x-tweet-fetcher.json +71 -0
- openmd/routing/cards/xyz-dl.json +60 -0
- openmd/routing/cards/yutto.json +68 -0
- openmd/routing/entry_shim.py +71 -0
- openmd/routing/lifecycle.py +459 -0
- openmd/routing/loader.py +138 -0
- openmd/routing/providers.py +158 -0
- openmd/routing/readers.py +108 -0
- openmd/routing/routes.json +504 -0
- openmd/routing/runner.py +192 -0
- openmd/routing/view.py +40 -0
- openmd/runtime/__init__.py +1 -0
- openmd/runtime/_windows_job.py +234 -0
- openmd/runtime/asr.py +217 -0
- openmd/runtime/asr_worker.py +119 -0
- openmd/runtime/gpu_lock.py +81 -0
- openmd/runtime/gpu_quarantine.py +85 -0
- openmd/runtime/interpreters.py +407 -0
- openmd/runtime/managed_tools.py +204 -0
- openmd/runtime/models.py +202 -0
- openmd/runtime/services.py +175 -0
- openmd/runtime/tempdirs.py +227 -0
- openmd_cli-0.1.0.dist-info/METADATA +168 -0
- openmd_cli-0.1.0.dist-info/RECORD +177 -0
- openmd_cli-0.1.0.dist-info/WHEEL +5 -0
- openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
openmd/browser_host.py
ADDED
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
"""Chrome Native Messaging host for the Openmd browser extension.
|
|
2
|
+
|
|
3
|
+
Chrome starts this process per message (``chrome.runtime.sendNativeMessage``) and talks over
|
|
4
|
+
stdin/stdout with 4-byte little-endian length-prefixed JSON. Only the extension id listed in the
|
|
5
|
+
host manifest may connect (see ``openmd browser-setup``). There is no network listener.
|
|
6
|
+
|
|
7
|
+
Saving runs as a detached background job so long transcriptions survive the popup and the
|
|
8
|
+
host process. Each job lives in ``<data>/browser-jobs/<id>/status.json``.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import struct
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
18
|
+
import time
|
|
19
|
+
import uuid
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
from urllib.parse import urlparse
|
|
23
|
+
|
|
24
|
+
from openmd.core.preferences import data_directory, load_preferences
|
|
25
|
+
from openmd.core.secrets import sanitize_error_summary
|
|
26
|
+
from openmd.runtime.interpreters import owned_popen, communicate_interruptibly, kill_process_tree, process_options, terminate_quietly
|
|
27
|
+
from openmd.runtime.services import ensure_opencli_daemon
|
|
28
|
+
|
|
29
|
+
HOST_NAME = "com.openmd.host"
|
|
30
|
+
MAX_MESSAGE = 1024 * 1024
|
|
31
|
+
RECENT_JOBS = 8
|
|
32
|
+
GO_FLAGS = ["--yes", "--enable-network-backends", "--enable-browser-backends", "--enable-media-download", "--enable-asr", "--json"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# ---------------------------------------------------------------- protocol
|
|
36
|
+
|
|
37
|
+
def read_message(stream) -> dict[str, Any] | None:
|
|
38
|
+
header = stream.read(4)
|
|
39
|
+
if len(header) < 4:
|
|
40
|
+
return None
|
|
41
|
+
(length,) = struct.unpack("<I", header)
|
|
42
|
+
if length > MAX_MESSAGE:
|
|
43
|
+
raise ValueError("message_too_large")
|
|
44
|
+
payload = stream.read(length)
|
|
45
|
+
value = json.loads(payload.decode("utf-8"))
|
|
46
|
+
if not isinstance(value, dict):
|
|
47
|
+
raise ValueError("message_not_object")
|
|
48
|
+
return value
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def write_message(stream, value: dict[str, Any]) -> None:
|
|
52
|
+
data = json.dumps(value, ensure_ascii=False).encode("utf-8")
|
|
53
|
+
stream.write(struct.pack("<I", len(data)) + data)
|
|
54
|
+
stream.flush()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# ---------------------------------------------------------------- jobs
|
|
58
|
+
|
|
59
|
+
def jobs_root() -> Path:
|
|
60
|
+
return data_directory() / "browser-jobs"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def default_save_directory() -> Path:
|
|
64
|
+
last = load_preferences().get("last_save_directory")
|
|
65
|
+
return Path(last) if last else Path.home() / "Openmd"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _status_path(job_id: str) -> Path:
|
|
69
|
+
if not job_id or any(ch not in "0123456789abcdef" for ch in job_id):
|
|
70
|
+
raise ValueError("invalid_job_id")
|
|
71
|
+
return jobs_root() / job_id / "status.json"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def read_status(job_id: str) -> dict[str, Any]:
|
|
75
|
+
try:
|
|
76
|
+
return json.loads(_status_path(job_id).read_text(encoding="utf-8"))
|
|
77
|
+
except FileNotFoundError:
|
|
78
|
+
return {"id": job_id, "state": "unknown"}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _write_status(job_dir: Path, status: dict[str, Any]) -> None:
|
|
82
|
+
# The queue runner and status pollers (_recover) may write the same job at
|
|
83
|
+
# once: each writer needs its own temp file, or their bytes can interleave.
|
|
84
|
+
tmp = job_dir / f"status.json.{os.getpid()}.{uuid.uuid4().hex}.tmp"
|
|
85
|
+
try:
|
|
86
|
+
tmp.write_text(json.dumps(status, ensure_ascii=False), encoding="utf-8")
|
|
87
|
+
os.replace(tmp, job_dir / "status.json")
|
|
88
|
+
finally:
|
|
89
|
+
tmp.unlink(missing_ok=True)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _validate_url(url: object) -> str:
|
|
93
|
+
if not isinstance(url, str) or len(url) > 4096:
|
|
94
|
+
raise ValueError("invalid_url")
|
|
95
|
+
parsed = urlparse(url.strip())
|
|
96
|
+
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
97
|
+
raise ValueError("unsupported_url: 只能保存 http/https 网页")
|
|
98
|
+
return url.strip()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _validate_destination(value: object) -> Path:
|
|
102
|
+
if value in (None, ""):
|
|
103
|
+
return default_save_directory()
|
|
104
|
+
if not isinstance(value, str):
|
|
105
|
+
raise ValueError("invalid_save_to")
|
|
106
|
+
path = Path(os.path.expandvars(value.strip().strip('"'))).expanduser()
|
|
107
|
+
if not path.is_absolute():
|
|
108
|
+
raise ValueError("save_to_must_be_absolute: 请填写完整的文件夹路径")
|
|
109
|
+
return path
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
MAX_BATCH = 200
|
|
113
|
+
MAX_ATTEMPTS = 2 # one automatic retry after an interruption (shutdown, crash)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _pid_alive(pid: object) -> bool:
|
|
117
|
+
if not isinstance(pid, int) or pid <= 0:
|
|
118
|
+
return False
|
|
119
|
+
if os.name == "nt":
|
|
120
|
+
import ctypes
|
|
121
|
+
|
|
122
|
+
handle = ctypes.windll.kernel32.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION
|
|
123
|
+
if not handle:
|
|
124
|
+
return False
|
|
125
|
+
code = ctypes.c_ulong()
|
|
126
|
+
ctypes.windll.kernel32.GetExitCodeProcess(handle, ctypes.byref(code))
|
|
127
|
+
ctypes.windll.kernel32.CloseHandle(handle)
|
|
128
|
+
return code.value == 259 # STILL_ACTIVE
|
|
129
|
+
try:
|
|
130
|
+
os.kill(pid, 0)
|
|
131
|
+
except OSError:
|
|
132
|
+
return False
|
|
133
|
+
return True
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _lock_path() -> Path:
|
|
137
|
+
return jobs_root() / ".runner.lock"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _runner_alive() -> bool:
|
|
141
|
+
try:
|
|
142
|
+
return _pid_alive(int(_lock_path().read_text(encoding="utf-8").strip() or 0))
|
|
143
|
+
except (OSError, ValueError):
|
|
144
|
+
return False
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _spawn_runner(spawn=subprocess.Popen) -> None:
|
|
148
|
+
flags = 0
|
|
149
|
+
if os.name == "nt":
|
|
150
|
+
# CREATE_NO_WINDOW gives the runner a hidden console that every child tool (openmd, opencli,
|
|
151
|
+
# ffmpeg, ASR worker) inherits, so nothing ever flashes on screen. DETACHED_PROCESS must not
|
|
152
|
+
# be combined with it: Windows then ignores CREATE_NO_WINDOW and each console child opens
|
|
153
|
+
# its own visible terminal window.
|
|
154
|
+
flags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
|
|
155
|
+
command = [sys.executable, "-I", "-m", "openmd.browser_host", "--runner"]
|
|
156
|
+
kwargs = dict(stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, close_fds=True, cwd=str(jobs_root()))
|
|
157
|
+
try:
|
|
158
|
+
# Escape Chrome's job object so closing the browser host does not kill a long save.
|
|
159
|
+
spawn(command, creationflags=flags | getattr(subprocess, "CREATE_BREAKAWAY_FROM_JOB", 0), **kwargs)
|
|
160
|
+
except OSError:
|
|
161
|
+
spawn(command, creationflags=flags, **kwargs)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
_last_seq = 0
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _next_seq() -> int:
|
|
168
|
+
"""Strictly increasing within a process, ordered across processes (ns clock)."""
|
|
169
|
+
global _last_seq
|
|
170
|
+
_last_seq = max(time.time_ns(), _last_seq + 1)
|
|
171
|
+
return _last_seq
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
VIDEO_MODES = ("note", "text", "file")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _validate_video(video: object) -> str:
|
|
178
|
+
if video in (None, ""):
|
|
179
|
+
return "note"
|
|
180
|
+
if video not in VIDEO_MODES:
|
|
181
|
+
raise ValueError("invalid_video_mode")
|
|
182
|
+
return str(video)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _enqueue(url: object, destination: Path, batch: str | None = None, video: str = "note") -> dict[str, Any]:
|
|
186
|
+
url = _validate_url(url)
|
|
187
|
+
job_id = uuid.uuid4().hex
|
|
188
|
+
job_dir = jobs_root() / job_id
|
|
189
|
+
job_dir.mkdir(parents=True)
|
|
190
|
+
status = {"id": job_id, "state": "queued", "url": url, "save_to": str(destination), "created": time.time(), "seq": _next_seq(), "attempts": 0, "video": video}
|
|
191
|
+
if batch:
|
|
192
|
+
status["batch"] = batch
|
|
193
|
+
_write_status(job_dir, status)
|
|
194
|
+
return status
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def start_job(url: object, save_to: object = None, *, video: object = None, spawn=subprocess.Popen) -> dict[str, Any]:
|
|
198
|
+
status = _enqueue(url, _validate_destination(save_to), video=_validate_video(video))
|
|
199
|
+
_spawn_runner(spawn)
|
|
200
|
+
_prune_old_jobs()
|
|
201
|
+
return status
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def start_batch(urls: object, save_to: object = None, *, video: object = None, spawn=subprocess.Popen) -> dict[str, Any]:
|
|
205
|
+
"""Queue several links; they run one after another so ASR never overloads the GPU."""
|
|
206
|
+
if not isinstance(urls, list) or not urls:
|
|
207
|
+
raise ValueError("no_urls")
|
|
208
|
+
destination = _validate_destination(save_to)
|
|
209
|
+
video = _validate_video(video)
|
|
210
|
+
batch = uuid.uuid4().hex
|
|
211
|
+
jobs, rejected, seen = [], [], set()
|
|
212
|
+
for url in urls[:MAX_BATCH]:
|
|
213
|
+
key = url.strip() if isinstance(url, str) else url
|
|
214
|
+
if key in seen:
|
|
215
|
+
continue
|
|
216
|
+
seen.add(key)
|
|
217
|
+
try:
|
|
218
|
+
jobs.append(_enqueue(url, destination, batch, video))
|
|
219
|
+
except ValueError as exc:
|
|
220
|
+
rejected.append({"url": str(url)[:200], "error": sanitize_error_summary(exc)})
|
|
221
|
+
if jobs:
|
|
222
|
+
_spawn_runner(spawn)
|
|
223
|
+
_prune_old_jobs()
|
|
224
|
+
return {"batch": batch, "jobs": jobs, "rejected": rejected, "truncated": len(urls) > MAX_BATCH}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _recover(status: dict[str, Any]) -> dict[str, Any]:
|
|
228
|
+
"""A job whose process vanished (shutdown, crash) is re-queued once, then marked failed."""
|
|
229
|
+
if status.get("state") != "running" or _pid_alive(status.get("pid")):
|
|
230
|
+
return status
|
|
231
|
+
attempts = int(status.get("attempts", 1))
|
|
232
|
+
status.pop("pid", None)
|
|
233
|
+
status.pop("child_pid", None)
|
|
234
|
+
if attempts < MAX_ATTEMPTS:
|
|
235
|
+
status.update(state="queued", note="上次被中断(关机或进程结束),已重新排队")
|
|
236
|
+
else:
|
|
237
|
+
status.update(state="failed", error="被中断(关机或进程结束),重试后仍未完成", finished=time.time())
|
|
238
|
+
_write_status(_status_path(status["id"]).parent, status)
|
|
239
|
+
return status
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _queued() -> list[Path]:
|
|
243
|
+
root = jobs_root()
|
|
244
|
+
if not root.is_dir():
|
|
245
|
+
return []
|
|
246
|
+
found = []
|
|
247
|
+
for path in root.glob("*/status.json"):
|
|
248
|
+
try:
|
|
249
|
+
status = _recover(json.loads(path.read_text(encoding="utf-8")))
|
|
250
|
+
except (OSError, ValueError, KeyError):
|
|
251
|
+
continue
|
|
252
|
+
if status.get("state") == "queued":
|
|
253
|
+
found.append((status.get("seq") or int(status.get("created", 0) * 1e9), path.parent))
|
|
254
|
+
return [job_dir for _created, job_dir in sorted(found)]
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def ensure_runner(spawn=subprocess.Popen) -> bool:
|
|
258
|
+
"""Resume the queue (e.g. after a reboot) when work is waiting and no runner holds the lock."""
|
|
259
|
+
if _queued() and not _runner_alive():
|
|
260
|
+
_spawn_runner(spawn)
|
|
261
|
+
return True
|
|
262
|
+
return False
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _acquire_lock() -> bool:
|
|
266
|
+
lock = _lock_path()
|
|
267
|
+
lock.parent.mkdir(parents=True, exist_ok=True)
|
|
268
|
+
for _ in range(2):
|
|
269
|
+
try:
|
|
270
|
+
fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
271
|
+
except FileExistsError:
|
|
272
|
+
if _runner_alive():
|
|
273
|
+
return False
|
|
274
|
+
lock.unlink(missing_ok=True) # stale lock from a previous boot
|
|
275
|
+
continue
|
|
276
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
277
|
+
handle.write(str(os.getpid()))
|
|
278
|
+
return True
|
|
279
|
+
return False
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def run_queue(*, runner=None) -> int:
|
|
283
|
+
"""Single runner: process queued jobs oldest-first until none remain."""
|
|
284
|
+
processed = 0
|
|
285
|
+
while True:
|
|
286
|
+
if not _acquire_lock():
|
|
287
|
+
return processed
|
|
288
|
+
try:
|
|
289
|
+
while queued := _queued():
|
|
290
|
+
run_job(queued[0], runner=runner)
|
|
291
|
+
processed += 1
|
|
292
|
+
finally:
|
|
293
|
+
_lock_path().unlink(missing_ok=True)
|
|
294
|
+
# A job queued between the last scan and the unlock is picked up here.
|
|
295
|
+
if not _queued():
|
|
296
|
+
return processed
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _child_env() -> dict[str, str]:
|
|
300
|
+
# openmd prints non-ASCII titles/paths; without this the child writes the ANSI code page
|
|
301
|
+
# (GBK on Chinese Windows) and the job records mojibake paths.
|
|
302
|
+
return {**os.environ, "PYTHONIOENCODING": "utf-8", "PYTHONUTF8": "1"}
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _run_openmd(command: list[str], job_dir: Path, status: dict[str, Any]) -> subprocess.CompletedProcess:
|
|
306
|
+
# This runner is outside the per-job owner. A daemon started by the child
|
|
307
|
+
# controller would inherit that outer Job even if detached from its own Job.
|
|
308
|
+
if "--enable-browser-backends" in command and any(command[index:index + 2] == ["-m", "openmd"] for index in range(len(command) - 1)):
|
|
309
|
+
ensure_opencli_daemon(["opencli", "browser-job-read"], env=_child_env(), cwd=str(job_dir))
|
|
310
|
+
process = owned_popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
311
|
+
text=True, encoding="utf-8", errors="replace", cwd=str(job_dir), env=_child_env(), **process_options())
|
|
312
|
+
def check_cancelled():
|
|
313
|
+
# The runner owns the Job/group handle. An external host only writes
|
|
314
|
+
# cancellation state; no live root PID is needed to find descendants.
|
|
315
|
+
if read_status(status["id"]).get("state") == "cancelled":
|
|
316
|
+
kill_process_tree(process)
|
|
317
|
+
|
|
318
|
+
try:
|
|
319
|
+
status["child_pid"] = process.pid
|
|
320
|
+
_write_status(job_dir, status)
|
|
321
|
+
stdout, stderr = communicate_interruptibly(process, on_poll=check_cancelled)
|
|
322
|
+
return subprocess.CompletedProcess(command, process.returncode, stdout, stderr)
|
|
323
|
+
finally:
|
|
324
|
+
terminate_quietly(process)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def run_job(job_dir: Path, *, runner=None) -> dict[str, Any]:
|
|
328
|
+
"""Run openmd once for a queued job and record the outcome."""
|
|
329
|
+
status = json.loads((job_dir / "status.json").read_text(encoding="utf-8"))
|
|
330
|
+
status.update(state="running", started=time.time(), pid=os.getpid(), attempts=int(status.get("attempts", 0)) + 1)
|
|
331
|
+
status.pop("note", None)
|
|
332
|
+
_write_status(job_dir, status)
|
|
333
|
+
command = [sys.executable, "-I", "-X", "utf8", "-m", "openmd", "go", status["url"], *GO_FLAGS, "--video", status.get("video") or "note", "--save-to", status["save_to"]]
|
|
334
|
+
try:
|
|
335
|
+
if runner is None:
|
|
336
|
+
done = _run_openmd(command, job_dir, status)
|
|
337
|
+
else:
|
|
338
|
+
done = runner(command, stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", cwd=str(job_dir), env=_child_env())
|
|
339
|
+
current = read_status(status["id"])
|
|
340
|
+
if current.get("state") == "cancelled":
|
|
341
|
+
return current
|
|
342
|
+
lines = [line for line in (done.stdout or "").splitlines() if line.strip().startswith("{")]
|
|
343
|
+
result = json.loads(lines[-1]) if lines else {}
|
|
344
|
+
if done.returncode == 0 and result.get("ok"):
|
|
345
|
+
saved = result.get("saved_to") or ""
|
|
346
|
+
status.update(state="done", saved_to=saved, title=Path(saved).stem if saved else "")
|
|
347
|
+
else:
|
|
348
|
+
hint = [line for line in (done.stderr or "").splitlines() if line.strip()]
|
|
349
|
+
status.update(state="failed", error=sanitize_error_summary(result.get("error") or (hint[0] if hint else f"exit {done.returncode}")))
|
|
350
|
+
except Exception as exc: # noqa: BLE001 - a background job must always end with a status
|
|
351
|
+
status.update(state="failed", error=sanitize_error_summary(exc) or "job_failed")
|
|
352
|
+
status["finished"] = time.time()
|
|
353
|
+
status.pop("pid", None)
|
|
354
|
+
status.pop("child_pid", None)
|
|
355
|
+
_write_status(job_dir, status)
|
|
356
|
+
return status
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def recent_jobs() -> list[dict[str, Any]]:
|
|
360
|
+
root = jobs_root()
|
|
361
|
+
if not root.is_dir():
|
|
362
|
+
return []
|
|
363
|
+
items = []
|
|
364
|
+
for path in sorted(root.glob("*/status.json"), key=lambda p: p.stat().st_mtime, reverse=True)[:RECENT_JOBS]:
|
|
365
|
+
try:
|
|
366
|
+
items.append(json.loads(path.read_text(encoding="utf-8")))
|
|
367
|
+
except (OSError, ValueError):
|
|
368
|
+
continue
|
|
369
|
+
return items
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _prune_old_jobs(keep: int = 50) -> None:
|
|
373
|
+
import shutil
|
|
374
|
+
|
|
375
|
+
root = jobs_root()
|
|
376
|
+
dirs = sorted((p for p in root.iterdir() if p.is_dir()), key=lambda p: p.stat().st_mtime, reverse=True)
|
|
377
|
+
for old in dirs[keep:]:
|
|
378
|
+
try:
|
|
379
|
+
state = json.loads((old / "status.json").read_text(encoding="utf-8")).get("state")
|
|
380
|
+
except (OSError, ValueError):
|
|
381
|
+
state = None
|
|
382
|
+
if state in {"done", "failed", "cancelled", None}:
|
|
383
|
+
shutil.rmtree(old, ignore_errors=True)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def cancel_job(job_id: str) -> dict[str, Any]:
|
|
387
|
+
"""Cancel one job. A running job loses only its own openmd tree; the queue keeps going."""
|
|
388
|
+
status = read_status(job_id)
|
|
389
|
+
if status.get("state") not in {"queued", "running"}:
|
|
390
|
+
return status
|
|
391
|
+
status.update(state="cancelled", finished=time.time())
|
|
392
|
+
status.pop("pid", None)
|
|
393
|
+
status.pop("child_pid", None)
|
|
394
|
+
_write_status(_status_path(job_id).parent, status)
|
|
395
|
+
return status
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def pick_folder(initial: object = None) -> dict[str, Any]:
|
|
399
|
+
"""Show the native folder picker (topmost) and return the chosen absolute path."""
|
|
400
|
+
import tkinter
|
|
401
|
+
from tkinter import filedialog
|
|
402
|
+
|
|
403
|
+
root = tkinter.Tk()
|
|
404
|
+
root.withdraw()
|
|
405
|
+
root.attributes("-topmost", True)
|
|
406
|
+
try:
|
|
407
|
+
start = str(initial) if isinstance(initial, str) and Path(initial).is_dir() else str(Path.home())
|
|
408
|
+
chosen = filedialog.askdirectory(parent=root, initialdir=start, title="选择 Openmd 保存位置", mustexist=False)
|
|
409
|
+
finally:
|
|
410
|
+
root.destroy()
|
|
411
|
+
if not chosen:
|
|
412
|
+
return {"ok": True, "path": None}
|
|
413
|
+
return {"ok": True, "path": str(Path(chosen).resolve())}
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def reveal(job_id: str) -> dict[str, Any]:
|
|
417
|
+
"""Open Explorer at a saved file. Only paths recorded by our own jobs are accepted."""
|
|
418
|
+
status = read_status(job_id)
|
|
419
|
+
saved = status.get("saved_to")
|
|
420
|
+
if status.get("state") != "done" or not saved or not Path(saved).exists():
|
|
421
|
+
raise ValueError("nothing_to_reveal")
|
|
422
|
+
if os.name == "nt":
|
|
423
|
+
subprocess.Popen(["explorer", "/select,", str(Path(saved))])
|
|
424
|
+
return {"ok": True}
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
# ---------------------------------------------------------------- dispatch
|
|
428
|
+
|
|
429
|
+
def handle(message: dict[str, Any]) -> dict[str, Any]:
|
|
430
|
+
kind = message.get("type")
|
|
431
|
+
try:
|
|
432
|
+
if kind == "ping":
|
|
433
|
+
return {"ok": True, "version": 1, "save_to": str(default_save_directory())}
|
|
434
|
+
if kind == "save":
|
|
435
|
+
return {"ok": True, "job": start_job(message.get("url"), message.get("save_to"), video=message.get("video"))}
|
|
436
|
+
if kind == "save_many":
|
|
437
|
+
return {"ok": True, **start_batch(message.get("urls"), message.get("save_to"), video=message.get("video"))}
|
|
438
|
+
if kind == "status":
|
|
439
|
+
status = _recover(read_status(str(message.get("id", ""))))
|
|
440
|
+
ensure_runner()
|
|
441
|
+
return {"ok": True, "job": status}
|
|
442
|
+
if kind == "statuses":
|
|
443
|
+
ids = message.get("ids")
|
|
444
|
+
if not isinstance(ids, list):
|
|
445
|
+
raise ValueError("invalid_ids")
|
|
446
|
+
jobs = {}
|
|
447
|
+
for job_id in ids[:MAX_BATCH]:
|
|
448
|
+
try:
|
|
449
|
+
jobs[str(job_id)] = _recover(read_status(str(job_id)))
|
|
450
|
+
except ValueError:
|
|
451
|
+
jobs[str(job_id)] = {"id": str(job_id), "state": "unknown"}
|
|
452
|
+
ensure_runner()
|
|
453
|
+
return {"ok": True, "jobs": jobs}
|
|
454
|
+
if kind == "recent":
|
|
455
|
+
ensure_runner()
|
|
456
|
+
return {"ok": True, "jobs": recent_jobs()}
|
|
457
|
+
if kind == "cancel":
|
|
458
|
+
return {"ok": True, "job": cancel_job(str(message.get("id", "")))}
|
|
459
|
+
if kind == "reveal":
|
|
460
|
+
return reveal(str(message.get("id", "")))
|
|
461
|
+
if kind == "pick_folder":
|
|
462
|
+
return pick_folder(message.get("initial"))
|
|
463
|
+
return {"ok": False, "error": "unknown_message_type"}
|
|
464
|
+
except ValueError as exc:
|
|
465
|
+
return {"ok": False, "error": sanitize_error_summary(exc)}
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def serve(stdin=None, stdout=None) -> int:
|
|
469
|
+
if os.name == "nt" and stdin is None:
|
|
470
|
+
import msvcrt
|
|
471
|
+
|
|
472
|
+
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
|
|
473
|
+
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
|
|
474
|
+
stdin = stdin or sys.stdin.buffer
|
|
475
|
+
stdout = stdout or sys.stdout.buffer
|
|
476
|
+
while True:
|
|
477
|
+
try:
|
|
478
|
+
message = read_message(stdin)
|
|
479
|
+
except ValueError as exc:
|
|
480
|
+
write_message(stdout, {"ok": False, "error": str(exc)})
|
|
481
|
+
return 1
|
|
482
|
+
if message is None:
|
|
483
|
+
return 0
|
|
484
|
+
write_message(stdout, handle(message))
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def main(argv: list[str] | None = None) -> int:
|
|
488
|
+
args = list(sys.argv[1:] if argv is None else argv)
|
|
489
|
+
from openmd.runtime.managed_tools import activate
|
|
490
|
+
|
|
491
|
+
activate() # the queue runner talks to the opencli daemon directly
|
|
492
|
+
if args == ["--runner"]:
|
|
493
|
+
run_queue()
|
|
494
|
+
return 0
|
|
495
|
+
if len(args) == 2 and args[0] == "--run-job": # jobs started by the first extension build
|
|
496
|
+
run_job(Path(args[1]))
|
|
497
|
+
return 0
|
|
498
|
+
# Chrome passes the caller origin (and on Windows a parent window handle) as arguments.
|
|
499
|
+
return serve()
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
if __name__ == "__main__":
|
|
503
|
+
raise SystemExit(main())
|
openmd/browser_setup.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Register (or remove) the Openmd native messaging host for Chrome and Edge on this user.
|
|
2
|
+
|
|
3
|
+
Writes only user-scope state: ``<data>/browser/`` (host manifest, launcher, extension copy) and the
|
|
4
|
+
``HKCU\\Software\\{Google\\Chrome,Microsoft\\Edge}\\NativeMessagingHosts\\com.openmd.host`` keys.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import base64
|
|
11
|
+
import contextlib
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import shutil
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from openmd.browser_host import HOST_NAME
|
|
21
|
+
from openmd.core.preferences import data_directory
|
|
22
|
+
from openmd.runtime.interpreters import run_interruptibly
|
|
23
|
+
|
|
24
|
+
EXTENSION_DIR = Path(__file__).resolve().parent / "browser_extension"
|
|
25
|
+
REGISTRY_KEYS = (
|
|
26
|
+
r"HKCU\Software\Google\Chrome\NativeMessagingHosts",
|
|
27
|
+
r"HKCU\Software\Microsoft\Edge\NativeMessagingHosts",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def extension_id(manifest_path: Path = EXTENSION_DIR / "manifest.json") -> str:
|
|
32
|
+
"""Chrome's id for an unpacked extension whose manifest pins ``key``."""
|
|
33
|
+
key = json.loads(manifest_path.read_text(encoding="utf-8"))["key"]
|
|
34
|
+
digest = hashlib.sha256(base64.b64decode(key)).hexdigest()[:32]
|
|
35
|
+
return "".join(chr(ord("a") + int(ch, 16)) for ch in digest)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def host_files(root: Path | None = None) -> tuple[Path, Path]:
|
|
39
|
+
root = root or data_directory() / "browser"
|
|
40
|
+
return root / f"{HOST_NAME}.json", root / "openmd-browser-host.cmd"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def write_host_files(root: Path | None = None, python: str = sys.executable) -> Path:
|
|
44
|
+
manifest_path, launcher = host_files(root)
|
|
45
|
+
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
launcher.write_text(f'@echo off\r\n"{python}" -I -m openmd.browser_host %*\r\n', encoding="utf-8")
|
|
47
|
+
manifest = {
|
|
48
|
+
"name": HOST_NAME,
|
|
49
|
+
"description": "Openmd: save the current page as clean Markdown",
|
|
50
|
+
"path": str(launcher),
|
|
51
|
+
"type": "stdio",
|
|
52
|
+
"allowed_origins": [f"chrome-extension://{extension_id()}/"],
|
|
53
|
+
}
|
|
54
|
+
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
55
|
+
return manifest_path
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _reg(*args: str) -> bool:
|
|
59
|
+
return run_interruptibly(["reg", *args], capture_output=True, check=False).returncode == 0
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def install() -> int:
|
|
63
|
+
if os.name != "nt":
|
|
64
|
+
print("目前只支持 Windows。", file=sys.stderr)
|
|
65
|
+
return 2
|
|
66
|
+
manifest_path = write_host_files()
|
|
67
|
+
registered = [key for key in REGISTRY_KEYS if _reg("add", f"{key}\\{HOST_NAME}", "/ve", "/t", "REG_SZ", "/d", str(manifest_path), "/f")]
|
|
68
|
+
if not registered:
|
|
69
|
+
print("注册失败:无法写入当前用户注册表。", file=sys.stderr)
|
|
70
|
+
return 1
|
|
71
|
+
folder = stable_extension_dir()
|
|
72
|
+
print("浏览器插件的本地连接已就绪。")
|
|
73
|
+
print("下一步(只需一次):打开 chrome://extensions(Edge 为 edge://extensions)→ 打开“开发者模式”")
|
|
74
|
+
print("→ “加载已解压的扩展程序” → 选择这个文件夹:")
|
|
75
|
+
print(f" {folder}")
|
|
76
|
+
print(f"插件 ID 应显示为:{extension_id()}。升级 openmd 后再运行一次 openmd browser-setup 即可同步插件。")
|
|
77
|
+
if sys.stdout.isatty():
|
|
78
|
+
with contextlib.suppress(OSError):
|
|
79
|
+
os.startfile(folder) # type: ignore[attr-defined] # Windows only, checked above
|
|
80
|
+
return 0
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def stable_extension_dir(root: Path | None = None) -> Path:
|
|
84
|
+
"""Copy the packaged extension to a path that survives reinstalls.
|
|
85
|
+
|
|
86
|
+
An unpacked extension is loaded by folder path; the site-packages folder
|
|
87
|
+
moves with every venv/pipx/Python change, ~/.openmd/browser/extension does
|
|
88
|
+
not. The manifest key keeps the extension ID identical.
|
|
89
|
+
"""
|
|
90
|
+
target = (root or data_directory() / "browser") / "extension"
|
|
91
|
+
shutil.copytree(EXTENSION_DIR, target, dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__", "*.py", "*.pyc"))
|
|
92
|
+
return target
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def uninstall() -> int:
|
|
96
|
+
for key in REGISTRY_KEYS:
|
|
97
|
+
_reg("delete", f"{key}\\{HOST_NAME}", "/f")
|
|
98
|
+
for path in host_files():
|
|
99
|
+
path.unlink(missing_ok=True)
|
|
100
|
+
shutil.rmtree(data_directory() / "browser" / "extension", ignore_errors=True)
|
|
101
|
+
print("已移除浏览器插件的本地连接(插件本身请在 chrome://extensions 里移除)。")
|
|
102
|
+
return 0
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def cmd_browser_setup(args: argparse.Namespace) -> int:
|
|
106
|
+
return uninstall() if args.uninstall else install()
|