openmd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openmd/__init__.py +3 -0
- openmd/__main__.py +5 -0
- openmd/adapters/__init__.py +1 -0
- openmd/adapters/base.py +95 -0
- openmd/adapters/bilibili/__init__.py +11 -0
- openmd/adapters/bilibili/adapter.py +104 -0
- openmd/adapters/bilibili/discovery.py +33 -0
- openmd/adapters/douyin/__init__.py +11 -0
- openmd/adapters/douyin/adapter.py +117 -0
- openmd/adapters/douyin/discovery.py +36 -0
- openmd/adapters/douyin/sidecar.py +77 -0
- openmd/adapters/generic_media/__init__.py +14 -0
- openmd/adapters/high_privacy.py +77 -0
- openmd/adapters/local_audio/__init__.py +14 -0
- openmd/adapters/local_audio/adapter.py +169 -0
- openmd/adapters/local_audio/asr.py +58 -0
- openmd/adapters/local_audio/backends/__init__.py +1 -0
- openmd/adapters/local_audio/backends/auto.py +177 -0
- openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
- openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
- openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
- openmd/adapters/opencli.py +770 -0
- openmd/adapters/pdf/__init__.py +11 -0
- openmd/adapters/pdf/adapter.py +153 -0
- openmd/adapters/pdf/backends/__init__.py +1 -0
- openmd/adapters/pdf/backends/markitdown.py +165 -0
- openmd/adapters/pdf/parser.py +56 -0
- openmd/adapters/qq_group/__init__.py +11 -0
- openmd/adapters/qq_group/adapter.py +289 -0
- openmd/adapters/qq_group/parsers/__init__.py +1 -0
- openmd/adapters/qq_group/parsers/html.py +60 -0
- openmd/adapters/qq_group/parsers/json.py +71 -0
- openmd/adapters/registry.py +149 -0
- openmd/adapters/screenshot/__init__.py +11 -0
- openmd/adapters/screenshot/adapter.py +154 -0
- openmd/adapters/screenshot/backends/__init__.py +1 -0
- openmd/adapters/screenshot/backends/paddle.py +15 -0
- openmd/adapters/screenshot/backends/rapidocr.py +176 -0
- openmd/adapters/screenshot/backends/tesseract.py +15 -0
- openmd/adapters/screenshot/ocr.py +84 -0
- openmd/adapters/wechat_group/__init__.py +11 -0
- openmd/adapters/wechat_group/adapter.py +160 -0
- openmd/adapters/wechat_group/wx_cli.py +268 -0
- openmd/adapters/wechat_moments/__init__.py +11 -0
- openmd/adapters/wechat_moments/adapter.py +167 -0
- openmd/adapters/wechat_moments/wx_cli.py +265 -0
- openmd/adapters/weixin/__init__.py +11 -0
- openmd/adapters/weixin/adapter.py +148 -0
- openmd/adapters/weixin/discovery.py +510 -0
- openmd/adapters/weixin/report.py +278 -0
- openmd/adapters/xiaohongshu.py +158 -0
- openmd/backends/__init__.py +1 -0
- openmd/backends/base.py +204 -0
- openmd/backends/bilibili.py +1480 -0
- openmd/backends/bilibili_wbi.py +277 -0
- openmd/backends/discourse.py +426 -0
- openmd/backends/douyin.py +868 -0
- openmd/backends/douyin_detail.py +124 -0
- openmd/backends/douyin_providers.py +602 -0
- openmd/backends/executor.py +546 -0
- openmd/backends/feishu.py +861 -0
- openmd/backends/generic_media.py +77 -0
- openmd/backends/interactive.py +60 -0
- openmd/backends/ocr.py +41 -0
- openmd/backends/opencli.py +82 -0
- openmd/backends/pdf.py +122 -0
- openmd/backends/registry.py +234 -0
- openmd/backends/screenshot.py +79 -0
- openmd/backends/video_metadata.py +167 -0
- openmd/backends/web_article.py +363 -0
- openmd/backends/wechat_group.py +35 -0
- openmd/backends/wechat_moments.py +35 -0
- openmd/backends/weixin.py +767 -0
- openmd/backends/x.py +455 -0
- openmd/backends/xiaoe.py +940 -0
- openmd/backends/xiaohongshu.py +911 -0
- openmd/backends/xiaohongshu_providers.py +876 -0
- openmd/backends/xiaoyuzhou.py +1017 -0
- openmd/backends/youtube.py +280 -0
- openmd/backends/youtube_discovery.py +1245 -0
- openmd/backends/youtube_transcript.py +213 -0
- openmd/browser_extension/background.js +231 -0
- openmd/browser_extension/icons/icon-128.png +0 -0
- openmd/browser_extension/icons/icon-16.png +0 -0
- openmd/browser_extension/icons/icon-32.png +0 -0
- openmd/browser_extension/icons/icon-48.png +0 -0
- openmd/browser_extension/icons/icon-512.png +0 -0
- openmd/browser_extension/idle.html +6 -0
- openmd/browser_extension/manifest.json +43 -0
- openmd/browser_extension/options.html +42 -0
- openmd/browser_extension/options.js +117 -0
- openmd/browser_extension/popup.css +71 -0
- openmd/browser_extension/popup.html +17 -0
- openmd/browser_extension/popup.js +59 -0
- openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
- openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
- openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
- openmd/browser_host.py +503 -0
- openmd/browser_setup.py +106 -0
- openmd/cli.py +2012 -0
- openmd/console.py +32 -0
- openmd/core/__init__.py +1 -0
- openmd/core/capabilities.py +162 -0
- openmd/core/content_validation.py +323 -0
- openmd/core/coverage.py +163 -0
- openmd/core/detector.py +247 -0
- openmd/core/doctor.py +104 -0
- openmd/core/execution_policy.py +164 -0
- openmd/core/filename.py +64 -0
- openmd/core/input_validation.py +91 -0
- openmd/core/local_config.py +96 -0
- openmd/core/manifest.py +156 -0
- openmd/core/models.py +168 -0
- openmd/core/opencli_output.py +34 -0
- openmd/core/operation_contract.py +60 -0
- openmd/core/output.py +194 -0
- openmd/core/package_contract.py +335 -0
- openmd/core/package_validator.py +489 -0
- openmd/core/package_writer.py +666 -0
- openmd/core/planner.py +345 -0
- openmd/core/preferences.py +61 -0
- openmd/core/readable.py +129 -0
- openmd/core/receipt.py +285 -0
- openmd/core/renderer.py +185 -0
- openmd/core/runlog.py +208 -0
- openmd/core/secrets.py +313 -0
- openmd/doctor_view.py +68 -0
- openmd/interactive.py +453 -0
- openmd/ocr/__init__.py +1 -0
- openmd/ocr/breakers.py +84 -0
- openmd/ocr/config.py +54 -0
- openmd/ocr/documents.py +522 -0
- openmd/ocr/endpoint.py +72 -0
- openmd/ocr/layout.py +92 -0
- openmd/ocr/layout_filter.py +118 -0
- openmd/ocr/layout_worker.py +89 -0
- openmd/ocr/native_math.py +220 -0
- openmd/ocr/office.py +305 -0
- openmd/ocr/ollama.py +200 -0
- openmd/ocr/otsl.py +51 -0
- openmd/ocr/ovis-prompt.txt +1 -0
- openmd/ocr/pipeline.py +384 -0
- openmd/ocr/settings.json +25 -0
- openmd/ocr/tiling.py +631 -0
- openmd/post_video.py +154 -0
- openmd/probe.py +397 -0
- openmd/routing/__init__.py +1 -0
- openmd/routing/cards/builtin.json +620 -0
- openmd/routing/cards/defuddle.json +70 -0
- openmd/routing/cards/x-tweet-fetcher.json +71 -0
- openmd/routing/cards/xyz-dl.json +60 -0
- openmd/routing/cards/yutto.json +68 -0
- openmd/routing/entry_shim.py +71 -0
- openmd/routing/lifecycle.py +459 -0
- openmd/routing/loader.py +138 -0
- openmd/routing/providers.py +158 -0
- openmd/routing/readers.py +108 -0
- openmd/routing/routes.json +504 -0
- openmd/routing/runner.py +192 -0
- openmd/routing/view.py +40 -0
- openmd/runtime/__init__.py +1 -0
- openmd/runtime/_windows_job.py +234 -0
- openmd/runtime/asr.py +217 -0
- openmd/runtime/asr_worker.py +119 -0
- openmd/runtime/gpu_lock.py +81 -0
- openmd/runtime/gpu_quarantine.py +85 -0
- openmd/runtime/interpreters.py +407 -0
- openmd/runtime/managed_tools.py +204 -0
- openmd/runtime/models.py +202 -0
- openmd/runtime/services.py +175 -0
- openmd/runtime/tempdirs.py +227 -0
- openmd_cli-0.1.0.dist-info/METADATA +168 -0
- openmd_cli-0.1.0.dist-info/RECORD +177 -0
- openmd_cli-0.1.0.dist-info/WHEEL +5 -0
- openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
openmd/post_video.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Optional video step for posts whose body is text with an attached video (X for now).
|
|
2
|
+
|
|
3
|
+
Modes (``openmd go --video``):
|
|
4
|
+
- ``note`` (default): probe metadata only and add one line saying the post has a video.
|
|
5
|
+
- ``text``: temporary audio download plus local ASR; the transcript is appended to the same Markdown.
|
|
6
|
+
- ``file``: like ``text`` and also keep the video file next to the Markdown (assets folder).
|
|
7
|
+
|
|
8
|
+
The video step never fails the save: problems become a short note inside the Markdown.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
import shutil
|
|
15
|
+
import subprocess
|
|
16
|
+
import tempfile
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from urllib.parse import urlparse
|
|
19
|
+
|
|
20
|
+
from openmd.runtime.tempdirs import owned_mkdtemp
|
|
21
|
+
|
|
22
|
+
VIDEO_MODES = ("note", "text", "file")
|
|
23
|
+
MAX_VIDEOS = 4
|
|
24
|
+
PROBE_TIMEOUT_S = 45
|
|
25
|
+
DOWNLOAD_TIMEOUT_S = 600
|
|
26
|
+
_X_HOSTS = {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.x.com", "mobile.twitter.com"}
|
|
27
|
+
_X_STATUS = re.compile(r"^/[^/]+/status/[0-9]+")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def applies_to(url: str) -> bool:
|
|
31
|
+
parsed = urlparse(url)
|
|
32
|
+
return parsed.scheme in {"http", "https"} and parsed.netloc.lower() in _X_HOSTS and bool(_X_STATUS.match(parsed.path))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _run(command: list[str], timeout: int) -> subprocess.CompletedProcess:
|
|
36
|
+
from openmd.runtime.interpreters import run_interruptibly
|
|
37
|
+
|
|
38
|
+
return run_interruptibly(command, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def probe(url: str, *, run=_run) -> list[float] | None:
|
|
42
|
+
"""Durations (seconds) of the post's videos; [] when it has none; None when unknown."""
|
|
43
|
+
if shutil.which("yt-dlp") is None:
|
|
44
|
+
return None
|
|
45
|
+
try:
|
|
46
|
+
done = run(["yt-dlp", "--simulate", "--no-warnings", "--playlist-items", f"1:{MAX_VIDEOS}", "--print", "%(duration)s", url], PROBE_TIMEOUT_S)
|
|
47
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
48
|
+
return None
|
|
49
|
+
if done.returncode != 0:
|
|
50
|
+
return [] if "no video" in done.stderr.lower() else None
|
|
51
|
+
durations = []
|
|
52
|
+
for line in done.stdout.splitlines():
|
|
53
|
+
try:
|
|
54
|
+
durations.append(float(line.strip()))
|
|
55
|
+
except ValueError:
|
|
56
|
+
continue
|
|
57
|
+
return durations
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _clock(seconds: float) -> str:
|
|
61
|
+
total = int(round(seconds))
|
|
62
|
+
minutes, sec = divmod(total, 60)
|
|
63
|
+
return f"{minutes} 分 {sec} 秒" if minutes else f"{sec} 秒"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _summary(durations: list[float]) -> str:
|
|
67
|
+
if len(durations) == 1:
|
|
68
|
+
return f"本帖含视频(约 {_clock(durations[0])})"
|
|
69
|
+
return f"本帖含 {len(durations)} 段视频(共约 {_clock(sum(durations))})"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _download(url: str, workdir: Path, *, keep_video: bool, run) -> list[Path]:
|
|
73
|
+
template = str(workdir / "video-%(autonumber)02d.%(ext)s")
|
|
74
|
+
# Progressive mp4 only: X's HLS audio-only streams stall behind proxies (verified 2026-11-08),
|
|
75
|
+
# while the single-file mp4 downloads in seconds. For text, a small rendition is enough.
|
|
76
|
+
command = ["yt-dlp", "--no-warnings", "--playlist-items", f"1:{MAX_VIDEOS}", "--socket-timeout", "30", "--retries", "3",
|
|
77
|
+
"-f", "b[ext=mp4][protocol^=http]/b[ext=mp4]/b", "-o", template, url]
|
|
78
|
+
if not keep_video:
|
|
79
|
+
command[1:1] = ["-S", "res:480"]
|
|
80
|
+
done = run(command, DOWNLOAD_TIMEOUT_S)
|
|
81
|
+
if done.returncode != 0:
|
|
82
|
+
raise RuntimeError("视频下载失败")
|
|
83
|
+
files = sorted(p for p in workdir.iterdir() if p.is_file() and p.suffix.lower() in {".mp4", ".wav", ".m4a", ".webm", ".mkv"})
|
|
84
|
+
if not files:
|
|
85
|
+
raise RuntimeError("视频下载后没有找到文件")
|
|
86
|
+
return files
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _transcribe(path: Path, transcribe) -> str:
|
|
90
|
+
from openmd.core.readable import format_transcript
|
|
91
|
+
|
|
92
|
+
result = transcribe(path)
|
|
93
|
+
if not getattr(result, "has_usable_text", False):
|
|
94
|
+
raise RuntimeError("语音识别没有得到文字")
|
|
95
|
+
return format_transcript(result.text, source_type="video", platform="x").strip()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def video_section(url: str, mode: str, markdown_dir: Path, *, run=_run, transcribe=None) -> str | None:
|
|
99
|
+
"""Markdown to append for ``url``; None when the post has no video or nothing can be said."""
|
|
100
|
+
if mode not in VIDEO_MODES or not applies_to(url):
|
|
101
|
+
return None
|
|
102
|
+
durations = probe(url, run=run)
|
|
103
|
+
if not durations:
|
|
104
|
+
return None
|
|
105
|
+
summary = _summary(durations)
|
|
106
|
+
if mode == "note":
|
|
107
|
+
return f"> {summary},未转文字。需要时用插件右键「用 Openmd 保存(含视频文字版)」或 `openmd go --video text`。\n"
|
|
108
|
+
if transcribe is None:
|
|
109
|
+
from openmd.runtime.asr import get_asr_backend
|
|
110
|
+
|
|
111
|
+
transcribe = get_asr_backend("asr.auto").transcribe
|
|
112
|
+
from openmd.runtime.interpreters import remove_tree_with_retry
|
|
113
|
+
|
|
114
|
+
workdir = Path(owned_mkdtemp(prefix="openmd-x-video-"))
|
|
115
|
+
try:
|
|
116
|
+
try:
|
|
117
|
+
files = _download(url, workdir, keep_video=mode == "file", run=run)
|
|
118
|
+
except (OSError, subprocess.TimeoutExpired, RuntimeError) as exc:
|
|
119
|
+
return f"> {summary},转文字未完成:{exc if isinstance(exc, RuntimeError) else '视频下载失败'}。\n"
|
|
120
|
+
parts = []
|
|
121
|
+
embeds = []
|
|
122
|
+
for index, path in enumerate(files, 1):
|
|
123
|
+
heading = "## 视频逐字稿" if len(files) == 1 else f"## 视频逐字稿(第 {index} 段)"
|
|
124
|
+
try:
|
|
125
|
+
parts.append(f"{heading}\n\n{_transcribe(path, transcribe)}\n")
|
|
126
|
+
except RuntimeError as exc:
|
|
127
|
+
parts.append(f"{heading}\n\n> 转文字未完成:{exc}。\n")
|
|
128
|
+
if mode == "file":
|
|
129
|
+
assets = markdown_dir / "assets"
|
|
130
|
+
assets.mkdir(exist_ok=True)
|
|
131
|
+
target = assets / path.name
|
|
132
|
+
shutil.move(str(path), target)
|
|
133
|
+
embeds.append(f"") # <> keeps paths with spaces valid
|
|
134
|
+
body = "\n".join(parts)
|
|
135
|
+
if embeds:
|
|
136
|
+
body = "\n\n".join(embeds) + "\n\n" + body
|
|
137
|
+
return f"> {summary};以下逐字稿由本机语音识别生成。\n\n{body}"
|
|
138
|
+
finally:
|
|
139
|
+
remove_tree_with_retry(workdir)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def append_video_section(markdown_path: Path, url: str, mode: str, **kwargs) -> bool:
|
|
143
|
+
section = video_section(url, mode, markdown_path.parent, **kwargs)
|
|
144
|
+
if not section:
|
|
145
|
+
return False
|
|
146
|
+
text = markdown_path.read_text(encoding="utf-8").rstrip("\n")
|
|
147
|
+
body, sep, last = text.rpartition("\n")
|
|
148
|
+
if sep and last.startswith("*来源"):
|
|
149
|
+
# Keep the source footer last; the video section belongs to the post body.
|
|
150
|
+
updated = f"{body.rstrip()}\n\n{section.rstrip()}\n\n{last}\n"
|
|
151
|
+
else:
|
|
152
|
+
updated = f"{text}\n\n{section.rstrip()}\n"
|
|
153
|
+
markdown_path.write_text(updated, encoding="utf-8")
|
|
154
|
+
return True
|
openmd/probe.py
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""Bounded platform health checks through existing extraction/discovery routes.
|
|
2
|
+
|
|
3
|
+
Samples stay in machine-local configuration. Reports contain no URLs, source
|
|
4
|
+
text, response payloads, or credentials, and never enable media/ASR/cloud work.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import contextlib
|
|
11
|
+
import io
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
18
|
+
from dataclasses import asdict, dataclass, field
|
|
19
|
+
from datetime import datetime, timezone
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from openmd.adapters.registry import create_default_registry
|
|
24
|
+
from openmd.backends.base import DiscoveryRequest
|
|
25
|
+
from openmd.backends.executor import BackendExecutor, DiscoveryExecutor
|
|
26
|
+
from openmd.backends.registry import (
|
|
27
|
+
BackendRuntimeOptions,
|
|
28
|
+
create_runtime_backend_provider_registry,
|
|
29
|
+
create_runtime_discovery_provider_registry,
|
|
30
|
+
)
|
|
31
|
+
from openmd.core.detector import PLATFORM_IDS, detect_platform
|
|
32
|
+
from openmd.core.local_config import checkout_root
|
|
33
|
+
from openmd.core.secrets import sanitize_error_summary
|
|
34
|
+
from openmd.routing.loader import load_catalog
|
|
35
|
+
from openmd.runtime.interpreters import run_interruptibly, worker_command
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
CONFIG_SCHEMA = "openmd.probe-config/v1"
|
|
39
|
+
REPORT_SCHEMA = "openmd.probe/v1"
|
|
40
|
+
_FORBIDDEN_GATES = frozenset({"media_download", "asr", "cloud", "local_compute", "local_export"})
|
|
41
|
+
# This legacy builtin only declares a browser gate, but downloads and sends
|
|
42
|
+
# media to Tingwu. A probe must never invoke it, even with browser permission.
|
|
43
|
+
_FORBIDDEN_BACKENDS = frozenset({"opencli.xiaoe.transcript"})
|
|
44
|
+
_BOUNDED_DISCOVERY = {
|
|
45
|
+
"bilibili": {"bilibili-harvest.discovery": "network", "opencli.bilibili.user-videos": "browser"},
|
|
46
|
+
"youtube": {"opencli.youtube.discovery": "browser", "opencli.youtube.playlist.discovery": "browser", "youtube.data-api.discovery": "network", "youtube.data-api.playlist.discovery": "network"},
|
|
47
|
+
"douyin": {"opencli.douyin.user-videos": "browser"},
|
|
48
|
+
"xiaohongshu": {"opencli.xiaohongshu.user": "browser"},
|
|
49
|
+
"xiaoyuzhou": {"opencli.xiaoyuzhou.podcast-episodes": "browser"},
|
|
50
|
+
"xiaoe": {"opencli.xiaoe.catalog": "browser"},
|
|
51
|
+
}
|
|
52
|
+
_URL_RE = re.compile(r"(?:https?|file)://[^\s<>\"'\])]+", re.I)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class ProbeSample:
|
|
57
|
+
url: str
|
|
58
|
+
mode: str = "extract"
|
|
59
|
+
discovery_backend: str | None = None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class ProbeResult:
|
|
64
|
+
platform: str
|
|
65
|
+
configured: bool
|
|
66
|
+
status: str = "broken"
|
|
67
|
+
mode: str = "extract"
|
|
68
|
+
branch: str | None = None
|
|
69
|
+
reason: str | None = None
|
|
70
|
+
error_code: str | None = None
|
|
71
|
+
extraction_status: str | None = None
|
|
72
|
+
source_level: str | None = None
|
|
73
|
+
text_chars: int = 0
|
|
74
|
+
item_count: int = 0
|
|
75
|
+
route_version: str | None = None
|
|
76
|
+
tool_version: str | None = None
|
|
77
|
+
elapsed_seconds: float = 0.0
|
|
78
|
+
live_route_attempted: bool = False
|
|
79
|
+
attempts: list[dict[str, Any]] = field(default_factory=list)
|
|
80
|
+
excluded_branches: list[dict[str, str]] = field(default_factory=list)
|
|
81
|
+
detail_readable: bool | None = None
|
|
82
|
+
transcription: str | None = None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def default_config_path() -> Path:
|
|
86
|
+
explicit = os.environ.get("OPENMD_PROBE_CONFIG")
|
|
87
|
+
if explicit:
|
|
88
|
+
return Path(explicit).expanduser()
|
|
89
|
+
root = checkout_root()
|
|
90
|
+
return root / "out/probe-config.json" if root else Path.home() / ".openmd/probe-config.json"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def load_probe_samples(path: Path, *, required: bool = False) -> dict[str, ProbeSample]:
|
|
94
|
+
"""Read one sample per platform; reject credentials and executable options."""
|
|
95
|
+
if not path.is_file():
|
|
96
|
+
if required:
|
|
97
|
+
raise ValueError("probe_config_missing")
|
|
98
|
+
return {}
|
|
99
|
+
if path.stat().st_size > 128 * 1024:
|
|
100
|
+
raise ValueError("probe_config_too_large")
|
|
101
|
+
try:
|
|
102
|
+
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
103
|
+
except (OSError, ValueError) as exc:
|
|
104
|
+
raise ValueError("probe_config_invalid_json") from exc
|
|
105
|
+
if not isinstance(data, dict) or data.get("schema") != CONFIG_SCHEMA or set(data) - {"schema", "samples"}:
|
|
106
|
+
raise ValueError("probe_config_invalid_schema")
|
|
107
|
+
samples = data.get("samples")
|
|
108
|
+
if not isinstance(samples, dict) or set(samples) - set(PLATFORM_IDS):
|
|
109
|
+
raise ValueError("probe_config_invalid_platforms")
|
|
110
|
+
result: dict[str, ProbeSample] = {}
|
|
111
|
+
for platform, value in samples.items():
|
|
112
|
+
if not isinstance(value, dict) or set(value) - {"url", "mode", "discovery_backend"}:
|
|
113
|
+
raise ValueError("probe_config_invalid_sample_fields")
|
|
114
|
+
url = value.get("url", "")
|
|
115
|
+
mode = value.get("mode", "extract")
|
|
116
|
+
backend = value.get("discovery_backend")
|
|
117
|
+
if not isinstance(url, str) or mode not in {"extract", "manifest"}:
|
|
118
|
+
raise ValueError("probe_config_invalid_sample")
|
|
119
|
+
if backend is not None and (not isinstance(backend, str) or not backend or mode != "manifest"):
|
|
120
|
+
raise ValueError("probe_config_invalid_discovery_backend")
|
|
121
|
+
if url.strip():
|
|
122
|
+
result[platform] = ProbeSample(url.strip(), mode, backend)
|
|
123
|
+
return result
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _safe_reason(value: object, sample: str = "") -> str | None:
|
|
127
|
+
if value is None:
|
|
128
|
+
return None
|
|
129
|
+
text = str(value)
|
|
130
|
+
if sample:
|
|
131
|
+
text = text.replace(sample, "[sample]")
|
|
132
|
+
text = _URL_RE.sub("[url]", text)
|
|
133
|
+
text = re.sub(
|
|
134
|
+
r"(?i)(?:cookie|token|authorization|pass|secret|credential|api[_-]?key)"
|
|
135
|
+
r"[\w-]*[\"']?\s*[:=]\s*[\"']?[^\s,;}]+",
|
|
136
|
+
"[redacted]", text,
|
|
137
|
+
)
|
|
138
|
+
return sanitize_error_summary(text)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _safe_chain(platform: str, chain: tuple[str, ...], options: BackendRuntimeOptions) -> tuple[tuple[str, ...], list[dict[str, str]]]:
|
|
142
|
+
catalog = load_catalog()
|
|
143
|
+
safe: list[str] = []
|
|
144
|
+
excluded: list[dict[str, str]] = []
|
|
145
|
+
for backend in chain:
|
|
146
|
+
cap = catalog.capabilities.get(backend)
|
|
147
|
+
reason = None
|
|
148
|
+
gates = set(cap.get("gates", [])) if cap else set()
|
|
149
|
+
if backend in _FORBIDDEN_BACKENDS or gates & _FORBIDDEN_GATES:
|
|
150
|
+
reason = "probe_disallows_media_asr_cloud_or_compute"
|
|
151
|
+
elif cap is None or platform not in cap.get("sources", []):
|
|
152
|
+
reason = "probe_unknown_route"
|
|
153
|
+
elif "browser" in gates and not options.enable_browser_backends:
|
|
154
|
+
reason = "browser_gate_closed"
|
|
155
|
+
elif "network" in gates and not options.enable_network_backends:
|
|
156
|
+
reason = "network_gate_closed"
|
|
157
|
+
if reason:
|
|
158
|
+
excluded.append({"branch": backend, "reason": reason})
|
|
159
|
+
else:
|
|
160
|
+
safe.append(backend)
|
|
161
|
+
return tuple(safe), excluded
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def probe_platform(platform: str, sample: ProbeSample, options: BackendRuntimeOptions) -> ProbeResult:
|
|
165
|
+
"""One bounded sample; exceptions are isolated by the outer report runner."""
|
|
166
|
+
row = ProbeResult(platform=platform, configured=True, mode=sample.mode)
|
|
167
|
+
if detect_platform(sample.url) != platform:
|
|
168
|
+
row.reason = row.error_code = "sample_platform_mismatch"
|
|
169
|
+
return row
|
|
170
|
+
# URL-based sample resolution and discovery require an explicit gate. In
|
|
171
|
+
# particular, do not let locally registered metadata providers fake health.
|
|
172
|
+
if not (options.enable_network_backends or options.enable_browser_backends):
|
|
173
|
+
row.reason = row.error_code = "probe_network_gates_closed"
|
|
174
|
+
return row
|
|
175
|
+
if sample.mode == "manifest":
|
|
176
|
+
runtime = BackendRuntimeOptions(
|
|
177
|
+
requested_discovery_provider=sample.discovery_backend,
|
|
178
|
+
enable_network_backends=options.enable_network_backends,
|
|
179
|
+
enable_browser_backends=options.enable_browser_backends,
|
|
180
|
+
)
|
|
181
|
+
providers = create_runtime_discovery_provider_registry(platform, runtime)
|
|
182
|
+
providers.allow_skeleton_fallback = False
|
|
183
|
+
request = DiscoveryRequest(input_url=sample.url, platform=platform, limit=1, allow_all=False)
|
|
184
|
+
provider = providers.provider_for(request, requested_provider=sample.discovery_backend)
|
|
185
|
+
provider_name = getattr(provider, "name", "not_selected")
|
|
186
|
+
gate = _BOUNDED_DISCOVERY.get(platform, {}).get(provider_name)
|
|
187
|
+
if gate is None or not getattr(options, f"enable_{gate}_backends", False):
|
|
188
|
+
row.branch = provider_name
|
|
189
|
+
row.reason = row.error_code = "probe_discovery_not_bounded_or_gate_closed"
|
|
190
|
+
return row
|
|
191
|
+
result = DiscoveryExecutor(provider_registry=providers, requested_provider=sample.discovery_backend).discover(
|
|
192
|
+
request
|
|
193
|
+
)
|
|
194
|
+
row.branch = result.provider
|
|
195
|
+
row.item_count = len(result.items)
|
|
196
|
+
row.error_code = _safe_reason(result.error_code, sample.url)
|
|
197
|
+
row.reason = _safe_reason(result.error_summary, sample.url)
|
|
198
|
+
row.status = "ok" if result.ok and result.items and result.provider not in {"mock.discovery", "skeleton.discovery"} else "broken"
|
|
199
|
+
invalid_sample = result.error_code in {
|
|
200
|
+
"douyin_single_video_not_supported_for_discovery", "douyin_short_link_resolution_required", "douyin_user_sec_uid_not_found",
|
|
201
|
+
}
|
|
202
|
+
row.live_route_attempted = not invalid_sample
|
|
203
|
+
if invalid_sample:
|
|
204
|
+
row.status = "degraded"
|
|
205
|
+
row.reason = "missing_valid_probe_sample; configured selector is unsupported for bounded discovery; live route not covered"
|
|
206
|
+
if row.status == "broken" and not row.reason:
|
|
207
|
+
row.reason = row.error_code = "probe_manifest_empty_or_non_live"
|
|
208
|
+
row.attempts.append({"branch": row.branch, "status": row.status, "error_code": row.error_code, "reason": row.reason})
|
|
209
|
+
return row
|
|
210
|
+
|
|
211
|
+
if platform == "douyin":
|
|
212
|
+
# Health probing reads one work's detail only. It never downloads media
|
|
213
|
+
# or invokes ASR, so detail availability cannot certify transcription.
|
|
214
|
+
from openmd.backends.douyin_detail import DouyinDetailError, DouyinDetailProvider
|
|
215
|
+
|
|
216
|
+
row.branch = "opencli.douyin.detail"
|
|
217
|
+
row.transcription = "not_probed_media_asr_disallowed"
|
|
218
|
+
if not options.enable_browser_backends:
|
|
219
|
+
row.reason = row.error_code = "browser_gate_closed"
|
|
220
|
+
return row
|
|
221
|
+
adapter = create_default_registry().get_adapter(platform)
|
|
222
|
+
items = adapter.resolve_items(sample.url, limit=None, all=False)
|
|
223
|
+
if len(items) != 1 or items[0].input_type != "single":
|
|
224
|
+
row.reason = row.error_code = "probe_requires_single_item_or_manifest_mode"
|
|
225
|
+
return row
|
|
226
|
+
row.live_route_attempted = True
|
|
227
|
+
try:
|
|
228
|
+
DouyinDetailProvider().resolve(items[0])
|
|
229
|
+
row.detail_readable = True
|
|
230
|
+
row.item_count = 1
|
|
231
|
+
row.status = "degraded"
|
|
232
|
+
row.reason = "detail_readable; transcription_not_probed_media_asr_disallowed"
|
|
233
|
+
except DouyinDetailError as exc:
|
|
234
|
+
row.detail_readable = False
|
|
235
|
+
row.error_code = exc.code
|
|
236
|
+
row.reason = str(exc)
|
|
237
|
+
row.attempts.append({"branch": row.branch, "status": row.status, "error_code": row.error_code, "reason": row.reason})
|
|
238
|
+
return row
|
|
239
|
+
|
|
240
|
+
adapters = create_default_registry()
|
|
241
|
+
chain, row.excluded_branches = _safe_chain(platform, adapters.backend_chain(platform), options)
|
|
242
|
+
if not chain:
|
|
243
|
+
row.reason = row.error_code = "probe_no_safe_enabled_route"
|
|
244
|
+
return row
|
|
245
|
+
providers = create_runtime_backend_provider_registry(platform, options)
|
|
246
|
+
providers.allow_skeleton_fallback = False
|
|
247
|
+
items = adapters.get_adapter(platform).resolve_items(sample.url, limit=None, all=False)
|
|
248
|
+
if len(items) != 1 or items[0].input_type == "collection":
|
|
249
|
+
row.reason = row.error_code = "probe_requires_single_item_or_manifest_mode"
|
|
250
|
+
return row
|
|
251
|
+
executor = BackendExecutor(provider_registry=providers)
|
|
252
|
+
for backend in chain:
|
|
253
|
+
row.live_route_attempted = row.live_route_attempted or (
|
|
254
|
+
backend not in {"douyin.metadata", "manual.markdown-export"} and providers.provider_for(backend) is not None
|
|
255
|
+
)
|
|
256
|
+
result = executor.extract(items[0], (backend,), allow_fallback=False)
|
|
257
|
+
row.branch = result.backend
|
|
258
|
+
row.extraction_status = result.status
|
|
259
|
+
row.source_level = result.source_level
|
|
260
|
+
row.route_version = result.metadata.get("route_version")
|
|
261
|
+
row.tool_version = result.metadata.get("tool_run", {}).get("tool_version")
|
|
262
|
+
row.error_code = _safe_reason(result.error_code, sample.url)
|
|
263
|
+
row.reason = _safe_reason(result.error_summary, sample.url)
|
|
264
|
+
row.text_chars = len(result.text)
|
|
265
|
+
row.attempts.append({"branch": backend, "actual_branch": result.backend, "status": result.status, "error_code": row.error_code, "reason": row.reason})
|
|
266
|
+
if result.has_usable_text:
|
|
267
|
+
row.item_count = 1
|
|
268
|
+
if result.metadata.get("metadata_only") or backend == "douyin.metadata":
|
|
269
|
+
row.status = "degraded"
|
|
270
|
+
row.error_code = "missing_valid_probe_sample"
|
|
271
|
+
row.reason = "missing_valid_probe_sample; metadata_only_not_live_extraction; live route not covered"
|
|
272
|
+
elif result.status == "partial" or result.has_terminal_gaps or len(row.attempts) > 1:
|
|
273
|
+
row.status = "degraded"
|
|
274
|
+
row.reason = row.reason or "partial_content_or_fallback_recovered"
|
|
275
|
+
else:
|
|
276
|
+
row.status = "ok"
|
|
277
|
+
return row
|
|
278
|
+
row.reason = row.reason or "probe_no_usable_content"
|
|
279
|
+
return row
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _write_report(path: Path, report: dict[str, Any]) -> None:
|
|
283
|
+
temporary = path.with_suffix(".tmp")
|
|
284
|
+
temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
285
|
+
temporary.replace(path)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _probe_isolated(platform: str, sample: ProbeSample, options: BackendRuntimeOptions, timeout_seconds: int) -> ProbeResult:
|
|
289
|
+
"""Own the full provider subtree so one slow platform cannot stall all probes."""
|
|
290
|
+
payload = {"platform": platform, "sample": asdict(sample), "network": options.enable_network_backends, "browser": options.enable_browser_backends}
|
|
291
|
+
try:
|
|
292
|
+
process = run_interruptibly(
|
|
293
|
+
worker_command(Path(sys.executable), module="openmd.probe"),
|
|
294
|
+
input=json.dumps(payload), capture_output=True, text=True,
|
|
295
|
+
encoding="utf-8", errors="replace", timeout=timeout_seconds, check=False,
|
|
296
|
+
)
|
|
297
|
+
except subprocess.TimeoutExpired:
|
|
298
|
+
return ProbeResult(platform=platform, configured=True, mode=sample.mode, error_code="probe_timeout", reason=f"platform exceeded {timeout_seconds}s; owned subprocess tree reclaimed")
|
|
299
|
+
if process.returncode:
|
|
300
|
+
return ProbeResult(platform=platform, configured=True, mode=sample.mode, error_code="probe_worker_failed", reason=f"probe worker exit {process.returncode}")
|
|
301
|
+
try:
|
|
302
|
+
return ProbeResult(**json.loads(process.stdout))
|
|
303
|
+
except (ValueError, TypeError):
|
|
304
|
+
return ProbeResult(platform=platform, configured=True, mode=sample.mode, error_code="probe_worker_invalid_result", reason="probe worker returned an invalid report")
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def run_probe(
|
|
308
|
+
samples: dict[str, ProbeSample], *, out: Path, options: BackendRuntimeOptions,
|
|
309
|
+
platforms: tuple[str, ...] = PLATFORM_IDS, interval_seconds: float = 1.0, timeout_seconds: int = 90,
|
|
310
|
+
) -> tuple[dict[str, Any], Path]:
|
|
311
|
+
now = datetime.now(timezone.utc)
|
|
312
|
+
directory = out / now.astimezone().date().isoformat()
|
|
313
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
314
|
+
path = directory / f"probe-{now.strftime('%H%M%S-%f')}.json"
|
|
315
|
+
report: dict[str, Any] = {
|
|
316
|
+
"schema": REPORT_SCHEMA, "started_at": now.isoformat(), "completed": False,
|
|
317
|
+
"route_version": load_catalog().version,
|
|
318
|
+
"gates": {"network": options.enable_network_backends, "browser": options.enable_browser_backends, "media_download": False, "asr": False, "cloud": False},
|
|
319
|
+
"sample_policy": "one configured sample per platform; no URL or source content persisted",
|
|
320
|
+
"rate_limit_seconds": interval_seconds, "timeout_seconds": timeout_seconds, "results": [],
|
|
321
|
+
}
|
|
322
|
+
_write_report(path, report)
|
|
323
|
+
attempted = 0
|
|
324
|
+
# Rebuild options instead of trusting a caller to leave the heavy gates shut.
|
|
325
|
+
bounded_options = BackendRuntimeOptions(enable_network_backends=options.enable_network_backends, enable_browser_backends=options.enable_browser_backends)
|
|
326
|
+
# Provider commands run inside owned platform workers. Start the one allowed
|
|
327
|
+
# persistent service here, before those outer Jobs can acquire descendants.
|
|
328
|
+
if bounded_options.enable_browser_backends and any(platform in samples for platform in platforms):
|
|
329
|
+
from openmd.runtime.services import ensure_opencli_daemon
|
|
330
|
+
|
|
331
|
+
try:
|
|
332
|
+
ensure_opencli_daemon(["opencli", "probe-platform-read"])
|
|
333
|
+
except OSError as exc:
|
|
334
|
+
# Preserve independent network-only routes if the browser daemon is
|
|
335
|
+
# unavailable. Each browser provider still reports its own failure.
|
|
336
|
+
report["browser_preflight_reason"] = _safe_reason(exc)
|
|
337
|
+
for platform in platforms:
|
|
338
|
+
sample = samples.get(platform)
|
|
339
|
+
if sample is None:
|
|
340
|
+
row = ProbeResult(platform=platform, configured=False, status="degraded", reason="sample_not_configured")
|
|
341
|
+
else:
|
|
342
|
+
if attempted:
|
|
343
|
+
time.sleep(max(0.0, interval_seconds))
|
|
344
|
+
attempted += 1
|
|
345
|
+
started = time.monotonic()
|
|
346
|
+
report["current_platform"] = platform
|
|
347
|
+
_write_report(path, report)
|
|
348
|
+
try:
|
|
349
|
+
row = _probe_isolated(platform, sample, bounded_options, timeout_seconds)
|
|
350
|
+
except Exception as exc:
|
|
351
|
+
row = ProbeResult(platform=platform, configured=True, mode=sample.mode, reason=_safe_reason(exc, sample.url), error_code="probe_platform_error")
|
|
352
|
+
row.elapsed_seconds = round(time.monotonic() - started, 3)
|
|
353
|
+
report["results"].append(asdict(row))
|
|
354
|
+
_write_report(path, report) # crash/cancel preserves completed platforms
|
|
355
|
+
report["summary"] = {status: sum(r["status"] == status for r in report["results"]) for status in ("ok", "degraded", "broken")}
|
|
356
|
+
report["configured_count"] = attempted
|
|
357
|
+
report["ok"] = attempted > 0 and all(row["status"] == "ok" for row in report["results"] if row["configured"])
|
|
358
|
+
report["completed"] = True
|
|
359
|
+
report.pop("current_platform", None)
|
|
360
|
+
report["finished_at"] = datetime.now(timezone.utc).isoformat()
|
|
361
|
+
_write_report(path, report)
|
|
362
|
+
return report, path
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def cmd_probe(args: argparse.Namespace) -> int:
|
|
366
|
+
config = Path(args.config).expanduser() if args.config else default_config_path()
|
|
367
|
+
samples = load_probe_samples(config, required=bool(args.config or os.environ.get("OPENMD_PROBE_CONFIG")))
|
|
368
|
+
platforms = (args.platform,) if args.platform else PLATFORM_IDS
|
|
369
|
+
report, path = run_probe(
|
|
370
|
+
samples, out=Path(args.out), platforms=platforms,
|
|
371
|
+
options=BackendRuntimeOptions(enable_network_backends=args.enable_network_backends, enable_browser_backends=args.enable_browser_backends),
|
|
372
|
+
timeout_seconds=args.timeout,
|
|
373
|
+
)
|
|
374
|
+
if args.json:
|
|
375
|
+
print(json.dumps({**report, "report_path": str(path)}, ensure_ascii=False))
|
|
376
|
+
else:
|
|
377
|
+
for row in report["results"]:
|
|
378
|
+
print(f"{row['platform']}: {row['status']} | {row['branch'] or '-'} | {row['reason'] or '-'}")
|
|
379
|
+
print(f"报告:{path}")
|
|
380
|
+
return 0 if report["ok"] else 1
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _worker_main() -> None:
|
|
384
|
+
payload = json.loads(sys.stdin.read())
|
|
385
|
+
sample = ProbeSample(**payload["sample"])
|
|
386
|
+
platform = payload["platform"]
|
|
387
|
+
try:
|
|
388
|
+
# Provider diagnostics never share stdout with the sanitized wire result.
|
|
389
|
+
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
390
|
+
row = probe_platform(platform, sample, BackendRuntimeOptions(enable_network_backends=payload["network"], enable_browser_backends=payload["browser"]))
|
|
391
|
+
except Exception as exc:
|
|
392
|
+
row = ProbeResult(platform=platform, configured=True, mode=sample.mode, error_code="probe_platform_error", reason=_safe_reason(exc, sample.url))
|
|
393
|
+
sys.stdout.buffer.write(json.dumps(asdict(row), ensure_ascii=False).encode("utf-8"))
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
if __name__ == "__main__":
|
|
397
|
+
_worker_main()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Declarative tool cards and source/capability routing."""
|