openmd-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. openmd/__init__.py +3 -0
  2. openmd/__main__.py +5 -0
  3. openmd/adapters/__init__.py +1 -0
  4. openmd/adapters/base.py +95 -0
  5. openmd/adapters/bilibili/__init__.py +11 -0
  6. openmd/adapters/bilibili/adapter.py +104 -0
  7. openmd/adapters/bilibili/discovery.py +33 -0
  8. openmd/adapters/douyin/__init__.py +11 -0
  9. openmd/adapters/douyin/adapter.py +117 -0
  10. openmd/adapters/douyin/discovery.py +36 -0
  11. openmd/adapters/douyin/sidecar.py +77 -0
  12. openmd/adapters/generic_media/__init__.py +14 -0
  13. openmd/adapters/high_privacy.py +77 -0
  14. openmd/adapters/local_audio/__init__.py +14 -0
  15. openmd/adapters/local_audio/adapter.py +169 -0
  16. openmd/adapters/local_audio/asr.py +58 -0
  17. openmd/adapters/local_audio/backends/__init__.py +1 -0
  18. openmd/adapters/local_audio/backends/auto.py +177 -0
  19. openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
  20. openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
  21. openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
  22. openmd/adapters/opencli.py +770 -0
  23. openmd/adapters/pdf/__init__.py +11 -0
  24. openmd/adapters/pdf/adapter.py +153 -0
  25. openmd/adapters/pdf/backends/__init__.py +1 -0
  26. openmd/adapters/pdf/backends/markitdown.py +165 -0
  27. openmd/adapters/pdf/parser.py +56 -0
  28. openmd/adapters/qq_group/__init__.py +11 -0
  29. openmd/adapters/qq_group/adapter.py +289 -0
  30. openmd/adapters/qq_group/parsers/__init__.py +1 -0
  31. openmd/adapters/qq_group/parsers/html.py +60 -0
  32. openmd/adapters/qq_group/parsers/json.py +71 -0
  33. openmd/adapters/registry.py +149 -0
  34. openmd/adapters/screenshot/__init__.py +11 -0
  35. openmd/adapters/screenshot/adapter.py +154 -0
  36. openmd/adapters/screenshot/backends/__init__.py +1 -0
  37. openmd/adapters/screenshot/backends/paddle.py +15 -0
  38. openmd/adapters/screenshot/backends/rapidocr.py +176 -0
  39. openmd/adapters/screenshot/backends/tesseract.py +15 -0
  40. openmd/adapters/screenshot/ocr.py +84 -0
  41. openmd/adapters/wechat_group/__init__.py +11 -0
  42. openmd/adapters/wechat_group/adapter.py +160 -0
  43. openmd/adapters/wechat_group/wx_cli.py +268 -0
  44. openmd/adapters/wechat_moments/__init__.py +11 -0
  45. openmd/adapters/wechat_moments/adapter.py +167 -0
  46. openmd/adapters/wechat_moments/wx_cli.py +265 -0
  47. openmd/adapters/weixin/__init__.py +11 -0
  48. openmd/adapters/weixin/adapter.py +148 -0
  49. openmd/adapters/weixin/discovery.py +510 -0
  50. openmd/adapters/weixin/report.py +278 -0
  51. openmd/adapters/xiaohongshu.py +158 -0
  52. openmd/backends/__init__.py +1 -0
  53. openmd/backends/base.py +204 -0
  54. openmd/backends/bilibili.py +1480 -0
  55. openmd/backends/bilibili_wbi.py +277 -0
  56. openmd/backends/discourse.py +426 -0
  57. openmd/backends/douyin.py +868 -0
  58. openmd/backends/douyin_detail.py +124 -0
  59. openmd/backends/douyin_providers.py +602 -0
  60. openmd/backends/executor.py +546 -0
  61. openmd/backends/feishu.py +861 -0
  62. openmd/backends/generic_media.py +77 -0
  63. openmd/backends/interactive.py +60 -0
  64. openmd/backends/ocr.py +41 -0
  65. openmd/backends/opencli.py +82 -0
  66. openmd/backends/pdf.py +122 -0
  67. openmd/backends/registry.py +234 -0
  68. openmd/backends/screenshot.py +79 -0
  69. openmd/backends/video_metadata.py +167 -0
  70. openmd/backends/web_article.py +363 -0
  71. openmd/backends/wechat_group.py +35 -0
  72. openmd/backends/wechat_moments.py +35 -0
  73. openmd/backends/weixin.py +767 -0
  74. openmd/backends/x.py +455 -0
  75. openmd/backends/xiaoe.py +940 -0
  76. openmd/backends/xiaohongshu.py +911 -0
  77. openmd/backends/xiaohongshu_providers.py +876 -0
  78. openmd/backends/xiaoyuzhou.py +1017 -0
  79. openmd/backends/youtube.py +280 -0
  80. openmd/backends/youtube_discovery.py +1245 -0
  81. openmd/backends/youtube_transcript.py +213 -0
  82. openmd/browser_extension/background.js +231 -0
  83. openmd/browser_extension/icons/icon-128.png +0 -0
  84. openmd/browser_extension/icons/icon-16.png +0 -0
  85. openmd/browser_extension/icons/icon-32.png +0 -0
  86. openmd/browser_extension/icons/icon-48.png +0 -0
  87. openmd/browser_extension/icons/icon-512.png +0 -0
  88. openmd/browser_extension/idle.html +6 -0
  89. openmd/browser_extension/manifest.json +43 -0
  90. openmd/browser_extension/options.html +42 -0
  91. openmd/browser_extension/options.js +117 -0
  92. openmd/browser_extension/popup.css +71 -0
  93. openmd/browser_extension/popup.html +17 -0
  94. openmd/browser_extension/popup.js +59 -0
  95. openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
  96. openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
  97. openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
  98. openmd/browser_host.py +503 -0
  99. openmd/browser_setup.py +106 -0
  100. openmd/cli.py +2012 -0
  101. openmd/console.py +32 -0
  102. openmd/core/__init__.py +1 -0
  103. openmd/core/capabilities.py +162 -0
  104. openmd/core/content_validation.py +323 -0
  105. openmd/core/coverage.py +163 -0
  106. openmd/core/detector.py +247 -0
  107. openmd/core/doctor.py +104 -0
  108. openmd/core/execution_policy.py +164 -0
  109. openmd/core/filename.py +64 -0
  110. openmd/core/input_validation.py +91 -0
  111. openmd/core/local_config.py +96 -0
  112. openmd/core/manifest.py +156 -0
  113. openmd/core/models.py +168 -0
  114. openmd/core/opencli_output.py +34 -0
  115. openmd/core/operation_contract.py +60 -0
  116. openmd/core/output.py +194 -0
  117. openmd/core/package_contract.py +335 -0
  118. openmd/core/package_validator.py +489 -0
  119. openmd/core/package_writer.py +666 -0
  120. openmd/core/planner.py +345 -0
  121. openmd/core/preferences.py +61 -0
  122. openmd/core/readable.py +129 -0
  123. openmd/core/receipt.py +285 -0
  124. openmd/core/renderer.py +185 -0
  125. openmd/core/runlog.py +208 -0
  126. openmd/core/secrets.py +313 -0
  127. openmd/doctor_view.py +68 -0
  128. openmd/interactive.py +453 -0
  129. openmd/ocr/__init__.py +1 -0
  130. openmd/ocr/breakers.py +84 -0
  131. openmd/ocr/config.py +54 -0
  132. openmd/ocr/documents.py +522 -0
  133. openmd/ocr/endpoint.py +72 -0
  134. openmd/ocr/layout.py +92 -0
  135. openmd/ocr/layout_filter.py +118 -0
  136. openmd/ocr/layout_worker.py +89 -0
  137. openmd/ocr/native_math.py +220 -0
  138. openmd/ocr/office.py +305 -0
  139. openmd/ocr/ollama.py +200 -0
  140. openmd/ocr/otsl.py +51 -0
  141. openmd/ocr/ovis-prompt.txt +1 -0
  142. openmd/ocr/pipeline.py +384 -0
  143. openmd/ocr/settings.json +25 -0
  144. openmd/ocr/tiling.py +631 -0
  145. openmd/post_video.py +154 -0
  146. openmd/probe.py +397 -0
  147. openmd/routing/__init__.py +1 -0
  148. openmd/routing/cards/builtin.json +620 -0
  149. openmd/routing/cards/defuddle.json +70 -0
  150. openmd/routing/cards/x-tweet-fetcher.json +71 -0
  151. openmd/routing/cards/xyz-dl.json +60 -0
  152. openmd/routing/cards/yutto.json +68 -0
  153. openmd/routing/entry_shim.py +71 -0
  154. openmd/routing/lifecycle.py +459 -0
  155. openmd/routing/loader.py +138 -0
  156. openmd/routing/providers.py +158 -0
  157. openmd/routing/readers.py +108 -0
  158. openmd/routing/routes.json +504 -0
  159. openmd/routing/runner.py +192 -0
  160. openmd/routing/view.py +40 -0
  161. openmd/runtime/__init__.py +1 -0
  162. openmd/runtime/_windows_job.py +234 -0
  163. openmd/runtime/asr.py +217 -0
  164. openmd/runtime/asr_worker.py +119 -0
  165. openmd/runtime/gpu_lock.py +81 -0
  166. openmd/runtime/gpu_quarantine.py +85 -0
  167. openmd/runtime/interpreters.py +407 -0
  168. openmd/runtime/managed_tools.py +204 -0
  169. openmd/runtime/models.py +202 -0
  170. openmd/runtime/services.py +175 -0
  171. openmd/runtime/tempdirs.py +227 -0
  172. openmd_cli-0.1.0.dist-info/METADATA +168 -0
  173. openmd_cli-0.1.0.dist-info/RECORD +177 -0
  174. openmd_cli-0.1.0.dist-info/WHEEL +5 -0
  175. openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
  176. openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
  177. openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
openmd/post_video.py ADDED
@@ -0,0 +1,154 @@
1
+ """Optional video step for posts whose body is text with an attached video (X for now).
2
+
3
+ Modes (``openmd go --video``):
4
+ - ``note`` (default): probe metadata only and add one line saying the post has a video.
5
+ - ``text``: temporary audio download plus local ASR; the transcript is appended to the same Markdown.
6
+ - ``file``: like ``text`` and also keep the video file next to the Markdown (assets folder).
7
+
8
+ The video step never fails the save: problems become a short note inside the Markdown.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ import shutil
15
+ import subprocess
16
+ import tempfile
17
+ from pathlib import Path
18
+ from urllib.parse import urlparse
19
+
20
+ from openmd.runtime.tempdirs import owned_mkdtemp
21
+
22
+ VIDEO_MODES = ("note", "text", "file")
23
+ MAX_VIDEOS = 4
24
+ PROBE_TIMEOUT_S = 45
25
+ DOWNLOAD_TIMEOUT_S = 600
26
+ _X_HOSTS = {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.x.com", "mobile.twitter.com"}
27
+ _X_STATUS = re.compile(r"^/[^/]+/status/[0-9]+")
28
+
29
+
30
+ def applies_to(url: str) -> bool:
31
+ parsed = urlparse(url)
32
+ return parsed.scheme in {"http", "https"} and parsed.netloc.lower() in _X_HOSTS and bool(_X_STATUS.match(parsed.path))
33
+
34
+
35
+ def _run(command: list[str], timeout: int) -> subprocess.CompletedProcess:
36
+ from openmd.runtime.interpreters import run_interruptibly
37
+
38
+ return run_interruptibly(command, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
39
+
40
+
41
+ def probe(url: str, *, run=_run) -> list[float] | None:
42
+ """Durations (seconds) of the post's videos; [] when it has none; None when unknown."""
43
+ if shutil.which("yt-dlp") is None:
44
+ return None
45
+ try:
46
+ done = run(["yt-dlp", "--simulate", "--no-warnings", "--playlist-items", f"1:{MAX_VIDEOS}", "--print", "%(duration)s", url], PROBE_TIMEOUT_S)
47
+ except (OSError, subprocess.TimeoutExpired):
48
+ return None
49
+ if done.returncode != 0:
50
+ return [] if "no video" in done.stderr.lower() else None
51
+ durations = []
52
+ for line in done.stdout.splitlines():
53
+ try:
54
+ durations.append(float(line.strip()))
55
+ except ValueError:
56
+ continue
57
+ return durations
58
+
59
+
60
+ def _clock(seconds: float) -> str:
61
+ total = int(round(seconds))
62
+ minutes, sec = divmod(total, 60)
63
+ return f"{minutes} 分 {sec} 秒" if minutes else f"{sec} 秒"
64
+
65
+
66
+ def _summary(durations: list[float]) -> str:
67
+ if len(durations) == 1:
68
+ return f"本帖含视频(约 {_clock(durations[0])})"
69
+ return f"本帖含 {len(durations)} 段视频(共约 {_clock(sum(durations))})"
70
+
71
+
72
+ def _download(url: str, workdir: Path, *, keep_video: bool, run) -> list[Path]:
73
+ template = str(workdir / "video-%(autonumber)02d.%(ext)s")
74
+ # Progressive mp4 only: X's HLS audio-only streams stall behind proxies (verified 2026-11-08),
75
+ # while the single-file mp4 downloads in seconds. For text, a small rendition is enough.
76
+ command = ["yt-dlp", "--no-warnings", "--playlist-items", f"1:{MAX_VIDEOS}", "--socket-timeout", "30", "--retries", "3",
77
+ "-f", "b[ext=mp4][protocol^=http]/b[ext=mp4]/b", "-o", template, url]
78
+ if not keep_video:
79
+ command[1:1] = ["-S", "res:480"]
80
+ done = run(command, DOWNLOAD_TIMEOUT_S)
81
+ if done.returncode != 0:
82
+ raise RuntimeError("视频下载失败")
83
+ files = sorted(p for p in workdir.iterdir() if p.is_file() and p.suffix.lower() in {".mp4", ".wav", ".m4a", ".webm", ".mkv"})
84
+ if not files:
85
+ raise RuntimeError("视频下载后没有找到文件")
86
+ return files
87
+
88
+
89
+ def _transcribe(path: Path, transcribe) -> str:
90
+ from openmd.core.readable import format_transcript
91
+
92
+ result = transcribe(path)
93
+ if not getattr(result, "has_usable_text", False):
94
+ raise RuntimeError("语音识别没有得到文字")
95
+ return format_transcript(result.text, source_type="video", platform="x").strip()
96
+
97
+
98
+ def video_section(url: str, mode: str, markdown_dir: Path, *, run=_run, transcribe=None) -> str | None:
99
+ """Markdown to append for ``url``; None when the post has no video or nothing can be said."""
100
+ if mode not in VIDEO_MODES or not applies_to(url):
101
+ return None
102
+ durations = probe(url, run=run)
103
+ if not durations:
104
+ return None
105
+ summary = _summary(durations)
106
+ if mode == "note":
107
+ return f"> {summary},未转文字。需要时用插件右键「用 Openmd 保存(含视频文字版)」或 `openmd go --video text`。\n"
108
+ if transcribe is None:
109
+ from openmd.runtime.asr import get_asr_backend
110
+
111
+ transcribe = get_asr_backend("asr.auto").transcribe
112
+ from openmd.runtime.interpreters import remove_tree_with_retry
113
+
114
+ workdir = Path(owned_mkdtemp(prefix="openmd-x-video-"))
115
+ try:
116
+ try:
117
+ files = _download(url, workdir, keep_video=mode == "file", run=run)
118
+ except (OSError, subprocess.TimeoutExpired, RuntimeError) as exc:
119
+ return f"> {summary},转文字未完成:{exc if isinstance(exc, RuntimeError) else '视频下载失败'}。\n"
120
+ parts = []
121
+ embeds = []
122
+ for index, path in enumerate(files, 1):
123
+ heading = "## 视频逐字稿" if len(files) == 1 else f"## 视频逐字稿(第 {index} 段)"
124
+ try:
125
+ parts.append(f"{heading}\n\n{_transcribe(path, transcribe)}\n")
126
+ except RuntimeError as exc:
127
+ parts.append(f"{heading}\n\n> 转文字未完成:{exc}。\n")
128
+ if mode == "file":
129
+ assets = markdown_dir / "assets"
130
+ assets.mkdir(exist_ok=True)
131
+ target = assets / path.name
132
+ shutil.move(str(path), target)
133
+ embeds.append(f"![视频](<assets/{target.name}>)") # <> keeps paths with spaces valid
134
+ body = "\n".join(parts)
135
+ if embeds:
136
+ body = "\n\n".join(embeds) + "\n\n" + body
137
+ return f"> {summary};以下逐字稿由本机语音识别生成。\n\n{body}"
138
+ finally:
139
+ remove_tree_with_retry(workdir)
140
+
141
+
142
+ def append_video_section(markdown_path: Path, url: str, mode: str, **kwargs) -> bool:
143
+ section = video_section(url, mode, markdown_path.parent, **kwargs)
144
+ if not section:
145
+ return False
146
+ text = markdown_path.read_text(encoding="utf-8").rstrip("\n")
147
+ body, sep, last = text.rpartition("\n")
148
+ if sep and last.startswith("*来源"):
149
+ # Keep the source footer last; the video section belongs to the post body.
150
+ updated = f"{body.rstrip()}\n\n{section.rstrip()}\n\n{last}\n"
151
+ else:
152
+ updated = f"{text}\n\n{section.rstrip()}\n"
153
+ markdown_path.write_text(updated, encoding="utf-8")
154
+ return True
openmd/probe.py ADDED
@@ -0,0 +1,397 @@
1
+ """Bounded platform health checks through existing extraction/discovery routes.
2
+
3
+ Samples stay in machine-local configuration. Reports contain no URLs, source
4
+ text, response payloads, or credentials, and never enable media/ASR/cloud work.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import contextlib
11
+ import io
12
+ import json
13
+ import os
14
+ import re
15
+ import subprocess
16
+ import sys
17
+ import time
18
+ from dataclasses import asdict, dataclass, field
19
+ from datetime import datetime, timezone
20
+ from pathlib import Path
21
+ from typing import Any
22
+
23
+ from openmd.adapters.registry import create_default_registry
24
+ from openmd.backends.base import DiscoveryRequest
25
+ from openmd.backends.executor import BackendExecutor, DiscoveryExecutor
26
+ from openmd.backends.registry import (
27
+ BackendRuntimeOptions,
28
+ create_runtime_backend_provider_registry,
29
+ create_runtime_discovery_provider_registry,
30
+ )
31
+ from openmd.core.detector import PLATFORM_IDS, detect_platform
32
+ from openmd.core.local_config import checkout_root
33
+ from openmd.core.secrets import sanitize_error_summary
34
+ from openmd.routing.loader import load_catalog
35
+ from openmd.runtime.interpreters import run_interruptibly, worker_command
36
+
37
+
38
+ CONFIG_SCHEMA = "openmd.probe-config/v1"
39
+ REPORT_SCHEMA = "openmd.probe/v1"
40
+ _FORBIDDEN_GATES = frozenset({"media_download", "asr", "cloud", "local_compute", "local_export"})
41
+ # This legacy builtin only declares a browser gate, but downloads and sends
42
+ # media to Tingwu. A probe must never invoke it, even with browser permission.
43
+ _FORBIDDEN_BACKENDS = frozenset({"opencli.xiaoe.transcript"})
44
+ _BOUNDED_DISCOVERY = {
45
+ "bilibili": {"bilibili-harvest.discovery": "network", "opencli.bilibili.user-videos": "browser"},
46
+ "youtube": {"opencli.youtube.discovery": "browser", "opencli.youtube.playlist.discovery": "browser", "youtube.data-api.discovery": "network", "youtube.data-api.playlist.discovery": "network"},
47
+ "douyin": {"opencli.douyin.user-videos": "browser"},
48
+ "xiaohongshu": {"opencli.xiaohongshu.user": "browser"},
49
+ "xiaoyuzhou": {"opencli.xiaoyuzhou.podcast-episodes": "browser"},
50
+ "xiaoe": {"opencli.xiaoe.catalog": "browser"},
51
+ }
52
+ _URL_RE = re.compile(r"(?:https?|file)://[^\s<>\"'\])]+", re.I)
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class ProbeSample:
57
+ url: str
58
+ mode: str = "extract"
59
+ discovery_backend: str | None = None
60
+
61
+
62
+ @dataclass
63
+ class ProbeResult:
64
+ platform: str
65
+ configured: bool
66
+ status: str = "broken"
67
+ mode: str = "extract"
68
+ branch: str | None = None
69
+ reason: str | None = None
70
+ error_code: str | None = None
71
+ extraction_status: str | None = None
72
+ source_level: str | None = None
73
+ text_chars: int = 0
74
+ item_count: int = 0
75
+ route_version: str | None = None
76
+ tool_version: str | None = None
77
+ elapsed_seconds: float = 0.0
78
+ live_route_attempted: bool = False
79
+ attempts: list[dict[str, Any]] = field(default_factory=list)
80
+ excluded_branches: list[dict[str, str]] = field(default_factory=list)
81
+ detail_readable: bool | None = None
82
+ transcription: str | None = None
83
+
84
+
85
+ def default_config_path() -> Path:
86
+ explicit = os.environ.get("OPENMD_PROBE_CONFIG")
87
+ if explicit:
88
+ return Path(explicit).expanduser()
89
+ root = checkout_root()
90
+ return root / "out/probe-config.json" if root else Path.home() / ".openmd/probe-config.json"
91
+
92
+
93
+ def load_probe_samples(path: Path, *, required: bool = False) -> dict[str, ProbeSample]:
94
+ """Read one sample per platform; reject credentials and executable options."""
95
+ if not path.is_file():
96
+ if required:
97
+ raise ValueError("probe_config_missing")
98
+ return {}
99
+ if path.stat().st_size > 128 * 1024:
100
+ raise ValueError("probe_config_too_large")
101
+ try:
102
+ data = json.loads(path.read_text(encoding="utf-8-sig"))
103
+ except (OSError, ValueError) as exc:
104
+ raise ValueError("probe_config_invalid_json") from exc
105
+ if not isinstance(data, dict) or data.get("schema") != CONFIG_SCHEMA or set(data) - {"schema", "samples"}:
106
+ raise ValueError("probe_config_invalid_schema")
107
+ samples = data.get("samples")
108
+ if not isinstance(samples, dict) or set(samples) - set(PLATFORM_IDS):
109
+ raise ValueError("probe_config_invalid_platforms")
110
+ result: dict[str, ProbeSample] = {}
111
+ for platform, value in samples.items():
112
+ if not isinstance(value, dict) or set(value) - {"url", "mode", "discovery_backend"}:
113
+ raise ValueError("probe_config_invalid_sample_fields")
114
+ url = value.get("url", "")
115
+ mode = value.get("mode", "extract")
116
+ backend = value.get("discovery_backend")
117
+ if not isinstance(url, str) or mode not in {"extract", "manifest"}:
118
+ raise ValueError("probe_config_invalid_sample")
119
+ if backend is not None and (not isinstance(backend, str) or not backend or mode != "manifest"):
120
+ raise ValueError("probe_config_invalid_discovery_backend")
121
+ if url.strip():
122
+ result[platform] = ProbeSample(url.strip(), mode, backend)
123
+ return result
124
+
125
+
126
+ def _safe_reason(value: object, sample: str = "") -> str | None:
127
+ if value is None:
128
+ return None
129
+ text = str(value)
130
+ if sample:
131
+ text = text.replace(sample, "[sample]")
132
+ text = _URL_RE.sub("[url]", text)
133
+ text = re.sub(
134
+ r"(?i)(?:cookie|token|authorization|pass|secret|credential|api[_-]?key)"
135
+ r"[\w-]*[\"']?\s*[:=]\s*[\"']?[^\s,;}]+",
136
+ "[redacted]", text,
137
+ )
138
+ return sanitize_error_summary(text)
139
+
140
+
141
+ def _safe_chain(platform: str, chain: tuple[str, ...], options: BackendRuntimeOptions) -> tuple[tuple[str, ...], list[dict[str, str]]]:
142
+ catalog = load_catalog()
143
+ safe: list[str] = []
144
+ excluded: list[dict[str, str]] = []
145
+ for backend in chain:
146
+ cap = catalog.capabilities.get(backend)
147
+ reason = None
148
+ gates = set(cap.get("gates", [])) if cap else set()
149
+ if backend in _FORBIDDEN_BACKENDS or gates & _FORBIDDEN_GATES:
150
+ reason = "probe_disallows_media_asr_cloud_or_compute"
151
+ elif cap is None or platform not in cap.get("sources", []):
152
+ reason = "probe_unknown_route"
153
+ elif "browser" in gates and not options.enable_browser_backends:
154
+ reason = "browser_gate_closed"
155
+ elif "network" in gates and not options.enable_network_backends:
156
+ reason = "network_gate_closed"
157
+ if reason:
158
+ excluded.append({"branch": backend, "reason": reason})
159
+ else:
160
+ safe.append(backend)
161
+ return tuple(safe), excluded
162
+
163
+
164
+ def probe_platform(platform: str, sample: ProbeSample, options: BackendRuntimeOptions) -> ProbeResult:
165
+ """One bounded sample; exceptions are isolated by the outer report runner."""
166
+ row = ProbeResult(platform=platform, configured=True, mode=sample.mode)
167
+ if detect_platform(sample.url) != platform:
168
+ row.reason = row.error_code = "sample_platform_mismatch"
169
+ return row
170
+ # URL-based sample resolution and discovery require an explicit gate. In
171
+ # particular, do not let locally registered metadata providers fake health.
172
+ if not (options.enable_network_backends or options.enable_browser_backends):
173
+ row.reason = row.error_code = "probe_network_gates_closed"
174
+ return row
175
+ if sample.mode == "manifest":
176
+ runtime = BackendRuntimeOptions(
177
+ requested_discovery_provider=sample.discovery_backend,
178
+ enable_network_backends=options.enable_network_backends,
179
+ enable_browser_backends=options.enable_browser_backends,
180
+ )
181
+ providers = create_runtime_discovery_provider_registry(platform, runtime)
182
+ providers.allow_skeleton_fallback = False
183
+ request = DiscoveryRequest(input_url=sample.url, platform=platform, limit=1, allow_all=False)
184
+ provider = providers.provider_for(request, requested_provider=sample.discovery_backend)
185
+ provider_name = getattr(provider, "name", "not_selected")
186
+ gate = _BOUNDED_DISCOVERY.get(platform, {}).get(provider_name)
187
+ if gate is None or not getattr(options, f"enable_{gate}_backends", False):
188
+ row.branch = provider_name
189
+ row.reason = row.error_code = "probe_discovery_not_bounded_or_gate_closed"
190
+ return row
191
+ result = DiscoveryExecutor(provider_registry=providers, requested_provider=sample.discovery_backend).discover(
192
+ request
193
+ )
194
+ row.branch = result.provider
195
+ row.item_count = len(result.items)
196
+ row.error_code = _safe_reason(result.error_code, sample.url)
197
+ row.reason = _safe_reason(result.error_summary, sample.url)
198
+ row.status = "ok" if result.ok and result.items and result.provider not in {"mock.discovery", "skeleton.discovery"} else "broken"
199
+ invalid_sample = result.error_code in {
200
+ "douyin_single_video_not_supported_for_discovery", "douyin_short_link_resolution_required", "douyin_user_sec_uid_not_found",
201
+ }
202
+ row.live_route_attempted = not invalid_sample
203
+ if invalid_sample:
204
+ row.status = "degraded"
205
+ row.reason = "missing_valid_probe_sample; configured selector is unsupported for bounded discovery; live route not covered"
206
+ if row.status == "broken" and not row.reason:
207
+ row.reason = row.error_code = "probe_manifest_empty_or_non_live"
208
+ row.attempts.append({"branch": row.branch, "status": row.status, "error_code": row.error_code, "reason": row.reason})
209
+ return row
210
+
211
+ if platform == "douyin":
212
+ # Health probing reads one work's detail only. It never downloads media
213
+ # or invokes ASR, so detail availability cannot certify transcription.
214
+ from openmd.backends.douyin_detail import DouyinDetailError, DouyinDetailProvider
215
+
216
+ row.branch = "opencli.douyin.detail"
217
+ row.transcription = "not_probed_media_asr_disallowed"
218
+ if not options.enable_browser_backends:
219
+ row.reason = row.error_code = "browser_gate_closed"
220
+ return row
221
+ adapter = create_default_registry().get_adapter(platform)
222
+ items = adapter.resolve_items(sample.url, limit=None, all=False)
223
+ if len(items) != 1 or items[0].input_type != "single":
224
+ row.reason = row.error_code = "probe_requires_single_item_or_manifest_mode"
225
+ return row
226
+ row.live_route_attempted = True
227
+ try:
228
+ DouyinDetailProvider().resolve(items[0])
229
+ row.detail_readable = True
230
+ row.item_count = 1
231
+ row.status = "degraded"
232
+ row.reason = "detail_readable; transcription_not_probed_media_asr_disallowed"
233
+ except DouyinDetailError as exc:
234
+ row.detail_readable = False
235
+ row.error_code = exc.code
236
+ row.reason = str(exc)
237
+ row.attempts.append({"branch": row.branch, "status": row.status, "error_code": row.error_code, "reason": row.reason})
238
+ return row
239
+
240
+ adapters = create_default_registry()
241
+ chain, row.excluded_branches = _safe_chain(platform, adapters.backend_chain(platform), options)
242
+ if not chain:
243
+ row.reason = row.error_code = "probe_no_safe_enabled_route"
244
+ return row
245
+ providers = create_runtime_backend_provider_registry(platform, options)
246
+ providers.allow_skeleton_fallback = False
247
+ items = adapters.get_adapter(platform).resolve_items(sample.url, limit=None, all=False)
248
+ if len(items) != 1 or items[0].input_type == "collection":
249
+ row.reason = row.error_code = "probe_requires_single_item_or_manifest_mode"
250
+ return row
251
+ executor = BackendExecutor(provider_registry=providers)
252
+ for backend in chain:
253
+ row.live_route_attempted = row.live_route_attempted or (
254
+ backend not in {"douyin.metadata", "manual.markdown-export"} and providers.provider_for(backend) is not None
255
+ )
256
+ result = executor.extract(items[0], (backend,), allow_fallback=False)
257
+ row.branch = result.backend
258
+ row.extraction_status = result.status
259
+ row.source_level = result.source_level
260
+ row.route_version = result.metadata.get("route_version")
261
+ row.tool_version = result.metadata.get("tool_run", {}).get("tool_version")
262
+ row.error_code = _safe_reason(result.error_code, sample.url)
263
+ row.reason = _safe_reason(result.error_summary, sample.url)
264
+ row.text_chars = len(result.text)
265
+ row.attempts.append({"branch": backend, "actual_branch": result.backend, "status": result.status, "error_code": row.error_code, "reason": row.reason})
266
+ if result.has_usable_text:
267
+ row.item_count = 1
268
+ if result.metadata.get("metadata_only") or backend == "douyin.metadata":
269
+ row.status = "degraded"
270
+ row.error_code = "missing_valid_probe_sample"
271
+ row.reason = "missing_valid_probe_sample; metadata_only_not_live_extraction; live route not covered"
272
+ elif result.status == "partial" or result.has_terminal_gaps or len(row.attempts) > 1:
273
+ row.status = "degraded"
274
+ row.reason = row.reason or "partial_content_or_fallback_recovered"
275
+ else:
276
+ row.status = "ok"
277
+ return row
278
+ row.reason = row.reason or "probe_no_usable_content"
279
+ return row
280
+
281
+
282
+ def _write_report(path: Path, report: dict[str, Any]) -> None:
283
+ temporary = path.with_suffix(".tmp")
284
+ temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
285
+ temporary.replace(path)
286
+
287
+
288
+ def _probe_isolated(platform: str, sample: ProbeSample, options: BackendRuntimeOptions, timeout_seconds: int) -> ProbeResult:
289
+ """Own the full provider subtree so one slow platform cannot stall all probes."""
290
+ payload = {"platform": platform, "sample": asdict(sample), "network": options.enable_network_backends, "browser": options.enable_browser_backends}
291
+ try:
292
+ process = run_interruptibly(
293
+ worker_command(Path(sys.executable), module="openmd.probe"),
294
+ input=json.dumps(payload), capture_output=True, text=True,
295
+ encoding="utf-8", errors="replace", timeout=timeout_seconds, check=False,
296
+ )
297
+ except subprocess.TimeoutExpired:
298
+ return ProbeResult(platform=platform, configured=True, mode=sample.mode, error_code="probe_timeout", reason=f"platform exceeded {timeout_seconds}s; owned subprocess tree reclaimed")
299
+ if process.returncode:
300
+ return ProbeResult(platform=platform, configured=True, mode=sample.mode, error_code="probe_worker_failed", reason=f"probe worker exit {process.returncode}")
301
+ try:
302
+ return ProbeResult(**json.loads(process.stdout))
303
+ except (ValueError, TypeError):
304
+ return ProbeResult(platform=platform, configured=True, mode=sample.mode, error_code="probe_worker_invalid_result", reason="probe worker returned an invalid report")
305
+
306
+
307
+ def run_probe(
308
+ samples: dict[str, ProbeSample], *, out: Path, options: BackendRuntimeOptions,
309
+ platforms: tuple[str, ...] = PLATFORM_IDS, interval_seconds: float = 1.0, timeout_seconds: int = 90,
310
+ ) -> tuple[dict[str, Any], Path]:
311
+ now = datetime.now(timezone.utc)
312
+ directory = out / now.astimezone().date().isoformat()
313
+ directory.mkdir(parents=True, exist_ok=True)
314
+ path = directory / f"probe-{now.strftime('%H%M%S-%f')}.json"
315
+ report: dict[str, Any] = {
316
+ "schema": REPORT_SCHEMA, "started_at": now.isoformat(), "completed": False,
317
+ "route_version": load_catalog().version,
318
+ "gates": {"network": options.enable_network_backends, "browser": options.enable_browser_backends, "media_download": False, "asr": False, "cloud": False},
319
+ "sample_policy": "one configured sample per platform; no URL or source content persisted",
320
+ "rate_limit_seconds": interval_seconds, "timeout_seconds": timeout_seconds, "results": [],
321
+ }
322
+ _write_report(path, report)
323
+ attempted = 0
324
+ # Rebuild options instead of trusting a caller to leave the heavy gates shut.
325
+ bounded_options = BackendRuntimeOptions(enable_network_backends=options.enable_network_backends, enable_browser_backends=options.enable_browser_backends)
326
+ # Provider commands run inside owned platform workers. Start the one allowed
327
+ # persistent service here, before those outer Jobs can acquire descendants.
328
+ if bounded_options.enable_browser_backends and any(platform in samples for platform in platforms):
329
+ from openmd.runtime.services import ensure_opencli_daemon
330
+
331
+ try:
332
+ ensure_opencli_daemon(["opencli", "probe-platform-read"])
333
+ except OSError as exc:
334
+ # Preserve independent network-only routes if the browser daemon is
335
+ # unavailable. Each browser provider still reports its own failure.
336
+ report["browser_preflight_reason"] = _safe_reason(exc)
337
+ for platform in platforms:
338
+ sample = samples.get(platform)
339
+ if sample is None:
340
+ row = ProbeResult(platform=platform, configured=False, status="degraded", reason="sample_not_configured")
341
+ else:
342
+ if attempted:
343
+ time.sleep(max(0.0, interval_seconds))
344
+ attempted += 1
345
+ started = time.monotonic()
346
+ report["current_platform"] = platform
347
+ _write_report(path, report)
348
+ try:
349
+ row = _probe_isolated(platform, sample, bounded_options, timeout_seconds)
350
+ except Exception as exc:
351
+ row = ProbeResult(platform=platform, configured=True, mode=sample.mode, reason=_safe_reason(exc, sample.url), error_code="probe_platform_error")
352
+ row.elapsed_seconds = round(time.monotonic() - started, 3)
353
+ report["results"].append(asdict(row))
354
+ _write_report(path, report) # crash/cancel preserves completed platforms
355
+ report["summary"] = {status: sum(r["status"] == status for r in report["results"]) for status in ("ok", "degraded", "broken")}
356
+ report["configured_count"] = attempted
357
+ report["ok"] = attempted > 0 and all(row["status"] == "ok" for row in report["results"] if row["configured"])
358
+ report["completed"] = True
359
+ report.pop("current_platform", None)
360
+ report["finished_at"] = datetime.now(timezone.utc).isoformat()
361
+ _write_report(path, report)
362
+ return report, path
363
+
364
+
365
+ def cmd_probe(args: argparse.Namespace) -> int:
366
+ config = Path(args.config).expanduser() if args.config else default_config_path()
367
+ samples = load_probe_samples(config, required=bool(args.config or os.environ.get("OPENMD_PROBE_CONFIG")))
368
+ platforms = (args.platform,) if args.platform else PLATFORM_IDS
369
+ report, path = run_probe(
370
+ samples, out=Path(args.out), platforms=platforms,
371
+ options=BackendRuntimeOptions(enable_network_backends=args.enable_network_backends, enable_browser_backends=args.enable_browser_backends),
372
+ timeout_seconds=args.timeout,
373
+ )
374
+ if args.json:
375
+ print(json.dumps({**report, "report_path": str(path)}, ensure_ascii=False))
376
+ else:
377
+ for row in report["results"]:
378
+ print(f"{row['platform']}: {row['status']} | {row['branch'] or '-'} | {row['reason'] or '-'}")
379
+ print(f"报告:{path}")
380
+ return 0 if report["ok"] else 1
381
+
382
+
383
+ def _worker_main() -> None:
384
+ payload = json.loads(sys.stdin.read())
385
+ sample = ProbeSample(**payload["sample"])
386
+ platform = payload["platform"]
387
+ try:
388
+ # Provider diagnostics never share stdout with the sanitized wire result.
389
+ with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
390
+ row = probe_platform(platform, sample, BackendRuntimeOptions(enable_network_backends=payload["network"], enable_browser_backends=payload["browser"]))
391
+ except Exception as exc:
392
+ row = ProbeResult(platform=platform, configured=True, mode=sample.mode, error_code="probe_platform_error", reason=_safe_reason(exc, sample.url))
393
+ sys.stdout.buffer.write(json.dumps(asdict(row), ensure_ascii=False).encode("utf-8"))
394
+
395
+
396
+ if __name__ == "__main__":
397
+ _worker_main()
@@ -0,0 +1 @@
1
+ """Declarative tool cards and source/capability routing."""