openmd-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. openmd/__init__.py +3 -0
  2. openmd/__main__.py +5 -0
  3. openmd/adapters/__init__.py +1 -0
  4. openmd/adapters/base.py +95 -0
  5. openmd/adapters/bilibili/__init__.py +11 -0
  6. openmd/adapters/bilibili/adapter.py +104 -0
  7. openmd/adapters/bilibili/discovery.py +33 -0
  8. openmd/adapters/douyin/__init__.py +11 -0
  9. openmd/adapters/douyin/adapter.py +117 -0
  10. openmd/adapters/douyin/discovery.py +36 -0
  11. openmd/adapters/douyin/sidecar.py +77 -0
  12. openmd/adapters/generic_media/__init__.py +14 -0
  13. openmd/adapters/high_privacy.py +77 -0
  14. openmd/adapters/local_audio/__init__.py +14 -0
  15. openmd/adapters/local_audio/adapter.py +169 -0
  16. openmd/adapters/local_audio/asr.py +58 -0
  17. openmd/adapters/local_audio/backends/__init__.py +1 -0
  18. openmd/adapters/local_audio/backends/auto.py +177 -0
  19. openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
  20. openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
  21. openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
  22. openmd/adapters/opencli.py +770 -0
  23. openmd/adapters/pdf/__init__.py +11 -0
  24. openmd/adapters/pdf/adapter.py +153 -0
  25. openmd/adapters/pdf/backends/__init__.py +1 -0
  26. openmd/adapters/pdf/backends/markitdown.py +165 -0
  27. openmd/adapters/pdf/parser.py +56 -0
  28. openmd/adapters/qq_group/__init__.py +11 -0
  29. openmd/adapters/qq_group/adapter.py +289 -0
  30. openmd/adapters/qq_group/parsers/__init__.py +1 -0
  31. openmd/adapters/qq_group/parsers/html.py +60 -0
  32. openmd/adapters/qq_group/parsers/json.py +71 -0
  33. openmd/adapters/registry.py +149 -0
  34. openmd/adapters/screenshot/__init__.py +11 -0
  35. openmd/adapters/screenshot/adapter.py +154 -0
  36. openmd/adapters/screenshot/backends/__init__.py +1 -0
  37. openmd/adapters/screenshot/backends/paddle.py +15 -0
  38. openmd/adapters/screenshot/backends/rapidocr.py +176 -0
  39. openmd/adapters/screenshot/backends/tesseract.py +15 -0
  40. openmd/adapters/screenshot/ocr.py +84 -0
  41. openmd/adapters/wechat_group/__init__.py +11 -0
  42. openmd/adapters/wechat_group/adapter.py +160 -0
  43. openmd/adapters/wechat_group/wx_cli.py +268 -0
  44. openmd/adapters/wechat_moments/__init__.py +11 -0
  45. openmd/adapters/wechat_moments/adapter.py +167 -0
  46. openmd/adapters/wechat_moments/wx_cli.py +265 -0
  47. openmd/adapters/weixin/__init__.py +11 -0
  48. openmd/adapters/weixin/adapter.py +148 -0
  49. openmd/adapters/weixin/discovery.py +510 -0
  50. openmd/adapters/weixin/report.py +278 -0
  51. openmd/adapters/xiaohongshu.py +158 -0
  52. openmd/backends/__init__.py +1 -0
  53. openmd/backends/base.py +204 -0
  54. openmd/backends/bilibili.py +1480 -0
  55. openmd/backends/bilibili_wbi.py +277 -0
  56. openmd/backends/discourse.py +426 -0
  57. openmd/backends/douyin.py +868 -0
  58. openmd/backends/douyin_detail.py +124 -0
  59. openmd/backends/douyin_providers.py +602 -0
  60. openmd/backends/executor.py +546 -0
  61. openmd/backends/feishu.py +861 -0
  62. openmd/backends/generic_media.py +77 -0
  63. openmd/backends/interactive.py +60 -0
  64. openmd/backends/ocr.py +41 -0
  65. openmd/backends/opencli.py +82 -0
  66. openmd/backends/pdf.py +122 -0
  67. openmd/backends/registry.py +234 -0
  68. openmd/backends/screenshot.py +79 -0
  69. openmd/backends/video_metadata.py +167 -0
  70. openmd/backends/web_article.py +363 -0
  71. openmd/backends/wechat_group.py +35 -0
  72. openmd/backends/wechat_moments.py +35 -0
  73. openmd/backends/weixin.py +767 -0
  74. openmd/backends/x.py +455 -0
  75. openmd/backends/xiaoe.py +940 -0
  76. openmd/backends/xiaohongshu.py +911 -0
  77. openmd/backends/xiaohongshu_providers.py +876 -0
  78. openmd/backends/xiaoyuzhou.py +1017 -0
  79. openmd/backends/youtube.py +280 -0
  80. openmd/backends/youtube_discovery.py +1245 -0
  81. openmd/backends/youtube_transcript.py +213 -0
  82. openmd/browser_extension/background.js +231 -0
  83. openmd/browser_extension/icons/icon-128.png +0 -0
  84. openmd/browser_extension/icons/icon-16.png +0 -0
  85. openmd/browser_extension/icons/icon-32.png +0 -0
  86. openmd/browser_extension/icons/icon-48.png +0 -0
  87. openmd/browser_extension/icons/icon-512.png +0 -0
  88. openmd/browser_extension/idle.html +6 -0
  89. openmd/browser_extension/manifest.json +43 -0
  90. openmd/browser_extension/options.html +42 -0
  91. openmd/browser_extension/options.js +117 -0
  92. openmd/browser_extension/popup.css +71 -0
  93. openmd/browser_extension/popup.html +17 -0
  94. openmd/browser_extension/popup.js +59 -0
  95. openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
  96. openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
  97. openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
  98. openmd/browser_host.py +503 -0
  99. openmd/browser_setup.py +106 -0
  100. openmd/cli.py +2012 -0
  101. openmd/console.py +32 -0
  102. openmd/core/__init__.py +1 -0
  103. openmd/core/capabilities.py +162 -0
  104. openmd/core/content_validation.py +323 -0
  105. openmd/core/coverage.py +163 -0
  106. openmd/core/detector.py +247 -0
  107. openmd/core/doctor.py +104 -0
  108. openmd/core/execution_policy.py +164 -0
  109. openmd/core/filename.py +64 -0
  110. openmd/core/input_validation.py +91 -0
  111. openmd/core/local_config.py +96 -0
  112. openmd/core/manifest.py +156 -0
  113. openmd/core/models.py +168 -0
  114. openmd/core/opencli_output.py +34 -0
  115. openmd/core/operation_contract.py +60 -0
  116. openmd/core/output.py +194 -0
  117. openmd/core/package_contract.py +335 -0
  118. openmd/core/package_validator.py +489 -0
  119. openmd/core/package_writer.py +666 -0
  120. openmd/core/planner.py +345 -0
  121. openmd/core/preferences.py +61 -0
  122. openmd/core/readable.py +129 -0
  123. openmd/core/receipt.py +285 -0
  124. openmd/core/renderer.py +185 -0
  125. openmd/core/runlog.py +208 -0
  126. openmd/core/secrets.py +313 -0
  127. openmd/doctor_view.py +68 -0
  128. openmd/interactive.py +453 -0
  129. openmd/ocr/__init__.py +1 -0
  130. openmd/ocr/breakers.py +84 -0
  131. openmd/ocr/config.py +54 -0
  132. openmd/ocr/documents.py +522 -0
  133. openmd/ocr/endpoint.py +72 -0
  134. openmd/ocr/layout.py +92 -0
  135. openmd/ocr/layout_filter.py +118 -0
  136. openmd/ocr/layout_worker.py +89 -0
  137. openmd/ocr/native_math.py +220 -0
  138. openmd/ocr/office.py +305 -0
  139. openmd/ocr/ollama.py +200 -0
  140. openmd/ocr/otsl.py +51 -0
  141. openmd/ocr/ovis-prompt.txt +1 -0
  142. openmd/ocr/pipeline.py +384 -0
  143. openmd/ocr/settings.json +25 -0
  144. openmd/ocr/tiling.py +631 -0
  145. openmd/post_video.py +154 -0
  146. openmd/probe.py +397 -0
  147. openmd/routing/__init__.py +1 -0
  148. openmd/routing/cards/builtin.json +620 -0
  149. openmd/routing/cards/defuddle.json +70 -0
  150. openmd/routing/cards/x-tweet-fetcher.json +71 -0
  151. openmd/routing/cards/xyz-dl.json +60 -0
  152. openmd/routing/cards/yutto.json +68 -0
  153. openmd/routing/entry_shim.py +71 -0
  154. openmd/routing/lifecycle.py +459 -0
  155. openmd/routing/loader.py +138 -0
  156. openmd/routing/providers.py +158 -0
  157. openmd/routing/readers.py +108 -0
  158. openmd/routing/routes.json +504 -0
  159. openmd/routing/runner.py +192 -0
  160. openmd/routing/view.py +40 -0
  161. openmd/runtime/__init__.py +1 -0
  162. openmd/runtime/_windows_job.py +234 -0
  163. openmd/runtime/asr.py +217 -0
  164. openmd/runtime/asr_worker.py +119 -0
  165. openmd/runtime/gpu_lock.py +81 -0
  166. openmd/runtime/gpu_quarantine.py +85 -0
  167. openmd/runtime/interpreters.py +407 -0
  168. openmd/runtime/managed_tools.py +204 -0
  169. openmd/runtime/models.py +202 -0
  170. openmd/runtime/services.py +175 -0
  171. openmd/runtime/tempdirs.py +227 -0
  172. openmd_cli-0.1.0.dist-info/METADATA +168 -0
  173. openmd_cli-0.1.0.dist-info/RECORD +177 -0
  174. openmd_cli-0.1.0.dist-info/WHEEL +5 -0
  175. openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
  176. openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
  177. openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
openmd/cli.py ADDED
@@ -0,0 +1,2012 @@
1
+ """Argparse-based Openmd CLI skeleton."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import shutil
8
+ import sys
9
+ import time
10
+ from dataclasses import asdict, dataclass, field, replace
11
+ from pathlib import Path
12
+
13
+ from openmd.adapters.registry import create_default_registry
14
+ from openmd.backends.base import DiscoveryRequest
15
+ from openmd.backends.douyin import DOUYIN_OPENMD_CREATOR_VIDEOS_BACKEND
16
+ from openmd.backends.douyin_providers import DOUYIN_TRANSCRIPT_BACKEND
17
+ from openmd.backends.xiaohongshu import OPENCLI_OPENMD_XIAOHONGSHU_CREATOR_NOTES_BACKEND
18
+ from openmd.backends.executor import (
19
+ BackendExecutor,
20
+ DiscoveryExecutor,
21
+ )
22
+ from openmd.backends.registry import (
23
+ BackendRuntimeOptions,
24
+ create_runtime_backend_provider_registry,
25
+ create_runtime_discovery_provider_registry,
26
+ normalize_discovery_backend,
27
+ )
28
+ from openmd.backends.youtube_discovery import (
29
+ YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND,
30
+ is_youtube_collection_candidate,
31
+ opencli_youtube_playlist_target,
32
+ youtube_data_api_playlist_id,
33
+ )
34
+ from openmd.core.detector import PLATFORM_IDS, detect_platform
35
+ from openmd.core.coverage import CollectionCoverage, collection_content_type_counts
36
+ from openmd.core.capabilities import filter_capabilities
37
+ from openmd.core.doctor import DOCTOR_PROFILES, build_doctor_report
38
+ from openmd.core.manifest import write_discovery_manifest
39
+ from openmd.core.models import ContentItem
40
+ from openmd.core.output import write_item_result
41
+ from openmd.core.package_contract import (
42
+ apply_platform_extraction_contract,
43
+ extraction_contract_version_for,
44
+ required_component_ids_for_item,
45
+ package_metadata_satisfies_extraction_contract,
46
+ package_source_level,
47
+ )
48
+ from openmd.core.package_writer import find_corrupt_package_for_item, find_existing_package_for_item, write_item_package
49
+ from openmd.core.package_validator import PackageValidationError, validate_package_or_raise
50
+ from openmd.core.planner import build_plan
51
+ from openmd.core.operation_contract import collect_all_allowed
52
+ from openmd.core.receipt import CompactReceiptRun
53
+ from openmd.core.execution_policy import CompactOutputLock, execution_fingerprint, require_matching_fingerprint
54
+ from openmd.core.secrets import sanitize_url
55
+ from openmd.core.runlog import (
56
+ append_runlog_entry,
57
+ build_item_runlog_entry,
58
+ build_runlog_entry,
59
+ default_runlog_path,
60
+ )
61
+ from openmd.core.secrets import sanitize_error_summary
62
+
63
+ DEFAULT_GUARDRAIL_ITEM_THRESHOLD = 100
64
+ DEFAULT_GUARDRAIL_FAILURE_STREAK = 10
65
+
66
+
67
+ @dataclass
68
+ class CliResult:
69
+ ok: bool
70
+ command: str
71
+ item_count: int = 0
72
+ success_count: int = 0
73
+ failed_count: int = 0
74
+ skipped_count: int = 0
75
+ outputs: list[str] = field(default_factory=list)
76
+ guardrail_signals: list[str] = field(default_factory=list)
77
+ source_level_counts: dict[str, int] = field(default_factory=dict)
78
+ failure_code_counts: dict[str, int] = field(default_factory=dict)
79
+ coverage: dict[str, object] = field(default_factory=dict)
80
+ message: str = ""
81
+
82
+
83
+ @dataclass(frozen=True)
84
+ class OutputWriteResult:
85
+ path: Path
86
+ package_id: str | None = None
87
+ package_operation: str | None = None
88
+
89
+
90
+ ADVANCED_COMMANDS = ("ingest", "collect", "doctor", "probe", "routes", "report", "capabilities", "plan", "browser-setup")
91
+ GO_DAILY_OPTIONS = {"help", "url", "save_to", "video", "raw", "json", "verbose"}
92
+
93
+
94
+ def build_parser() -> argparse.ArgumentParser:
95
+ parser = argparse.ArgumentParser(
96
+ prog="openmd",
97
+ usage="openmd [链接] | openmd batch <链接清单.txt> --out <目录>",
98
+ description="把链接保存成干净的 Markdown 原文 / 逐字稿。",
99
+ formatter_class=argparse.RawDescriptionHelpFormatter,
100
+ epilog=(
101
+ "日常用法:\n"
102
+ " openmd 粘贴链接,回车,选保存位置\n"
103
+ " openmd \"https://...\" 直接处理这个链接\n"
104
+ " openmd \"https://...\" -o <目录> 脚本 / Agent:不提问,直接保存\n"
105
+ " openmd batch urls.txt --out <目录> 一次处理多个链接\n"
106
+ " openmd doctor 能处理哪些内容、缺什么怎么装\n"
107
+ "\n"
108
+ "浏览器插件:运行 openmd browser-setup,按提示在 Chrome / Edge 加载它打开的文件夹。\n"
109
+ f"高级命令(自动化 / 排障):{', '.join(ADVANCED_COMMANDS)};用 openmd <命令> -h 查看。"
110
+ ),
111
+ )
112
+ subparsers = parser.add_subparsers(dest="command", metavar="{go,batch}", prog="openmd")
113
+
114
+ # Advanced commands are registered without help text so the daily help stays short;
115
+ # they keep working exactly as before.
116
+ ingest = subparsers.add_parser("ingest", description="ingest one input")
117
+ ingest.add_argument("url")
118
+ _add_shared_options(ingest)
119
+ ingest.set_defaults(func=cmd_ingest)
120
+
121
+ from openmd.interactive import cmd_go
122
+
123
+ go = subparsers.add_parser("go", help="处理一个链接,然后问保存位置(默认命令)", description="处理一个链接,然后问保存位置。")
124
+ go.add_argument("url", nargs="?", help="链接或本地文件;不填则提示粘贴")
125
+ _add_shared_options(go, output_required=False, default_limit=1)
126
+ go.add_argument("-o", "--save-to", dest="save_to", help="保存目录;不填则询问(回车用上次目录)")
127
+ go.add_argument("--yes", action="store_true", help="compatibility no-op: a destination is the grant; capability flags narrow it")
128
+ go.add_argument("--json", action="store_true", help="输出机器可读的 JSON 结果")
129
+ go.add_argument("--verbose", action="store_true", help="显示用了哪条工具路线和诊断信息")
130
+ go.add_argument("--video", choices=("note", "text", "file"), default="note", help="帖子带视频时(目前 X):note 只注明有视频(默认);text 附上视频逐字稿;file 再保留视频文件")
131
+ for action in go._actions:
132
+ if action.dest == "raw":
133
+ action.help = "保留原始逐行时间戳 / 表格,不合并段落"
134
+ elif action.dest not in GO_DAILY_OPTIONS:
135
+ action.help = argparse.SUPPRESS # still accepted; hidden from daily help
136
+ go.set_defaults(func=cmd_go)
137
+
138
+ batch = subparsers.add_parser("batch", help="一次处理清单文件里的多个链接", description="ingest inputs from a text file")
139
+ batch.add_argument("urls_file")
140
+ _add_shared_options(batch)
141
+ batch.set_defaults(func=cmd_batch)
142
+
143
+ collect = subparsers.add_parser("collect", description="discover a target, then optionally ingest its items")
144
+ collect.add_argument("url")
145
+ _add_shared_options(
146
+ collect,
147
+ default_limit=20,
148
+ include_manifest_only=True,
149
+ include_discovery_backend=True,
150
+ )
151
+ collect.add_argument(
152
+ "--max-items",
153
+ type=int,
154
+ default=0,
155
+ help="controlled extraction slice size after full discovery; 0 processes the remaining collection",
156
+ )
157
+ collect.add_argument(
158
+ "--start-index",
159
+ type=int,
160
+ default=0,
161
+ help="zero-based extraction slice start after full discovery; default: 0",
162
+ )
163
+ collect.add_argument(
164
+ "--progress",
165
+ action="store_true",
166
+ help="write sanitized per-item progress heartbeats to stderr",
167
+ )
168
+ collect.set_defaults(func=cmd_collect)
169
+
170
+ capabilities = subparsers.add_parser("capabilities", description="show read-only capability bindings")
171
+ capabilities.add_argument("--source", help="filter by normalized source id")
172
+ capabilities.add_argument("--format", choices=("json", "table"), default="table")
173
+ capabilities.set_defaults(func=cmd_capabilities)
174
+
175
+ plan = subparsers.add_parser("plan", description="build a read-only capability plan")
176
+ plan.add_argument("input")
177
+ plan.add_argument("--input-kind", choices=("auto", "single", "list", "collection", "file", "folder"), default="auto")
178
+ plan.add_argument("--all", action="store_true", help="request source-exhausted planning only")
179
+ plan.add_argument("--gate", action="append", default=[], help="declare a planning gate without executing a provider")
180
+ plan.add_argument("--format", choices=("json", "table"), default="table")
181
+ plan.set_defaults(func=cmd_plan)
182
+
183
+ doctor = subparsers.add_parser(
184
+ "doctor",
185
+ description="print local entry and safe session-boundary diagnostics",
186
+ )
187
+ doctor.add_argument("--profile", choices=DOCTOR_PROFILES, default="all")
188
+ doctor.add_argument("--json", action="store_true", help="machine-readable report (default when stdout is not a terminal)")
189
+ doctor.add_argument("--probe-commands", action="store_true", help="explicitly probe installed tool versions")
190
+ doctor.add_argument("--clear-gpu-quarantine", action="store_true", help="explicitly clear only the local GPU quarantine marker; never unload models or contact Ollama")
191
+ doctor.set_defaults(func=cmd_doctor)
192
+
193
+ from openmd.probe import cmd_probe
194
+
195
+ probe = subparsers.add_parser("probe", description="用本机配置的单条样本探活平台;不下载媒体或运行 ASR。")
196
+ probe.add_argument("--platform", choices=PLATFORM_IDS)
197
+ probe.add_argument("--config", help="本机样本 JSON;默认 OPENMD_PROBE_CONFIG 或 out/probe-config.json")
198
+ probe.add_argument("--out", default="out/probe", help="脱敏 JSON 报告目录(默认 out/probe/<date>/)")
199
+ probe.add_argument("--timeout", type=int, choices=range(1, 301), metavar="1..300", default=90, help="每个平台的最长秒数;默认 90,超时回收该平台调用子树并继续")
200
+ probe.add_argument("--enable-network-backends", action="store_true", help="显式允许现有公开网络读取路线")
201
+ probe.add_argument("--enable-browser-backends", action="store_true", help="显式允许现有浏览器/登录态读取路线")
202
+ probe.add_argument("--json", action="store_true", help="输出脱敏 JSON 回执")
203
+ probe.set_defaults(func=cmd_probe)
204
+
205
+ routes = subparsers.add_parser("routes", description="show effective routes or manage benchmark evidence")
206
+ routes.add_argument("action", nargs="?", choices=("bench", "demote", "restore"))
207
+ routes.add_argument("capability_id", nargs="?")
208
+ routes.add_argument("--source")
209
+ routes.add_argument("--capability")
210
+ routes.add_argument("--sample", help="fixed sample identifier for this bounded benchmark")
211
+ routes.add_argument("--samples-file", help="explicit sample registry, including existing local PDF/image fixtures")
212
+ routes.add_argument("--artifacts", help="benchmark artifacts directory")
213
+ routes.add_argument("--format", choices=("json", "table", "markdown"), default="table")
214
+ routes.add_argument("--candidates", help="comma-separated backend capability identifiers")
215
+ routes.add_argument("--evidence", action="append", default=[])
216
+ routes.add_argument("--ledger")
217
+ routes.add_argument("--overlay")
218
+ routes.add_argument("--note", default="")
219
+ for gate in ("network-backends", "browser-backends", "media-download", "asr", "cloud-backends"):
220
+ routes.add_argument(f"--enable-{gate}", action="store_true")
221
+ routes.set_defaults(func=cmd_routes)
222
+
223
+ from openmd.browser_setup import cmd_browser_setup
224
+
225
+ browser_setup = subparsers.add_parser("browser-setup", description="连接 Chrome / Edge 插件与本机 openmd(只写当前用户设置)")
226
+ browser_setup.add_argument("--uninstall", action="store_true", help="移除本地连接")
227
+ browser_setup.set_defaults(func=cmd_browser_setup)
228
+
229
+ report = subparsers.add_parser("report", description="build reports from sanitized runlogs")
230
+ report_subparsers = report.add_subparsers(dest="report_command")
231
+ weixin_author_batch = report_subparsers.add_parser(
232
+ "weixin-author-batch",
233
+ help="summarize Weixin author-batch coverage, recovered fallback, and missing items",
234
+ )
235
+ weixin_author_batch.add_argument(
236
+ "--runlog",
237
+ action="append",
238
+ required=True,
239
+ help="sanitized runlog JSONL path; repeat to merge multiple runs",
240
+ )
241
+ weixin_author_batch.add_argument("--out", required=True, help="report output directory")
242
+ weixin_author_batch.add_argument(
243
+ "--format",
244
+ choices=("both", "json", "markdown"),
245
+ default="both",
246
+ help="report artifact format",
247
+ )
248
+ weixin_author_batch.set_defaults(func=cmd_report_weixin_author_batch)
249
+ return parser
250
+
251
+
252
+ def _add_shared_options(
253
+ parser: argparse.ArgumentParser,
254
+ *,
255
+ default_limit: int | None = None,
256
+ include_manifest_only: bool = False,
257
+ include_discovery_backend: bool = False,
258
+ output_required: bool = True,
259
+ ) -> None:
260
+ parser.add_argument("--out", required=output_required, help="output directory")
261
+ parser.add_argument("--backend", help="backend name override")
262
+ parser.add_argument("--fallback", dest="fallback", action="store_true", default=True)
263
+ parser.add_argument("--no-fallback", dest="fallback", action="store_false")
264
+ parser.add_argument("--limit", type=int, default=default_limit)
265
+ parser.add_argument(
266
+ "--all",
267
+ action="store_true",
268
+ help="full collection; only runs for explicitly gated providers",
269
+ )
270
+ parser.add_argument("--overwrite", action="store_true")
271
+ parser.add_argument(
272
+ "--package",
273
+ action="store_true",
274
+ help="write V1 item.openmd/ package artifacts instead of V0 single Markdown",
275
+ )
276
+ parser.add_argument(
277
+ "--package-root",
278
+ help="package library root; defaults to <out>/packages when --package is used",
279
+ )
280
+ parser.add_argument(
281
+ "--skip-existing",
282
+ action="store_true",
283
+ help="skip existing package_id matches; package mode uses this behavior by default",
284
+ )
285
+ parser.add_argument(
286
+ "--update",
287
+ action="store_true",
288
+ help="update an existing package in place when package_id already exists",
289
+ )
290
+ parser.add_argument(
291
+ "--new-version",
292
+ action="store_true",
293
+ help="create a versioned package snapshot instead of updating/skipping",
294
+ )
295
+ parser.add_argument("--download-assets", action="store_true")
296
+ parser.add_argument("--raw", action="store_true", help="retain original transcript lines and timestamps without readable grouping")
297
+ parser.add_argument("--keep-raw", action="store_true")
298
+ parser.add_argument("--keep-media", action="store_true")
299
+ parser.add_argument(
300
+ "--enable-media-download",
301
+ action="store_true",
302
+ help="opt in to media download fallback backends; raw media is still deleted unless --keep-media is set",
303
+ )
304
+ parser.add_argument(
305
+ "--enable-asr",
306
+ action="store_true",
307
+ help="opt in to ASR fallback backends after native text/subtitle paths fail",
308
+ )
309
+ parser.add_argument(
310
+ "--enable-cloud-backends",
311
+ action="store_true",
312
+ help="explicitly allow cloud ASR/OCR providers; --enable-asr alone does not allow upload",
313
+ )
314
+ parser.add_argument(
315
+ "--guardrail-item-threshold",
316
+ type=int,
317
+ default=DEFAULT_GUARDRAIL_ITEM_THRESHOLD,
318
+ help="emit a runlog guardrail signal when a batch/collection reaches this item count; 0 disables",
319
+ )
320
+ parser.add_argument(
321
+ "--guardrail-failure-streak",
322
+ type=int,
323
+ default=DEFAULT_GUARDRAIL_FAILURE_STREAK,
324
+ help="pause batch/collection after this many consecutive failed items; 0 disables",
325
+ )
326
+ if include_manifest_only:
327
+ parser.add_argument(
328
+ "--manifest-only",
329
+ action="store_true",
330
+ help="write discovery manifest and skip item extraction",
331
+ )
332
+ if include_discovery_backend:
333
+ parser.add_argument(
334
+ "--discovery-backend",
335
+ help="discovery provider override for collect, independent from --backend",
336
+ )
337
+ parser.add_argument("--runlog", help="runlog JSONL path; defaults to <out>/runlog.jsonl")
338
+ parser.add_argument(
339
+ "--receipt",
340
+ choices=("compact",),
341
+ help="write a bounded per-run receipt and print one compact JSON line",
342
+ )
343
+ parser.add_argument(
344
+ "--enable-network-backends",
345
+ action="store_true",
346
+ help="opt in to real network backends and LibreOffice fetching Office external resources",
347
+ )
348
+ parser.add_argument(
349
+ "--enable-browser-backends",
350
+ action="store_true",
351
+ help="opt in to browser-context backends such as OpenCLI for matching collection inputs",
352
+ )
353
+
354
+
355
+ def _runlog_path(args: argparse.Namespace) -> Path:
356
+ return Path(args.runlog) if args.runlog else default_runlog_path(args.out)
357
+
358
+
359
+ def _print_result(result: CliResult) -> None:
360
+ # Keep CLI status JSON ASCII-safe for Windows consoles with narrow encodings.
361
+ print(json.dumps(asdict(result), ensure_ascii=True, sort_keys=True))
362
+
363
+
364
+ def _start_compact_receipt(args: argparse.Namespace, command: str) -> str | None:
365
+ if getattr(args, "receipt", None) != "compact":
366
+ return None
367
+ try:
368
+ run = CompactReceiptRun.create(args.out, command)
369
+ except OSError:
370
+ args._compact_receipt_storage_unavailable = True
371
+ return "compact receipt storage is unavailable"
372
+ runlog_path = _runlog_path(args)
373
+ args._compact_receipt_run = run
374
+ args._compact_runlog_path = runlog_path
375
+ args._compact_runlog_offset = runlog_path.stat().st_size if runlog_path.exists() else 0
376
+ args._compact_requires_fresh_stdin = (
377
+ getattr(args, "url", None) == "-" or getattr(args, "urls_file", None) == "-"
378
+ )
379
+ run.requires_fresh_stdin = args._compact_requires_fresh_stdin
380
+ run.write_checkpoint()
381
+ if getattr(args, "package", False):
382
+ try:
383
+ args._compact_output_lock = CompactOutputLock.acquire(args.out, run.run_id)
384
+ except FileExistsError:
385
+ return "compact package output is locked; inspect .openmd-compact.lock before retrying"
386
+ return None
387
+
388
+
389
+ def _compact_failure_payload() -> dict[str, object]:
390
+ """Return the bounded receipt used when the compact run cannot be persisted."""
391
+ counts = {status: 0 for status in CompactReceiptRun.TERMINAL_STATUSES}
392
+ counts["failed"] = 1
393
+ counts["discovered"] = 1
394
+ counts["skipped"] = 0
395
+ counts["input_occurrences"] = 1
396
+ counts["duplicate_inputs"] = 0
397
+ return {
398
+ "schema_version": "openmd.receipt.v1",
399
+ "status": "failed",
400
+ "counts": counts,
401
+ "summary_ref": None,
402
+ "error_counts": {"unknown": 1},
403
+ "other_error_count": 0,
404
+ }
405
+
406
+
407
+ def _print_compact_failure() -> int:
408
+ print(json.dumps(_compact_failure_payload(), ensure_ascii=True, separators=(",", ":")))
409
+ return 1
410
+
411
+
412
+ def _register_compact_inputs(args: argparse.Namespace, identities: dict[str, int]) -> None:
413
+ run = getattr(args, "_compact_receipt_run", None)
414
+ if run is None:
415
+ return
416
+ args._compact_inputs_registered = True
417
+ args._compact_pending_identities = identities
418
+ run.replace_expected(identities)
419
+
420
+
421
+ def _sync_compact_rows(args: argparse.Namespace) -> None:
422
+ run = getattr(args, "_compact_receipt_run", None)
423
+ if run is None:
424
+ return
425
+ path = args._compact_runlog_path
426
+ rows = _read_new_runlog_rows(path, args._compact_runlog_offset)
427
+ _record_compact_rows(run, rows, args.out, args=args)
428
+ args._compact_runlog_offset = path.stat().st_size if path.exists() else 0
429
+
430
+
431
+ def _append_execution_entry(args: argparse.Namespace, entry: dict[str, object]) -> None:
432
+ append_runlog_entry(_runlog_path(args), entry)
433
+ # Receipt/checkpoint must be durable before the next provider call.
434
+ _sync_compact_rows(args)
435
+
436
+
437
+ def _prepare_compact_collection(args: argparse.Namespace, items: list[ContentItem]) -> list[ContentItem]:
438
+ if getattr(args, "receipt", None) != "compact":
439
+ return items
440
+ unique: dict[str, ContentItem] = {}
441
+ occurrences: dict[str, int] = {}
442
+ by_url: dict[str, int] = {}
443
+ for item in items:
444
+ identity = _compact_item_identity(item)
445
+ unique.setdefault(identity, item)
446
+ occurrences[identity] = occurrences.get(identity, 0) + 1
447
+ by_url[sanitize_url(item.url)] = occurrences[identity]
448
+ args._compact_input_occurrences_by_url = by_url
449
+ _register_compact_inputs(args, occurrences)
450
+ return list(unique.values())
451
+
452
+
453
+ def _reconcile_compact_identity(args: argparse.Namespace, original: ContentItem, resolved: ContentItem) -> None:
454
+ """A resolver may replace a short locator with the canonical resource URL."""
455
+ if getattr(args, "_compact_receipt_run", None) is None:
456
+ return
457
+ previous, current = _compact_item_identity(original), _compact_item_identity(resolved)
458
+ if previous == current:
459
+ return
460
+ pending = getattr(args, "_compact_pending_identities", {})
461
+ occurrence = pending.pop(previous, 1)
462
+ pending[current] = pending.get(current, 0) + occurrence
463
+ run = args._compact_receipt_run
464
+ expected = dict(run.expected)
465
+ old_occurrence = expected.pop(previous, occurrence)
466
+ expected[current] = expected.get(current, 0) + old_occurrence
467
+ run.add_occurrences(current, old_occurrence)
468
+ run.replace_expected(expected)
469
+ by_url = getattr(args, "_compact_input_occurrences_by_url", {})
470
+ by_url[sanitize_url(resolved.url)] = expected[current]
471
+ args._compact_input_occurrences_by_url = by_url
472
+
473
+
474
+ def _register_resolved_batch_items(args: argparse.Namespace, input_url: str, items: list[ContentItem]) -> None:
475
+ run = getattr(args, "_compact_receipt_run", None)
476
+ if run is None:
477
+ return
478
+ previous = _compact_url_identity(input_url)
479
+ pending = args._compact_pending_identities
480
+ occurrence = pending.get(previous, 1)
481
+ if not items:
482
+ if not run.has_item(previous):
483
+ run.record_item(stable_identity=previous, input_occurrence=occurrence,
484
+ status="blocked", error_code="blocked", recovery_type="empty_scope")
485
+ return
486
+ pending.pop(previous, None)
487
+ expected = dict(run.expected)
488
+ expected.pop(previous, None)
489
+ by_url = args._compact_input_occurrences_by_url
490
+ for item in items:
491
+ identity = _compact_item_identity(item)
492
+ pending[identity] = pending.get(identity, 0) + occurrence
493
+ expected[identity] = expected.get(identity, 0) + occurrence
494
+ run.add_occurrences(identity, occurrence)
495
+ by_url[sanitize_url(item.url)] = expected[identity]
496
+ run.replace_expected(expected)
497
+
498
+
499
+ def _emit_result(
500
+ args: argparse.Namespace,
501
+ result: CliResult,
502
+ *,
503
+ discovery_complete: bool = True,
504
+ manifest_only: bool = False,
505
+ ) -> int:
506
+ run = getattr(args, "_compact_receipt_run", None)
507
+ if run is None:
508
+ if getattr(args, "_compact_receipt_storage_unavailable", False):
509
+ return _print_compact_failure()
510
+ _print_result(result)
511
+ return 0 if result.ok else 1
512
+ try:
513
+ if not manifest_only:
514
+ _sync_compact_rows(args)
515
+ for identity, occurrence in getattr(args, "_compact_pending_identities", {}).items():
516
+ if not run.has_item(identity):
517
+ run.record_item(
518
+ stable_identity=identity, input_occurrence=occurrence,
519
+ status="unattempted", quality_status="not_evaluated",
520
+ recovery_type="manifest_only" if manifest_only else "rerun_input",
521
+ )
522
+ if not run.has_items and not result.ok and not getattr(args, "_compact_inputs_registered", False):
523
+ run.record_item(
524
+ stable_identity=CompactReceiptRun.stable_identity(result.command, "preflight"),
525
+ input_occurrence=1, status="blocked", error_code="blocked",
526
+ recovery_type="inspect_runlog",
527
+ )
528
+ payload, exit_code = run.finalize(
529
+ discovery_complete=discovery_complete,
530
+ manifest_only=manifest_only,
531
+ resume_supported=bool(getattr(args, "package", False)),
532
+ requires_fresh_stdin=bool(getattr(args, "_compact_requires_fresh_stdin", False)),
533
+ failed=bool(getattr(args, "_compact_execution_failed", False)),
534
+ )
535
+ lock = getattr(args, "_compact_output_lock", None)
536
+ if lock is not None:
537
+ lock.release()
538
+ args._compact_output_lock = None
539
+ print(json.dumps(payload, ensure_ascii=True, separators=(",", ":")))
540
+ return exit_code
541
+ except (OSError, ValueError):
542
+ return _print_compact_failure()
543
+
544
+
545
+ def _read_new_runlog_rows(path: Path, offset: int) -> list[dict[str, object]]:
546
+ if not path.exists():
547
+ return []
548
+ with path.open("rb") as handle:
549
+ handle.seek(offset)
550
+ lines = handle.read().splitlines()
551
+ rows: list[dict[str, object]] = []
552
+ for line in lines:
553
+ try:
554
+ value = json.loads(line)
555
+ except json.JSONDecodeError as exc:
556
+ raise ValueError("malformed_runlog_line") from exc
557
+ if isinstance(value, dict) and value.get("event_type") == "item":
558
+ rows.append(value)
559
+ return rows
560
+
561
+
562
+ def _compact_item_identity(item: ContentItem) -> str:
563
+ return CompactReceiptRun.stable_identity(item.platform, sanitize_url(item.url))
564
+
565
+
566
+ def _compact_url_identity(url: str) -> str:
567
+ return CompactReceiptRun.stable_identity(detect_platform(url), sanitize_url(url))
568
+
569
+
570
+ def _prepare_resolved_item(
571
+ args: argparse.Namespace,
572
+ original: ContentItem,
573
+ resolved: ContentItem,
574
+ provider_chain: tuple[str, ...],
575
+ ) -> tuple[ContentItem, str | None] | None:
576
+ """Return the contract-applied item and its fingerprint, or None if the receipt already has it."""
577
+ item = apply_platform_extraction_contract(resolved)
578
+ if item.platform == "douyin" and getattr(args, "backend", None) == "douyin.metadata":
579
+ item = replace(item, raw_meta={**item.raw_meta, "extraction_target": "metadata"})
580
+ _reconcile_compact_identity(args, original, item)
581
+ run = getattr(args, "_compact_receipt_run", None)
582
+ if run is not None and run.has_item(_compact_item_identity(item)):
583
+ return None
584
+ return item, _compact_execution_fingerprint(item, args, provider_chain)
585
+
586
+
587
+ def _record_compact_rows(
588
+ run: CompactReceiptRun,
589
+ rows: list[dict[str, object]],
590
+ out_dir: str,
591
+ *,
592
+ args: argparse.Namespace,
593
+ ) -> set[str]:
594
+ out_root = Path(out_dir).resolve()
595
+ grouped: dict[str, list[dict[str, object]]] = {}
596
+ for row in rows:
597
+ identity = CompactReceiptRun.stable_identity(
598
+ str(row.get("platform") or ""),
599
+ sanitize_url(str(row.get("input_url") or "")),
600
+ )
601
+ grouped.setdefault(identity, []).append(row)
602
+ for identity, grouped_rows in grouped.items():
603
+ if run.has_item(identity):
604
+ continue
605
+ row = grouped_rows[-1]
606
+ status = _compact_terminal_status(row)
607
+ output_path = str(row.get("output_path") or "")
608
+ artifact_ref: str | None = None
609
+ if output_path:
610
+ try:
611
+ artifact_ref = Path(output_path).resolve().relative_to(out_root).as_posix()
612
+ except ValueError:
613
+ artifact_ref = None
614
+ run.record_item(
615
+ stable_identity=identity,
616
+ input_occurrence=max(
617
+ len(grouped_rows),
618
+ int(getattr(args, "_compact_input_occurrences_by_url", {}).get(str(row.get("input_url") or ""), 1)),
619
+ ),
620
+ status=status,
621
+ skipped=str(row.get("package_operation") or "") == "skipped_existing",
622
+ artifact_ref=artifact_ref,
623
+ source_level=str(row.get("source_level") or "unknown"),
624
+ quality_status="passed" if status == "verified" else "not_evaluated",
625
+ error_code=str(row.get("error_code") or "") or None,
626
+ failure_class=str(row.get("failure_class") or "") or None,
627
+ recovery_type=str(row.get("resume_pointer") or "") or None,
628
+ route_trace=row.get("route_trace") if isinstance(row.get("route_trace"), dict) else None,
629
+ )
630
+ return set(grouped)
631
+
632
+
633
+ def _compact_terminal_status(row: dict[str, object]) -> str:
634
+ if int(row.get("terminal_gap_count") or 0) > 0:
635
+ return "terminal_gap"
636
+ status = str(row.get("status") or "failed")
637
+ if status in {"verified", "partial"}:
638
+ return status
639
+ failure_class = str(row.get("failure_class") or "")
640
+ if str(row.get("error_code") or "") in {
641
+ "execution_fingerprint_missing",
642
+ "execution_fingerprint_mismatch",
643
+ "extraction_contract_mismatch",
644
+ "package_validation_failed",
645
+ }:
646
+ return "blocked"
647
+ if failure_class == "runtime_transient":
648
+ return "retryable_failure"
649
+ if failure_class == "source_unavailable":
650
+ return "terminal_gap"
651
+ if failure_class == "user_resolvable":
652
+ return "blocked"
653
+ return "failed"
654
+
655
+
656
+ def _compact_execution_fingerprint(
657
+ item: ContentItem,
658
+ args: argparse.Namespace,
659
+ provider_chain: tuple[str, ...],
660
+ ) -> str | None:
661
+ if getattr(args, "receipt", None) != "compact" or not getattr(args, "package", False):
662
+ return None
663
+ raw_meta = item.raw_meta if isinstance(item.raw_meta, dict) else {}
664
+ modalities_value = raw_meta.get("modalities")
665
+ modalities = tuple(str(value) for value in modalities_value) if isinstance(modalities_value, list) else (item.source_type,)
666
+ effective_config: dict[str, object] = {}
667
+ if item.platform in {"pdf", "screenshot"} and any(backend.startswith(("ocr.", "pdf.")) for backend in provider_chain):
668
+ from openmd.ocr.config import settings
669
+
670
+ effective_config["ocr_router"] = settings()
671
+ from openmd.runtime.asr import backend_execution_config
672
+
673
+ if "asr.faster_whisper" in provider_chain:
674
+ effective_config["faster_whisper"] = backend_execution_config("asr.faster_whisper")
675
+ from openmd.routing.loader import load_catalog
676
+
677
+ composed_asr = any(load_catalog().capabilities.get(backend, {}).get("then") == "asr.auto" for backend in provider_chain)
678
+ if "asr.auto" in provider_chain or composed_asr or "douyin.transcript" in provider_chain:
679
+ effective_config["asr_auto"] = backend_execution_config("asr.auto")
680
+ if "asr.qwen3" in provider_chain:
681
+ effective_config["qwen3"] = backend_execution_config("asr.qwen3")
682
+ if item.platform in {"douyin", "xiaohongshu"}:
683
+ from openmd.backends.douyin_providers import SILICONFLOW_MODEL
684
+
685
+ effective_config["cloud_asr_model"] = SILICONFLOW_MODEL
686
+ return execution_fingerprint(
687
+ requested_profile=str(raw_meta.get("profile") or "") or None,
688
+ requested_modalities=modalities,
689
+ provider_chain=provider_chain,
690
+ backend_override=getattr(args, "backend", None),
691
+ model=str(raw_meta.get("asr_model") or "") or None,
692
+ language=str(raw_meta.get("language") or "") or None,
693
+ extraction_contract={
694
+ "version": extraction_contract_version_for(item),
695
+ "required_components": list(required_component_ids_for_item(item)),
696
+ },
697
+ effective_config=effective_config,
698
+ )
699
+
700
+
701
+ def _read_urls_file(path: str) -> list[str]:
702
+ lines = sys.stdin.read().splitlines() if path == "-" else Path(path).read_text(encoding="utf-8").splitlines()
703
+ return [line.strip() for line in lines if line.strip() and not line.lstrip().startswith("#")]
704
+
705
+
706
+ def _read_single_input(value: str) -> str:
707
+ if value != "-":
708
+ return value
709
+ inputs = [line.strip() for line in sys.stdin.read().splitlines() if line.strip() and not line.lstrip().startswith("#")]
710
+ if len(inputs) != 1:
711
+ raise ValueError("stdin must contain exactly one non-comment input")
712
+ return inputs[0]
713
+
714
+
715
+ def _log_all_guard(input_url: str, args: argparse.Namespace, *, command: str) -> CliResult:
716
+ platform = detect_platform(input_url)
717
+ entry = build_runlog_entry(
718
+ input_url=input_url,
719
+ platform=platform,
720
+ backend=args.backend or "not_selected",
721
+ status="failed",
722
+ source_type="mixed",
723
+ source_level="partial_extraction",
724
+ item_count=0,
725
+ success_count=0,
726
+ failed_count=1,
727
+ event_type="guardrail",
728
+ failure_class="user_resolvable",
729
+ guardrail_signals=["all_requires_explicit_gate"],
730
+ resume_pointer="rerun with an explicitly supported provider gate",
731
+ error_code="all_requires_confirmation",
732
+ error_summary="--all requires an explicitly supported provider gate before collection work starts.",
733
+ )
734
+ _append_execution_entry(args, entry)
735
+ return CliResult(
736
+ ok=False,
737
+ command=command,
738
+ failed_count=1,
739
+ guardrail_signals=["all_requires_explicit_gate"],
740
+ message="--all requires explicit confirmation before any collection work.",
741
+ )
742
+
743
+
744
+ def _allow_scoped_all_collect(input_url: str, args: argparse.Namespace) -> bool:
745
+ """Allow full collection only for explicitly gated lanes that support it."""
746
+ if not getattr(args, "all", False):
747
+ return True
748
+ return collect_all_allowed(
749
+ _detect_collect_platform(input_url),
750
+ browser_enabled=bool(getattr(args, "enable_browser_backends", False)),
751
+ network_enabled=bool(getattr(args, "enable_network_backends", False)),
752
+ discovery_provider=normalize_discovery_backend(getattr(args, "discovery_backend", None)),
753
+ )
754
+
755
+
756
+ def _build_backend_executor(args: argparse.Namespace, *, platform: str) -> BackendExecutor:
757
+ from openmd.backends.video_metadata import enrich_video_result
758
+
759
+ provider_registry = create_runtime_backend_provider_registry(
760
+ platform,
761
+ _backend_runtime_options(args),
762
+ )
763
+ authorizer = getattr(args, "_browser_authorizer", None)
764
+ if authorizer is not None:
765
+ from openmd.backends.interactive import DeferredBrowserRegistry
766
+
767
+ provider_registry = DeferredBrowserRegistry(provider_registry, platform, args, authorizer)
768
+ return BackendExecutor(
769
+ provider_registry=provider_registry,
770
+ result_enricher=lambda item, result: enrich_video_result(
771
+ item, result, network_enabled=bool(getattr(args, "enable_network_backends", False))
772
+ ),
773
+ )
774
+
775
+
776
+ def _build_discovery_executor(args: argparse.Namespace, *, platform: str) -> DiscoveryExecutor:
777
+ options = _backend_runtime_options(args)
778
+ provider_registry = create_runtime_discovery_provider_registry(platform, options)
779
+ return DiscoveryExecutor(
780
+ provider_registry=provider_registry,
781
+ requested_provider=normalize_discovery_backend(options.requested_discovery_provider),
782
+ )
783
+
784
+
785
+ def _backend_runtime_options(args: argparse.Namespace) -> BackendRuntimeOptions:
786
+ keep_media_artifacts = getattr(args, "keep_media", False)
787
+ return BackendRuntimeOptions(
788
+ backend_override=getattr(args, "backend", None),
789
+ requested_discovery_provider=getattr(args, "discovery_backend", None),
790
+ enable_network_backends=getattr(args, "enable_network_backends", False),
791
+ enable_browser_backends=getattr(args, "enable_browser_backends", False),
792
+ enable_media_download_backends=getattr(args, "enable_media_download", False),
793
+ enable_asr_backends=getattr(args, "enable_asr", False),
794
+ enable_cloud_backends=getattr(args, "enable_cloud_backends", False),
795
+ keep_media_artifacts=keep_media_artifacts,
796
+ media_artifact_dir=str(Path(args.out) / "raw-media") if keep_media_artifacts else None,
797
+ )
798
+
799
+
800
+ def _select_collect_discovery_backend(input_url: str, args: argparse.Namespace) -> None:
801
+ if getattr(args, "discovery_backend", None):
802
+ return
803
+ if getattr(args, "enable_network_backends", False) and youtube_data_api_playlist_id(input_url) is not None:
804
+ args.discovery_backend = YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND
805
+ return
806
+ if not getattr(args, "enable_browser_backends", False):
807
+ return
808
+ if _detect_collect_platform(input_url) == "douyin" and getattr(args, "all", False):
809
+ args.discovery_backend = DOUYIN_OPENMD_CREATOR_VIDEOS_BACKEND
810
+ if not getattr(args, "backend", None):
811
+ args.backend = DOUYIN_TRANSCRIPT_BACKEND
812
+ return
813
+ if _detect_collect_platform(input_url) == "xiaohongshu" and getattr(args, "all", False):
814
+ args.discovery_backend = OPENCLI_OPENMD_XIAOHONGSHU_CREATOR_NOTES_BACKEND
815
+ return
816
+ if opencli_youtube_playlist_target(input_url) is not None:
817
+ args.discovery_backend = "youtube-playlist"
818
+
819
+
820
+ def _requested_backend(platform: str, args: argparse.Namespace) -> str | None:
821
+ """Explicit --backend wins; local media under --enable-asr picks its model by language."""
822
+ if getattr(args, "backend", None):
823
+ return args.backend
824
+ if platform == "generic_media" and getattr(args, "enable_asr", False):
825
+ return "asr.auto"
826
+ return None
827
+
828
+
829
+ def _detect_collect_platform(input_url: str) -> str:
830
+ platform = detect_platform(input_url)
831
+ if platform == "web_article" and is_youtube_collection_candidate(input_url):
832
+ return "youtube"
833
+ return platform
834
+
835
+
836
+ def _pace_douyin_extraction(item: ContentItem, args: argparse.Namespace) -> None:
837
+ """Share one bounded cadence across collection and multi-URL batch items."""
838
+ if item.platform != "douyin" or getattr(args, "backend", None) in {"mock", "douyin.metadata"}:
839
+ return
840
+ previous = getattr(args, "_douyin_last_extraction_at", None)
841
+ if previous is not None:
842
+ time.sleep(max(0.0, 1.0 - (time.monotonic() - previous)))
843
+ args._douyin_last_extraction_at = time.monotonic()
844
+
845
+
846
+ def _process_items(
847
+ items: list[ContentItem],
848
+ args: argparse.Namespace,
849
+ *,
850
+ command: str,
851
+ ) -> CliResult:
852
+ registry = create_default_registry()
853
+ outputs: list[str] = []
854
+ success_count = 0
855
+ failed_count = 0
856
+ processed_count = 0
857
+ consecutive_failures = 0
858
+ executors: dict[str, BackendExecutor] = {}
859
+ skipped_count = 0
860
+ source_level_counts: dict[str, int] = {}
861
+ failure_code_counts: dict[str, int] = {}
862
+ coverage = CollectionCoverage()
863
+
864
+ for index, item in enumerate(items, start=1):
865
+ processed_count += 1
866
+ selection = registry.select_backend(
867
+ item.platform,
868
+ _requested_backend(item.platform, args),
869
+ allow_fallback=args.fallback,
870
+ )
871
+ executor = executors.get(item.platform)
872
+ if executor is None:
873
+ executor = _build_backend_executor(args, platform=item.platform)
874
+ executors[item.platform] = executor
875
+ identity = executor.resolve_item_identity(item, selection.chain)
876
+ if not identity.ok:
877
+ failed_count += 1
878
+ consecutive_failures += 1
879
+ coverage.record_failure()
880
+ code = identity.error_code or "identity_resolution_failed"
881
+ failure_code_counts[code] = failure_code_counts.get(code, 0) + 1
882
+ _append_execution_entry(
883
+ args,
884
+ _build_identity_resolution_failure_entry(item, identity),
885
+ )
886
+ if _guardrail_failure_streak_reached(args, consecutive_failures):
887
+ _append_guardrail_stop_entry(args, command=command, index=index, total=len(items))
888
+ break
889
+ continue
890
+ prepared = _prepare_resolved_item(args, item, identity.item, selection.chain)
891
+ if prepared is None:
892
+ continue
893
+ item, fingerprint = prepared
894
+ try:
895
+ skipped_output = _maybe_skip_existing_package(item, args, execution_fingerprint=fingerprint)
896
+ except PackageValidationError as exc:
897
+ failed_count += 1
898
+ consecutive_failures += 1
899
+ coverage.record_failure()
900
+ entry = _build_package_validation_failure_entry_for_item(item, exc)
901
+ _append_execution_entry(args, entry)
902
+ if _guardrail_failure_streak_reached(args, consecutive_failures):
903
+ _append_guardrail_stop_entry(args, command=command, index=index, total=len(items))
904
+ break
905
+ continue
906
+ if skipped_output is not None:
907
+ outputs.append(str(skipped_output.path))
908
+ success_count += 1
909
+ skipped_count += 1
910
+ skipped_metadata = _read_package_metadata(skipped_output.path)
911
+ coverage.record_metadata(skipped_metadata, skipped_existing=True)
912
+ skipped_source_level = str(skipped_metadata.get("source_level") or "unknown")
913
+ source_level_counts[skipped_source_level] = source_level_counts.get(skipped_source_level, 0) + 1
914
+ consecutive_failures = 0
915
+ _append_execution_entry(
916
+ args,
917
+ _build_skipped_package_runlog_entry(item, skipped_output),
918
+ )
919
+ if getattr(args, "progress", False):
920
+ print(
921
+ json.dumps(
922
+ {
923
+ "event": "item_progress",
924
+ "index": index,
925
+ "total": len(items),
926
+ "video_id": item.id,
927
+ "status": "verified",
928
+ "source_level": skipped_source_level,
929
+ "package_operation": "skipped_existing",
930
+ },
931
+ ensure_ascii=True,
932
+ sort_keys=True,
933
+ ),
934
+ file=sys.stderr,
935
+ flush=True,
936
+ )
937
+ continue
938
+
939
+ _pace_douyin_extraction(item, args)
940
+ result = executor.extract(
941
+ item,
942
+ selection.chain,
943
+ allow_fallback=args.fallback,
944
+ )
945
+ output: OutputWriteResult | None = None
946
+ if result.has_usable_text:
947
+ try:
948
+ output = _write_extraction_output(item, result, args, execution_fingerprint=fingerprint)
949
+ except PackageValidationError as exc:
950
+ failed_count += 1
951
+ consecutive_failures += 1
952
+ coverage.record_result(result, force_failed=True)
953
+ entry = _build_package_validation_failure_entry(item, result, exc)
954
+ _append_execution_entry(args, entry)
955
+ if _guardrail_failure_streak_reached(args, consecutive_failures):
956
+ _append_guardrail_stop_entry(args, command=command, index=index, total=len(items))
957
+ break
958
+ continue
959
+ else:
960
+ outputs.append(str(output.path))
961
+ success_count += 1
962
+ source_level_counts[result.source_level] = source_level_counts.get(result.source_level, 0) + 1
963
+ consecutive_failures = 0
964
+ else:
965
+ failed_count += 1
966
+ consecutive_failures += 1
967
+ code = result.error_code or "unknown_failure"
968
+ failure_code_counts[code] = failure_code_counts.get(code, 0) + 1
969
+
970
+ coverage.record_result(result)
971
+
972
+ entry = build_item_runlog_entry(
973
+ item,
974
+ result,
975
+ output_path=output.path if output else None,
976
+ package_id=output.package_id if output else None,
977
+ package_operation=output.package_operation if output else None,
978
+ )
979
+ _append_execution_entry(args, entry)
980
+ if getattr(args, "progress", False):
981
+ print(
982
+ json.dumps(
983
+ {
984
+ "event": "item_progress",
985
+ "index": index,
986
+ "total": len(items),
987
+ "video_id": item.id,
988
+ "status": result.status,
989
+ "source_level": result.source_level,
990
+ "error_code": result.error_code,
991
+ },
992
+ ensure_ascii=True,
993
+ sort_keys=True,
994
+ ),
995
+ file=sys.stderr,
996
+ flush=True,
997
+ )
998
+ if _guardrail_failure_streak_reached(args, consecutive_failures):
999
+ _append_guardrail_stop_entry(args, command=command, index=index, total=len(items))
1000
+ break
1001
+
1002
+ ok = failed_count == 0 and bool(items)
1003
+ return CliResult(
1004
+ ok=ok,
1005
+ command=command,
1006
+ item_count=processed_count,
1007
+ success_count=success_count,
1008
+ failed_count=failed_count,
1009
+ skipped_count=skipped_count,
1010
+ outputs=outputs,
1011
+ guardrail_signals=["consecutive_failure_stop"] if processed_count < len(items) else [],
1012
+ source_level_counts=source_level_counts,
1013
+ failure_code_counts=failure_code_counts,
1014
+ coverage=coverage.as_dict(),
1015
+ message="completed" if ok else ("guardrail stopped" if processed_count < len(items) else "completed with failures"),
1016
+ )
1017
+
1018
+
1019
+ def _process_input(input_url: str, args: argparse.Namespace, *, command: str) -> CliResult:
1020
+ if getattr(args, "all", False):
1021
+ return _log_all_guard(input_url, args, command=command)
1022
+
1023
+ registry = create_default_registry()
1024
+ adapter = registry.adapter_for(input_url)
1025
+ selection = registry.select_backend(
1026
+ adapter.id,
1027
+ _requested_backend(adapter.id, args),
1028
+ allow_fallback=args.fallback,
1029
+ )
1030
+ if getattr(args, "enable_network_backends", False) and any(b.startswith("opencli.") for b in selection.chain):
1031
+ from openmd.runtime.managed_tools import ensure_opencli
1032
+
1033
+ # One `pip install openmd-cli` is enough: fetch the pinned opencli on first need.
1034
+ ensure_opencli()
1035
+ items = [
1036
+ apply_platform_extraction_contract(item)
1037
+ for item in adapter.resolve_items(input_url, limit=args.limit, all=False)
1038
+ ]
1039
+ if command == "ingest":
1040
+ items = _prepare_compact_collection(args, items)
1041
+ elif command == "batch":
1042
+ _register_resolved_batch_items(args, input_url, items)
1043
+ executor = _build_backend_executor(args, platform=adapter.id)
1044
+ outputs: list[str] = []
1045
+ success_count = 0
1046
+ failed_count = 0
1047
+ skipped_count = 0
1048
+
1049
+ for item in items:
1050
+ identity = executor.resolve_item_identity(item, selection.chain)
1051
+ if not identity.ok:
1052
+ failed_count += 1
1053
+ _append_execution_entry(
1054
+ args,
1055
+ _build_identity_resolution_failure_entry(item, identity),
1056
+ )
1057
+ continue
1058
+ prepared = _prepare_resolved_item(args, item, identity.item, selection.chain)
1059
+ if prepared is None:
1060
+ continue
1061
+ item, fingerprint = prepared
1062
+ try:
1063
+ skipped_output = _maybe_skip_existing_package(item, args, execution_fingerprint=fingerprint)
1064
+ except PackageValidationError as exc:
1065
+ failed_count += 1
1066
+ entry = _build_package_validation_failure_entry_for_item(item, exc)
1067
+ _append_execution_entry(args, entry)
1068
+ continue
1069
+ if skipped_output is not None:
1070
+ outputs.append(str(skipped_output.path))
1071
+ success_count += 1
1072
+ skipped_count += 1
1073
+ _append_execution_entry(
1074
+ args,
1075
+ _build_skipped_package_runlog_entry(item, skipped_output),
1076
+ )
1077
+ continue
1078
+
1079
+ _pace_douyin_extraction(item, args)
1080
+ result = executor.extract(
1081
+ item,
1082
+ selection.chain,
1083
+ allow_fallback=args.fallback,
1084
+ )
1085
+ output: OutputWriteResult | None = None
1086
+ if result.has_usable_text:
1087
+ try:
1088
+ output = _write_extraction_output(item, result, args, execution_fingerprint=fingerprint)
1089
+ except PackageValidationError as exc:
1090
+ failed_count += 1
1091
+ entry = _build_package_validation_failure_entry(item, result, exc)
1092
+ _append_execution_entry(args, entry)
1093
+ continue
1094
+ else:
1095
+ outputs.append(str(output.path))
1096
+ success_count += 1
1097
+ else:
1098
+ failed_count += 1
1099
+
1100
+ entry = build_item_runlog_entry(
1101
+ item,
1102
+ result,
1103
+ output_path=output.path if output else None,
1104
+ package_id=output.package_id if output else None,
1105
+ package_operation=output.package_operation if output else None,
1106
+ )
1107
+ _append_execution_entry(args, entry)
1108
+
1109
+ ok = failed_count == 0 and bool(items)
1110
+ return CliResult(
1111
+ ok=ok,
1112
+ command=command,
1113
+ item_count=len(items),
1114
+ success_count=success_count,
1115
+ failed_count=failed_count,
1116
+ skipped_count=skipped_count,
1117
+ outputs=outputs,
1118
+ message="completed" if ok else "completed with failures",
1119
+ )
1120
+
1121
+
1122
+ def _maybe_skip_existing_package(
1123
+ item: ContentItem,
1124
+ args: argparse.Namespace,
1125
+ *,
1126
+ execution_fingerprint: str | None = None,
1127
+ ) -> OutputWriteResult | None:
1128
+ if not getattr(args, "package", False):
1129
+ return None
1130
+ if (
1131
+ getattr(args, "update", False)
1132
+ or getattr(args, "overwrite", False)
1133
+ or getattr(args, "new_version", False)
1134
+ ):
1135
+ return None
1136
+ if getattr(args, "receipt", None) == "compact":
1137
+ corrupt = find_corrupt_package_for_item(
1138
+ item,
1139
+ args.out,
1140
+ package_root=getattr(args, "package_root", None),
1141
+ )
1142
+ if corrupt is not None:
1143
+ raise PackageValidationError("package_validation_failed")
1144
+ existing = find_existing_package_for_item(
1145
+ item,
1146
+ args.out,
1147
+ package_root=getattr(args, "package_root", None),
1148
+ )
1149
+ if existing is None:
1150
+ return None
1151
+ validate_package_or_raise(existing.package_path)
1152
+ metadata = _read_package_metadata(existing.package_path)
1153
+ if execution_fingerprint:
1154
+ mismatch = require_matching_fingerprint(metadata, execution_fingerprint)
1155
+ if mismatch:
1156
+ raise PackageValidationError(mismatch)
1157
+ if not package_metadata_satisfies_extraction_contract(metadata, item):
1158
+ if getattr(args, "receipt", None) == "compact":
1159
+ raise PackageValidationError("extraction_contract_mismatch")
1160
+ return None
1161
+ return OutputWriteResult(
1162
+ path=existing.package_path,
1163
+ package_id=existing.package_id,
1164
+ package_operation=existing.operation,
1165
+ )
1166
+
1167
+
1168
+ def _write_extraction_output(
1169
+ item: ContentItem,
1170
+ result,
1171
+ args: argparse.Namespace,
1172
+ *,
1173
+ execution_fingerprint: str | None = None,
1174
+ ) -> OutputWriteResult:
1175
+ if getattr(args, "package", False):
1176
+ package_result = write_item_package(
1177
+ item,
1178
+ result,
1179
+ args.out,
1180
+ package_root=getattr(args, "package_root", None),
1181
+ skip_existing=True,
1182
+ update=getattr(args, "update", False),
1183
+ overwrite=getattr(args, "overwrite", False),
1184
+ new_version=getattr(args, "new_version", False),
1185
+ execution_fingerprint=execution_fingerprint,
1186
+ )
1187
+ try:
1188
+ validate_package_or_raise(package_result.package_path)
1189
+ except PackageValidationError:
1190
+ if package_result.operation in {"created", "new_version"}:
1191
+ shutil.rmtree(package_result.package_path, ignore_errors=True)
1192
+ raise
1193
+ return OutputWriteResult(
1194
+ path=package_result.package_path,
1195
+ package_id=package_result.package_id,
1196
+ package_operation=package_result.operation,
1197
+ )
1198
+
1199
+ return OutputWriteResult(
1200
+ path=write_item_result(
1201
+ item,
1202
+ result,
1203
+ args.out,
1204
+ overwrite=getattr(args, "overwrite", False),
1205
+ download_assets=getattr(args, "download_assets", False),
1206
+ raw=getattr(args, "raw", False),
1207
+ )
1208
+ )
1209
+
1210
+
1211
+ def _build_package_validation_failure_entry(
1212
+ item: ContentItem,
1213
+ result,
1214
+ exc: PackageValidationError,
1215
+ ) -> dict:
1216
+ return build_runlog_entry(
1217
+ input_url=item.url,
1218
+ platform=item.platform,
1219
+ backend=result.backend,
1220
+ status="failed",
1221
+ source_type=item.source_type,
1222
+ source_level=result.source_level,
1223
+ error_code="package_validation_failed",
1224
+ error_summary=exc,
1225
+ )
1226
+
1227
+
1228
+ def _build_identity_resolution_failure_entry(item: ContentItem, resolution) -> dict:
1229
+ return build_runlog_entry(
1230
+ input_url=item.url,
1231
+ platform=item.platform,
1232
+ backend=str(resolution.provider or "identity.not_available"),
1233
+ status="failed",
1234
+ source_type=item.source_type,
1235
+ source_level="partial_extraction",
1236
+ error_code=resolution.error_code or "identity_resolution_failed",
1237
+ error_summary=resolution.error_summary or "Durable item identity resolution failed.",
1238
+ resume_pointer="rerun after the identity resolver is available; unresolved identity was not skipped or extracted",
1239
+ )
1240
+
1241
+
1242
+ def _build_package_validation_failure_entry_for_item(
1243
+ item: ContentItem,
1244
+ exc: PackageValidationError,
1245
+ ) -> dict:
1246
+ error_code = str(exc)
1247
+ if error_code not in {"execution_fingerprint_missing", "execution_fingerprint_mismatch"}:
1248
+ error_code = "package_validation_failed"
1249
+ return build_runlog_entry(
1250
+ input_url=item.url,
1251
+ platform=item.platform,
1252
+ backend="existing_package",
1253
+ status="failed",
1254
+ source_type=item.source_type,
1255
+ source_level="partial_extraction",
1256
+ error_code=error_code,
1257
+ error_summary=exc,
1258
+ )
1259
+
1260
+
1261
+ def _build_skipped_package_runlog_entry(
1262
+ item: ContentItem,
1263
+ output: OutputWriteResult,
1264
+ ) -> dict:
1265
+ metadata = _read_package_metadata(output.path)
1266
+ backend = metadata.get("backend")
1267
+ backend_name = "existing_package"
1268
+ if isinstance(backend, dict) and backend.get("name"):
1269
+ backend_name = str(backend["name"])
1270
+ elif isinstance(backend, str) and backend.strip():
1271
+ backend_name = backend
1272
+ return build_runlog_entry(
1273
+ input_url=item.url,
1274
+ platform=item.platform,
1275
+ backend=backend_name,
1276
+ status=str(metadata.get("status") or "verified"),
1277
+ source_type=item.source_type,
1278
+ source_level=str(metadata.get("source_level") or "partial_extraction"),
1279
+ output_path=output.path,
1280
+ package_id=output.package_id,
1281
+ package_operation=output.package_operation,
1282
+ )
1283
+
1284
+
1285
+ def _read_package_metadata(package_path: Path) -> dict:
1286
+ try:
1287
+ value = json.loads((package_path / "metadata.json").read_text(encoding="utf-8"))
1288
+ except (OSError, json.JSONDecodeError):
1289
+ return {}
1290
+ return value if isinstance(value, dict) else {}
1291
+
1292
+
1293
+ def cmd_doctor(args: argparse.Namespace) -> int:
1294
+ if getattr(args, "clear_gpu_quarantine", False):
1295
+ from openmd.runtime.gpu_quarantine import clear_quarantine_manually
1296
+
1297
+ print(json.dumps({"command": "doctor", "ok": True, "gpu_quarantine": clear_quarantine_manually()}, sort_keys=True))
1298
+ return 0
1299
+ from openmd.runtime.tempdirs import sweep_tempdirs
1300
+
1301
+ cleanup = sweep_tempdirs()
1302
+ profile = getattr(args, "profile", "core")
1303
+ try:
1304
+ diagnostics = build_doctor_report(profile)
1305
+ from openmd.routing.loader import effective_routes
1306
+
1307
+ effective_routes()
1308
+ if profile in {"all", "tools"}:
1309
+ from openmd.routing.providers import doctor_tools
1310
+
1311
+ diagnostics["tools"] = doctor_tools(probe_commands=getattr(args, "probe_commands", False))
1312
+ diagnostics["ready"] = bool(diagnostics.get("ready") and diagnostics["tools"]["ok"])
1313
+ except (OSError, ValueError) as exc:
1314
+ diagnostics = {"profile": profile, "ready": False, "error_code": "local_configuration_invalid", "error_summary": sanitize_error_summary(exc), "next_action": "Repair the explicitly selected local position configuration or route overlay; no source extraction was started."}
1315
+ ready = diagnostics.get("ready", diagnostics.get("manifest_valid", True))
1316
+ payload = {
1317
+ "ok": ready,
1318
+ "command": "doctor",
1319
+ "temp_cleanup": cleanup,
1320
+ "official_command": "openmd",
1321
+ "module_entry": "python -m openmd",
1322
+ "repo_wrapper": str(Path("scripts/openmd.ps1")),
1323
+ "openmd_on_path": shutil.which("openmd") is not None,
1324
+ "safe_session_boundary": {
1325
+ "core_owns_platform_sessions": False,
1326
+ "reads_private_browser_profiles": False,
1327
+ "imports_or_exports_platform_sessions": False,
1328
+ "default_raw_artifact_retention": False,
1329
+ },
1330
+ "structured_states": [
1331
+ "auth_required",
1332
+ "session_expired",
1333
+ "permission_required",
1334
+ "cache_stale",
1335
+ "refresh_needed",
1336
+ "no_delta",
1337
+ "sidecar_unreachable",
1338
+ ],
1339
+ "next_action": diagnostics.get("next_action") or (
1340
+ "Use platform or sidecar UI to repair login/session/cache state, "
1341
+ "then rerun Openmd. Core records status only and does not store private session material."
1342
+ ),
1343
+ "profile": profile,
1344
+ "profile_diagnostics": diagnostics,
1345
+ }
1346
+ if not getattr(args, "json", False) and sys.stdout.isatty():
1347
+ from openmd.doctor_view import render
1348
+
1349
+ print(render(payload))
1350
+ # Optional capabilities never make the human view an error.
1351
+ return 0 if diagnostics.get("manifest_valid", True) and "error_code" not in diagnostics else 1
1352
+ print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
1353
+ return 0 if ready else 1
1354
+
1355
+
1356
+ def cmd_routes(args: argparse.Namespace) -> int:
1357
+ from openmd.routing.loader import route_view
1358
+
1359
+ action = args.action
1360
+ if action is None:
1361
+ view = route_view(source=args.source)
1362
+ if args.format == "json":
1363
+ print(json.dumps(view, ensure_ascii=True, sort_keys=True))
1364
+ elif args.format == "markdown":
1365
+ from openmd.routing.view import render_routes_markdown
1366
+
1367
+ print(render_routes_markdown(source=args.source, effective=True), end="")
1368
+ else:
1369
+ rows = view["routes"]
1370
+ for row in rows:
1371
+ values = (row["source"], row["capability"], " → ".join(row["chain"]))
1372
+ print("\t".join(values))
1373
+ return 0
1374
+ if not args.source:
1375
+ raise ValueError("routes_source_required")
1376
+ from openmd.routing.lifecycle import demote, restore, run_benchmark
1377
+
1378
+ if action == "bench":
1379
+ if not args.capability:
1380
+ raise ValueError("routes_capability_required")
1381
+ payload = run_benchmark(source=args.source, capability=args.capability,
1382
+ candidates=args.candidates.split(",") if args.candidates else None,
1383
+ options=_backend_runtime_options(args), ledger_path=args.ledger, sample_id=args.sample,
1384
+ samples_path=getattr(args, "samples_file", None), artifact_dir=getattr(args, "artifacts", None))
1385
+ else:
1386
+ if not args.capability_id:
1387
+ raise ValueError("routes_capability_id_required")
1388
+ if action == "demote":
1389
+ if not args.ledger:
1390
+ raise ValueError("routes_ledger_required")
1391
+ payload = demote(args.capability_id, source=args.source, evidence_ids=args.evidence,
1392
+ ledger_path=args.ledger, overlay_path=args.overlay, note=args.note)
1393
+ else:
1394
+ payload = restore(args.capability_id, source=args.source, overlay_path=args.overlay, note=args.note)
1395
+ print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
1396
+ return 0
1397
+
1398
+
1399
+ def cmd_capabilities(args: argparse.Namespace) -> int:
1400
+ records = filter_capabilities(source_id=args.source)
1401
+ if args.source and not records:
1402
+ print(json.dumps({"count": 0, "capabilities": []}, ensure_ascii=True, sort_keys=True))
1403
+ return 1
1404
+ if args.format == "json":
1405
+ print(json.dumps({"count": len(records), "capabilities": [record.as_dict() for record in records]}, ensure_ascii=True, sort_keys=True))
1406
+ return 0
1407
+ print("source_id\ttier\tstate\tplanned_provider\truntime_provider\tbinding_state\texecution_ready")
1408
+ for record in records:
1409
+ print("\t".join((record.source_id, record.tier, record.state, record.planned_provider or "", record.runtime_provider or "", record.binding_state, str(record.execution_ready).lower())))
1410
+ return 0
1411
+
1412
+
1413
+ def cmd_plan(args: argparse.Namespace) -> int:
1414
+ result = build_plan(
1415
+ args.input,
1416
+ input_kind=args.input_kind,
1417
+ request_all=args.all,
1418
+ available_gates=set(args.gate),
1419
+ )
1420
+ payload = result.as_dict()
1421
+ if args.format == "json":
1422
+ print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
1423
+ else:
1424
+ for key, value in payload.items():
1425
+ print(f"{key}\t{value}")
1426
+ return 0 if result.ok else 1
1427
+
1428
+
1429
+ def cmd_report_weixin_author_batch(args: argparse.Namespace) -> int:
1430
+ from openmd.adapters.weixin.report import load_runlog_entries, write_weixin_author_batch_report
1431
+
1432
+ write_json = args.format in {"both", "json"}
1433
+ write_markdown = args.format in {"both", "markdown"}
1434
+ entries = load_runlog_entries(args.runlog)
1435
+ result = write_weixin_author_batch_report(
1436
+ entries,
1437
+ args.out,
1438
+ write_json=write_json,
1439
+ write_markdown=write_markdown,
1440
+ )
1441
+ outputs = [str(path) for path in (result.json_path, result.markdown_path) if path is not None]
1442
+ summary = result.report.get("summary", {})
1443
+ _print_result(
1444
+ CliResult(
1445
+ ok=True,
1446
+ command="report",
1447
+ item_count=int(summary.get("total_items", 0)),
1448
+ success_count=int(summary.get("covered_count", 0)),
1449
+ failed_count=int(summary.get("failed_count", 0)),
1450
+ outputs=outputs,
1451
+ message="report written",
1452
+ )
1453
+ )
1454
+ return 0
1455
+
1456
+
1457
+ def cmd_ingest(args: argparse.Namespace) -> int:
1458
+ start_error = _start_compact_receipt(args, "ingest")
1459
+ if start_error:
1460
+ return _emit_result(args, CliResult(ok=False, command="ingest", failed_count=1, message=start_error))
1461
+ try:
1462
+ args.url = _read_single_input(args.url)
1463
+ except ValueError as exc:
1464
+ result = CliResult(ok=False, command="ingest", failed_count=1, message=sanitize_error_summary(exc))
1465
+ if getattr(args, "receipt", None) == "compact":
1466
+ return _emit_result(args, result)
1467
+ print(f"openmd ingest: {sanitize_error_summary(exc)}", file=sys.stderr)
1468
+ return 1
1469
+ result = _process_input(args.url, args, command="ingest")
1470
+ return _emit_result(args, result)
1471
+
1472
+
1473
+ def cmd_batch(args: argparse.Namespace) -> int:
1474
+ start_error = _start_compact_receipt(args, "batch")
1475
+ if start_error:
1476
+ return _emit_result(args, CliResult(ok=False, command="batch", failed_count=1, message=start_error))
1477
+ try:
1478
+ urls = _read_urls_file(args.urls_file)
1479
+ except OSError as exc:
1480
+ result = CliResult(ok=False, command="batch", failed_count=1, message=sanitize_error_summary(exc))
1481
+ if getattr(args, "receipt", None) == "compact":
1482
+ return _emit_result(args, result)
1483
+ print(f"openmd batch: {sanitize_error_summary(exc)}", file=sys.stderr)
1484
+ return 1
1485
+
1486
+ _append_large_batch_guardrail_if_needed(args, command="batch", total=len(urls))
1487
+
1488
+ urls_to_process = urls
1489
+ if getattr(args, "receipt", None) == "compact":
1490
+ occurrences: dict[str, int] = {}
1491
+ unique_urls: list[str] = []
1492
+ for url in urls:
1493
+ identity = sanitize_url(url)
1494
+ occurrences[identity] = occurrences.get(identity, 0) + 1
1495
+ if occurrences[identity] == 1:
1496
+ unique_urls.append(url)
1497
+ args._compact_input_occurrences_by_url = occurrences
1498
+ urls_to_process = unique_urls
1499
+ _register_compact_inputs(args, {
1500
+ _compact_url_identity(url): occurrences[sanitize_url(url)]
1501
+ for url in unique_urls
1502
+ })
1503
+
1504
+ aggregate = CliResult(ok=True, command="batch", message="completed")
1505
+ consecutive_failures = 0
1506
+ for line_number, url in enumerate(urls_to_process, start=1):
1507
+ result = _process_input(url, args, command="batch")
1508
+ _sync_compact_rows(args)
1509
+ if getattr(args, "receipt", None) == "compact":
1510
+ args._compact_pending_identities.pop(_compact_url_identity(url), None)
1511
+ aggregate.item_count += result.item_count
1512
+ aggregate.success_count += result.success_count
1513
+ aggregate.failed_count += result.failed_count
1514
+ aggregate.outputs.extend(result.outputs)
1515
+ for signal in result.guardrail_signals:
1516
+ if signal not in aggregate.guardrail_signals:
1517
+ aggregate.guardrail_signals.append(signal)
1518
+ if not result.ok:
1519
+ aggregate.ok = False
1520
+ aggregate.message = "completed with failures"
1521
+ consecutive_failures += 1
1522
+ else:
1523
+ consecutive_failures = 0
1524
+ if _guardrail_failure_streak_reached(args, consecutive_failures):
1525
+ _append_guardrail_stop_entry(args, command="batch", index=line_number, total=len(urls))
1526
+ if "consecutive_failure_stop" not in aggregate.guardrail_signals:
1527
+ aggregate.guardrail_signals.append("consecutive_failure_stop")
1528
+ aggregate.message = "guardrail stopped"
1529
+ break
1530
+ return _emit_result(args, aggregate)
1531
+
1532
+
1533
+ def _write_youtube_discovery_summary(out_dir: str, discovery_result, **fields: object) -> Path:
1534
+ summary = {
1535
+ "page_count": discovery_result.metadata.get("page_count", 0),
1536
+ "request_count": discovery_result.metadata.get("request_count", 0),
1537
+ **fields,
1538
+ }
1539
+ summary_path = Path(out_dir) / "youtube-discovery-summary.json"
1540
+ summary_path.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
1541
+ return summary_path
1542
+
1543
+
1544
+ def cmd_collect(args: argparse.Namespace) -> int:
1545
+ start_error = _start_compact_receipt(args, "collect")
1546
+ if start_error:
1547
+ return _emit_result(args, CliResult(ok=False, command="collect", failed_count=1, message=start_error), discovery_complete=False)
1548
+ try:
1549
+ args.url = _read_single_input(args.url)
1550
+ except ValueError as exc:
1551
+ result = CliResult(ok=False, command="collect", failed_count=1, message=sanitize_error_summary(exc))
1552
+ if getattr(args, "receipt", None) == "compact":
1553
+ return _emit_result(args, result, discovery_complete=False)
1554
+ print(f"openmd collect: {sanitize_error_summary(exc)}", file=sys.stderr)
1555
+ return 1
1556
+ if args.all and not _allow_scoped_all_collect(args.url, args):
1557
+ result = _log_all_guard(args.url, args, command="collect")
1558
+ return _emit_result(args, result, discovery_complete=False)
1559
+
1560
+ platform = _detect_collect_platform(args.url)
1561
+ _select_collect_discovery_backend(args.url, args)
1562
+ discovery_executor = _build_discovery_executor(args, platform=platform)
1563
+ discovery_result = discovery_executor.discover(
1564
+ DiscoveryRequest(
1565
+ input_url=args.url,
1566
+ platform=platform,
1567
+ limit=None if args.all else args.limit,
1568
+ allow_all=bool(args.all),
1569
+ )
1570
+ )
1571
+
1572
+ if not discovery_result.ok:
1573
+ partial_outputs: list[str] = []
1574
+ partial_count = 0
1575
+ if platform == "youtube":
1576
+ # Failed discovery must preserve known inputs without extracting or
1577
+ # claiming them verified, including compact expected-item accounting.
1578
+ _prepare_compact_collection(args, discovery_result.items)
1579
+ args._compact_execution_failed = True
1580
+ partial_count = len(discovery_result.items)
1581
+ partial_manifest = write_discovery_manifest(
1582
+ out_dir=args.out, platform=platform,
1583
+ collection_url=str(discovery_result.metadata.get("normalized_url") or args.url),
1584
+ provider=discovery_result.provider, items=discovery_result.items,
1585
+ limit=None if args.all else args.limit,
1586
+ )
1587
+ partial_outputs.append(str(partial_manifest))
1588
+ summary_path = _write_youtube_discovery_summary(
1589
+ args.out, discovery_result,
1590
+ status="failed", source_exhausted=False,
1591
+ item_count=partial_count, error_code=discovery_result.error_code,
1592
+ stop_reason=discovery_result.error_code,
1593
+ resume_policy="restart_discovery_and_reuse_verified_packages",
1594
+ )
1595
+ partial_outputs.append(str(summary_path))
1596
+ entry = build_runlog_entry(
1597
+ input_url=args.url,
1598
+ platform=platform,
1599
+ backend=discovery_result.provider,
1600
+ status="failed",
1601
+ source_type="mixed",
1602
+ source_level="partial_extraction",
1603
+ item_count=partial_count,
1604
+ success_count=0,
1605
+ failed_count=1,
1606
+ error_code=discovery_result.error_code,
1607
+ error_summary=discovery_result.error_summary,
1608
+ event_type="discovery" if platform == "youtube" else "item",
1609
+ )
1610
+ _append_execution_entry(args, entry)
1611
+ result = CliResult(
1612
+ ok=False,
1613
+ command="collect",
1614
+ item_count=partial_count,
1615
+ success_count=0,
1616
+ failed_count=1,
1617
+ outputs=partial_outputs,
1618
+ message=discovery_result.error_summary or "collection discovery failed",
1619
+ )
1620
+ return _emit_result(args, result, discovery_complete=False)
1621
+
1622
+ discovered_items = _prepare_compact_collection(args, discovery_result.items)
1623
+ discovery_complete = not args.all or bool(discovery_result.metadata.get("source_exhausted"))
1624
+ collection_url = str(discovery_result.metadata.get("normalized_url") or args.url)
1625
+ manifest_path = write_discovery_manifest(
1626
+ out_dir=args.out,
1627
+ platform=platform,
1628
+ collection_url=collection_url,
1629
+ provider=discovery_result.provider,
1630
+ items=discovered_items,
1631
+ limit=None if args.all else args.limit,
1632
+ )
1633
+ _append_large_batch_guardrail_if_needed(args, command="collect", total=len(discovery_result.items))
1634
+ if platform == "youtube" and discovery_result.provider.startswith("youtube.data-api"):
1635
+ metadata = discovery_result.metadata
1636
+ _write_youtube_discovery_summary(
1637
+ args.out, discovery_result,
1638
+ status="ok", source_exhausted=bool(metadata.get("source_exhausted")),
1639
+ item_count=len(discovery_result.items),
1640
+ playlist_item_count=metadata.get("playlist_item_count", 0),
1641
+ duplicate_item_count=metadata.get("duplicate_item_count", 0),
1642
+ stop_reason=metadata.get("stop_reason"),
1643
+ )
1644
+
1645
+ if getattr(args, "manifest_only", False):
1646
+ entry = build_runlog_entry(
1647
+ input_url=collection_url,
1648
+ platform=platform,
1649
+ backend=discovery_result.provider,
1650
+ status="verified",
1651
+ source_type="mixed",
1652
+ source_level="third_party_extractor",
1653
+ output_path=manifest_path,
1654
+ item_count=len(discovery_result.items),
1655
+ success_count=len(discovery_result.items),
1656
+ failed_count=0,
1657
+ event_type="discovery",
1658
+ )
1659
+ _append_execution_entry(args, entry)
1660
+ result = CliResult(
1661
+ ok=bool(discovery_result.items) or (platform == "youtube" and args.all and discovery_complete),
1662
+ command="collect",
1663
+ item_count=len(discovery_result.items),
1664
+ success_count=len(discovery_result.items),
1665
+ failed_count=0,
1666
+ outputs=[str(manifest_path)],
1667
+ message="manifest written",
1668
+ )
1669
+ return _emit_result(args, result, discovery_complete=discovery_complete and result.ok, manifest_only=True)
1670
+
1671
+ start_index = max(0, int(getattr(args, "start_index", 0) or 0))
1672
+ max_items = max(0, int(getattr(args, "max_items", 0) or 0))
1673
+ remaining_items = discovered_items[start_index:]
1674
+ extraction_items = remaining_items[:max_items] if max_items else remaining_items
1675
+ result = _process_items(extraction_items, args, command="collect")
1676
+ if platform == "douyin" and args.all:
1677
+ coverage_path = _write_douyin_coverage_summary(
1678
+ args=args,
1679
+ result=result,
1680
+ manifest_path=manifest_path,
1681
+ discovered_count=len(discovery_result.items),
1682
+ source_exhausted=bool(discovery_result.metadata.get("source_exhausted")),
1683
+ discovery_metadata=dict(discovery_result.metadata),
1684
+ start_index=start_index,
1685
+ max_items=max_items,
1686
+ )
1687
+ result.outputs.append(str(coverage_path))
1688
+ elif platform == "xiaohongshu" and args.all:
1689
+ coverage_path = _write_xiaohongshu_coverage_summary(
1690
+ args=args,
1691
+ result=result,
1692
+ discovered_items=discovery_result.items,
1693
+ discovery_metadata=dict(discovery_result.metadata),
1694
+ start_index=start_index,
1695
+ max_items=max_items,
1696
+ )
1697
+ result.outputs.append(str(coverage_path))
1698
+ return _emit_result(
1699
+ args,
1700
+ result,
1701
+ discovery_complete=discovery_complete,
1702
+ )
1703
+
1704
+
1705
+ def main(argv: list[str] | None = None) -> int:
1706
+ from openmd.console import configure_console
1707
+
1708
+ configure_console()
1709
+ actual = list(sys.argv[1:] if argv is None else argv)
1710
+ if not actual:
1711
+ from openmd.interactive import is_interactive
1712
+
1713
+ if is_interactive():
1714
+ actual = ["go"]
1715
+ elif actual[0] not in {"ingest", "go", "batch", "collect", "capabilities", "plan", "doctor", "probe", "routes", "report", "browser-setup"} and not actual[0].startswith("-"):
1716
+ actual.insert(0, "go")
1717
+ parser = build_parser()
1718
+ try:
1719
+ args = parser.parse_args(actual)
1720
+ except SystemExit as exc:
1721
+ return int(exc.code) if isinstance(exc.code, int) else 1
1722
+ from openmd.runtime.managed_tools import activate as activate_managed_tools
1723
+
1724
+ activate_managed_tools() # put Openmd's own opencli/node on PATH if it was installed
1725
+ if getattr(args, "command", None) != "doctor":
1726
+ from openmd.runtime.tempdirs import sweep_at_startup
1727
+
1728
+ sweep_at_startup()
1729
+ if not hasattr(args, "func"):
1730
+ parser.print_help()
1731
+ return 0
1732
+ if getattr(args, "package", False):
1733
+ if getattr(args, "receipt", None) == "compact" and any(
1734
+ (
1735
+ getattr(args, "update", False),
1736
+ getattr(args, "overwrite", False),
1737
+ getattr(args, "new_version", False),
1738
+ )
1739
+ ):
1740
+ print(
1741
+ "openmd: --receipt compact cannot be combined with --update, --overwrite, or --new-version",
1742
+ file=sys.stderr,
1743
+ )
1744
+ return 1
1745
+ if getattr(args, "new_version", False) and (
1746
+ getattr(args, "update", False) or getattr(args, "overwrite", False)
1747
+ ):
1748
+ print(
1749
+ "openmd: --new-version cannot be combined with --update or --overwrite",
1750
+ file=sys.stderr,
1751
+ )
1752
+ return 1
1753
+ try:
1754
+ return args.func(args)
1755
+ except (Exception, KeyboardInterrupt) as exc:
1756
+ if isinstance(exc, KeyboardInterrupt):
1757
+ print("已取消。", file=sys.stderr)
1758
+ if getattr(args, "receipt", None) != "compact":
1759
+ return 130
1760
+ if getattr(args, "receipt", None) == "compact":
1761
+ args._compact_execution_failed = not isinstance(exc, KeyboardInterrupt)
1762
+ if getattr(args, "_compact_receipt_run", None) is not None:
1763
+ return _emit_result(args, CliResult(ok=False, command=args.command, message="execution_interrupted"), discovery_complete=False)
1764
+ return _print_compact_failure()
1765
+ print(f"openmd: {sanitize_error_summary(exc)}", file=sys.stderr)
1766
+ return 1
1767
+ finally:
1768
+ lock = getattr(args, "_compact_output_lock", None)
1769
+ if lock is not None:
1770
+ try:
1771
+ lock.release()
1772
+ except OSError:
1773
+ pass # The bounded failure path already reports failure; leave residual lock for inspection.
1774
+
1775
+
1776
+ def _write_douyin_coverage_summary(
1777
+ *,
1778
+ args: argparse.Namespace,
1779
+ result: CliResult,
1780
+ manifest_path: Path,
1781
+ discovered_count: int,
1782
+ source_exhausted: bool,
1783
+ discovery_metadata: dict[str, object],
1784
+ start_index: int,
1785
+ max_items: int,
1786
+ ) -> Path:
1787
+ path = Path(args.out) / "coverage-summary.json"
1788
+ payload = {
1789
+ "platform": "douyin",
1790
+ "creator_input": args.url,
1791
+ "discovery_provider": getattr(args, "discovery_backend", None),
1792
+ "transcript_backend": getattr(args, "backend", None),
1793
+ "source_exhausted": source_exhausted,
1794
+ "page_count": discovery_metadata.get("page_count"),
1795
+ "empty_page_count": discovery_metadata.get("empty_page_count"),
1796
+ "duplicate_page_count": discovery_metadata.get("duplicate_page_count"),
1797
+ "pagination_retry_count": discovery_metadata.get("pagination_retry_count"),
1798
+ "video_count": discovery_metadata.get("video_count"),
1799
+ "image_post_count": discovery_metadata.get("image_post_count"),
1800
+ "discovered": discovered_count,
1801
+ "attempted": result.item_count,
1802
+ "start_index": start_index,
1803
+ "slice_size": max_items,
1804
+ "success": result.success_count,
1805
+ "failed": result.failed_count,
1806
+ "skipped_existing": result.skipped_count,
1807
+ "remaining_unattempted": max(0, discovered_count - start_index - result.item_count),
1808
+ "complete_run": start_index == 0 and max_items == 0 and result.item_count == discovered_count,
1809
+ "source_level_counts": _package_coverage_source_level_counts(result.source_level_counts),
1810
+ "failure_code_counts": result.failure_code_counts,
1811
+ "guardrail_signals": result.guardrail_signals,
1812
+ "manifest": str(manifest_path),
1813
+ "runlog": str(_runlog_path(args)),
1814
+ "media_retention": "none",
1815
+ }
1816
+ path.parent.mkdir(parents=True, exist_ok=True)
1817
+ path.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
1818
+ return path
1819
+
1820
+
1821
+ def _package_coverage_source_level_counts(counts: dict[str, int]) -> dict[str, int]:
1822
+ normalized: dict[str, int] = {}
1823
+ for source_level, count in counts.items():
1824
+ key = package_source_level(source_level)
1825
+ normalized[key] = normalized.get(key, 0) + int(count)
1826
+ return normalized
1827
+
1828
+
1829
+ def _write_xiaohongshu_coverage_summary(
1830
+ *,
1831
+ args: argparse.Namespace,
1832
+ result: CliResult,
1833
+ discovered_items: list[ContentItem],
1834
+ discovery_metadata: dict[str, object],
1835
+ start_index: int,
1836
+ max_items: int,
1837
+ ) -> Path:
1838
+ """Write a path-free, count-only Xiaohongshu full-collection summary."""
1839
+ path = Path(args.out) / "coverage-summary.json"
1840
+ discovered = len(discovered_items)
1841
+ remaining_unattempted = max(0, discovered - start_index - result.item_count)
1842
+ coverage = dict(result.coverage)
1843
+ retryable_items = _non_negative_int(coverage.get("retryable_items"))
1844
+ terminal_gaps = _non_negative_int(coverage.get("terminal_gaps"))
1845
+ remaining = remaining_unattempted + retryable_items
1846
+ component_counts, component_source_counts = _xiaohongshu_component_coverage(coverage)
1847
+
1848
+ payload = {
1849
+ "platform": "xiaohongshu",
1850
+ "package_mode": bool(getattr(args, "package", False)),
1851
+ "discovered": discovered,
1852
+ **collection_content_type_counts(discovered_items),
1853
+ "attempted": result.item_count,
1854
+ "verified": _non_negative_int(coverage.get("verified")),
1855
+ "partial": _non_negative_int(coverage.get("partial")),
1856
+ "terminal_gaps": terminal_gaps,
1857
+ "terminal_gap_items": _non_negative_int(coverage.get("terminal_gap_items")),
1858
+ "retryable": _non_negative_int(coverage.get("retryable")),
1859
+ "retryable_items": retryable_items,
1860
+ "failed": result.failed_count,
1861
+ "skipped_existing": result.skipped_count,
1862
+ "remaining": remaining,
1863
+ "remaining_unattempted": remaining_unattempted,
1864
+ "start_index": start_index,
1865
+ "slice_size": max_items,
1866
+ "complete_run": start_index == 0 and max_items == 0 and result.item_count == discovered,
1867
+ "source_exhausted": discovery_metadata.get("source_exhausted") is True,
1868
+ "explicit_exhaustion": discovery_metadata.get("explicit_exhaustion") is True,
1869
+ "page_count": _optional_non_negative_int(discovery_metadata.get("page_count")),
1870
+ "scroll_count": _optional_non_negative_int(discovery_metadata.get("scroll_count")),
1871
+ "anomaly_count": _optional_non_negative_int(discovery_metadata.get("anomaly_count")),
1872
+ "stable_round_count": _optional_non_negative_int(discovery_metadata.get("stable_round_count")),
1873
+ "empty_snapshot_count": _optional_non_negative_int(discovery_metadata.get("empty_snapshot_count")),
1874
+ "duplicate_snapshot_count": _optional_non_negative_int(
1875
+ discovery_metadata.get("duplicate_snapshot_count")
1876
+ ),
1877
+ "component_counts": component_counts,
1878
+ "component_source_counts": component_source_counts,
1879
+ "source_counts": _safe_count_mapping(coverage.get("source_counts")),
1880
+ "failure_code_counts": _safe_count_mapping(result.failure_code_counts),
1881
+ "guardrail_signals": list(result.guardrail_signals),
1882
+ "media_retention": "none",
1883
+ }
1884
+ payload["outcome"] = (
1885
+ "failed"
1886
+ if result.failed_count or remaining
1887
+ else "complete_with_gaps"
1888
+ if terminal_gaps
1889
+ else "verified"
1890
+ )
1891
+ path.parent.mkdir(parents=True, exist_ok=True)
1892
+ path.write_text(
1893
+ json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
1894
+ encoding="utf-8",
1895
+ )
1896
+ return path
1897
+
1898
+
1899
+ def _xiaohongshu_component_coverage(
1900
+ coverage: dict[str, object],
1901
+ ) -> tuple[dict[str, dict[str, int]], dict[str, dict[str, int]]]:
1902
+ raw_counts = coverage.get("component_counts")
1903
+ raw_sources = coverage.get("component_source_counts")
1904
+ count_mapping = raw_counts if isinstance(raw_counts, dict) else {}
1905
+ source_mapping = raw_sources if isinstance(raw_sources, dict) else {}
1906
+ component_counts: dict[str, dict[str, int]] = {}
1907
+ component_source_counts: dict[str, dict[str, int]] = {}
1908
+ for component_id in ("native_text", "image_ocr", "video_transcript"):
1909
+ component_counts[component_id] = _safe_count_mapping(count_mapping.get(component_id))
1910
+ component_counts[component_id].setdefault("total", 0)
1911
+ component_source_counts[component_id] = _safe_count_mapping(source_mapping.get(component_id))
1912
+ return component_counts, component_source_counts
1913
+
1914
+
1915
+ def _safe_count_mapping(value: object) -> dict[str, int]:
1916
+ if not isinstance(value, dict):
1917
+ return {}
1918
+ return {
1919
+ str(key): _non_negative_int(count)
1920
+ for key, count in value.items()
1921
+ if isinstance(key, str)
1922
+ }
1923
+
1924
+
1925
+ def _optional_non_negative_int(value: object) -> int | None:
1926
+ if value is None:
1927
+ return None
1928
+ return _non_negative_int(value)
1929
+
1930
+
1931
+ def _non_negative_int(value: object) -> int:
1932
+ try:
1933
+ return max(0, int(value or 0))
1934
+ except (TypeError, ValueError):
1935
+ return 0
1936
+
1937
+
1938
+ def _guardrail_item_threshold(args: argparse.Namespace) -> int:
1939
+ return max(0, int(getattr(args, "guardrail_item_threshold", DEFAULT_GUARDRAIL_ITEM_THRESHOLD) or 0))
1940
+
1941
+
1942
+ def _guardrail_failure_streak(args: argparse.Namespace) -> int:
1943
+ return max(0, int(getattr(args, "guardrail_failure_streak", DEFAULT_GUARDRAIL_FAILURE_STREAK) or 0))
1944
+
1945
+
1946
+ def _guardrail_failure_streak_reached(args: argparse.Namespace, consecutive_failures: int) -> bool:
1947
+ threshold = _guardrail_failure_streak(args)
1948
+ return threshold > 0 and consecutive_failures >= threshold
1949
+
1950
+
1951
+ def _append_large_batch_guardrail_if_needed(
1952
+ args: argparse.Namespace,
1953
+ *,
1954
+ command: str,
1955
+ total: int,
1956
+ ) -> None:
1957
+ threshold = _guardrail_item_threshold(args)
1958
+ if threshold <= 0 or total < threshold:
1959
+ return
1960
+ append_runlog_entry(
1961
+ _runlog_path(args),
1962
+ build_runlog_entry(
1963
+ input_url=getattr(args, "urls_file", None) or getattr(args, "url", "") or "",
1964
+ platform=_detect_collect_platform(getattr(args, "url", "")) if command == "collect" else "mixed",
1965
+ backend="openmd.batch.guardrail",
1966
+ status="partial",
1967
+ source_type="mixed",
1968
+ source_level="partial_extraction",
1969
+ item_count=total,
1970
+ success_count=0,
1971
+ failed_count=0,
1972
+ event_type="guardrail",
1973
+ guardrail_signals=["large_item_count"],
1974
+ resume_pointer="runlog records per-item progress; rerun uses skip-existing/package dedupe where applicable",
1975
+ ),
1976
+ )
1977
+
1978
+
1979
+ def _append_guardrail_stop_entry(
1980
+ args: argparse.Namespace,
1981
+ *,
1982
+ command: str,
1983
+ index: int,
1984
+ total: int,
1985
+ ) -> None:
1986
+ append_runlog_entry(
1987
+ _runlog_path(args),
1988
+ build_runlog_entry(
1989
+ input_url=getattr(args, "urls_file", None) or getattr(args, "url", "") or "",
1990
+ platform=_detect_collect_platform(getattr(args, "url", "")) if command == "collect" else "mixed",
1991
+ backend="openmd.batch.guardrail",
1992
+ status="failed",
1993
+ source_type="mixed",
1994
+ source_level="partial_extraction",
1995
+ item_count=total,
1996
+ success_count=0,
1997
+ failed_count=0,
1998
+ event_type="guardrail",
1999
+ failure_class="runtime_transient",
2000
+ guardrail_signals=["consecutive_failure_stop"],
2001
+ resume_pointer=f"resume from item {index + 1} of {total}",
2002
+ error_code="guardrail_consecutive_failures",
2003
+ error_summary=(
2004
+ "Batch paused after consecutive failures. Inspect recent runlog rows, "
2005
+ "fix backend/runtime state, then resume from the recorded pointer."
2006
+ ),
2007
+ ),
2008
+ )
2009
+
2010
+
2011
+ if __name__ == "__main__":
2012
+ raise SystemExit(main())