openmd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openmd/__init__.py +3 -0
- openmd/__main__.py +5 -0
- openmd/adapters/__init__.py +1 -0
- openmd/adapters/base.py +95 -0
- openmd/adapters/bilibili/__init__.py +11 -0
- openmd/adapters/bilibili/adapter.py +104 -0
- openmd/adapters/bilibili/discovery.py +33 -0
- openmd/adapters/douyin/__init__.py +11 -0
- openmd/adapters/douyin/adapter.py +117 -0
- openmd/adapters/douyin/discovery.py +36 -0
- openmd/adapters/douyin/sidecar.py +77 -0
- openmd/adapters/generic_media/__init__.py +14 -0
- openmd/adapters/high_privacy.py +77 -0
- openmd/adapters/local_audio/__init__.py +14 -0
- openmd/adapters/local_audio/adapter.py +169 -0
- openmd/adapters/local_audio/asr.py +58 -0
- openmd/adapters/local_audio/backends/__init__.py +1 -0
- openmd/adapters/local_audio/backends/auto.py +177 -0
- openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
- openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
- openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
- openmd/adapters/opencli.py +770 -0
- openmd/adapters/pdf/__init__.py +11 -0
- openmd/adapters/pdf/adapter.py +153 -0
- openmd/adapters/pdf/backends/__init__.py +1 -0
- openmd/adapters/pdf/backends/markitdown.py +165 -0
- openmd/adapters/pdf/parser.py +56 -0
- openmd/adapters/qq_group/__init__.py +11 -0
- openmd/adapters/qq_group/adapter.py +289 -0
- openmd/adapters/qq_group/parsers/__init__.py +1 -0
- openmd/adapters/qq_group/parsers/html.py +60 -0
- openmd/adapters/qq_group/parsers/json.py +71 -0
- openmd/adapters/registry.py +149 -0
- openmd/adapters/screenshot/__init__.py +11 -0
- openmd/adapters/screenshot/adapter.py +154 -0
- openmd/adapters/screenshot/backends/__init__.py +1 -0
- openmd/adapters/screenshot/backends/paddle.py +15 -0
- openmd/adapters/screenshot/backends/rapidocr.py +176 -0
- openmd/adapters/screenshot/backends/tesseract.py +15 -0
- openmd/adapters/screenshot/ocr.py +84 -0
- openmd/adapters/wechat_group/__init__.py +11 -0
- openmd/adapters/wechat_group/adapter.py +160 -0
- openmd/adapters/wechat_group/wx_cli.py +268 -0
- openmd/adapters/wechat_moments/__init__.py +11 -0
- openmd/adapters/wechat_moments/adapter.py +167 -0
- openmd/adapters/wechat_moments/wx_cli.py +265 -0
- openmd/adapters/weixin/__init__.py +11 -0
- openmd/adapters/weixin/adapter.py +148 -0
- openmd/adapters/weixin/discovery.py +510 -0
- openmd/adapters/weixin/report.py +278 -0
- openmd/adapters/xiaohongshu.py +158 -0
- openmd/backends/__init__.py +1 -0
- openmd/backends/base.py +204 -0
- openmd/backends/bilibili.py +1480 -0
- openmd/backends/bilibili_wbi.py +277 -0
- openmd/backends/discourse.py +426 -0
- openmd/backends/douyin.py +868 -0
- openmd/backends/douyin_detail.py +124 -0
- openmd/backends/douyin_providers.py +602 -0
- openmd/backends/executor.py +546 -0
- openmd/backends/feishu.py +861 -0
- openmd/backends/generic_media.py +77 -0
- openmd/backends/interactive.py +60 -0
- openmd/backends/ocr.py +41 -0
- openmd/backends/opencli.py +82 -0
- openmd/backends/pdf.py +122 -0
- openmd/backends/registry.py +234 -0
- openmd/backends/screenshot.py +79 -0
- openmd/backends/video_metadata.py +167 -0
- openmd/backends/web_article.py +363 -0
- openmd/backends/wechat_group.py +35 -0
- openmd/backends/wechat_moments.py +35 -0
- openmd/backends/weixin.py +767 -0
- openmd/backends/x.py +455 -0
- openmd/backends/xiaoe.py +940 -0
- openmd/backends/xiaohongshu.py +911 -0
- openmd/backends/xiaohongshu_providers.py +876 -0
- openmd/backends/xiaoyuzhou.py +1017 -0
- openmd/backends/youtube.py +280 -0
- openmd/backends/youtube_discovery.py +1245 -0
- openmd/backends/youtube_transcript.py +213 -0
- openmd/browser_extension/background.js +231 -0
- openmd/browser_extension/icons/icon-128.png +0 -0
- openmd/browser_extension/icons/icon-16.png +0 -0
- openmd/browser_extension/icons/icon-32.png +0 -0
- openmd/browser_extension/icons/icon-48.png +0 -0
- openmd/browser_extension/icons/icon-512.png +0 -0
- openmd/browser_extension/idle.html +6 -0
- openmd/browser_extension/manifest.json +43 -0
- openmd/browser_extension/options.html +42 -0
- openmd/browser_extension/options.js +117 -0
- openmd/browser_extension/popup.css +71 -0
- openmd/browser_extension/popup.html +17 -0
- openmd/browser_extension/popup.js +59 -0
- openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
- openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
- openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
- openmd/browser_host.py +503 -0
- openmd/browser_setup.py +106 -0
- openmd/cli.py +2012 -0
- openmd/console.py +32 -0
- openmd/core/__init__.py +1 -0
- openmd/core/capabilities.py +162 -0
- openmd/core/content_validation.py +323 -0
- openmd/core/coverage.py +163 -0
- openmd/core/detector.py +247 -0
- openmd/core/doctor.py +104 -0
- openmd/core/execution_policy.py +164 -0
- openmd/core/filename.py +64 -0
- openmd/core/input_validation.py +91 -0
- openmd/core/local_config.py +96 -0
- openmd/core/manifest.py +156 -0
- openmd/core/models.py +168 -0
- openmd/core/opencli_output.py +34 -0
- openmd/core/operation_contract.py +60 -0
- openmd/core/output.py +194 -0
- openmd/core/package_contract.py +335 -0
- openmd/core/package_validator.py +489 -0
- openmd/core/package_writer.py +666 -0
- openmd/core/planner.py +345 -0
- openmd/core/preferences.py +61 -0
- openmd/core/readable.py +129 -0
- openmd/core/receipt.py +285 -0
- openmd/core/renderer.py +185 -0
- openmd/core/runlog.py +208 -0
- openmd/core/secrets.py +313 -0
- openmd/doctor_view.py +68 -0
- openmd/interactive.py +453 -0
- openmd/ocr/__init__.py +1 -0
- openmd/ocr/breakers.py +84 -0
- openmd/ocr/config.py +54 -0
- openmd/ocr/documents.py +522 -0
- openmd/ocr/endpoint.py +72 -0
- openmd/ocr/layout.py +92 -0
- openmd/ocr/layout_filter.py +118 -0
- openmd/ocr/layout_worker.py +89 -0
- openmd/ocr/native_math.py +220 -0
- openmd/ocr/office.py +305 -0
- openmd/ocr/ollama.py +200 -0
- openmd/ocr/otsl.py +51 -0
- openmd/ocr/ovis-prompt.txt +1 -0
- openmd/ocr/pipeline.py +384 -0
- openmd/ocr/settings.json +25 -0
- openmd/ocr/tiling.py +631 -0
- openmd/post_video.py +154 -0
- openmd/probe.py +397 -0
- openmd/routing/__init__.py +1 -0
- openmd/routing/cards/builtin.json +620 -0
- openmd/routing/cards/defuddle.json +70 -0
- openmd/routing/cards/x-tweet-fetcher.json +71 -0
- openmd/routing/cards/xyz-dl.json +60 -0
- openmd/routing/cards/yutto.json +68 -0
- openmd/routing/entry_shim.py +71 -0
- openmd/routing/lifecycle.py +459 -0
- openmd/routing/loader.py +138 -0
- openmd/routing/providers.py +158 -0
- openmd/routing/readers.py +108 -0
- openmd/routing/routes.json +504 -0
- openmd/routing/runner.py +192 -0
- openmd/routing/view.py +40 -0
- openmd/runtime/__init__.py +1 -0
- openmd/runtime/_windows_job.py +234 -0
- openmd/runtime/asr.py +217 -0
- openmd/runtime/asr_worker.py +119 -0
- openmd/runtime/gpu_lock.py +81 -0
- openmd/runtime/gpu_quarantine.py +85 -0
- openmd/runtime/interpreters.py +407 -0
- openmd/runtime/managed_tools.py +204 -0
- openmd/runtime/models.py +202 -0
- openmd/runtime/services.py +175 -0
- openmd/runtime/tempdirs.py +227 -0
- openmd_cli-0.1.0.dist-info/METADATA +168 -0
- openmd_cli-0.1.0.dist-info/RECORD +177 -0
- openmd_cli-0.1.0.dist-info/WHEEL +5 -0
- openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
openmd/cli.py
ADDED
|
@@ -0,0 +1,2012 @@
|
|
|
1
|
+
"""Argparse-based Openmd CLI skeleton."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import shutil
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
from dataclasses import asdict, dataclass, field, replace
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from openmd.adapters.registry import create_default_registry
|
|
14
|
+
from openmd.backends.base import DiscoveryRequest
|
|
15
|
+
from openmd.backends.douyin import DOUYIN_OPENMD_CREATOR_VIDEOS_BACKEND
|
|
16
|
+
from openmd.backends.douyin_providers import DOUYIN_TRANSCRIPT_BACKEND
|
|
17
|
+
from openmd.backends.xiaohongshu import OPENCLI_OPENMD_XIAOHONGSHU_CREATOR_NOTES_BACKEND
|
|
18
|
+
from openmd.backends.executor import (
|
|
19
|
+
BackendExecutor,
|
|
20
|
+
DiscoveryExecutor,
|
|
21
|
+
)
|
|
22
|
+
from openmd.backends.registry import (
|
|
23
|
+
BackendRuntimeOptions,
|
|
24
|
+
create_runtime_backend_provider_registry,
|
|
25
|
+
create_runtime_discovery_provider_registry,
|
|
26
|
+
normalize_discovery_backend,
|
|
27
|
+
)
|
|
28
|
+
from openmd.backends.youtube_discovery import (
|
|
29
|
+
YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND,
|
|
30
|
+
is_youtube_collection_candidate,
|
|
31
|
+
opencli_youtube_playlist_target,
|
|
32
|
+
youtube_data_api_playlist_id,
|
|
33
|
+
)
|
|
34
|
+
from openmd.core.detector import PLATFORM_IDS, detect_platform
|
|
35
|
+
from openmd.core.coverage import CollectionCoverage, collection_content_type_counts
|
|
36
|
+
from openmd.core.capabilities import filter_capabilities
|
|
37
|
+
from openmd.core.doctor import DOCTOR_PROFILES, build_doctor_report
|
|
38
|
+
from openmd.core.manifest import write_discovery_manifest
|
|
39
|
+
from openmd.core.models import ContentItem
|
|
40
|
+
from openmd.core.output import write_item_result
|
|
41
|
+
from openmd.core.package_contract import (
|
|
42
|
+
apply_platform_extraction_contract,
|
|
43
|
+
extraction_contract_version_for,
|
|
44
|
+
required_component_ids_for_item,
|
|
45
|
+
package_metadata_satisfies_extraction_contract,
|
|
46
|
+
package_source_level,
|
|
47
|
+
)
|
|
48
|
+
from openmd.core.package_writer import find_corrupt_package_for_item, find_existing_package_for_item, write_item_package
|
|
49
|
+
from openmd.core.package_validator import PackageValidationError, validate_package_or_raise
|
|
50
|
+
from openmd.core.planner import build_plan
|
|
51
|
+
from openmd.core.operation_contract import collect_all_allowed
|
|
52
|
+
from openmd.core.receipt import CompactReceiptRun
|
|
53
|
+
from openmd.core.execution_policy import CompactOutputLock, execution_fingerprint, require_matching_fingerprint
|
|
54
|
+
from openmd.core.secrets import sanitize_url
|
|
55
|
+
from openmd.core.runlog import (
|
|
56
|
+
append_runlog_entry,
|
|
57
|
+
build_item_runlog_entry,
|
|
58
|
+
build_runlog_entry,
|
|
59
|
+
default_runlog_path,
|
|
60
|
+
)
|
|
61
|
+
from openmd.core.secrets import sanitize_error_summary
|
|
62
|
+
|
|
63
|
+
DEFAULT_GUARDRAIL_ITEM_THRESHOLD = 100
|
|
64
|
+
DEFAULT_GUARDRAIL_FAILURE_STREAK = 10
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class CliResult:
|
|
69
|
+
ok: bool
|
|
70
|
+
command: str
|
|
71
|
+
item_count: int = 0
|
|
72
|
+
success_count: int = 0
|
|
73
|
+
failed_count: int = 0
|
|
74
|
+
skipped_count: int = 0
|
|
75
|
+
outputs: list[str] = field(default_factory=list)
|
|
76
|
+
guardrail_signals: list[str] = field(default_factory=list)
|
|
77
|
+
source_level_counts: dict[str, int] = field(default_factory=dict)
|
|
78
|
+
failure_code_counts: dict[str, int] = field(default_factory=dict)
|
|
79
|
+
coverage: dict[str, object] = field(default_factory=dict)
|
|
80
|
+
message: str = ""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class OutputWriteResult:
|
|
85
|
+
path: Path
|
|
86
|
+
package_id: str | None = None
|
|
87
|
+
package_operation: str | None = None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
ADVANCED_COMMANDS = ("ingest", "collect", "doctor", "probe", "routes", "report", "capabilities", "plan", "browser-setup")
|
|
91
|
+
GO_DAILY_OPTIONS = {"help", "url", "save_to", "video", "raw", "json", "verbose"}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
95
|
+
parser = argparse.ArgumentParser(
|
|
96
|
+
prog="openmd",
|
|
97
|
+
usage="openmd [链接] | openmd batch <链接清单.txt> --out <目录>",
|
|
98
|
+
description="把链接保存成干净的 Markdown 原文 / 逐字稿。",
|
|
99
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
100
|
+
epilog=(
|
|
101
|
+
"日常用法:\n"
|
|
102
|
+
" openmd 粘贴链接,回车,选保存位置\n"
|
|
103
|
+
" openmd \"https://...\" 直接处理这个链接\n"
|
|
104
|
+
" openmd \"https://...\" -o <目录> 脚本 / Agent:不提问,直接保存\n"
|
|
105
|
+
" openmd batch urls.txt --out <目录> 一次处理多个链接\n"
|
|
106
|
+
" openmd doctor 能处理哪些内容、缺什么怎么装\n"
|
|
107
|
+
"\n"
|
|
108
|
+
"浏览器插件:运行 openmd browser-setup,按提示在 Chrome / Edge 加载它打开的文件夹。\n"
|
|
109
|
+
f"高级命令(自动化 / 排障):{', '.join(ADVANCED_COMMANDS)};用 openmd <命令> -h 查看。"
|
|
110
|
+
),
|
|
111
|
+
)
|
|
112
|
+
subparsers = parser.add_subparsers(dest="command", metavar="{go,batch}", prog="openmd")
|
|
113
|
+
|
|
114
|
+
# Advanced commands are registered without help text so the daily help stays short;
|
|
115
|
+
# they keep working exactly as before.
|
|
116
|
+
ingest = subparsers.add_parser("ingest", description="ingest one input")
|
|
117
|
+
ingest.add_argument("url")
|
|
118
|
+
_add_shared_options(ingest)
|
|
119
|
+
ingest.set_defaults(func=cmd_ingest)
|
|
120
|
+
|
|
121
|
+
from openmd.interactive import cmd_go
|
|
122
|
+
|
|
123
|
+
go = subparsers.add_parser("go", help="处理一个链接,然后问保存位置(默认命令)", description="处理一个链接,然后问保存位置。")
|
|
124
|
+
go.add_argument("url", nargs="?", help="链接或本地文件;不填则提示粘贴")
|
|
125
|
+
_add_shared_options(go, output_required=False, default_limit=1)
|
|
126
|
+
go.add_argument("-o", "--save-to", dest="save_to", help="保存目录;不填则询问(回车用上次目录)")
|
|
127
|
+
go.add_argument("--yes", action="store_true", help="compatibility no-op: a destination is the grant; capability flags narrow it")
|
|
128
|
+
go.add_argument("--json", action="store_true", help="输出机器可读的 JSON 结果")
|
|
129
|
+
go.add_argument("--verbose", action="store_true", help="显示用了哪条工具路线和诊断信息")
|
|
130
|
+
go.add_argument("--video", choices=("note", "text", "file"), default="note", help="帖子带视频时(目前 X):note 只注明有视频(默认);text 附上视频逐字稿;file 再保留视频文件")
|
|
131
|
+
for action in go._actions:
|
|
132
|
+
if action.dest == "raw":
|
|
133
|
+
action.help = "保留原始逐行时间戳 / 表格,不合并段落"
|
|
134
|
+
elif action.dest not in GO_DAILY_OPTIONS:
|
|
135
|
+
action.help = argparse.SUPPRESS # still accepted; hidden from daily help
|
|
136
|
+
go.set_defaults(func=cmd_go)
|
|
137
|
+
|
|
138
|
+
batch = subparsers.add_parser("batch", help="一次处理清单文件里的多个链接", description="ingest inputs from a text file")
|
|
139
|
+
batch.add_argument("urls_file")
|
|
140
|
+
_add_shared_options(batch)
|
|
141
|
+
batch.set_defaults(func=cmd_batch)
|
|
142
|
+
|
|
143
|
+
collect = subparsers.add_parser("collect", description="discover a target, then optionally ingest its items")
|
|
144
|
+
collect.add_argument("url")
|
|
145
|
+
_add_shared_options(
|
|
146
|
+
collect,
|
|
147
|
+
default_limit=20,
|
|
148
|
+
include_manifest_only=True,
|
|
149
|
+
include_discovery_backend=True,
|
|
150
|
+
)
|
|
151
|
+
collect.add_argument(
|
|
152
|
+
"--max-items",
|
|
153
|
+
type=int,
|
|
154
|
+
default=0,
|
|
155
|
+
help="controlled extraction slice size after full discovery; 0 processes the remaining collection",
|
|
156
|
+
)
|
|
157
|
+
collect.add_argument(
|
|
158
|
+
"--start-index",
|
|
159
|
+
type=int,
|
|
160
|
+
default=0,
|
|
161
|
+
help="zero-based extraction slice start after full discovery; default: 0",
|
|
162
|
+
)
|
|
163
|
+
collect.add_argument(
|
|
164
|
+
"--progress",
|
|
165
|
+
action="store_true",
|
|
166
|
+
help="write sanitized per-item progress heartbeats to stderr",
|
|
167
|
+
)
|
|
168
|
+
collect.set_defaults(func=cmd_collect)
|
|
169
|
+
|
|
170
|
+
capabilities = subparsers.add_parser("capabilities", description="show read-only capability bindings")
|
|
171
|
+
capabilities.add_argument("--source", help="filter by normalized source id")
|
|
172
|
+
capabilities.add_argument("--format", choices=("json", "table"), default="table")
|
|
173
|
+
capabilities.set_defaults(func=cmd_capabilities)
|
|
174
|
+
|
|
175
|
+
plan = subparsers.add_parser("plan", description="build a read-only capability plan")
|
|
176
|
+
plan.add_argument("input")
|
|
177
|
+
plan.add_argument("--input-kind", choices=("auto", "single", "list", "collection", "file", "folder"), default="auto")
|
|
178
|
+
plan.add_argument("--all", action="store_true", help="request source-exhausted planning only")
|
|
179
|
+
plan.add_argument("--gate", action="append", default=[], help="declare a planning gate without executing a provider")
|
|
180
|
+
plan.add_argument("--format", choices=("json", "table"), default="table")
|
|
181
|
+
plan.set_defaults(func=cmd_plan)
|
|
182
|
+
|
|
183
|
+
doctor = subparsers.add_parser(
|
|
184
|
+
"doctor",
|
|
185
|
+
description="print local entry and safe session-boundary diagnostics",
|
|
186
|
+
)
|
|
187
|
+
doctor.add_argument("--profile", choices=DOCTOR_PROFILES, default="all")
|
|
188
|
+
doctor.add_argument("--json", action="store_true", help="machine-readable report (default when stdout is not a terminal)")
|
|
189
|
+
doctor.add_argument("--probe-commands", action="store_true", help="explicitly probe installed tool versions")
|
|
190
|
+
doctor.add_argument("--clear-gpu-quarantine", action="store_true", help="explicitly clear only the local GPU quarantine marker; never unload models or contact Ollama")
|
|
191
|
+
doctor.set_defaults(func=cmd_doctor)
|
|
192
|
+
|
|
193
|
+
from openmd.probe import cmd_probe
|
|
194
|
+
|
|
195
|
+
probe = subparsers.add_parser("probe", description="用本机配置的单条样本探活平台;不下载媒体或运行 ASR。")
|
|
196
|
+
probe.add_argument("--platform", choices=PLATFORM_IDS)
|
|
197
|
+
probe.add_argument("--config", help="本机样本 JSON;默认 OPENMD_PROBE_CONFIG 或 out/probe-config.json")
|
|
198
|
+
probe.add_argument("--out", default="out/probe", help="脱敏 JSON 报告目录(默认 out/probe/<date>/)")
|
|
199
|
+
probe.add_argument("--timeout", type=int, choices=range(1, 301), metavar="1..300", default=90, help="每个平台的最长秒数;默认 90,超时回收该平台调用子树并继续")
|
|
200
|
+
probe.add_argument("--enable-network-backends", action="store_true", help="显式允许现有公开网络读取路线")
|
|
201
|
+
probe.add_argument("--enable-browser-backends", action="store_true", help="显式允许现有浏览器/登录态读取路线")
|
|
202
|
+
probe.add_argument("--json", action="store_true", help="输出脱敏 JSON 回执")
|
|
203
|
+
probe.set_defaults(func=cmd_probe)
|
|
204
|
+
|
|
205
|
+
routes = subparsers.add_parser("routes", description="show effective routes or manage benchmark evidence")
|
|
206
|
+
routes.add_argument("action", nargs="?", choices=("bench", "demote", "restore"))
|
|
207
|
+
routes.add_argument("capability_id", nargs="?")
|
|
208
|
+
routes.add_argument("--source")
|
|
209
|
+
routes.add_argument("--capability")
|
|
210
|
+
routes.add_argument("--sample", help="fixed sample identifier for this bounded benchmark")
|
|
211
|
+
routes.add_argument("--samples-file", help="explicit sample registry, including existing local PDF/image fixtures")
|
|
212
|
+
routes.add_argument("--artifacts", help="benchmark artifacts directory")
|
|
213
|
+
routes.add_argument("--format", choices=("json", "table", "markdown"), default="table")
|
|
214
|
+
routes.add_argument("--candidates", help="comma-separated backend capability identifiers")
|
|
215
|
+
routes.add_argument("--evidence", action="append", default=[])
|
|
216
|
+
routes.add_argument("--ledger")
|
|
217
|
+
routes.add_argument("--overlay")
|
|
218
|
+
routes.add_argument("--note", default="")
|
|
219
|
+
for gate in ("network-backends", "browser-backends", "media-download", "asr", "cloud-backends"):
|
|
220
|
+
routes.add_argument(f"--enable-{gate}", action="store_true")
|
|
221
|
+
routes.set_defaults(func=cmd_routes)
|
|
222
|
+
|
|
223
|
+
from openmd.browser_setup import cmd_browser_setup
|
|
224
|
+
|
|
225
|
+
browser_setup = subparsers.add_parser("browser-setup", description="连接 Chrome / Edge 插件与本机 openmd(只写当前用户设置)")
|
|
226
|
+
browser_setup.add_argument("--uninstall", action="store_true", help="移除本地连接")
|
|
227
|
+
browser_setup.set_defaults(func=cmd_browser_setup)
|
|
228
|
+
|
|
229
|
+
report = subparsers.add_parser("report", description="build reports from sanitized runlogs")
|
|
230
|
+
report_subparsers = report.add_subparsers(dest="report_command")
|
|
231
|
+
weixin_author_batch = report_subparsers.add_parser(
|
|
232
|
+
"weixin-author-batch",
|
|
233
|
+
help="summarize Weixin author-batch coverage, recovered fallback, and missing items",
|
|
234
|
+
)
|
|
235
|
+
weixin_author_batch.add_argument(
|
|
236
|
+
"--runlog",
|
|
237
|
+
action="append",
|
|
238
|
+
required=True,
|
|
239
|
+
help="sanitized runlog JSONL path; repeat to merge multiple runs",
|
|
240
|
+
)
|
|
241
|
+
weixin_author_batch.add_argument("--out", required=True, help="report output directory")
|
|
242
|
+
weixin_author_batch.add_argument(
|
|
243
|
+
"--format",
|
|
244
|
+
choices=("both", "json", "markdown"),
|
|
245
|
+
default="both",
|
|
246
|
+
help="report artifact format",
|
|
247
|
+
)
|
|
248
|
+
weixin_author_batch.set_defaults(func=cmd_report_weixin_author_batch)
|
|
249
|
+
return parser
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _add_shared_options(
|
|
253
|
+
parser: argparse.ArgumentParser,
|
|
254
|
+
*,
|
|
255
|
+
default_limit: int | None = None,
|
|
256
|
+
include_manifest_only: bool = False,
|
|
257
|
+
include_discovery_backend: bool = False,
|
|
258
|
+
output_required: bool = True,
|
|
259
|
+
) -> None:
|
|
260
|
+
parser.add_argument("--out", required=output_required, help="output directory")
|
|
261
|
+
parser.add_argument("--backend", help="backend name override")
|
|
262
|
+
parser.add_argument("--fallback", dest="fallback", action="store_true", default=True)
|
|
263
|
+
parser.add_argument("--no-fallback", dest="fallback", action="store_false")
|
|
264
|
+
parser.add_argument("--limit", type=int, default=default_limit)
|
|
265
|
+
parser.add_argument(
|
|
266
|
+
"--all",
|
|
267
|
+
action="store_true",
|
|
268
|
+
help="full collection; only runs for explicitly gated providers",
|
|
269
|
+
)
|
|
270
|
+
parser.add_argument("--overwrite", action="store_true")
|
|
271
|
+
parser.add_argument(
|
|
272
|
+
"--package",
|
|
273
|
+
action="store_true",
|
|
274
|
+
help="write V1 item.openmd/ package artifacts instead of V0 single Markdown",
|
|
275
|
+
)
|
|
276
|
+
parser.add_argument(
|
|
277
|
+
"--package-root",
|
|
278
|
+
help="package library root; defaults to <out>/packages when --package is used",
|
|
279
|
+
)
|
|
280
|
+
parser.add_argument(
|
|
281
|
+
"--skip-existing",
|
|
282
|
+
action="store_true",
|
|
283
|
+
help="skip existing package_id matches; package mode uses this behavior by default",
|
|
284
|
+
)
|
|
285
|
+
parser.add_argument(
|
|
286
|
+
"--update",
|
|
287
|
+
action="store_true",
|
|
288
|
+
help="update an existing package in place when package_id already exists",
|
|
289
|
+
)
|
|
290
|
+
parser.add_argument(
|
|
291
|
+
"--new-version",
|
|
292
|
+
action="store_true",
|
|
293
|
+
help="create a versioned package snapshot instead of updating/skipping",
|
|
294
|
+
)
|
|
295
|
+
parser.add_argument("--download-assets", action="store_true")
|
|
296
|
+
parser.add_argument("--raw", action="store_true", help="retain original transcript lines and timestamps without readable grouping")
|
|
297
|
+
parser.add_argument("--keep-raw", action="store_true")
|
|
298
|
+
parser.add_argument("--keep-media", action="store_true")
|
|
299
|
+
parser.add_argument(
|
|
300
|
+
"--enable-media-download",
|
|
301
|
+
action="store_true",
|
|
302
|
+
help="opt in to media download fallback backends; raw media is still deleted unless --keep-media is set",
|
|
303
|
+
)
|
|
304
|
+
parser.add_argument(
|
|
305
|
+
"--enable-asr",
|
|
306
|
+
action="store_true",
|
|
307
|
+
help="opt in to ASR fallback backends after native text/subtitle paths fail",
|
|
308
|
+
)
|
|
309
|
+
parser.add_argument(
|
|
310
|
+
"--enable-cloud-backends",
|
|
311
|
+
action="store_true",
|
|
312
|
+
help="explicitly allow cloud ASR/OCR providers; --enable-asr alone does not allow upload",
|
|
313
|
+
)
|
|
314
|
+
parser.add_argument(
|
|
315
|
+
"--guardrail-item-threshold",
|
|
316
|
+
type=int,
|
|
317
|
+
default=DEFAULT_GUARDRAIL_ITEM_THRESHOLD,
|
|
318
|
+
help="emit a runlog guardrail signal when a batch/collection reaches this item count; 0 disables",
|
|
319
|
+
)
|
|
320
|
+
parser.add_argument(
|
|
321
|
+
"--guardrail-failure-streak",
|
|
322
|
+
type=int,
|
|
323
|
+
default=DEFAULT_GUARDRAIL_FAILURE_STREAK,
|
|
324
|
+
help="pause batch/collection after this many consecutive failed items; 0 disables",
|
|
325
|
+
)
|
|
326
|
+
if include_manifest_only:
|
|
327
|
+
parser.add_argument(
|
|
328
|
+
"--manifest-only",
|
|
329
|
+
action="store_true",
|
|
330
|
+
help="write discovery manifest and skip item extraction",
|
|
331
|
+
)
|
|
332
|
+
if include_discovery_backend:
|
|
333
|
+
parser.add_argument(
|
|
334
|
+
"--discovery-backend",
|
|
335
|
+
help="discovery provider override for collect, independent from --backend",
|
|
336
|
+
)
|
|
337
|
+
parser.add_argument("--runlog", help="runlog JSONL path; defaults to <out>/runlog.jsonl")
|
|
338
|
+
parser.add_argument(
|
|
339
|
+
"--receipt",
|
|
340
|
+
choices=("compact",),
|
|
341
|
+
help="write a bounded per-run receipt and print one compact JSON line",
|
|
342
|
+
)
|
|
343
|
+
parser.add_argument(
|
|
344
|
+
"--enable-network-backends",
|
|
345
|
+
action="store_true",
|
|
346
|
+
help="opt in to real network backends and LibreOffice fetching Office external resources",
|
|
347
|
+
)
|
|
348
|
+
parser.add_argument(
|
|
349
|
+
"--enable-browser-backends",
|
|
350
|
+
action="store_true",
|
|
351
|
+
help="opt in to browser-context backends such as OpenCLI for matching collection inputs",
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _runlog_path(args: argparse.Namespace) -> Path:
|
|
356
|
+
return Path(args.runlog) if args.runlog else default_runlog_path(args.out)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _print_result(result: CliResult) -> None:
|
|
360
|
+
# Keep CLI status JSON ASCII-safe for Windows consoles with narrow encodings.
|
|
361
|
+
print(json.dumps(asdict(result), ensure_ascii=True, sort_keys=True))
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _start_compact_receipt(args: argparse.Namespace, command: str) -> str | None:
|
|
365
|
+
if getattr(args, "receipt", None) != "compact":
|
|
366
|
+
return None
|
|
367
|
+
try:
|
|
368
|
+
run = CompactReceiptRun.create(args.out, command)
|
|
369
|
+
except OSError:
|
|
370
|
+
args._compact_receipt_storage_unavailable = True
|
|
371
|
+
return "compact receipt storage is unavailable"
|
|
372
|
+
runlog_path = _runlog_path(args)
|
|
373
|
+
args._compact_receipt_run = run
|
|
374
|
+
args._compact_runlog_path = runlog_path
|
|
375
|
+
args._compact_runlog_offset = runlog_path.stat().st_size if runlog_path.exists() else 0
|
|
376
|
+
args._compact_requires_fresh_stdin = (
|
|
377
|
+
getattr(args, "url", None) == "-" or getattr(args, "urls_file", None) == "-"
|
|
378
|
+
)
|
|
379
|
+
run.requires_fresh_stdin = args._compact_requires_fresh_stdin
|
|
380
|
+
run.write_checkpoint()
|
|
381
|
+
if getattr(args, "package", False):
|
|
382
|
+
try:
|
|
383
|
+
args._compact_output_lock = CompactOutputLock.acquire(args.out, run.run_id)
|
|
384
|
+
except FileExistsError:
|
|
385
|
+
return "compact package output is locked; inspect .openmd-compact.lock before retrying"
|
|
386
|
+
return None
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _compact_failure_payload() -> dict[str, object]:
|
|
390
|
+
"""Return the bounded receipt used when the compact run cannot be persisted."""
|
|
391
|
+
counts = {status: 0 for status in CompactReceiptRun.TERMINAL_STATUSES}
|
|
392
|
+
counts["failed"] = 1
|
|
393
|
+
counts["discovered"] = 1
|
|
394
|
+
counts["skipped"] = 0
|
|
395
|
+
counts["input_occurrences"] = 1
|
|
396
|
+
counts["duplicate_inputs"] = 0
|
|
397
|
+
return {
|
|
398
|
+
"schema_version": "openmd.receipt.v1",
|
|
399
|
+
"status": "failed",
|
|
400
|
+
"counts": counts,
|
|
401
|
+
"summary_ref": None,
|
|
402
|
+
"error_counts": {"unknown": 1},
|
|
403
|
+
"other_error_count": 0,
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _print_compact_failure() -> int:
|
|
408
|
+
print(json.dumps(_compact_failure_payload(), ensure_ascii=True, separators=(",", ":")))
|
|
409
|
+
return 1
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _register_compact_inputs(args: argparse.Namespace, identities: dict[str, int]) -> None:
|
|
413
|
+
run = getattr(args, "_compact_receipt_run", None)
|
|
414
|
+
if run is None:
|
|
415
|
+
return
|
|
416
|
+
args._compact_inputs_registered = True
|
|
417
|
+
args._compact_pending_identities = identities
|
|
418
|
+
run.replace_expected(identities)
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _sync_compact_rows(args: argparse.Namespace) -> None:
|
|
422
|
+
run = getattr(args, "_compact_receipt_run", None)
|
|
423
|
+
if run is None:
|
|
424
|
+
return
|
|
425
|
+
path = args._compact_runlog_path
|
|
426
|
+
rows = _read_new_runlog_rows(path, args._compact_runlog_offset)
|
|
427
|
+
_record_compact_rows(run, rows, args.out, args=args)
|
|
428
|
+
args._compact_runlog_offset = path.stat().st_size if path.exists() else 0
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _append_execution_entry(args: argparse.Namespace, entry: dict[str, object]) -> None:
|
|
432
|
+
append_runlog_entry(_runlog_path(args), entry)
|
|
433
|
+
# Receipt/checkpoint must be durable before the next provider call.
|
|
434
|
+
_sync_compact_rows(args)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _prepare_compact_collection(args: argparse.Namespace, items: list[ContentItem]) -> list[ContentItem]:
|
|
438
|
+
if getattr(args, "receipt", None) != "compact":
|
|
439
|
+
return items
|
|
440
|
+
unique: dict[str, ContentItem] = {}
|
|
441
|
+
occurrences: dict[str, int] = {}
|
|
442
|
+
by_url: dict[str, int] = {}
|
|
443
|
+
for item in items:
|
|
444
|
+
identity = _compact_item_identity(item)
|
|
445
|
+
unique.setdefault(identity, item)
|
|
446
|
+
occurrences[identity] = occurrences.get(identity, 0) + 1
|
|
447
|
+
by_url[sanitize_url(item.url)] = occurrences[identity]
|
|
448
|
+
args._compact_input_occurrences_by_url = by_url
|
|
449
|
+
_register_compact_inputs(args, occurrences)
|
|
450
|
+
return list(unique.values())
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _reconcile_compact_identity(args: argparse.Namespace, original: ContentItem, resolved: ContentItem) -> None:
|
|
454
|
+
"""A resolver may replace a short locator with the canonical resource URL."""
|
|
455
|
+
if getattr(args, "_compact_receipt_run", None) is None:
|
|
456
|
+
return
|
|
457
|
+
previous, current = _compact_item_identity(original), _compact_item_identity(resolved)
|
|
458
|
+
if previous == current:
|
|
459
|
+
return
|
|
460
|
+
pending = getattr(args, "_compact_pending_identities", {})
|
|
461
|
+
occurrence = pending.pop(previous, 1)
|
|
462
|
+
pending[current] = pending.get(current, 0) + occurrence
|
|
463
|
+
run = args._compact_receipt_run
|
|
464
|
+
expected = dict(run.expected)
|
|
465
|
+
old_occurrence = expected.pop(previous, occurrence)
|
|
466
|
+
expected[current] = expected.get(current, 0) + old_occurrence
|
|
467
|
+
run.add_occurrences(current, old_occurrence)
|
|
468
|
+
run.replace_expected(expected)
|
|
469
|
+
by_url = getattr(args, "_compact_input_occurrences_by_url", {})
|
|
470
|
+
by_url[sanitize_url(resolved.url)] = expected[current]
|
|
471
|
+
args._compact_input_occurrences_by_url = by_url
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _register_resolved_batch_items(args: argparse.Namespace, input_url: str, items: list[ContentItem]) -> None:
|
|
475
|
+
run = getattr(args, "_compact_receipt_run", None)
|
|
476
|
+
if run is None:
|
|
477
|
+
return
|
|
478
|
+
previous = _compact_url_identity(input_url)
|
|
479
|
+
pending = args._compact_pending_identities
|
|
480
|
+
occurrence = pending.get(previous, 1)
|
|
481
|
+
if not items:
|
|
482
|
+
if not run.has_item(previous):
|
|
483
|
+
run.record_item(stable_identity=previous, input_occurrence=occurrence,
|
|
484
|
+
status="blocked", error_code="blocked", recovery_type="empty_scope")
|
|
485
|
+
return
|
|
486
|
+
pending.pop(previous, None)
|
|
487
|
+
expected = dict(run.expected)
|
|
488
|
+
expected.pop(previous, None)
|
|
489
|
+
by_url = args._compact_input_occurrences_by_url
|
|
490
|
+
for item in items:
|
|
491
|
+
identity = _compact_item_identity(item)
|
|
492
|
+
pending[identity] = pending.get(identity, 0) + occurrence
|
|
493
|
+
expected[identity] = expected.get(identity, 0) + occurrence
|
|
494
|
+
run.add_occurrences(identity, occurrence)
|
|
495
|
+
by_url[sanitize_url(item.url)] = expected[identity]
|
|
496
|
+
run.replace_expected(expected)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _emit_result(
|
|
500
|
+
args: argparse.Namespace,
|
|
501
|
+
result: CliResult,
|
|
502
|
+
*,
|
|
503
|
+
discovery_complete: bool = True,
|
|
504
|
+
manifest_only: bool = False,
|
|
505
|
+
) -> int:
|
|
506
|
+
run = getattr(args, "_compact_receipt_run", None)
|
|
507
|
+
if run is None:
|
|
508
|
+
if getattr(args, "_compact_receipt_storage_unavailable", False):
|
|
509
|
+
return _print_compact_failure()
|
|
510
|
+
_print_result(result)
|
|
511
|
+
return 0 if result.ok else 1
|
|
512
|
+
try:
|
|
513
|
+
if not manifest_only:
|
|
514
|
+
_sync_compact_rows(args)
|
|
515
|
+
for identity, occurrence in getattr(args, "_compact_pending_identities", {}).items():
|
|
516
|
+
if not run.has_item(identity):
|
|
517
|
+
run.record_item(
|
|
518
|
+
stable_identity=identity, input_occurrence=occurrence,
|
|
519
|
+
status="unattempted", quality_status="not_evaluated",
|
|
520
|
+
recovery_type="manifest_only" if manifest_only else "rerun_input",
|
|
521
|
+
)
|
|
522
|
+
if not run.has_items and not result.ok and not getattr(args, "_compact_inputs_registered", False):
|
|
523
|
+
run.record_item(
|
|
524
|
+
stable_identity=CompactReceiptRun.stable_identity(result.command, "preflight"),
|
|
525
|
+
input_occurrence=1, status="blocked", error_code="blocked",
|
|
526
|
+
recovery_type="inspect_runlog",
|
|
527
|
+
)
|
|
528
|
+
payload, exit_code = run.finalize(
|
|
529
|
+
discovery_complete=discovery_complete,
|
|
530
|
+
manifest_only=manifest_only,
|
|
531
|
+
resume_supported=bool(getattr(args, "package", False)),
|
|
532
|
+
requires_fresh_stdin=bool(getattr(args, "_compact_requires_fresh_stdin", False)),
|
|
533
|
+
failed=bool(getattr(args, "_compact_execution_failed", False)),
|
|
534
|
+
)
|
|
535
|
+
lock = getattr(args, "_compact_output_lock", None)
|
|
536
|
+
if lock is not None:
|
|
537
|
+
lock.release()
|
|
538
|
+
args._compact_output_lock = None
|
|
539
|
+
print(json.dumps(payload, ensure_ascii=True, separators=(",", ":")))
|
|
540
|
+
return exit_code
|
|
541
|
+
except (OSError, ValueError):
|
|
542
|
+
return _print_compact_failure()
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def _read_new_runlog_rows(path: Path, offset: int) -> list[dict[str, object]]:
|
|
546
|
+
if not path.exists():
|
|
547
|
+
return []
|
|
548
|
+
with path.open("rb") as handle:
|
|
549
|
+
handle.seek(offset)
|
|
550
|
+
lines = handle.read().splitlines()
|
|
551
|
+
rows: list[dict[str, object]] = []
|
|
552
|
+
for line in lines:
|
|
553
|
+
try:
|
|
554
|
+
value = json.loads(line)
|
|
555
|
+
except json.JSONDecodeError as exc:
|
|
556
|
+
raise ValueError("malformed_runlog_line") from exc
|
|
557
|
+
if isinstance(value, dict) and value.get("event_type") == "item":
|
|
558
|
+
rows.append(value)
|
|
559
|
+
return rows
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _compact_item_identity(item: ContentItem) -> str:
|
|
563
|
+
return CompactReceiptRun.stable_identity(item.platform, sanitize_url(item.url))
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _compact_url_identity(url: str) -> str:
|
|
567
|
+
return CompactReceiptRun.stable_identity(detect_platform(url), sanitize_url(url))
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _prepare_resolved_item(
|
|
571
|
+
args: argparse.Namespace,
|
|
572
|
+
original: ContentItem,
|
|
573
|
+
resolved: ContentItem,
|
|
574
|
+
provider_chain: tuple[str, ...],
|
|
575
|
+
) -> tuple[ContentItem, str | None] | None:
|
|
576
|
+
"""Return the contract-applied item and its fingerprint, or None if the receipt already has it."""
|
|
577
|
+
item = apply_platform_extraction_contract(resolved)
|
|
578
|
+
if item.platform == "douyin" and getattr(args, "backend", None) == "douyin.metadata":
|
|
579
|
+
item = replace(item, raw_meta={**item.raw_meta, "extraction_target": "metadata"})
|
|
580
|
+
_reconcile_compact_identity(args, original, item)
|
|
581
|
+
run = getattr(args, "_compact_receipt_run", None)
|
|
582
|
+
if run is not None and run.has_item(_compact_item_identity(item)):
|
|
583
|
+
return None
|
|
584
|
+
return item, _compact_execution_fingerprint(item, args, provider_chain)
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def _record_compact_rows(
|
|
588
|
+
run: CompactReceiptRun,
|
|
589
|
+
rows: list[dict[str, object]],
|
|
590
|
+
out_dir: str,
|
|
591
|
+
*,
|
|
592
|
+
args: argparse.Namespace,
|
|
593
|
+
) -> set[str]:
|
|
594
|
+
out_root = Path(out_dir).resolve()
|
|
595
|
+
grouped: dict[str, list[dict[str, object]]] = {}
|
|
596
|
+
for row in rows:
|
|
597
|
+
identity = CompactReceiptRun.stable_identity(
|
|
598
|
+
str(row.get("platform") or ""),
|
|
599
|
+
sanitize_url(str(row.get("input_url") or "")),
|
|
600
|
+
)
|
|
601
|
+
grouped.setdefault(identity, []).append(row)
|
|
602
|
+
for identity, grouped_rows in grouped.items():
|
|
603
|
+
if run.has_item(identity):
|
|
604
|
+
continue
|
|
605
|
+
row = grouped_rows[-1]
|
|
606
|
+
status = _compact_terminal_status(row)
|
|
607
|
+
output_path = str(row.get("output_path") or "")
|
|
608
|
+
artifact_ref: str | None = None
|
|
609
|
+
if output_path:
|
|
610
|
+
try:
|
|
611
|
+
artifact_ref = Path(output_path).resolve().relative_to(out_root).as_posix()
|
|
612
|
+
except ValueError:
|
|
613
|
+
artifact_ref = None
|
|
614
|
+
run.record_item(
|
|
615
|
+
stable_identity=identity,
|
|
616
|
+
input_occurrence=max(
|
|
617
|
+
len(grouped_rows),
|
|
618
|
+
int(getattr(args, "_compact_input_occurrences_by_url", {}).get(str(row.get("input_url") or ""), 1)),
|
|
619
|
+
),
|
|
620
|
+
status=status,
|
|
621
|
+
skipped=str(row.get("package_operation") or "") == "skipped_existing",
|
|
622
|
+
artifact_ref=artifact_ref,
|
|
623
|
+
source_level=str(row.get("source_level") or "unknown"),
|
|
624
|
+
quality_status="passed" if status == "verified" else "not_evaluated",
|
|
625
|
+
error_code=str(row.get("error_code") or "") or None,
|
|
626
|
+
failure_class=str(row.get("failure_class") or "") or None,
|
|
627
|
+
recovery_type=str(row.get("resume_pointer") or "") or None,
|
|
628
|
+
route_trace=row.get("route_trace") if isinstance(row.get("route_trace"), dict) else None,
|
|
629
|
+
)
|
|
630
|
+
return set(grouped)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _compact_terminal_status(row: dict[str, object]) -> str:
|
|
634
|
+
if int(row.get("terminal_gap_count") or 0) > 0:
|
|
635
|
+
return "terminal_gap"
|
|
636
|
+
status = str(row.get("status") or "failed")
|
|
637
|
+
if status in {"verified", "partial"}:
|
|
638
|
+
return status
|
|
639
|
+
failure_class = str(row.get("failure_class") or "")
|
|
640
|
+
if str(row.get("error_code") or "") in {
|
|
641
|
+
"execution_fingerprint_missing",
|
|
642
|
+
"execution_fingerprint_mismatch",
|
|
643
|
+
"extraction_contract_mismatch",
|
|
644
|
+
"package_validation_failed",
|
|
645
|
+
}:
|
|
646
|
+
return "blocked"
|
|
647
|
+
if failure_class == "runtime_transient":
|
|
648
|
+
return "retryable_failure"
|
|
649
|
+
if failure_class == "source_unavailable":
|
|
650
|
+
return "terminal_gap"
|
|
651
|
+
if failure_class == "user_resolvable":
|
|
652
|
+
return "blocked"
|
|
653
|
+
return "failed"
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _compact_execution_fingerprint(
|
|
657
|
+
item: ContentItem,
|
|
658
|
+
args: argparse.Namespace,
|
|
659
|
+
provider_chain: tuple[str, ...],
|
|
660
|
+
) -> str | None:
|
|
661
|
+
if getattr(args, "receipt", None) != "compact" or not getattr(args, "package", False):
|
|
662
|
+
return None
|
|
663
|
+
raw_meta = item.raw_meta if isinstance(item.raw_meta, dict) else {}
|
|
664
|
+
modalities_value = raw_meta.get("modalities")
|
|
665
|
+
modalities = tuple(str(value) for value in modalities_value) if isinstance(modalities_value, list) else (item.source_type,)
|
|
666
|
+
effective_config: dict[str, object] = {}
|
|
667
|
+
if item.platform in {"pdf", "screenshot"} and any(backend.startswith(("ocr.", "pdf.")) for backend in provider_chain):
|
|
668
|
+
from openmd.ocr.config import settings
|
|
669
|
+
|
|
670
|
+
effective_config["ocr_router"] = settings()
|
|
671
|
+
from openmd.runtime.asr import backend_execution_config
|
|
672
|
+
|
|
673
|
+
if "asr.faster_whisper" in provider_chain:
|
|
674
|
+
effective_config["faster_whisper"] = backend_execution_config("asr.faster_whisper")
|
|
675
|
+
from openmd.routing.loader import load_catalog
|
|
676
|
+
|
|
677
|
+
composed_asr = any(load_catalog().capabilities.get(backend, {}).get("then") == "asr.auto" for backend in provider_chain)
|
|
678
|
+
if "asr.auto" in provider_chain or composed_asr or "douyin.transcript" in provider_chain:
|
|
679
|
+
effective_config["asr_auto"] = backend_execution_config("asr.auto")
|
|
680
|
+
if "asr.qwen3" in provider_chain:
|
|
681
|
+
effective_config["qwen3"] = backend_execution_config("asr.qwen3")
|
|
682
|
+
if item.platform in {"douyin", "xiaohongshu"}:
|
|
683
|
+
from openmd.backends.douyin_providers import SILICONFLOW_MODEL
|
|
684
|
+
|
|
685
|
+
effective_config["cloud_asr_model"] = SILICONFLOW_MODEL
|
|
686
|
+
return execution_fingerprint(
|
|
687
|
+
requested_profile=str(raw_meta.get("profile") or "") or None,
|
|
688
|
+
requested_modalities=modalities,
|
|
689
|
+
provider_chain=provider_chain,
|
|
690
|
+
backend_override=getattr(args, "backend", None),
|
|
691
|
+
model=str(raw_meta.get("asr_model") or "") or None,
|
|
692
|
+
language=str(raw_meta.get("language") or "") or None,
|
|
693
|
+
extraction_contract={
|
|
694
|
+
"version": extraction_contract_version_for(item),
|
|
695
|
+
"required_components": list(required_component_ids_for_item(item)),
|
|
696
|
+
},
|
|
697
|
+
effective_config=effective_config,
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _read_urls_file(path: str) -> list[str]:
|
|
702
|
+
lines = sys.stdin.read().splitlines() if path == "-" else Path(path).read_text(encoding="utf-8").splitlines()
|
|
703
|
+
return [line.strip() for line in lines if line.strip() and not line.lstrip().startswith("#")]
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def _read_single_input(value: str) -> str:
|
|
707
|
+
if value != "-":
|
|
708
|
+
return value
|
|
709
|
+
inputs = [line.strip() for line in sys.stdin.read().splitlines() if line.strip() and not line.lstrip().startswith("#")]
|
|
710
|
+
if len(inputs) != 1:
|
|
711
|
+
raise ValueError("stdin must contain exactly one non-comment input")
|
|
712
|
+
return inputs[0]
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def _log_all_guard(input_url: str, args: argparse.Namespace, *, command: str) -> CliResult:
|
|
716
|
+
platform = detect_platform(input_url)
|
|
717
|
+
entry = build_runlog_entry(
|
|
718
|
+
input_url=input_url,
|
|
719
|
+
platform=platform,
|
|
720
|
+
backend=args.backend or "not_selected",
|
|
721
|
+
status="failed",
|
|
722
|
+
source_type="mixed",
|
|
723
|
+
source_level="partial_extraction",
|
|
724
|
+
item_count=0,
|
|
725
|
+
success_count=0,
|
|
726
|
+
failed_count=1,
|
|
727
|
+
event_type="guardrail",
|
|
728
|
+
failure_class="user_resolvable",
|
|
729
|
+
guardrail_signals=["all_requires_explicit_gate"],
|
|
730
|
+
resume_pointer="rerun with an explicitly supported provider gate",
|
|
731
|
+
error_code="all_requires_confirmation",
|
|
732
|
+
error_summary="--all requires an explicitly supported provider gate before collection work starts.",
|
|
733
|
+
)
|
|
734
|
+
_append_execution_entry(args, entry)
|
|
735
|
+
return CliResult(
|
|
736
|
+
ok=False,
|
|
737
|
+
command=command,
|
|
738
|
+
failed_count=1,
|
|
739
|
+
guardrail_signals=["all_requires_explicit_gate"],
|
|
740
|
+
message="--all requires explicit confirmation before any collection work.",
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
def _allow_scoped_all_collect(input_url: str, args: argparse.Namespace) -> bool:
|
|
745
|
+
"""Allow full collection only for explicitly gated lanes that support it."""
|
|
746
|
+
if not getattr(args, "all", False):
|
|
747
|
+
return True
|
|
748
|
+
return collect_all_allowed(
|
|
749
|
+
_detect_collect_platform(input_url),
|
|
750
|
+
browser_enabled=bool(getattr(args, "enable_browser_backends", False)),
|
|
751
|
+
network_enabled=bool(getattr(args, "enable_network_backends", False)),
|
|
752
|
+
discovery_provider=normalize_discovery_backend(getattr(args, "discovery_backend", None)),
|
|
753
|
+
)
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def _build_backend_executor(args: argparse.Namespace, *, platform: str) -> BackendExecutor:
|
|
757
|
+
from openmd.backends.video_metadata import enrich_video_result
|
|
758
|
+
|
|
759
|
+
provider_registry = create_runtime_backend_provider_registry(
|
|
760
|
+
platform,
|
|
761
|
+
_backend_runtime_options(args),
|
|
762
|
+
)
|
|
763
|
+
authorizer = getattr(args, "_browser_authorizer", None)
|
|
764
|
+
if authorizer is not None:
|
|
765
|
+
from openmd.backends.interactive import DeferredBrowserRegistry
|
|
766
|
+
|
|
767
|
+
provider_registry = DeferredBrowserRegistry(provider_registry, platform, args, authorizer)
|
|
768
|
+
return BackendExecutor(
|
|
769
|
+
provider_registry=provider_registry,
|
|
770
|
+
result_enricher=lambda item, result: enrich_video_result(
|
|
771
|
+
item, result, network_enabled=bool(getattr(args, "enable_network_backends", False))
|
|
772
|
+
),
|
|
773
|
+
)
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
def _build_discovery_executor(args: argparse.Namespace, *, platform: str) -> DiscoveryExecutor:
|
|
777
|
+
options = _backend_runtime_options(args)
|
|
778
|
+
provider_registry = create_runtime_discovery_provider_registry(platform, options)
|
|
779
|
+
return DiscoveryExecutor(
|
|
780
|
+
provider_registry=provider_registry,
|
|
781
|
+
requested_provider=normalize_discovery_backend(options.requested_discovery_provider),
|
|
782
|
+
)
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def _backend_runtime_options(args: argparse.Namespace) -> BackendRuntimeOptions:
|
|
786
|
+
keep_media_artifacts = getattr(args, "keep_media", False)
|
|
787
|
+
return BackendRuntimeOptions(
|
|
788
|
+
backend_override=getattr(args, "backend", None),
|
|
789
|
+
requested_discovery_provider=getattr(args, "discovery_backend", None),
|
|
790
|
+
enable_network_backends=getattr(args, "enable_network_backends", False),
|
|
791
|
+
enable_browser_backends=getattr(args, "enable_browser_backends", False),
|
|
792
|
+
enable_media_download_backends=getattr(args, "enable_media_download", False),
|
|
793
|
+
enable_asr_backends=getattr(args, "enable_asr", False),
|
|
794
|
+
enable_cloud_backends=getattr(args, "enable_cloud_backends", False),
|
|
795
|
+
keep_media_artifacts=keep_media_artifacts,
|
|
796
|
+
media_artifact_dir=str(Path(args.out) / "raw-media") if keep_media_artifacts else None,
|
|
797
|
+
)
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _select_collect_discovery_backend(input_url: str, args: argparse.Namespace) -> None:
|
|
801
|
+
if getattr(args, "discovery_backend", None):
|
|
802
|
+
return
|
|
803
|
+
if getattr(args, "enable_network_backends", False) and youtube_data_api_playlist_id(input_url) is not None:
|
|
804
|
+
args.discovery_backend = YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND
|
|
805
|
+
return
|
|
806
|
+
if not getattr(args, "enable_browser_backends", False):
|
|
807
|
+
return
|
|
808
|
+
if _detect_collect_platform(input_url) == "douyin" and getattr(args, "all", False):
|
|
809
|
+
args.discovery_backend = DOUYIN_OPENMD_CREATOR_VIDEOS_BACKEND
|
|
810
|
+
if not getattr(args, "backend", None):
|
|
811
|
+
args.backend = DOUYIN_TRANSCRIPT_BACKEND
|
|
812
|
+
return
|
|
813
|
+
if _detect_collect_platform(input_url) == "xiaohongshu" and getattr(args, "all", False):
|
|
814
|
+
args.discovery_backend = OPENCLI_OPENMD_XIAOHONGSHU_CREATOR_NOTES_BACKEND
|
|
815
|
+
return
|
|
816
|
+
if opencli_youtube_playlist_target(input_url) is not None:
|
|
817
|
+
args.discovery_backend = "youtube-playlist"
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _requested_backend(platform: str, args: argparse.Namespace) -> str | None:
|
|
821
|
+
"""Explicit --backend wins; local media under --enable-asr picks its model by language."""
|
|
822
|
+
if getattr(args, "backend", None):
|
|
823
|
+
return args.backend
|
|
824
|
+
if platform == "generic_media" and getattr(args, "enable_asr", False):
|
|
825
|
+
return "asr.auto"
|
|
826
|
+
return None
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def _detect_collect_platform(input_url: str) -> str:
|
|
830
|
+
platform = detect_platform(input_url)
|
|
831
|
+
if platform == "web_article" and is_youtube_collection_candidate(input_url):
|
|
832
|
+
return "youtube"
|
|
833
|
+
return platform
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def _pace_douyin_extraction(item: ContentItem, args: argparse.Namespace) -> None:
|
|
837
|
+
"""Share one bounded cadence across collection and multi-URL batch items."""
|
|
838
|
+
if item.platform != "douyin" or getattr(args, "backend", None) in {"mock", "douyin.metadata"}:
|
|
839
|
+
return
|
|
840
|
+
previous = getattr(args, "_douyin_last_extraction_at", None)
|
|
841
|
+
if previous is not None:
|
|
842
|
+
time.sleep(max(0.0, 1.0 - (time.monotonic() - previous)))
|
|
843
|
+
args._douyin_last_extraction_at = time.monotonic()
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def _process_items(
|
|
847
|
+
items: list[ContentItem],
|
|
848
|
+
args: argparse.Namespace,
|
|
849
|
+
*,
|
|
850
|
+
command: str,
|
|
851
|
+
) -> CliResult:
|
|
852
|
+
registry = create_default_registry()
|
|
853
|
+
outputs: list[str] = []
|
|
854
|
+
success_count = 0
|
|
855
|
+
failed_count = 0
|
|
856
|
+
processed_count = 0
|
|
857
|
+
consecutive_failures = 0
|
|
858
|
+
executors: dict[str, BackendExecutor] = {}
|
|
859
|
+
skipped_count = 0
|
|
860
|
+
source_level_counts: dict[str, int] = {}
|
|
861
|
+
failure_code_counts: dict[str, int] = {}
|
|
862
|
+
coverage = CollectionCoverage()
|
|
863
|
+
|
|
864
|
+
for index, item in enumerate(items, start=1):
|
|
865
|
+
processed_count += 1
|
|
866
|
+
selection = registry.select_backend(
|
|
867
|
+
item.platform,
|
|
868
|
+
_requested_backend(item.platform, args),
|
|
869
|
+
allow_fallback=args.fallback,
|
|
870
|
+
)
|
|
871
|
+
executor = executors.get(item.platform)
|
|
872
|
+
if executor is None:
|
|
873
|
+
executor = _build_backend_executor(args, platform=item.platform)
|
|
874
|
+
executors[item.platform] = executor
|
|
875
|
+
identity = executor.resolve_item_identity(item, selection.chain)
|
|
876
|
+
if not identity.ok:
|
|
877
|
+
failed_count += 1
|
|
878
|
+
consecutive_failures += 1
|
|
879
|
+
coverage.record_failure()
|
|
880
|
+
code = identity.error_code or "identity_resolution_failed"
|
|
881
|
+
failure_code_counts[code] = failure_code_counts.get(code, 0) + 1
|
|
882
|
+
_append_execution_entry(
|
|
883
|
+
args,
|
|
884
|
+
_build_identity_resolution_failure_entry(item, identity),
|
|
885
|
+
)
|
|
886
|
+
if _guardrail_failure_streak_reached(args, consecutive_failures):
|
|
887
|
+
_append_guardrail_stop_entry(args, command=command, index=index, total=len(items))
|
|
888
|
+
break
|
|
889
|
+
continue
|
|
890
|
+
prepared = _prepare_resolved_item(args, item, identity.item, selection.chain)
|
|
891
|
+
if prepared is None:
|
|
892
|
+
continue
|
|
893
|
+
item, fingerprint = prepared
|
|
894
|
+
try:
|
|
895
|
+
skipped_output = _maybe_skip_existing_package(item, args, execution_fingerprint=fingerprint)
|
|
896
|
+
except PackageValidationError as exc:
|
|
897
|
+
failed_count += 1
|
|
898
|
+
consecutive_failures += 1
|
|
899
|
+
coverage.record_failure()
|
|
900
|
+
entry = _build_package_validation_failure_entry_for_item(item, exc)
|
|
901
|
+
_append_execution_entry(args, entry)
|
|
902
|
+
if _guardrail_failure_streak_reached(args, consecutive_failures):
|
|
903
|
+
_append_guardrail_stop_entry(args, command=command, index=index, total=len(items))
|
|
904
|
+
break
|
|
905
|
+
continue
|
|
906
|
+
if skipped_output is not None:
|
|
907
|
+
outputs.append(str(skipped_output.path))
|
|
908
|
+
success_count += 1
|
|
909
|
+
skipped_count += 1
|
|
910
|
+
skipped_metadata = _read_package_metadata(skipped_output.path)
|
|
911
|
+
coverage.record_metadata(skipped_metadata, skipped_existing=True)
|
|
912
|
+
skipped_source_level = str(skipped_metadata.get("source_level") or "unknown")
|
|
913
|
+
source_level_counts[skipped_source_level] = source_level_counts.get(skipped_source_level, 0) + 1
|
|
914
|
+
consecutive_failures = 0
|
|
915
|
+
_append_execution_entry(
|
|
916
|
+
args,
|
|
917
|
+
_build_skipped_package_runlog_entry(item, skipped_output),
|
|
918
|
+
)
|
|
919
|
+
if getattr(args, "progress", False):
|
|
920
|
+
print(
|
|
921
|
+
json.dumps(
|
|
922
|
+
{
|
|
923
|
+
"event": "item_progress",
|
|
924
|
+
"index": index,
|
|
925
|
+
"total": len(items),
|
|
926
|
+
"video_id": item.id,
|
|
927
|
+
"status": "verified",
|
|
928
|
+
"source_level": skipped_source_level,
|
|
929
|
+
"package_operation": "skipped_existing",
|
|
930
|
+
},
|
|
931
|
+
ensure_ascii=True,
|
|
932
|
+
sort_keys=True,
|
|
933
|
+
),
|
|
934
|
+
file=sys.stderr,
|
|
935
|
+
flush=True,
|
|
936
|
+
)
|
|
937
|
+
continue
|
|
938
|
+
|
|
939
|
+
_pace_douyin_extraction(item, args)
|
|
940
|
+
result = executor.extract(
|
|
941
|
+
item,
|
|
942
|
+
selection.chain,
|
|
943
|
+
allow_fallback=args.fallback,
|
|
944
|
+
)
|
|
945
|
+
output: OutputWriteResult | None = None
|
|
946
|
+
if result.has_usable_text:
|
|
947
|
+
try:
|
|
948
|
+
output = _write_extraction_output(item, result, args, execution_fingerprint=fingerprint)
|
|
949
|
+
except PackageValidationError as exc:
|
|
950
|
+
failed_count += 1
|
|
951
|
+
consecutive_failures += 1
|
|
952
|
+
coverage.record_result(result, force_failed=True)
|
|
953
|
+
entry = _build_package_validation_failure_entry(item, result, exc)
|
|
954
|
+
_append_execution_entry(args, entry)
|
|
955
|
+
if _guardrail_failure_streak_reached(args, consecutive_failures):
|
|
956
|
+
_append_guardrail_stop_entry(args, command=command, index=index, total=len(items))
|
|
957
|
+
break
|
|
958
|
+
continue
|
|
959
|
+
else:
|
|
960
|
+
outputs.append(str(output.path))
|
|
961
|
+
success_count += 1
|
|
962
|
+
source_level_counts[result.source_level] = source_level_counts.get(result.source_level, 0) + 1
|
|
963
|
+
consecutive_failures = 0
|
|
964
|
+
else:
|
|
965
|
+
failed_count += 1
|
|
966
|
+
consecutive_failures += 1
|
|
967
|
+
code = result.error_code or "unknown_failure"
|
|
968
|
+
failure_code_counts[code] = failure_code_counts.get(code, 0) + 1
|
|
969
|
+
|
|
970
|
+
coverage.record_result(result)
|
|
971
|
+
|
|
972
|
+
entry = build_item_runlog_entry(
|
|
973
|
+
item,
|
|
974
|
+
result,
|
|
975
|
+
output_path=output.path if output else None,
|
|
976
|
+
package_id=output.package_id if output else None,
|
|
977
|
+
package_operation=output.package_operation if output else None,
|
|
978
|
+
)
|
|
979
|
+
_append_execution_entry(args, entry)
|
|
980
|
+
if getattr(args, "progress", False):
|
|
981
|
+
print(
|
|
982
|
+
json.dumps(
|
|
983
|
+
{
|
|
984
|
+
"event": "item_progress",
|
|
985
|
+
"index": index,
|
|
986
|
+
"total": len(items),
|
|
987
|
+
"video_id": item.id,
|
|
988
|
+
"status": result.status,
|
|
989
|
+
"source_level": result.source_level,
|
|
990
|
+
"error_code": result.error_code,
|
|
991
|
+
},
|
|
992
|
+
ensure_ascii=True,
|
|
993
|
+
sort_keys=True,
|
|
994
|
+
),
|
|
995
|
+
file=sys.stderr,
|
|
996
|
+
flush=True,
|
|
997
|
+
)
|
|
998
|
+
if _guardrail_failure_streak_reached(args, consecutive_failures):
|
|
999
|
+
_append_guardrail_stop_entry(args, command=command, index=index, total=len(items))
|
|
1000
|
+
break
|
|
1001
|
+
|
|
1002
|
+
ok = failed_count == 0 and bool(items)
|
|
1003
|
+
return CliResult(
|
|
1004
|
+
ok=ok,
|
|
1005
|
+
command=command,
|
|
1006
|
+
item_count=processed_count,
|
|
1007
|
+
success_count=success_count,
|
|
1008
|
+
failed_count=failed_count,
|
|
1009
|
+
skipped_count=skipped_count,
|
|
1010
|
+
outputs=outputs,
|
|
1011
|
+
guardrail_signals=["consecutive_failure_stop"] if processed_count < len(items) else [],
|
|
1012
|
+
source_level_counts=source_level_counts,
|
|
1013
|
+
failure_code_counts=failure_code_counts,
|
|
1014
|
+
coverage=coverage.as_dict(),
|
|
1015
|
+
message="completed" if ok else ("guardrail stopped" if processed_count < len(items) else "completed with failures"),
|
|
1016
|
+
)
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def _process_input(input_url: str, args: argparse.Namespace, *, command: str) -> CliResult:
|
|
1020
|
+
if getattr(args, "all", False):
|
|
1021
|
+
return _log_all_guard(input_url, args, command=command)
|
|
1022
|
+
|
|
1023
|
+
registry = create_default_registry()
|
|
1024
|
+
adapter = registry.adapter_for(input_url)
|
|
1025
|
+
selection = registry.select_backend(
|
|
1026
|
+
adapter.id,
|
|
1027
|
+
_requested_backend(adapter.id, args),
|
|
1028
|
+
allow_fallback=args.fallback,
|
|
1029
|
+
)
|
|
1030
|
+
if getattr(args, "enable_network_backends", False) and any(b.startswith("opencli.") for b in selection.chain):
|
|
1031
|
+
from openmd.runtime.managed_tools import ensure_opencli
|
|
1032
|
+
|
|
1033
|
+
# One `pip install openmd-cli` is enough: fetch the pinned opencli on first need.
|
|
1034
|
+
ensure_opencli()
|
|
1035
|
+
items = [
|
|
1036
|
+
apply_platform_extraction_contract(item)
|
|
1037
|
+
for item in adapter.resolve_items(input_url, limit=args.limit, all=False)
|
|
1038
|
+
]
|
|
1039
|
+
if command == "ingest":
|
|
1040
|
+
items = _prepare_compact_collection(args, items)
|
|
1041
|
+
elif command == "batch":
|
|
1042
|
+
_register_resolved_batch_items(args, input_url, items)
|
|
1043
|
+
executor = _build_backend_executor(args, platform=adapter.id)
|
|
1044
|
+
outputs: list[str] = []
|
|
1045
|
+
success_count = 0
|
|
1046
|
+
failed_count = 0
|
|
1047
|
+
skipped_count = 0
|
|
1048
|
+
|
|
1049
|
+
for item in items:
|
|
1050
|
+
identity = executor.resolve_item_identity(item, selection.chain)
|
|
1051
|
+
if not identity.ok:
|
|
1052
|
+
failed_count += 1
|
|
1053
|
+
_append_execution_entry(
|
|
1054
|
+
args,
|
|
1055
|
+
_build_identity_resolution_failure_entry(item, identity),
|
|
1056
|
+
)
|
|
1057
|
+
continue
|
|
1058
|
+
prepared = _prepare_resolved_item(args, item, identity.item, selection.chain)
|
|
1059
|
+
if prepared is None:
|
|
1060
|
+
continue
|
|
1061
|
+
item, fingerprint = prepared
|
|
1062
|
+
try:
|
|
1063
|
+
skipped_output = _maybe_skip_existing_package(item, args, execution_fingerprint=fingerprint)
|
|
1064
|
+
except PackageValidationError as exc:
|
|
1065
|
+
failed_count += 1
|
|
1066
|
+
entry = _build_package_validation_failure_entry_for_item(item, exc)
|
|
1067
|
+
_append_execution_entry(args, entry)
|
|
1068
|
+
continue
|
|
1069
|
+
if skipped_output is not None:
|
|
1070
|
+
outputs.append(str(skipped_output.path))
|
|
1071
|
+
success_count += 1
|
|
1072
|
+
skipped_count += 1
|
|
1073
|
+
_append_execution_entry(
|
|
1074
|
+
args,
|
|
1075
|
+
_build_skipped_package_runlog_entry(item, skipped_output),
|
|
1076
|
+
)
|
|
1077
|
+
continue
|
|
1078
|
+
|
|
1079
|
+
_pace_douyin_extraction(item, args)
|
|
1080
|
+
result = executor.extract(
|
|
1081
|
+
item,
|
|
1082
|
+
selection.chain,
|
|
1083
|
+
allow_fallback=args.fallback,
|
|
1084
|
+
)
|
|
1085
|
+
output: OutputWriteResult | None = None
|
|
1086
|
+
if result.has_usable_text:
|
|
1087
|
+
try:
|
|
1088
|
+
output = _write_extraction_output(item, result, args, execution_fingerprint=fingerprint)
|
|
1089
|
+
except PackageValidationError as exc:
|
|
1090
|
+
failed_count += 1
|
|
1091
|
+
entry = _build_package_validation_failure_entry(item, result, exc)
|
|
1092
|
+
_append_execution_entry(args, entry)
|
|
1093
|
+
continue
|
|
1094
|
+
else:
|
|
1095
|
+
outputs.append(str(output.path))
|
|
1096
|
+
success_count += 1
|
|
1097
|
+
else:
|
|
1098
|
+
failed_count += 1
|
|
1099
|
+
|
|
1100
|
+
entry = build_item_runlog_entry(
|
|
1101
|
+
item,
|
|
1102
|
+
result,
|
|
1103
|
+
output_path=output.path if output else None,
|
|
1104
|
+
package_id=output.package_id if output else None,
|
|
1105
|
+
package_operation=output.package_operation if output else None,
|
|
1106
|
+
)
|
|
1107
|
+
_append_execution_entry(args, entry)
|
|
1108
|
+
|
|
1109
|
+
ok = failed_count == 0 and bool(items)
|
|
1110
|
+
return CliResult(
|
|
1111
|
+
ok=ok,
|
|
1112
|
+
command=command,
|
|
1113
|
+
item_count=len(items),
|
|
1114
|
+
success_count=success_count,
|
|
1115
|
+
failed_count=failed_count,
|
|
1116
|
+
skipped_count=skipped_count,
|
|
1117
|
+
outputs=outputs,
|
|
1118
|
+
message="completed" if ok else "completed with failures",
|
|
1119
|
+
)
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def _maybe_skip_existing_package(
|
|
1123
|
+
item: ContentItem,
|
|
1124
|
+
args: argparse.Namespace,
|
|
1125
|
+
*,
|
|
1126
|
+
execution_fingerprint: str | None = None,
|
|
1127
|
+
) -> OutputWriteResult | None:
|
|
1128
|
+
if not getattr(args, "package", False):
|
|
1129
|
+
return None
|
|
1130
|
+
if (
|
|
1131
|
+
getattr(args, "update", False)
|
|
1132
|
+
or getattr(args, "overwrite", False)
|
|
1133
|
+
or getattr(args, "new_version", False)
|
|
1134
|
+
):
|
|
1135
|
+
return None
|
|
1136
|
+
if getattr(args, "receipt", None) == "compact":
|
|
1137
|
+
corrupt = find_corrupt_package_for_item(
|
|
1138
|
+
item,
|
|
1139
|
+
args.out,
|
|
1140
|
+
package_root=getattr(args, "package_root", None),
|
|
1141
|
+
)
|
|
1142
|
+
if corrupt is not None:
|
|
1143
|
+
raise PackageValidationError("package_validation_failed")
|
|
1144
|
+
existing = find_existing_package_for_item(
|
|
1145
|
+
item,
|
|
1146
|
+
args.out,
|
|
1147
|
+
package_root=getattr(args, "package_root", None),
|
|
1148
|
+
)
|
|
1149
|
+
if existing is None:
|
|
1150
|
+
return None
|
|
1151
|
+
validate_package_or_raise(existing.package_path)
|
|
1152
|
+
metadata = _read_package_metadata(existing.package_path)
|
|
1153
|
+
if execution_fingerprint:
|
|
1154
|
+
mismatch = require_matching_fingerprint(metadata, execution_fingerprint)
|
|
1155
|
+
if mismatch:
|
|
1156
|
+
raise PackageValidationError(mismatch)
|
|
1157
|
+
if not package_metadata_satisfies_extraction_contract(metadata, item):
|
|
1158
|
+
if getattr(args, "receipt", None) == "compact":
|
|
1159
|
+
raise PackageValidationError("extraction_contract_mismatch")
|
|
1160
|
+
return None
|
|
1161
|
+
return OutputWriteResult(
|
|
1162
|
+
path=existing.package_path,
|
|
1163
|
+
package_id=existing.package_id,
|
|
1164
|
+
package_operation=existing.operation,
|
|
1165
|
+
)
|
|
1166
|
+
|
|
1167
|
+
|
|
1168
|
+
def _write_extraction_output(
|
|
1169
|
+
item: ContentItem,
|
|
1170
|
+
result,
|
|
1171
|
+
args: argparse.Namespace,
|
|
1172
|
+
*,
|
|
1173
|
+
execution_fingerprint: str | None = None,
|
|
1174
|
+
) -> OutputWriteResult:
|
|
1175
|
+
if getattr(args, "package", False):
|
|
1176
|
+
package_result = write_item_package(
|
|
1177
|
+
item,
|
|
1178
|
+
result,
|
|
1179
|
+
args.out,
|
|
1180
|
+
package_root=getattr(args, "package_root", None),
|
|
1181
|
+
skip_existing=True,
|
|
1182
|
+
update=getattr(args, "update", False),
|
|
1183
|
+
overwrite=getattr(args, "overwrite", False),
|
|
1184
|
+
new_version=getattr(args, "new_version", False),
|
|
1185
|
+
execution_fingerprint=execution_fingerprint,
|
|
1186
|
+
)
|
|
1187
|
+
try:
|
|
1188
|
+
validate_package_or_raise(package_result.package_path)
|
|
1189
|
+
except PackageValidationError:
|
|
1190
|
+
if package_result.operation in {"created", "new_version"}:
|
|
1191
|
+
shutil.rmtree(package_result.package_path, ignore_errors=True)
|
|
1192
|
+
raise
|
|
1193
|
+
return OutputWriteResult(
|
|
1194
|
+
path=package_result.package_path,
|
|
1195
|
+
package_id=package_result.package_id,
|
|
1196
|
+
package_operation=package_result.operation,
|
|
1197
|
+
)
|
|
1198
|
+
|
|
1199
|
+
return OutputWriteResult(
|
|
1200
|
+
path=write_item_result(
|
|
1201
|
+
item,
|
|
1202
|
+
result,
|
|
1203
|
+
args.out,
|
|
1204
|
+
overwrite=getattr(args, "overwrite", False),
|
|
1205
|
+
download_assets=getattr(args, "download_assets", False),
|
|
1206
|
+
raw=getattr(args, "raw", False),
|
|
1207
|
+
)
|
|
1208
|
+
)
|
|
1209
|
+
|
|
1210
|
+
|
|
1211
|
+
def _build_package_validation_failure_entry(
|
|
1212
|
+
item: ContentItem,
|
|
1213
|
+
result,
|
|
1214
|
+
exc: PackageValidationError,
|
|
1215
|
+
) -> dict:
|
|
1216
|
+
return build_runlog_entry(
|
|
1217
|
+
input_url=item.url,
|
|
1218
|
+
platform=item.platform,
|
|
1219
|
+
backend=result.backend,
|
|
1220
|
+
status="failed",
|
|
1221
|
+
source_type=item.source_type,
|
|
1222
|
+
source_level=result.source_level,
|
|
1223
|
+
error_code="package_validation_failed",
|
|
1224
|
+
error_summary=exc,
|
|
1225
|
+
)
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def _build_identity_resolution_failure_entry(item: ContentItem, resolution) -> dict:
|
|
1229
|
+
return build_runlog_entry(
|
|
1230
|
+
input_url=item.url,
|
|
1231
|
+
platform=item.platform,
|
|
1232
|
+
backend=str(resolution.provider or "identity.not_available"),
|
|
1233
|
+
status="failed",
|
|
1234
|
+
source_type=item.source_type,
|
|
1235
|
+
source_level="partial_extraction",
|
|
1236
|
+
error_code=resolution.error_code or "identity_resolution_failed",
|
|
1237
|
+
error_summary=resolution.error_summary or "Durable item identity resolution failed.",
|
|
1238
|
+
resume_pointer="rerun after the identity resolver is available; unresolved identity was not skipped or extracted",
|
|
1239
|
+
)
|
|
1240
|
+
|
|
1241
|
+
|
|
1242
|
+
def _build_package_validation_failure_entry_for_item(
|
|
1243
|
+
item: ContentItem,
|
|
1244
|
+
exc: PackageValidationError,
|
|
1245
|
+
) -> dict:
|
|
1246
|
+
error_code = str(exc)
|
|
1247
|
+
if error_code not in {"execution_fingerprint_missing", "execution_fingerprint_mismatch"}:
|
|
1248
|
+
error_code = "package_validation_failed"
|
|
1249
|
+
return build_runlog_entry(
|
|
1250
|
+
input_url=item.url,
|
|
1251
|
+
platform=item.platform,
|
|
1252
|
+
backend="existing_package",
|
|
1253
|
+
status="failed",
|
|
1254
|
+
source_type=item.source_type,
|
|
1255
|
+
source_level="partial_extraction",
|
|
1256
|
+
error_code=error_code,
|
|
1257
|
+
error_summary=exc,
|
|
1258
|
+
)
|
|
1259
|
+
|
|
1260
|
+
|
|
1261
|
+
def _build_skipped_package_runlog_entry(
|
|
1262
|
+
item: ContentItem,
|
|
1263
|
+
output: OutputWriteResult,
|
|
1264
|
+
) -> dict:
|
|
1265
|
+
metadata = _read_package_metadata(output.path)
|
|
1266
|
+
backend = metadata.get("backend")
|
|
1267
|
+
backend_name = "existing_package"
|
|
1268
|
+
if isinstance(backend, dict) and backend.get("name"):
|
|
1269
|
+
backend_name = str(backend["name"])
|
|
1270
|
+
elif isinstance(backend, str) and backend.strip():
|
|
1271
|
+
backend_name = backend
|
|
1272
|
+
return build_runlog_entry(
|
|
1273
|
+
input_url=item.url,
|
|
1274
|
+
platform=item.platform,
|
|
1275
|
+
backend=backend_name,
|
|
1276
|
+
status=str(metadata.get("status") or "verified"),
|
|
1277
|
+
source_type=item.source_type,
|
|
1278
|
+
source_level=str(metadata.get("source_level") or "partial_extraction"),
|
|
1279
|
+
output_path=output.path,
|
|
1280
|
+
package_id=output.package_id,
|
|
1281
|
+
package_operation=output.package_operation,
|
|
1282
|
+
)
|
|
1283
|
+
|
|
1284
|
+
|
|
1285
|
+
def _read_package_metadata(package_path: Path) -> dict:
|
|
1286
|
+
try:
|
|
1287
|
+
value = json.loads((package_path / "metadata.json").read_text(encoding="utf-8"))
|
|
1288
|
+
except (OSError, json.JSONDecodeError):
|
|
1289
|
+
return {}
|
|
1290
|
+
return value if isinstance(value, dict) else {}
|
|
1291
|
+
|
|
1292
|
+
|
|
1293
|
+
def cmd_doctor(args: argparse.Namespace) -> int:
|
|
1294
|
+
if getattr(args, "clear_gpu_quarantine", False):
|
|
1295
|
+
from openmd.runtime.gpu_quarantine import clear_quarantine_manually
|
|
1296
|
+
|
|
1297
|
+
print(json.dumps({"command": "doctor", "ok": True, "gpu_quarantine": clear_quarantine_manually()}, sort_keys=True))
|
|
1298
|
+
return 0
|
|
1299
|
+
from openmd.runtime.tempdirs import sweep_tempdirs
|
|
1300
|
+
|
|
1301
|
+
cleanup = sweep_tempdirs()
|
|
1302
|
+
profile = getattr(args, "profile", "core")
|
|
1303
|
+
try:
|
|
1304
|
+
diagnostics = build_doctor_report(profile)
|
|
1305
|
+
from openmd.routing.loader import effective_routes
|
|
1306
|
+
|
|
1307
|
+
effective_routes()
|
|
1308
|
+
if profile in {"all", "tools"}:
|
|
1309
|
+
from openmd.routing.providers import doctor_tools
|
|
1310
|
+
|
|
1311
|
+
diagnostics["tools"] = doctor_tools(probe_commands=getattr(args, "probe_commands", False))
|
|
1312
|
+
diagnostics["ready"] = bool(diagnostics.get("ready") and diagnostics["tools"]["ok"])
|
|
1313
|
+
except (OSError, ValueError) as exc:
|
|
1314
|
+
diagnostics = {"profile": profile, "ready": False, "error_code": "local_configuration_invalid", "error_summary": sanitize_error_summary(exc), "next_action": "Repair the explicitly selected local position configuration or route overlay; no source extraction was started."}
|
|
1315
|
+
ready = diagnostics.get("ready", diagnostics.get("manifest_valid", True))
|
|
1316
|
+
payload = {
|
|
1317
|
+
"ok": ready,
|
|
1318
|
+
"command": "doctor",
|
|
1319
|
+
"temp_cleanup": cleanup,
|
|
1320
|
+
"official_command": "openmd",
|
|
1321
|
+
"module_entry": "python -m openmd",
|
|
1322
|
+
"repo_wrapper": str(Path("scripts/openmd.ps1")),
|
|
1323
|
+
"openmd_on_path": shutil.which("openmd") is not None,
|
|
1324
|
+
"safe_session_boundary": {
|
|
1325
|
+
"core_owns_platform_sessions": False,
|
|
1326
|
+
"reads_private_browser_profiles": False,
|
|
1327
|
+
"imports_or_exports_platform_sessions": False,
|
|
1328
|
+
"default_raw_artifact_retention": False,
|
|
1329
|
+
},
|
|
1330
|
+
"structured_states": [
|
|
1331
|
+
"auth_required",
|
|
1332
|
+
"session_expired",
|
|
1333
|
+
"permission_required",
|
|
1334
|
+
"cache_stale",
|
|
1335
|
+
"refresh_needed",
|
|
1336
|
+
"no_delta",
|
|
1337
|
+
"sidecar_unreachable",
|
|
1338
|
+
],
|
|
1339
|
+
"next_action": diagnostics.get("next_action") or (
|
|
1340
|
+
"Use platform or sidecar UI to repair login/session/cache state, "
|
|
1341
|
+
"then rerun Openmd. Core records status only and does not store private session material."
|
|
1342
|
+
),
|
|
1343
|
+
"profile": profile,
|
|
1344
|
+
"profile_diagnostics": diagnostics,
|
|
1345
|
+
}
|
|
1346
|
+
if not getattr(args, "json", False) and sys.stdout.isatty():
|
|
1347
|
+
from openmd.doctor_view import render
|
|
1348
|
+
|
|
1349
|
+
print(render(payload))
|
|
1350
|
+
# Optional capabilities never make the human view an error.
|
|
1351
|
+
return 0 if diagnostics.get("manifest_valid", True) and "error_code" not in diagnostics else 1
|
|
1352
|
+
print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
|
|
1353
|
+
return 0 if ready else 1
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
def cmd_routes(args: argparse.Namespace) -> int:
|
|
1357
|
+
from openmd.routing.loader import route_view
|
|
1358
|
+
|
|
1359
|
+
action = args.action
|
|
1360
|
+
if action is None:
|
|
1361
|
+
view = route_view(source=args.source)
|
|
1362
|
+
if args.format == "json":
|
|
1363
|
+
print(json.dumps(view, ensure_ascii=True, sort_keys=True))
|
|
1364
|
+
elif args.format == "markdown":
|
|
1365
|
+
from openmd.routing.view import render_routes_markdown
|
|
1366
|
+
|
|
1367
|
+
print(render_routes_markdown(source=args.source, effective=True), end="")
|
|
1368
|
+
else:
|
|
1369
|
+
rows = view["routes"]
|
|
1370
|
+
for row in rows:
|
|
1371
|
+
values = (row["source"], row["capability"], " → ".join(row["chain"]))
|
|
1372
|
+
print("\t".join(values))
|
|
1373
|
+
return 0
|
|
1374
|
+
if not args.source:
|
|
1375
|
+
raise ValueError("routes_source_required")
|
|
1376
|
+
from openmd.routing.lifecycle import demote, restore, run_benchmark
|
|
1377
|
+
|
|
1378
|
+
if action == "bench":
|
|
1379
|
+
if not args.capability:
|
|
1380
|
+
raise ValueError("routes_capability_required")
|
|
1381
|
+
payload = run_benchmark(source=args.source, capability=args.capability,
|
|
1382
|
+
candidates=args.candidates.split(",") if args.candidates else None,
|
|
1383
|
+
options=_backend_runtime_options(args), ledger_path=args.ledger, sample_id=args.sample,
|
|
1384
|
+
samples_path=getattr(args, "samples_file", None), artifact_dir=getattr(args, "artifacts", None))
|
|
1385
|
+
else:
|
|
1386
|
+
if not args.capability_id:
|
|
1387
|
+
raise ValueError("routes_capability_id_required")
|
|
1388
|
+
if action == "demote":
|
|
1389
|
+
if not args.ledger:
|
|
1390
|
+
raise ValueError("routes_ledger_required")
|
|
1391
|
+
payload = demote(args.capability_id, source=args.source, evidence_ids=args.evidence,
|
|
1392
|
+
ledger_path=args.ledger, overlay_path=args.overlay, note=args.note)
|
|
1393
|
+
else:
|
|
1394
|
+
payload = restore(args.capability_id, source=args.source, overlay_path=args.overlay, note=args.note)
|
|
1395
|
+
print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
|
|
1396
|
+
return 0
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
def cmd_capabilities(args: argparse.Namespace) -> int:
|
|
1400
|
+
records = filter_capabilities(source_id=args.source)
|
|
1401
|
+
if args.source and not records:
|
|
1402
|
+
print(json.dumps({"count": 0, "capabilities": []}, ensure_ascii=True, sort_keys=True))
|
|
1403
|
+
return 1
|
|
1404
|
+
if args.format == "json":
|
|
1405
|
+
print(json.dumps({"count": len(records), "capabilities": [record.as_dict() for record in records]}, ensure_ascii=True, sort_keys=True))
|
|
1406
|
+
return 0
|
|
1407
|
+
print("source_id\ttier\tstate\tplanned_provider\truntime_provider\tbinding_state\texecution_ready")
|
|
1408
|
+
for record in records:
|
|
1409
|
+
print("\t".join((record.source_id, record.tier, record.state, record.planned_provider or "", record.runtime_provider or "", record.binding_state, str(record.execution_ready).lower())))
|
|
1410
|
+
return 0
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def cmd_plan(args: argparse.Namespace) -> int:
|
|
1414
|
+
result = build_plan(
|
|
1415
|
+
args.input,
|
|
1416
|
+
input_kind=args.input_kind,
|
|
1417
|
+
request_all=args.all,
|
|
1418
|
+
available_gates=set(args.gate),
|
|
1419
|
+
)
|
|
1420
|
+
payload = result.as_dict()
|
|
1421
|
+
if args.format == "json":
|
|
1422
|
+
print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
|
|
1423
|
+
else:
|
|
1424
|
+
for key, value in payload.items():
|
|
1425
|
+
print(f"{key}\t{value}")
|
|
1426
|
+
return 0 if result.ok else 1
|
|
1427
|
+
|
|
1428
|
+
|
|
1429
|
+
def cmd_report_weixin_author_batch(args: argparse.Namespace) -> int:
|
|
1430
|
+
from openmd.adapters.weixin.report import load_runlog_entries, write_weixin_author_batch_report
|
|
1431
|
+
|
|
1432
|
+
write_json = args.format in {"both", "json"}
|
|
1433
|
+
write_markdown = args.format in {"both", "markdown"}
|
|
1434
|
+
entries = load_runlog_entries(args.runlog)
|
|
1435
|
+
result = write_weixin_author_batch_report(
|
|
1436
|
+
entries,
|
|
1437
|
+
args.out,
|
|
1438
|
+
write_json=write_json,
|
|
1439
|
+
write_markdown=write_markdown,
|
|
1440
|
+
)
|
|
1441
|
+
outputs = [str(path) for path in (result.json_path, result.markdown_path) if path is not None]
|
|
1442
|
+
summary = result.report.get("summary", {})
|
|
1443
|
+
_print_result(
|
|
1444
|
+
CliResult(
|
|
1445
|
+
ok=True,
|
|
1446
|
+
command="report",
|
|
1447
|
+
item_count=int(summary.get("total_items", 0)),
|
|
1448
|
+
success_count=int(summary.get("covered_count", 0)),
|
|
1449
|
+
failed_count=int(summary.get("failed_count", 0)),
|
|
1450
|
+
outputs=outputs,
|
|
1451
|
+
message="report written",
|
|
1452
|
+
)
|
|
1453
|
+
)
|
|
1454
|
+
return 0
|
|
1455
|
+
|
|
1456
|
+
|
|
1457
|
+
def cmd_ingest(args: argparse.Namespace) -> int:
|
|
1458
|
+
start_error = _start_compact_receipt(args, "ingest")
|
|
1459
|
+
if start_error:
|
|
1460
|
+
return _emit_result(args, CliResult(ok=False, command="ingest", failed_count=1, message=start_error))
|
|
1461
|
+
try:
|
|
1462
|
+
args.url = _read_single_input(args.url)
|
|
1463
|
+
except ValueError as exc:
|
|
1464
|
+
result = CliResult(ok=False, command="ingest", failed_count=1, message=sanitize_error_summary(exc))
|
|
1465
|
+
if getattr(args, "receipt", None) == "compact":
|
|
1466
|
+
return _emit_result(args, result)
|
|
1467
|
+
print(f"openmd ingest: {sanitize_error_summary(exc)}", file=sys.stderr)
|
|
1468
|
+
return 1
|
|
1469
|
+
result = _process_input(args.url, args, command="ingest")
|
|
1470
|
+
return _emit_result(args, result)
|
|
1471
|
+
|
|
1472
|
+
|
|
1473
|
+
def cmd_batch(args: argparse.Namespace) -> int:
|
|
1474
|
+
start_error = _start_compact_receipt(args, "batch")
|
|
1475
|
+
if start_error:
|
|
1476
|
+
return _emit_result(args, CliResult(ok=False, command="batch", failed_count=1, message=start_error))
|
|
1477
|
+
try:
|
|
1478
|
+
urls = _read_urls_file(args.urls_file)
|
|
1479
|
+
except OSError as exc:
|
|
1480
|
+
result = CliResult(ok=False, command="batch", failed_count=1, message=sanitize_error_summary(exc))
|
|
1481
|
+
if getattr(args, "receipt", None) == "compact":
|
|
1482
|
+
return _emit_result(args, result)
|
|
1483
|
+
print(f"openmd batch: {sanitize_error_summary(exc)}", file=sys.stderr)
|
|
1484
|
+
return 1
|
|
1485
|
+
|
|
1486
|
+
_append_large_batch_guardrail_if_needed(args, command="batch", total=len(urls))
|
|
1487
|
+
|
|
1488
|
+
urls_to_process = urls
|
|
1489
|
+
if getattr(args, "receipt", None) == "compact":
|
|
1490
|
+
occurrences: dict[str, int] = {}
|
|
1491
|
+
unique_urls: list[str] = []
|
|
1492
|
+
for url in urls:
|
|
1493
|
+
identity = sanitize_url(url)
|
|
1494
|
+
occurrences[identity] = occurrences.get(identity, 0) + 1
|
|
1495
|
+
if occurrences[identity] == 1:
|
|
1496
|
+
unique_urls.append(url)
|
|
1497
|
+
args._compact_input_occurrences_by_url = occurrences
|
|
1498
|
+
urls_to_process = unique_urls
|
|
1499
|
+
_register_compact_inputs(args, {
|
|
1500
|
+
_compact_url_identity(url): occurrences[sanitize_url(url)]
|
|
1501
|
+
for url in unique_urls
|
|
1502
|
+
})
|
|
1503
|
+
|
|
1504
|
+
aggregate = CliResult(ok=True, command="batch", message="completed")
|
|
1505
|
+
consecutive_failures = 0
|
|
1506
|
+
for line_number, url in enumerate(urls_to_process, start=1):
|
|
1507
|
+
result = _process_input(url, args, command="batch")
|
|
1508
|
+
_sync_compact_rows(args)
|
|
1509
|
+
if getattr(args, "receipt", None) == "compact":
|
|
1510
|
+
args._compact_pending_identities.pop(_compact_url_identity(url), None)
|
|
1511
|
+
aggregate.item_count += result.item_count
|
|
1512
|
+
aggregate.success_count += result.success_count
|
|
1513
|
+
aggregate.failed_count += result.failed_count
|
|
1514
|
+
aggregate.outputs.extend(result.outputs)
|
|
1515
|
+
for signal in result.guardrail_signals:
|
|
1516
|
+
if signal not in aggregate.guardrail_signals:
|
|
1517
|
+
aggregate.guardrail_signals.append(signal)
|
|
1518
|
+
if not result.ok:
|
|
1519
|
+
aggregate.ok = False
|
|
1520
|
+
aggregate.message = "completed with failures"
|
|
1521
|
+
consecutive_failures += 1
|
|
1522
|
+
else:
|
|
1523
|
+
consecutive_failures = 0
|
|
1524
|
+
if _guardrail_failure_streak_reached(args, consecutive_failures):
|
|
1525
|
+
_append_guardrail_stop_entry(args, command="batch", index=line_number, total=len(urls))
|
|
1526
|
+
if "consecutive_failure_stop" not in aggregate.guardrail_signals:
|
|
1527
|
+
aggregate.guardrail_signals.append("consecutive_failure_stop")
|
|
1528
|
+
aggregate.message = "guardrail stopped"
|
|
1529
|
+
break
|
|
1530
|
+
return _emit_result(args, aggregate)
|
|
1531
|
+
|
|
1532
|
+
|
|
1533
|
+
def _write_youtube_discovery_summary(out_dir: str, discovery_result, **fields: object) -> Path:
|
|
1534
|
+
summary = {
|
|
1535
|
+
"page_count": discovery_result.metadata.get("page_count", 0),
|
|
1536
|
+
"request_count": discovery_result.metadata.get("request_count", 0),
|
|
1537
|
+
**fields,
|
|
1538
|
+
}
|
|
1539
|
+
summary_path = Path(out_dir) / "youtube-discovery-summary.json"
|
|
1540
|
+
summary_path.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
|
1541
|
+
return summary_path
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
def cmd_collect(args: argparse.Namespace) -> int:
|
|
1545
|
+
start_error = _start_compact_receipt(args, "collect")
|
|
1546
|
+
if start_error:
|
|
1547
|
+
return _emit_result(args, CliResult(ok=False, command="collect", failed_count=1, message=start_error), discovery_complete=False)
|
|
1548
|
+
try:
|
|
1549
|
+
args.url = _read_single_input(args.url)
|
|
1550
|
+
except ValueError as exc:
|
|
1551
|
+
result = CliResult(ok=False, command="collect", failed_count=1, message=sanitize_error_summary(exc))
|
|
1552
|
+
if getattr(args, "receipt", None) == "compact":
|
|
1553
|
+
return _emit_result(args, result, discovery_complete=False)
|
|
1554
|
+
print(f"openmd collect: {sanitize_error_summary(exc)}", file=sys.stderr)
|
|
1555
|
+
return 1
|
|
1556
|
+
if args.all and not _allow_scoped_all_collect(args.url, args):
|
|
1557
|
+
result = _log_all_guard(args.url, args, command="collect")
|
|
1558
|
+
return _emit_result(args, result, discovery_complete=False)
|
|
1559
|
+
|
|
1560
|
+
platform = _detect_collect_platform(args.url)
|
|
1561
|
+
_select_collect_discovery_backend(args.url, args)
|
|
1562
|
+
discovery_executor = _build_discovery_executor(args, platform=platform)
|
|
1563
|
+
discovery_result = discovery_executor.discover(
|
|
1564
|
+
DiscoveryRequest(
|
|
1565
|
+
input_url=args.url,
|
|
1566
|
+
platform=platform,
|
|
1567
|
+
limit=None if args.all else args.limit,
|
|
1568
|
+
allow_all=bool(args.all),
|
|
1569
|
+
)
|
|
1570
|
+
)
|
|
1571
|
+
|
|
1572
|
+
if not discovery_result.ok:
|
|
1573
|
+
partial_outputs: list[str] = []
|
|
1574
|
+
partial_count = 0
|
|
1575
|
+
if platform == "youtube":
|
|
1576
|
+
# Failed discovery must preserve known inputs without extracting or
|
|
1577
|
+
# claiming them verified, including compact expected-item accounting.
|
|
1578
|
+
_prepare_compact_collection(args, discovery_result.items)
|
|
1579
|
+
args._compact_execution_failed = True
|
|
1580
|
+
partial_count = len(discovery_result.items)
|
|
1581
|
+
partial_manifest = write_discovery_manifest(
|
|
1582
|
+
out_dir=args.out, platform=platform,
|
|
1583
|
+
collection_url=str(discovery_result.metadata.get("normalized_url") or args.url),
|
|
1584
|
+
provider=discovery_result.provider, items=discovery_result.items,
|
|
1585
|
+
limit=None if args.all else args.limit,
|
|
1586
|
+
)
|
|
1587
|
+
partial_outputs.append(str(partial_manifest))
|
|
1588
|
+
summary_path = _write_youtube_discovery_summary(
|
|
1589
|
+
args.out, discovery_result,
|
|
1590
|
+
status="failed", source_exhausted=False,
|
|
1591
|
+
item_count=partial_count, error_code=discovery_result.error_code,
|
|
1592
|
+
stop_reason=discovery_result.error_code,
|
|
1593
|
+
resume_policy="restart_discovery_and_reuse_verified_packages",
|
|
1594
|
+
)
|
|
1595
|
+
partial_outputs.append(str(summary_path))
|
|
1596
|
+
entry = build_runlog_entry(
|
|
1597
|
+
input_url=args.url,
|
|
1598
|
+
platform=platform,
|
|
1599
|
+
backend=discovery_result.provider,
|
|
1600
|
+
status="failed",
|
|
1601
|
+
source_type="mixed",
|
|
1602
|
+
source_level="partial_extraction",
|
|
1603
|
+
item_count=partial_count,
|
|
1604
|
+
success_count=0,
|
|
1605
|
+
failed_count=1,
|
|
1606
|
+
error_code=discovery_result.error_code,
|
|
1607
|
+
error_summary=discovery_result.error_summary,
|
|
1608
|
+
event_type="discovery" if platform == "youtube" else "item",
|
|
1609
|
+
)
|
|
1610
|
+
_append_execution_entry(args, entry)
|
|
1611
|
+
result = CliResult(
|
|
1612
|
+
ok=False,
|
|
1613
|
+
command="collect",
|
|
1614
|
+
item_count=partial_count,
|
|
1615
|
+
success_count=0,
|
|
1616
|
+
failed_count=1,
|
|
1617
|
+
outputs=partial_outputs,
|
|
1618
|
+
message=discovery_result.error_summary or "collection discovery failed",
|
|
1619
|
+
)
|
|
1620
|
+
return _emit_result(args, result, discovery_complete=False)
|
|
1621
|
+
|
|
1622
|
+
discovered_items = _prepare_compact_collection(args, discovery_result.items)
|
|
1623
|
+
discovery_complete = not args.all or bool(discovery_result.metadata.get("source_exhausted"))
|
|
1624
|
+
collection_url = str(discovery_result.metadata.get("normalized_url") or args.url)
|
|
1625
|
+
manifest_path = write_discovery_manifest(
|
|
1626
|
+
out_dir=args.out,
|
|
1627
|
+
platform=platform,
|
|
1628
|
+
collection_url=collection_url,
|
|
1629
|
+
provider=discovery_result.provider,
|
|
1630
|
+
items=discovered_items,
|
|
1631
|
+
limit=None if args.all else args.limit,
|
|
1632
|
+
)
|
|
1633
|
+
_append_large_batch_guardrail_if_needed(args, command="collect", total=len(discovery_result.items))
|
|
1634
|
+
if platform == "youtube" and discovery_result.provider.startswith("youtube.data-api"):
|
|
1635
|
+
metadata = discovery_result.metadata
|
|
1636
|
+
_write_youtube_discovery_summary(
|
|
1637
|
+
args.out, discovery_result,
|
|
1638
|
+
status="ok", source_exhausted=bool(metadata.get("source_exhausted")),
|
|
1639
|
+
item_count=len(discovery_result.items),
|
|
1640
|
+
playlist_item_count=metadata.get("playlist_item_count", 0),
|
|
1641
|
+
duplicate_item_count=metadata.get("duplicate_item_count", 0),
|
|
1642
|
+
stop_reason=metadata.get("stop_reason"),
|
|
1643
|
+
)
|
|
1644
|
+
|
|
1645
|
+
if getattr(args, "manifest_only", False):
|
|
1646
|
+
entry = build_runlog_entry(
|
|
1647
|
+
input_url=collection_url,
|
|
1648
|
+
platform=platform,
|
|
1649
|
+
backend=discovery_result.provider,
|
|
1650
|
+
status="verified",
|
|
1651
|
+
source_type="mixed",
|
|
1652
|
+
source_level="third_party_extractor",
|
|
1653
|
+
output_path=manifest_path,
|
|
1654
|
+
item_count=len(discovery_result.items),
|
|
1655
|
+
success_count=len(discovery_result.items),
|
|
1656
|
+
failed_count=0,
|
|
1657
|
+
event_type="discovery",
|
|
1658
|
+
)
|
|
1659
|
+
_append_execution_entry(args, entry)
|
|
1660
|
+
result = CliResult(
|
|
1661
|
+
ok=bool(discovery_result.items) or (platform == "youtube" and args.all and discovery_complete),
|
|
1662
|
+
command="collect",
|
|
1663
|
+
item_count=len(discovery_result.items),
|
|
1664
|
+
success_count=len(discovery_result.items),
|
|
1665
|
+
failed_count=0,
|
|
1666
|
+
outputs=[str(manifest_path)],
|
|
1667
|
+
message="manifest written",
|
|
1668
|
+
)
|
|
1669
|
+
return _emit_result(args, result, discovery_complete=discovery_complete and result.ok, manifest_only=True)
|
|
1670
|
+
|
|
1671
|
+
start_index = max(0, int(getattr(args, "start_index", 0) or 0))
|
|
1672
|
+
max_items = max(0, int(getattr(args, "max_items", 0) or 0))
|
|
1673
|
+
remaining_items = discovered_items[start_index:]
|
|
1674
|
+
extraction_items = remaining_items[:max_items] if max_items else remaining_items
|
|
1675
|
+
result = _process_items(extraction_items, args, command="collect")
|
|
1676
|
+
if platform == "douyin" and args.all:
|
|
1677
|
+
coverage_path = _write_douyin_coverage_summary(
|
|
1678
|
+
args=args,
|
|
1679
|
+
result=result,
|
|
1680
|
+
manifest_path=manifest_path,
|
|
1681
|
+
discovered_count=len(discovery_result.items),
|
|
1682
|
+
source_exhausted=bool(discovery_result.metadata.get("source_exhausted")),
|
|
1683
|
+
discovery_metadata=dict(discovery_result.metadata),
|
|
1684
|
+
start_index=start_index,
|
|
1685
|
+
max_items=max_items,
|
|
1686
|
+
)
|
|
1687
|
+
result.outputs.append(str(coverage_path))
|
|
1688
|
+
elif platform == "xiaohongshu" and args.all:
|
|
1689
|
+
coverage_path = _write_xiaohongshu_coverage_summary(
|
|
1690
|
+
args=args,
|
|
1691
|
+
result=result,
|
|
1692
|
+
discovered_items=discovery_result.items,
|
|
1693
|
+
discovery_metadata=dict(discovery_result.metadata),
|
|
1694
|
+
start_index=start_index,
|
|
1695
|
+
max_items=max_items,
|
|
1696
|
+
)
|
|
1697
|
+
result.outputs.append(str(coverage_path))
|
|
1698
|
+
return _emit_result(
|
|
1699
|
+
args,
|
|
1700
|
+
result,
|
|
1701
|
+
discovery_complete=discovery_complete,
|
|
1702
|
+
)
|
|
1703
|
+
|
|
1704
|
+
|
|
1705
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1706
|
+
from openmd.console import configure_console
|
|
1707
|
+
|
|
1708
|
+
configure_console()
|
|
1709
|
+
actual = list(sys.argv[1:] if argv is None else argv)
|
|
1710
|
+
if not actual:
|
|
1711
|
+
from openmd.interactive import is_interactive
|
|
1712
|
+
|
|
1713
|
+
if is_interactive():
|
|
1714
|
+
actual = ["go"]
|
|
1715
|
+
elif actual[0] not in {"ingest", "go", "batch", "collect", "capabilities", "plan", "doctor", "probe", "routes", "report", "browser-setup"} and not actual[0].startswith("-"):
|
|
1716
|
+
actual.insert(0, "go")
|
|
1717
|
+
parser = build_parser()
|
|
1718
|
+
try:
|
|
1719
|
+
args = parser.parse_args(actual)
|
|
1720
|
+
except SystemExit as exc:
|
|
1721
|
+
return int(exc.code) if isinstance(exc.code, int) else 1
|
|
1722
|
+
from openmd.runtime.managed_tools import activate as activate_managed_tools
|
|
1723
|
+
|
|
1724
|
+
activate_managed_tools() # put Openmd's own opencli/node on PATH if it was installed
|
|
1725
|
+
if getattr(args, "command", None) != "doctor":
|
|
1726
|
+
from openmd.runtime.tempdirs import sweep_at_startup
|
|
1727
|
+
|
|
1728
|
+
sweep_at_startup()
|
|
1729
|
+
if not hasattr(args, "func"):
|
|
1730
|
+
parser.print_help()
|
|
1731
|
+
return 0
|
|
1732
|
+
if getattr(args, "package", False):
|
|
1733
|
+
if getattr(args, "receipt", None) == "compact" and any(
|
|
1734
|
+
(
|
|
1735
|
+
getattr(args, "update", False),
|
|
1736
|
+
getattr(args, "overwrite", False),
|
|
1737
|
+
getattr(args, "new_version", False),
|
|
1738
|
+
)
|
|
1739
|
+
):
|
|
1740
|
+
print(
|
|
1741
|
+
"openmd: --receipt compact cannot be combined with --update, --overwrite, or --new-version",
|
|
1742
|
+
file=sys.stderr,
|
|
1743
|
+
)
|
|
1744
|
+
return 1
|
|
1745
|
+
if getattr(args, "new_version", False) and (
|
|
1746
|
+
getattr(args, "update", False) or getattr(args, "overwrite", False)
|
|
1747
|
+
):
|
|
1748
|
+
print(
|
|
1749
|
+
"openmd: --new-version cannot be combined with --update or --overwrite",
|
|
1750
|
+
file=sys.stderr,
|
|
1751
|
+
)
|
|
1752
|
+
return 1
|
|
1753
|
+
try:
|
|
1754
|
+
return args.func(args)
|
|
1755
|
+
except (Exception, KeyboardInterrupt) as exc:
|
|
1756
|
+
if isinstance(exc, KeyboardInterrupt):
|
|
1757
|
+
print("已取消。", file=sys.stderr)
|
|
1758
|
+
if getattr(args, "receipt", None) != "compact":
|
|
1759
|
+
return 130
|
|
1760
|
+
if getattr(args, "receipt", None) == "compact":
|
|
1761
|
+
args._compact_execution_failed = not isinstance(exc, KeyboardInterrupt)
|
|
1762
|
+
if getattr(args, "_compact_receipt_run", None) is not None:
|
|
1763
|
+
return _emit_result(args, CliResult(ok=False, command=args.command, message="execution_interrupted"), discovery_complete=False)
|
|
1764
|
+
return _print_compact_failure()
|
|
1765
|
+
print(f"openmd: {sanitize_error_summary(exc)}", file=sys.stderr)
|
|
1766
|
+
return 1
|
|
1767
|
+
finally:
|
|
1768
|
+
lock = getattr(args, "_compact_output_lock", None)
|
|
1769
|
+
if lock is not None:
|
|
1770
|
+
try:
|
|
1771
|
+
lock.release()
|
|
1772
|
+
except OSError:
|
|
1773
|
+
pass # The bounded failure path already reports failure; leave residual lock for inspection.
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
def _write_douyin_coverage_summary(
|
|
1777
|
+
*,
|
|
1778
|
+
args: argparse.Namespace,
|
|
1779
|
+
result: CliResult,
|
|
1780
|
+
manifest_path: Path,
|
|
1781
|
+
discovered_count: int,
|
|
1782
|
+
source_exhausted: bool,
|
|
1783
|
+
discovery_metadata: dict[str, object],
|
|
1784
|
+
start_index: int,
|
|
1785
|
+
max_items: int,
|
|
1786
|
+
) -> Path:
|
|
1787
|
+
path = Path(args.out) / "coverage-summary.json"
|
|
1788
|
+
payload = {
|
|
1789
|
+
"platform": "douyin",
|
|
1790
|
+
"creator_input": args.url,
|
|
1791
|
+
"discovery_provider": getattr(args, "discovery_backend", None),
|
|
1792
|
+
"transcript_backend": getattr(args, "backend", None),
|
|
1793
|
+
"source_exhausted": source_exhausted,
|
|
1794
|
+
"page_count": discovery_metadata.get("page_count"),
|
|
1795
|
+
"empty_page_count": discovery_metadata.get("empty_page_count"),
|
|
1796
|
+
"duplicate_page_count": discovery_metadata.get("duplicate_page_count"),
|
|
1797
|
+
"pagination_retry_count": discovery_metadata.get("pagination_retry_count"),
|
|
1798
|
+
"video_count": discovery_metadata.get("video_count"),
|
|
1799
|
+
"image_post_count": discovery_metadata.get("image_post_count"),
|
|
1800
|
+
"discovered": discovered_count,
|
|
1801
|
+
"attempted": result.item_count,
|
|
1802
|
+
"start_index": start_index,
|
|
1803
|
+
"slice_size": max_items,
|
|
1804
|
+
"success": result.success_count,
|
|
1805
|
+
"failed": result.failed_count,
|
|
1806
|
+
"skipped_existing": result.skipped_count,
|
|
1807
|
+
"remaining_unattempted": max(0, discovered_count - start_index - result.item_count),
|
|
1808
|
+
"complete_run": start_index == 0 and max_items == 0 and result.item_count == discovered_count,
|
|
1809
|
+
"source_level_counts": _package_coverage_source_level_counts(result.source_level_counts),
|
|
1810
|
+
"failure_code_counts": result.failure_code_counts,
|
|
1811
|
+
"guardrail_signals": result.guardrail_signals,
|
|
1812
|
+
"manifest": str(manifest_path),
|
|
1813
|
+
"runlog": str(_runlog_path(args)),
|
|
1814
|
+
"media_retention": "none",
|
|
1815
|
+
}
|
|
1816
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1817
|
+
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
1818
|
+
return path
|
|
1819
|
+
|
|
1820
|
+
|
|
1821
|
+
def _package_coverage_source_level_counts(counts: dict[str, int]) -> dict[str, int]:
|
|
1822
|
+
normalized: dict[str, int] = {}
|
|
1823
|
+
for source_level, count in counts.items():
|
|
1824
|
+
key = package_source_level(source_level)
|
|
1825
|
+
normalized[key] = normalized.get(key, 0) + int(count)
|
|
1826
|
+
return normalized
|
|
1827
|
+
|
|
1828
|
+
|
|
1829
|
+
def _write_xiaohongshu_coverage_summary(
|
|
1830
|
+
*,
|
|
1831
|
+
args: argparse.Namespace,
|
|
1832
|
+
result: CliResult,
|
|
1833
|
+
discovered_items: list[ContentItem],
|
|
1834
|
+
discovery_metadata: dict[str, object],
|
|
1835
|
+
start_index: int,
|
|
1836
|
+
max_items: int,
|
|
1837
|
+
) -> Path:
|
|
1838
|
+
"""Write a path-free, count-only Xiaohongshu full-collection summary."""
|
|
1839
|
+
path = Path(args.out) / "coverage-summary.json"
|
|
1840
|
+
discovered = len(discovered_items)
|
|
1841
|
+
remaining_unattempted = max(0, discovered - start_index - result.item_count)
|
|
1842
|
+
coverage = dict(result.coverage)
|
|
1843
|
+
retryable_items = _non_negative_int(coverage.get("retryable_items"))
|
|
1844
|
+
terminal_gaps = _non_negative_int(coverage.get("terminal_gaps"))
|
|
1845
|
+
remaining = remaining_unattempted + retryable_items
|
|
1846
|
+
component_counts, component_source_counts = _xiaohongshu_component_coverage(coverage)
|
|
1847
|
+
|
|
1848
|
+
payload = {
|
|
1849
|
+
"platform": "xiaohongshu",
|
|
1850
|
+
"package_mode": bool(getattr(args, "package", False)),
|
|
1851
|
+
"discovered": discovered,
|
|
1852
|
+
**collection_content_type_counts(discovered_items),
|
|
1853
|
+
"attempted": result.item_count,
|
|
1854
|
+
"verified": _non_negative_int(coverage.get("verified")),
|
|
1855
|
+
"partial": _non_negative_int(coverage.get("partial")),
|
|
1856
|
+
"terminal_gaps": terminal_gaps,
|
|
1857
|
+
"terminal_gap_items": _non_negative_int(coverage.get("terminal_gap_items")),
|
|
1858
|
+
"retryable": _non_negative_int(coverage.get("retryable")),
|
|
1859
|
+
"retryable_items": retryable_items,
|
|
1860
|
+
"failed": result.failed_count,
|
|
1861
|
+
"skipped_existing": result.skipped_count,
|
|
1862
|
+
"remaining": remaining,
|
|
1863
|
+
"remaining_unattempted": remaining_unattempted,
|
|
1864
|
+
"start_index": start_index,
|
|
1865
|
+
"slice_size": max_items,
|
|
1866
|
+
"complete_run": start_index == 0 and max_items == 0 and result.item_count == discovered,
|
|
1867
|
+
"source_exhausted": discovery_metadata.get("source_exhausted") is True,
|
|
1868
|
+
"explicit_exhaustion": discovery_metadata.get("explicit_exhaustion") is True,
|
|
1869
|
+
"page_count": _optional_non_negative_int(discovery_metadata.get("page_count")),
|
|
1870
|
+
"scroll_count": _optional_non_negative_int(discovery_metadata.get("scroll_count")),
|
|
1871
|
+
"anomaly_count": _optional_non_negative_int(discovery_metadata.get("anomaly_count")),
|
|
1872
|
+
"stable_round_count": _optional_non_negative_int(discovery_metadata.get("stable_round_count")),
|
|
1873
|
+
"empty_snapshot_count": _optional_non_negative_int(discovery_metadata.get("empty_snapshot_count")),
|
|
1874
|
+
"duplicate_snapshot_count": _optional_non_negative_int(
|
|
1875
|
+
discovery_metadata.get("duplicate_snapshot_count")
|
|
1876
|
+
),
|
|
1877
|
+
"component_counts": component_counts,
|
|
1878
|
+
"component_source_counts": component_source_counts,
|
|
1879
|
+
"source_counts": _safe_count_mapping(coverage.get("source_counts")),
|
|
1880
|
+
"failure_code_counts": _safe_count_mapping(result.failure_code_counts),
|
|
1881
|
+
"guardrail_signals": list(result.guardrail_signals),
|
|
1882
|
+
"media_retention": "none",
|
|
1883
|
+
}
|
|
1884
|
+
payload["outcome"] = (
|
|
1885
|
+
"failed"
|
|
1886
|
+
if result.failed_count or remaining
|
|
1887
|
+
else "complete_with_gaps"
|
|
1888
|
+
if terminal_gaps
|
|
1889
|
+
else "verified"
|
|
1890
|
+
)
|
|
1891
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1892
|
+
path.write_text(
|
|
1893
|
+
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
1894
|
+
encoding="utf-8",
|
|
1895
|
+
)
|
|
1896
|
+
return path
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
def _xiaohongshu_component_coverage(
|
|
1900
|
+
coverage: dict[str, object],
|
|
1901
|
+
) -> tuple[dict[str, dict[str, int]], dict[str, dict[str, int]]]:
|
|
1902
|
+
raw_counts = coverage.get("component_counts")
|
|
1903
|
+
raw_sources = coverage.get("component_source_counts")
|
|
1904
|
+
count_mapping = raw_counts if isinstance(raw_counts, dict) else {}
|
|
1905
|
+
source_mapping = raw_sources if isinstance(raw_sources, dict) else {}
|
|
1906
|
+
component_counts: dict[str, dict[str, int]] = {}
|
|
1907
|
+
component_source_counts: dict[str, dict[str, int]] = {}
|
|
1908
|
+
for component_id in ("native_text", "image_ocr", "video_transcript"):
|
|
1909
|
+
component_counts[component_id] = _safe_count_mapping(count_mapping.get(component_id))
|
|
1910
|
+
component_counts[component_id].setdefault("total", 0)
|
|
1911
|
+
component_source_counts[component_id] = _safe_count_mapping(source_mapping.get(component_id))
|
|
1912
|
+
return component_counts, component_source_counts
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _safe_count_mapping(value: object) -> dict[str, int]:
|
|
1916
|
+
if not isinstance(value, dict):
|
|
1917
|
+
return {}
|
|
1918
|
+
return {
|
|
1919
|
+
str(key): _non_negative_int(count)
|
|
1920
|
+
for key, count in value.items()
|
|
1921
|
+
if isinstance(key, str)
|
|
1922
|
+
}
|
|
1923
|
+
|
|
1924
|
+
|
|
1925
|
+
def _optional_non_negative_int(value: object) -> int | None:
|
|
1926
|
+
if value is None:
|
|
1927
|
+
return None
|
|
1928
|
+
return _non_negative_int(value)
|
|
1929
|
+
|
|
1930
|
+
|
|
1931
|
+
def _non_negative_int(value: object) -> int:
|
|
1932
|
+
try:
|
|
1933
|
+
return max(0, int(value or 0))
|
|
1934
|
+
except (TypeError, ValueError):
|
|
1935
|
+
return 0
|
|
1936
|
+
|
|
1937
|
+
|
|
1938
|
+
def _guardrail_item_threshold(args: argparse.Namespace) -> int:
|
|
1939
|
+
return max(0, int(getattr(args, "guardrail_item_threshold", DEFAULT_GUARDRAIL_ITEM_THRESHOLD) or 0))
|
|
1940
|
+
|
|
1941
|
+
|
|
1942
|
+
def _guardrail_failure_streak(args: argparse.Namespace) -> int:
|
|
1943
|
+
return max(0, int(getattr(args, "guardrail_failure_streak", DEFAULT_GUARDRAIL_FAILURE_STREAK) or 0))
|
|
1944
|
+
|
|
1945
|
+
|
|
1946
|
+
def _guardrail_failure_streak_reached(args: argparse.Namespace, consecutive_failures: int) -> bool:
|
|
1947
|
+
threshold = _guardrail_failure_streak(args)
|
|
1948
|
+
return threshold > 0 and consecutive_failures >= threshold
|
|
1949
|
+
|
|
1950
|
+
|
|
1951
|
+
def _append_large_batch_guardrail_if_needed(
|
|
1952
|
+
args: argparse.Namespace,
|
|
1953
|
+
*,
|
|
1954
|
+
command: str,
|
|
1955
|
+
total: int,
|
|
1956
|
+
) -> None:
|
|
1957
|
+
threshold = _guardrail_item_threshold(args)
|
|
1958
|
+
if threshold <= 0 or total < threshold:
|
|
1959
|
+
return
|
|
1960
|
+
append_runlog_entry(
|
|
1961
|
+
_runlog_path(args),
|
|
1962
|
+
build_runlog_entry(
|
|
1963
|
+
input_url=getattr(args, "urls_file", None) or getattr(args, "url", "") or "",
|
|
1964
|
+
platform=_detect_collect_platform(getattr(args, "url", "")) if command == "collect" else "mixed",
|
|
1965
|
+
backend="openmd.batch.guardrail",
|
|
1966
|
+
status="partial",
|
|
1967
|
+
source_type="mixed",
|
|
1968
|
+
source_level="partial_extraction",
|
|
1969
|
+
item_count=total,
|
|
1970
|
+
success_count=0,
|
|
1971
|
+
failed_count=0,
|
|
1972
|
+
event_type="guardrail",
|
|
1973
|
+
guardrail_signals=["large_item_count"],
|
|
1974
|
+
resume_pointer="runlog records per-item progress; rerun uses skip-existing/package dedupe where applicable",
|
|
1975
|
+
),
|
|
1976
|
+
)
|
|
1977
|
+
|
|
1978
|
+
|
|
1979
|
+
def _append_guardrail_stop_entry(
|
|
1980
|
+
args: argparse.Namespace,
|
|
1981
|
+
*,
|
|
1982
|
+
command: str,
|
|
1983
|
+
index: int,
|
|
1984
|
+
total: int,
|
|
1985
|
+
) -> None:
|
|
1986
|
+
append_runlog_entry(
|
|
1987
|
+
_runlog_path(args),
|
|
1988
|
+
build_runlog_entry(
|
|
1989
|
+
input_url=getattr(args, "urls_file", None) or getattr(args, "url", "") or "",
|
|
1990
|
+
platform=_detect_collect_platform(getattr(args, "url", "")) if command == "collect" else "mixed",
|
|
1991
|
+
backend="openmd.batch.guardrail",
|
|
1992
|
+
status="failed",
|
|
1993
|
+
source_type="mixed",
|
|
1994
|
+
source_level="partial_extraction",
|
|
1995
|
+
item_count=total,
|
|
1996
|
+
success_count=0,
|
|
1997
|
+
failed_count=0,
|
|
1998
|
+
event_type="guardrail",
|
|
1999
|
+
failure_class="runtime_transient",
|
|
2000
|
+
guardrail_signals=["consecutive_failure_stop"],
|
|
2001
|
+
resume_pointer=f"resume from item {index + 1} of {total}",
|
|
2002
|
+
error_code="guardrail_consecutive_failures",
|
|
2003
|
+
error_summary=(
|
|
2004
|
+
"Batch paused after consecutive failures. Inspect recent runlog rows, "
|
|
2005
|
+
"fix backend/runtime state, then resume from the recorded pointer."
|
|
2006
|
+
),
|
|
2007
|
+
),
|
|
2008
|
+
)
|
|
2009
|
+
|
|
2010
|
+
|
|
2011
|
+
if __name__ == "__main__":
|
|
2012
|
+
raise SystemExit(main())
|