web-forager 0.0.0-stage → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +112 -0
- package/CHANGELOG.md +454 -0
- package/LICENSE +21 -0
- package/README.md +477 -3
- package/README_zh.md +477 -0
- package/SKILL.md +351 -0
- package/THIRD_PARTY_LICENSES.txt +35 -0
- package/browser/bridges/playwright-bridge.js +375 -0
- package/browser/launch_chrome.py +144 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +79 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +115 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +116 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +56 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +272 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +105 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +432 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +308 -0
- package/docs/REFERENCE_zh.md +309 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +97 -0
- package/mcp_server.py +597 -0
- package/package.json +80 -6
- package/requirements.txt +12 -0
- package/run.py +852 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +874 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +4220 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +135 -0
- package/scripts/scholar.py +354 -0
- package/scripts/sitemap.py +180 -0
- package/scripts/untrusted.py +95 -0
package/run.py
ADDED
|
@@ -0,0 +1,852 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""web-forager command-line entry point.
|
|
3
|
+
|
|
4
|
+
Actions
|
|
5
|
+
-------
|
|
6
|
+
fetch
|
|
7
|
+
``--url URL`` fetch one page and return metadata, Markdown and tables
|
|
8
|
+
``--query TEXT`` search (360+Bing by default), then fetch the top results
|
|
9
|
+
|
|
10
|
+
clean
|
|
11
|
+
``--input file`` clean a local HTML/PDF/DOCX/XLSX/EPUB/CSV file
|
|
12
|
+
``--url URL`` fetch a page/document and clean it
|
|
13
|
+
(no input) read HTML/PDF/document bytes from stdin
|
|
14
|
+
|
|
15
|
+
bili
|
|
16
|
+
``--url BV...`` / ``--query TEXT`` / neither Bilibili video, search or feed
|
|
17
|
+
|
|
18
|
+
surf
|
|
19
|
+
``--url URL`` or ``--query TEXT`` open/search, then follow ranked links
|
|
20
|
+
``--depth N --follow same-site|all|none --session NAME``
|
|
21
|
+
|
|
22
|
+
scholar
|
|
23
|
+
``--source arxiv|crossref|pubmed|doi --query TEXT`` bibliographic records
|
|
24
|
+
|
|
25
|
+
map
|
|
26
|
+
``--url SITE`` discover page URLs from robots.txt / sitemaps
|
|
27
|
+
|
|
28
|
+
Global
|
|
29
|
+
``--engine auto|all|360|bing|sogou|weixin|duckduckgo``
|
|
30
|
+
``--cache --chunk-size N --search-pages N``
|
|
31
|
+
``--agent`` OpenCLI-style ``{ok,data}`` / ``{ok,error}`` envelope for agents
|
|
32
|
+
|
|
33
|
+
MCP
|
|
34
|
+
``python mcp_server.py`` stdio JSON-RPC server exposing the same engine.
|
|
35
|
+
|
|
36
|
+
本插件仅供学习与信息收集使用。使用者需遵守目标网站的 Terms of Service 及相关版权法律。
|
|
37
|
+
禁止用于恶意并发攻击。严禁收集、聚合或推断特定自然人的个人信息,或将其用于人肉搜索、骚扰、跟踪等侵害他人隐私的活动;此类用法产生的法律责任由使用者自行承担。
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import argparse
|
|
43
|
+
import hashlib
|
|
44
|
+
import json
|
|
45
|
+
import re
|
|
46
|
+
import sys
|
|
47
|
+
from pathlib import Path
|
|
48
|
+
|
|
49
|
+
_PLUGIN_ROOT = Path(__file__).resolve().parent
|
|
50
|
+
if str(_PLUGIN_ROOT) not in sys.path:
|
|
51
|
+
sys.path.insert(0, str(_PLUGIN_ROOT))
|
|
52
|
+
|
|
53
|
+
from scripts import bilibili, documents, engine, pdf as pdf_mod, scholar, sitemap # noqa: E402
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _reconfigure_stdio() -> None:
|
|
57
|
+
"""Force UTF-8 so Chinese metadata survives Windows code pages."""
|
|
58
|
+
|
|
59
|
+
for stream in (sys.stdout, sys.stderr):
|
|
60
|
+
try:
|
|
61
|
+
stream.reconfigure(encoding="utf-8") # type: ignore[attr-defined]
|
|
62
|
+
except Exception:
|
|
63
|
+
pass
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
67
|
+
parser = argparse.ArgumentParser(
|
|
68
|
+
prog="web-forager",
|
|
69
|
+
description=(
|
|
70
|
+
"Polite, robots-aware web information gathering and "
|
|
71
|
+
"HTML-to-Markdown cleaning for public pages."
|
|
72
|
+
|
|
73
|
+
),
|
|
74
|
+
epilog=engine.DISCLAIMER,
|
|
75
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
76
|
+
)
|
|
77
|
+
parser.add_argument(
|
|
78
|
+
"--action",
|
|
79
|
+
required=True,
|
|
80
|
+
choices=["fetch", "find", "feed", "clean", "bili", "surf", "scholar", "map", "doctor"],
|
|
81
|
+
help="operation to run",
|
|
82
|
+
)
|
|
83
|
+
parser.add_argument("--url", help="target URL, or a Bilibili BV/av id (fetch/clean/bili/surf/map)")
|
|
84
|
+
parser.add_argument("--query", help="keyword query (fetch/bili/surf/scholar)")
|
|
85
|
+
parser.add_argument("--site", help="domain or base URL to search inside (find)")
|
|
86
|
+
parser.add_argument("--input", dest="input_path", help="local HTML/PDF/DOCX/XLSX/EPUB/CSV file (clean)")
|
|
87
|
+
parser.add_argument(
|
|
88
|
+
"--source",
|
|
89
|
+
choices=list(scholar.SCHOLAR_SOURCES),
|
|
90
|
+
default="arxiv",
|
|
91
|
+
help="scholarly source for --action scholar (default: arxiv)",
|
|
92
|
+
)
|
|
93
|
+
parser.add_argument(
|
|
94
|
+
"--sort",
|
|
95
|
+
choices=["relevance", "submittedDate", "lastUpdatedDate"],
|
|
96
|
+
default="relevance",
|
|
97
|
+
help="arXiv sort order (default: relevance)",
|
|
98
|
+
)
|
|
99
|
+
parser.add_argument("--limit", type=int, default=500, help="map: maximum URLs to return (default: 500)")
|
|
100
|
+
parser.add_argument("--max-sitemaps", type=int, default=5, help="map: sitemap files to read (default: 5)")
|
|
101
|
+
parser.add_argument(
|
|
102
|
+
"--engine",
|
|
103
|
+
choices=list(engine.SEARCH_ENGINES),
|
|
104
|
+
default="auto",
|
|
105
|
+
help="search backend: auto=360+Bing (mostly-CJK queries: 360+Sogou+WeChat); all=every engine (default: auto)",
|
|
106
|
+
)
|
|
107
|
+
parser.add_argument("--num-results", type=int, default=8, help="search results to request (default: 8)")
|
|
108
|
+
parser.add_argument(
|
|
109
|
+
"--serial-engines",
|
|
110
|
+
action="store_true",
|
|
111
|
+
help="ask search engines one after another (default: parallel across different hosts)",
|
|
112
|
+
)
|
|
113
|
+
parser.add_argument(
|
|
114
|
+
"--max-pages",
|
|
115
|
+
type=int,
|
|
116
|
+
default=3,
|
|
117
|
+
help="result pages to fetch (default: 3; 0 = search results only)",
|
|
118
|
+
)
|
|
119
|
+
parser.add_argument("--search-pages", type=int, default=1, help="search result pages to read per engine (default: 1)")
|
|
120
|
+
parser.add_argument(
|
|
121
|
+
"--mode",
|
|
122
|
+
choices=["auto", "feed", "video", "search", "related", "surf"],
|
|
123
|
+
default="auto",
|
|
124
|
+
help="Bilibili mode (default auto: query->search, url->video, neither->feed)",
|
|
125
|
+
)
|
|
126
|
+
parser.add_argument("--depth", type=int, default=1, help="Bilibili/surf depth (default: 1)")
|
|
127
|
+
parser.add_argument(
|
|
128
|
+
"--follow",
|
|
129
|
+
choices=["same-site", "all", "none"],
|
|
130
|
+
default="same-site",
|
|
131
|
+
help="surf link policy: same-site (default), all, none",
|
|
132
|
+
)
|
|
133
|
+
parser.add_argument(
|
|
134
|
+
"--session",
|
|
135
|
+
help="surf session name; visited URLs and the trail are saved under ~/.web-forager/sessions/",
|
|
136
|
+
)
|
|
137
|
+
parser.add_argument(
|
|
138
|
+
"--direct-search",
|
|
139
|
+
action="store_true",
|
|
140
|
+
help=(
|
|
141
|
+
"Bilibili only: query search.bilibili.com directly. That host disallows "
|
|
142
|
+
"generic agents in robots.txt, so this is an explicit opt-in override; "
|
|
143
|
+
"video pages are still fetched with robots checks enabled."
|
|
144
|
+
),
|
|
145
|
+
)
|
|
146
|
+
parser.add_argument(
|
|
147
|
+
"--render",
|
|
148
|
+
choices=["auto", "never", "always"],
|
|
149
|
+
default="auto",
|
|
150
|
+
help="dynamic rendering with Playwright: auto detects JS shells (default), never, always",
|
|
151
|
+
)
|
|
152
|
+
parser.add_argument("--no-tables", action="store_true", help="do not extract tables")
|
|
153
|
+
parser.add_argument("--keep-links", action="store_true", help="preserve hyperlinks in Markdown")
|
|
154
|
+
parser.add_argument("--images", action="store_true", help="preserve image references in Markdown")
|
|
155
|
+
parser.add_argument("--max-chars", type=int, default=20_000, help="truncate Markdown to N characters (0 = no limit)")
|
|
156
|
+
parser.add_argument(
|
|
157
|
+
"--chunk-size",
|
|
158
|
+
type=int,
|
|
159
|
+
default=0,
|
|
160
|
+
help="also emit Markdown chunks of about N characters (0 = off)",
|
|
161
|
+
)
|
|
162
|
+
parser.add_argument("--cache", action="store_true", help="reuse responses cached under ~/.web-forager/cache")
|
|
163
|
+
parser.add_argument("--cache-ttl", type=int, default=3600, help="cache lifetime in seconds (default: 3600)")
|
|
164
|
+
parser.add_argument("--cache-dir", default="", help="override the cache directory")
|
|
165
|
+
parser.add_argument("--wait-for", default="", help="Playwright: wait for this CSS selector before reading the page")
|
|
166
|
+
parser.add_argument("--scroll", type=int, default=0, help="Playwright: wheel-scroll N times to load lazy content")
|
|
167
|
+
parser.add_argument("--timeout", type=float, default=20.0, help="per-request timeout in seconds")
|
|
168
|
+
parser.add_argument("--delay", type=float, default=1.0, help="politeness delay per host in seconds")
|
|
169
|
+
parser.add_argument(
|
|
170
|
+
"--robots-policy",
|
|
171
|
+
choices=("respect", "warn", "ignore"),
|
|
172
|
+
default="respect",
|
|
173
|
+
help=(
|
|
174
|
+
"respect: refuse a disallowed path (default); warn: read it and mark "
|
|
175
|
+
"the record with compliance.robots_overridden; ignore: do not consult robots.txt"
|
|
176
|
+
),
|
|
177
|
+
)
|
|
178
|
+
parser.add_argument(
|
|
179
|
+
"--ignore-robots",
|
|
180
|
+
action="store_true",
|
|
181
|
+
help="DANGEROUS: same as --robots-policy ignore (only for resources you are authorised to access)",
|
|
182
|
+
)
|
|
183
|
+
parser.add_argument(
|
|
184
|
+
"--allow-private",
|
|
185
|
+
action="store_true",
|
|
186
|
+
help=(
|
|
187
|
+
"DANGEROUS: allow loopback, private and link-local targets. "
|
|
188
|
+
"For local fixtures and trusted intranet tools only"
|
|
189
|
+
),
|
|
190
|
+
)
|
|
191
|
+
parser.add_argument(
|
|
192
|
+
"--input-gate",
|
|
193
|
+
choices=["off", "mark", "strict"],
|
|
194
|
+
default="mark",
|
|
195
|
+
help=(
|
|
196
|
+
"treat fetched text as untrusted: mark instruction-shaped content "
|
|
197
|
+
"(default), withhold the body on a hard hit (strict), or skip the scan"
|
|
198
|
+
),
|
|
199
|
+
)
|
|
200
|
+
parser.add_argument("--user-agent", default=engine.DEFAULT_USER_AGENT, help="explicit User-Agent string")
|
|
201
|
+
parser.add_argument(
|
|
202
|
+
"--probe",
|
|
203
|
+
action="store_true",
|
|
204
|
+
help="doctor: ask each search engine once and report which ones answer",
|
|
205
|
+
)
|
|
206
|
+
parser.add_argument(
|
|
207
|
+
"--proxy",
|
|
208
|
+
default="",
|
|
209
|
+
help="route every request through this proxy (default: HTTPS_PROXY / ALL_PROXY from the environment)",
|
|
210
|
+
)
|
|
211
|
+
parser.add_argument("--format", choices=["json", "markdown"], default="json", help="stdout format")
|
|
212
|
+
parser.add_argument(
|
|
213
|
+
"--full-results",
|
|
214
|
+
action="store_true",
|
|
215
|
+
help=(
|
|
216
|
+
"repeat every page record inside results as well as pages "
|
|
217
|
+
"(1.0.1 behaviour; off by default because it doubles the response)"
|
|
218
|
+
),
|
|
219
|
+
)
|
|
220
|
+
parser.add_argument(
|
|
221
|
+
"--core",
|
|
222
|
+
action="store_true",
|
|
223
|
+
help="report results in the shared core-1 schema that browser mode also emits",
|
|
224
|
+
)
|
|
225
|
+
parser.add_argument(
|
|
226
|
+
"--agent",
|
|
227
|
+
action="store_true",
|
|
228
|
+
help="OpenCLI-style envelope: {ok:true,data:...} or {ok:false,error:{code,message,hint}}",
|
|
229
|
+
)
|
|
230
|
+
parser.add_argument("--out", help="write the result to this file instead of stdout")
|
|
231
|
+
parser.add_argument(
|
|
232
|
+
"--root",
|
|
233
|
+
help=(
|
|
234
|
+
"Chelate workspace root; writes <root>/process/web-forager/ and a "
|
|
235
|
+
"<root>/_broadcast/ index entry. Ignored when --out is given."
|
|
236
|
+
),
|
|
237
|
+
)
|
|
238
|
+
parser.add_argument("--verbose", "-v", action="store_true", help="diagnostics on stderr")
|
|
239
|
+
parser.add_argument("--version", action="version", version=f"web-forager {engine.__version__}")
|
|
240
|
+
return parser
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def build_config(args: argparse.Namespace) -> engine.FetchConfig:
|
|
244
|
+
config = engine.FetchConfig(
|
|
245
|
+
user_agent=args.user_agent,
|
|
246
|
+
timeout=args.timeout,
|
|
247
|
+
delay=args.delay,
|
|
248
|
+
max_chars=args.max_chars,
|
|
249
|
+
respect_robots=not args.ignore_robots,
|
|
250
|
+
robots_policy="ignore" if args.ignore_robots else args.robots_policy,
|
|
251
|
+
render=args.render,
|
|
252
|
+
include_tables=not args.no_tables,
|
|
253
|
+
keep_links=args.keep_links,
|
|
254
|
+
include_images=args.images,
|
|
255
|
+
max_pages=max(0, args.max_pages),
|
|
256
|
+
num_results=max(1, args.num_results),
|
|
257
|
+
verbose=args.verbose,
|
|
258
|
+
extract_links=args.action == "surf",
|
|
259
|
+
follow=args.follow,
|
|
260
|
+
search_engine=args.engine,
|
|
261
|
+
parallel_engines=not args.serial_engines,
|
|
262
|
+
proxy=args.proxy,
|
|
263
|
+
search_pages=max(1, args.search_pages),
|
|
264
|
+
cache=args.cache,
|
|
265
|
+
cache_ttl=max(0, args.cache_ttl),
|
|
266
|
+
cache_dir=args.cache_dir,
|
|
267
|
+
chunk_size=max(0, args.chunk_size),
|
|
268
|
+
wait_for=args.wait_for,
|
|
269
|
+
scroll=max(0, args.scroll),
|
|
270
|
+
allow_private=bool(args.allow_private) or engine._allow_private_from_env(),
|
|
271
|
+
input_gate=args.input_gate,
|
|
272
|
+
)
|
|
273
|
+
config.validate()
|
|
274
|
+
return config
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# ---------------------------------------------------------------------------
|
|
278
|
+
# Output helpers
|
|
279
|
+
# ---------------------------------------------------------------------------
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def document_to_markdown(document: dict) -> str:
|
|
283
|
+
"""Render a result envelope as a human-readable Markdown document."""
|
|
284
|
+
|
|
285
|
+
if document.get("action") == "clean":
|
|
286
|
+
return document.get("markdown", "")
|
|
287
|
+
if document.get("action") == "scholar":
|
|
288
|
+
return scholar.to_markdown(document)
|
|
289
|
+
if document.get("action") == "map":
|
|
290
|
+
return sitemap.to_markdown(document)
|
|
291
|
+
|
|
292
|
+
blocks: list[str] = []
|
|
293
|
+
for record in document.get("results", []):
|
|
294
|
+
if record.get("error"):
|
|
295
|
+
blocks.append(f"## {record.get('url', 'unknown')}\n\n**Error:** {record['error']}")
|
|
296
|
+
continue
|
|
297
|
+
title = record.get("title") or record.get("final_url") or record.get("url", "")
|
|
298
|
+
if not record.get("markdown"):
|
|
299
|
+
blocks.append(f"- [{title}]({record.get('url', '')})")
|
|
300
|
+
continue
|
|
301
|
+
lines = [f"# {title}", "", f"> Source: {record.get('final_url') or record.get('url', '')}", ""]
|
|
302
|
+
metadata = record.get("metadata") or {}
|
|
303
|
+
if metadata.get("authors"):
|
|
304
|
+
lines.append(f"**Authors:** {', '.join(metadata['authors'])}")
|
|
305
|
+
if metadata.get("published"):
|
|
306
|
+
lines.append(f"**Published:** {metadata['published']}")
|
|
307
|
+
if metadata.get("doi"):
|
|
308
|
+
lines.append(f"**DOI:** {metadata['doi']}")
|
|
309
|
+
if metadata.get("journal"):
|
|
310
|
+
lines.append(f"**Journal:** {metadata['journal']}")
|
|
311
|
+
if metadata.get("authors") or metadata.get("published") or metadata.get("doi") or metadata.get("journal"):
|
|
312
|
+
lines.append("")
|
|
313
|
+
lines.append(record.get("markdown", "").strip())
|
|
314
|
+
blocks.append("\n".join(lines).strip())
|
|
315
|
+
return "\n\n---\n\n".join(blocks)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def emit(text: str, output_path: str | None) -> None:
|
|
319
|
+
if output_path:
|
|
320
|
+
target = Path(output_path)
|
|
321
|
+
if target.parent and not target.parent.exists():
|
|
322
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
323
|
+
target.write_text(text, encoding="utf-8", newline="\n")
|
|
324
|
+
print(f"[web-forager] wrote {target}", file=sys.stderr)
|
|
325
|
+
else:
|
|
326
|
+
sys.stdout.write(text)
|
|
327
|
+
if not text.endswith("\n"):
|
|
328
|
+
sys.stdout.write("\n")
|
|
329
|
+
sys.stdout.flush()
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
# ---------------------------------------------------------------------------
|
|
333
|
+
# Chelate workspace integration
|
|
334
|
+
# ---------------------------------------------------------------------------
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _workspace_slug(args: argparse.Namespace) -> str:
|
|
338
|
+
raw = args.query or args.url or (Path(args.input_path).stem if args.input_path else "stdin")
|
|
339
|
+
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", str(raw)).strip("-._")
|
|
340
|
+
if not slug:
|
|
341
|
+
slug = "result-" + hashlib.sha1(str(raw).encode("utf-8")).hexdigest()[:8]
|
|
342
|
+
return slug[:48]
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _workspace_timestamp(document: dict) -> str:
|
|
346
|
+
stamp = str(document.get("generated_at") or engine.utc_now())
|
|
347
|
+
return re.sub(r"[^0-9TZ]", "", stamp) or "00000000T000000Z"
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _document_summary(document: dict) -> str:
|
|
351
|
+
action = document.get("action")
|
|
352
|
+
if action == "clean":
|
|
353
|
+
return f"cleaned {document.get('source', 'input')}"
|
|
354
|
+
if action == "scholar":
|
|
355
|
+
return f"{document.get('count', 0)} record(s) from {document.get('source', 'scholar')}"
|
|
356
|
+
if action == "map":
|
|
357
|
+
return f"{document.get('count', 0)} URL(s) from {document.get('origin', 'sitemap')}"
|
|
358
|
+
results = document.get("results") or []
|
|
359
|
+
if action == "bili":
|
|
360
|
+
first = results[0] if results else {}
|
|
361
|
+
title = first.get("title") or document.get("query") or document.get("seed") or ""
|
|
362
|
+
mode = document.get("mode", "bili")
|
|
363
|
+
return f"bilibili {mode} · {len(results)} item(s)" + (f" · {title}" if title else "")
|
|
364
|
+
first = results[0] if results else {}
|
|
365
|
+
title = first.get("title") or first.get("url") or ""
|
|
366
|
+
return f"{len(results)} page(s)" + (f" · {title}" if title else "")
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def write_workspace(
|
|
370
|
+
args: argparse.Namespace,
|
|
371
|
+
document: dict,
|
|
372
|
+
json_text: str,
|
|
373
|
+
markdown_text: str,
|
|
374
|
+
) -> list[Path]:
|
|
375
|
+
"""Write the run into a Chelate four-zone workspace.
|
|
376
|
+
|
|
377
|
+
Layout follows the main library:
|
|
378
|
+
|
|
379
|
+
* ``<root>/process/web-forager/`` - intermediate JSON + Markdown artefacts
|
|
380
|
+
* ``<root>/_broadcast/`` - one read-only index entry per run
|
|
381
|
+
"""
|
|
382
|
+
|
|
383
|
+
root = Path(args.root).expanduser().resolve()
|
|
384
|
+
if root.exists() and not root.is_dir():
|
|
385
|
+
raise ValueError(f"--root is not a directory: {root}")
|
|
386
|
+
|
|
387
|
+
process_dir = root / "process" / "web-forager"
|
|
388
|
+
broadcast_dir = root / "_broadcast"
|
|
389
|
+
process_dir.mkdir(parents=True, exist_ok=True)
|
|
390
|
+
broadcast_dir.mkdir(parents=True, exist_ok=True)
|
|
391
|
+
|
|
392
|
+
base = f"web-forager_{_workspace_timestamp(document)}_{_workspace_slug(args)}"
|
|
393
|
+
json_path = process_dir / f"{base}.json"
|
|
394
|
+
markdown_path = process_dir / f"{base}.md"
|
|
395
|
+
index_path = broadcast_dir / f"{base}.json"
|
|
396
|
+
|
|
397
|
+
json_path.write_text(json_text + "\n", encoding="utf-8", newline="\n")
|
|
398
|
+
markdown_path.write_text(markdown_text + "\n", encoding="utf-8", newline="\n")
|
|
399
|
+
|
|
400
|
+
index = {
|
|
401
|
+
"tool": "web-forager",
|
|
402
|
+
"version": engine.__version__,
|
|
403
|
+
"action": document.get("action", ""),
|
|
404
|
+
"generated_at": document.get("generated_at", ""),
|
|
405
|
+
"summary": _document_summary(document),
|
|
406
|
+
"source": document.get("source") or document.get("query") or document.get("seed") or "",
|
|
407
|
+
"files": {
|
|
408
|
+
"json": json_path.relative_to(root).as_posix(),
|
|
409
|
+
"markdown": markdown_path.relative_to(root).as_posix(),
|
|
410
|
+
},
|
|
411
|
+
}
|
|
412
|
+
index_path.write_text(json.dumps(index, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", newline="\n")
|
|
413
|
+
|
|
414
|
+
for path in (json_path, markdown_path, index_path):
|
|
415
|
+
print(f"[web-forager] workspace: {path}", file=sys.stderr)
|
|
416
|
+
return [json_path, markdown_path, index_path]
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
# ---------------------------------------------------------------------------
|
|
420
|
+
# Actions
|
|
421
|
+
# ---------------------------------------------------------------------------
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _envelope(action: str, payload: dict, args: argparse.Namespace) -> dict:
|
|
425
|
+
"""Build the response envelope, honouring --full-results."""
|
|
426
|
+
|
|
427
|
+
return engine.build_envelope(action, payload, full_results=bool(getattr(args, "full_results", False)))
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def cmd_find(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
431
|
+
if not args.site:
|
|
432
|
+
raise ValueError("--action find needs --site")
|
|
433
|
+
if not args.query:
|
|
434
|
+
raise ValueError("--action find needs --query")
|
|
435
|
+
payload = engine.find_within_site(
|
|
436
|
+
args.site, args.query, config, max_pages=args.max_pages
|
|
437
|
+
)
|
|
438
|
+
return _envelope("find", payload, args)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def cmd_feed(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
442
|
+
if not args.url:
|
|
443
|
+
raise ValueError("--action feed needs --url")
|
|
444
|
+
payload = engine.discover_feed(args.url, config)
|
|
445
|
+
payload["results"] = [
|
|
446
|
+
{
|
|
447
|
+
"url": item.get("url", ""),
|
|
448
|
+
"title": item.get("title", ""),
|
|
449
|
+
"published": item.get("published", ""),
|
|
450
|
+
}
|
|
451
|
+
for item in payload.get("items") or []
|
|
452
|
+
]
|
|
453
|
+
return _envelope("feed", payload, args)
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def cmd_fetch(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
457
|
+
if bool(args.url) == bool(args.query):
|
|
458
|
+
raise ValueError("--action fetch needs exactly one of --url or --query")
|
|
459
|
+
|
|
460
|
+
if args.url:
|
|
461
|
+
page = engine.fetch_page(args.url, config)
|
|
462
|
+
return _envelope("fetch", page, args)
|
|
463
|
+
|
|
464
|
+
payload = engine.fetch_by_query(args.query, config, engine=args.engine)
|
|
465
|
+
return _envelope("fetch", payload, args)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _clean_chunks(payload: dict, config: engine.FetchConfig) -> dict:
|
|
469
|
+
if config.chunk_size:
|
|
470
|
+
payload["chunks"] = engine.chunk_markdown(payload.get("markdown", ""), config.chunk_size)
|
|
471
|
+
return payload
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _pdf_document(
|
|
475
|
+
data: bytes,
|
|
476
|
+
source: str,
|
|
477
|
+
url: str,
|
|
478
|
+
config: engine.FetchConfig,
|
|
479
|
+
) -> dict:
|
|
480
|
+
"""Build a clean-style document from PDF bytes.
|
|
481
|
+
|
|
482
|
+
``pdf_mod`` raises ``PdfUnavailable`` when pypdf is missing and
|
|
483
|
+
``PdfExtractionError`` when the file cannot be parsed; ``main`` maps both
|
|
484
|
+
to stable exit codes instead of letting a traceback escape.
|
|
485
|
+
"""
|
|
486
|
+
|
|
487
|
+
extracted = pdf_mod.extract(data)
|
|
488
|
+
markdown = pdf_mod.to_markdown(extracted, url=url or "")
|
|
489
|
+
if config.max_chars and len(markdown) > config.max_chars:
|
|
490
|
+
markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
|
|
491
|
+
|
|
492
|
+
return {
|
|
493
|
+
"tool": "web-forager",
|
|
494
|
+
"version": engine.__version__,
|
|
495
|
+
"action": "clean",
|
|
496
|
+
"generated_at": engine.utc_now(),
|
|
497
|
+
"source": source,
|
|
498
|
+
"content_type": "application/pdf",
|
|
499
|
+
"page_count": extracted.get("page_count", 0),
|
|
500
|
+
"empty_pages": extracted.get("empty_pages", 0),
|
|
501
|
+
"metadata": pdf_mod.to_metadata(extracted, url=url or ""),
|
|
502
|
+
"tables": [],
|
|
503
|
+
"markdown": markdown,
|
|
504
|
+
"chunks": engine.chunk_markdown(markdown, config.chunk_size) if config.chunk_size else [],
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _document_document(
|
|
509
|
+
data: bytes,
|
|
510
|
+
kind: str,
|
|
511
|
+
source: str,
|
|
512
|
+
url: str,
|
|
513
|
+
config: engine.FetchConfig,
|
|
514
|
+
) -> dict:
|
|
515
|
+
extracted = documents.extract(data, kind, url=url or source)
|
|
516
|
+
markdown = extracted["markdown"]
|
|
517
|
+
if config.max_chars and len(markdown) > config.max_chars:
|
|
518
|
+
markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
|
|
519
|
+
payload = {
|
|
520
|
+
"tool": "web-forager",
|
|
521
|
+
"version": engine.__version__,
|
|
522
|
+
"action": "clean",
|
|
523
|
+
"generated_at": engine.utc_now(),
|
|
524
|
+
"source": source,
|
|
525
|
+
"content_type": kind,
|
|
526
|
+
"metadata": documents.to_metadata(extracted, url=url or ""),
|
|
527
|
+
"tables": extracted.get("tables") or [],
|
|
528
|
+
"markdown": markdown,
|
|
529
|
+
}
|
|
530
|
+
return _clean_chunks(payload, config)
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def cmd_clean(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
534
|
+
source = ""
|
|
535
|
+
base_url: str | None = None
|
|
536
|
+
fetcher: engine.HttpFetcher | None = None
|
|
537
|
+
|
|
538
|
+
try:
|
|
539
|
+
if args.input_path and args.url:
|
|
540
|
+
raise ValueError("--action clean accepts --input or --url, not both")
|
|
541
|
+
|
|
542
|
+
if args.input_path:
|
|
543
|
+
path = Path(args.input_path)
|
|
544
|
+
if not path.is_file():
|
|
545
|
+
raise ValueError(f"input file not found: {path}")
|
|
546
|
+
data = path.read_bytes()
|
|
547
|
+
source = str(path)
|
|
548
|
+
if pdf_mod.is_pdf(data):
|
|
549
|
+
return _pdf_document(data, source, "", config)
|
|
550
|
+
kind = documents.detect(data, url=source)
|
|
551
|
+
if kind:
|
|
552
|
+
return _document_document(data, kind, source, "", config)
|
|
553
|
+
html = engine.decode_bytes(data)
|
|
554
|
+
base_url = args.url if args.url else None
|
|
555
|
+
elif args.url:
|
|
556
|
+
fetcher = engine.HttpFetcher(config)
|
|
557
|
+
response = fetcher.fetch(args.url)
|
|
558
|
+
source = response.final_url
|
|
559
|
+
base_url = response.final_url
|
|
560
|
+
if "application/pdf" in response.content_type or pdf_mod.is_pdf(response.content):
|
|
561
|
+
return _pdf_document(response.content, source, response.final_url, config)
|
|
562
|
+
kind = documents.detect(response.content, response.content_type, response.final_url)
|
|
563
|
+
if kind:
|
|
564
|
+
return _document_document(
|
|
565
|
+
response.content, kind, source, response.final_url, config
|
|
566
|
+
)
|
|
567
|
+
html = response.text
|
|
568
|
+
else:
|
|
569
|
+
if sys.stdin.isatty():
|
|
570
|
+
raise ValueError("--action clean needs --input, --url, or HTML on stdin")
|
|
571
|
+
data = sys.stdin.buffer.read()
|
|
572
|
+
source = "stdin"
|
|
573
|
+
if pdf_mod.is_pdf(data):
|
|
574
|
+
return _pdf_document(data, source, "", config)
|
|
575
|
+
kind = documents.detect(data)
|
|
576
|
+
if kind:
|
|
577
|
+
return _document_document(data, kind, source, "", config)
|
|
578
|
+
html = engine.decode_bytes(data)
|
|
579
|
+
|
|
580
|
+
markdown = engine.html_to_markdown(
|
|
581
|
+
html,
|
|
582
|
+
url=base_url,
|
|
583
|
+
include_tables=config.include_tables,
|
|
584
|
+
keep_links=config.keep_links,
|
|
585
|
+
include_images=config.include_images,
|
|
586
|
+
)
|
|
587
|
+
if not markdown.strip():
|
|
588
|
+
raise engine.ExtractionError("no readable content found in the supplied HTML")
|
|
589
|
+
if config.max_chars and len(markdown) > config.max_chars:
|
|
590
|
+
markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
|
|
591
|
+
|
|
592
|
+
payload = {
|
|
593
|
+
"tool": "web-forager",
|
|
594
|
+
"version": engine.__version__,
|
|
595
|
+
"action": "clean",
|
|
596
|
+
"generated_at": engine.utc_now(),
|
|
597
|
+
"source": source,
|
|
598
|
+
"metadata": engine.extract_metadata(html, url=base_url),
|
|
599
|
+
"tables": engine.extract_tables(html) if config.include_tables else [],
|
|
600
|
+
"markdown": markdown,
|
|
601
|
+
}
|
|
602
|
+
return _clean_chunks(payload, config)
|
|
603
|
+
finally:
|
|
604
|
+
if fetcher is not None:
|
|
605
|
+
fetcher.close()
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
# ---------------------------------------------------------------------------
|
|
609
|
+
# Entry point
|
|
610
|
+
# ---------------------------------------------------------------------------
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def cmd_surf(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
614
|
+
"""Search (or open) a page, then follow ranked links like a human browsing."""
|
|
615
|
+
|
|
616
|
+
if bool(args.url) == bool(args.query):
|
|
617
|
+
raise ValueError("--action surf needs exactly one of --url or --query")
|
|
618
|
+
|
|
619
|
+
seed = args.url or args.query
|
|
620
|
+
payload = engine.surf_web(
|
|
621
|
+
seed,
|
|
622
|
+
config,
|
|
623
|
+
depth=max(0, args.depth),
|
|
624
|
+
follow=args.follow,
|
|
625
|
+
max_pages=max(1, args.max_pages),
|
|
626
|
+
engine_name=args.engine,
|
|
627
|
+
session=args.session or None,
|
|
628
|
+
)
|
|
629
|
+
return _envelope("surf", payload, args)
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def cmd_scholar(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
633
|
+
if not args.query:
|
|
634
|
+
raise ValueError("--action scholar needs --query (or a DOI with --source doi)")
|
|
635
|
+
payload = scholar.search(
|
|
636
|
+
args.source,
|
|
637
|
+
args.query,
|
|
638
|
+
config,
|
|
639
|
+
limit=max(1, args.num_results),
|
|
640
|
+
sort=args.sort,
|
|
641
|
+
)
|
|
642
|
+
return _envelope("scholar", payload, args)
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def cmd_map(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
646
|
+
if not args.url:
|
|
647
|
+
raise ValueError("--action map needs --url (a site, sitemap or robots.txt URL)")
|
|
648
|
+
payload = sitemap.discover(
|
|
649
|
+
args.url,
|
|
650
|
+
config,
|
|
651
|
+
limit=max(1, args.limit),
|
|
652
|
+
max_sitemaps=max(1, args.max_sitemaps),
|
|
653
|
+
)
|
|
654
|
+
return _envelope("map", payload, args)
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def cmd_bili(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
658
|
+
mode = args.mode
|
|
659
|
+
if mode == "auto":
|
|
660
|
+
mode = "search" if args.query else "video" if args.url else "feed"
|
|
661
|
+
|
|
662
|
+
if mode == "feed":
|
|
663
|
+
payload = bilibili.feed(
|
|
664
|
+
config,
|
|
665
|
+
limit=args.num_results,
|
|
666
|
+
enrich=max(0, args.max_pages),
|
|
667
|
+
url=args.url or bilibili.BILI_HOME,
|
|
668
|
+
)
|
|
669
|
+
elif mode == "video":
|
|
670
|
+
if not args.url:
|
|
671
|
+
raise ValueError("--mode video needs --url with a BV id or a video URL")
|
|
672
|
+
record = bilibili.fetch_video(args.url, config)
|
|
673
|
+
payload = {"mode": "video", "videos": [record], "results": [record]}
|
|
674
|
+
elif mode == "search":
|
|
675
|
+
if not args.query:
|
|
676
|
+
raise ValueError("--mode search needs --query")
|
|
677
|
+
payload = bilibili.search(args.query, config, allow_direct=args.direct_search)
|
|
678
|
+
elif mode == "related":
|
|
679
|
+
if not args.url:
|
|
680
|
+
raise ValueError("--mode related needs --url with a BV id or a video URL")
|
|
681
|
+
payload = bilibili.related(
|
|
682
|
+
args.url, config, limit=args.num_results, enrich=max(0, args.max_pages)
|
|
683
|
+
)
|
|
684
|
+
elif mode == "surf":
|
|
685
|
+
seed = args.url or args.query
|
|
686
|
+
if not seed:
|
|
687
|
+
raise ValueError("--mode surf needs --url (a video) or --query (a keyword)")
|
|
688
|
+
payload = bilibili.surf(
|
|
689
|
+
seed,
|
|
690
|
+
config,
|
|
691
|
+
depth=max(0, args.depth),
|
|
692
|
+
per_level=max(1, args.max_pages),
|
|
693
|
+
allow_direct=args.direct_search,
|
|
694
|
+
)
|
|
695
|
+
else: # pragma: no cover - argparse prevents this
|
|
696
|
+
raise ValueError(f"unsupported Bilibili mode: {mode}")
|
|
697
|
+
|
|
698
|
+
return _envelope("bili", payload, args)
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def cmd_doctor(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
702
|
+
"""Report optional dependencies so an agent can install what is missing."""
|
|
703
|
+
|
|
704
|
+
payload = engine.doctor_report(config)
|
|
705
|
+
if getattr(args, "probe", False):
|
|
706
|
+
payload["search_probe"] = engine.probe_engines(config)
|
|
707
|
+
return _envelope("doctor", payload, args)
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def _fail(
|
|
711
|
+
args: argparse.Namespace,
|
|
712
|
+
code: str,
|
|
713
|
+
message: str,
|
|
714
|
+
hint: str,
|
|
715
|
+
exit_code: int,
|
|
716
|
+
) -> int:
|
|
717
|
+
"""Report a failure either as an OpenCLI-style envelope or on stderr."""
|
|
718
|
+
|
|
719
|
+
if getattr(args, "agent", False):
|
|
720
|
+
payload = {
|
|
721
|
+
"ok": False,
|
|
722
|
+
"error": {"code": code, "message": message, "hint": hint},
|
|
723
|
+
}
|
|
724
|
+
print(json.dumps(payload, ensure_ascii=False))
|
|
725
|
+
else:
|
|
726
|
+
suffix = f" ({hint})" if hint else ""
|
|
727
|
+
print(f"error: {message}{suffix}", file=sys.stderr)
|
|
728
|
+
return exit_code
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def main(argv: list[str] | None = None) -> int:
|
|
732
|
+
_reconfigure_stdio()
|
|
733
|
+
parser = build_parser()
|
|
734
|
+
args = parser.parse_args(argv)
|
|
735
|
+
|
|
736
|
+
try:
|
|
737
|
+
config = build_config(args)
|
|
738
|
+
if args.action == "fetch":
|
|
739
|
+
document = cmd_fetch(args, config)
|
|
740
|
+
elif args.action == "find":
|
|
741
|
+
document = cmd_find(args, config)
|
|
742
|
+
elif args.action == "feed":
|
|
743
|
+
document = cmd_feed(args, config)
|
|
744
|
+
elif args.action == "clean":
|
|
745
|
+
document = cmd_clean(args, config)
|
|
746
|
+
elif args.action == "surf":
|
|
747
|
+
document = cmd_surf(args, config)
|
|
748
|
+
elif args.action == "scholar":
|
|
749
|
+
document = cmd_scholar(args, config)
|
|
750
|
+
elif args.action == "map":
|
|
751
|
+
document = cmd_map(args, config)
|
|
752
|
+
elif args.action == "bili":
|
|
753
|
+
document = cmd_bili(args, config)
|
|
754
|
+
else:
|
|
755
|
+
document = cmd_doctor(args, config)
|
|
756
|
+
except engine.ProhibitedQuery as exc:
|
|
757
|
+
return _fail(
|
|
758
|
+
args,
|
|
759
|
+
"prohibited_query",
|
|
760
|
+
str(exc),
|
|
761
|
+
"search a topic, a paper title or an organisation; person profiling is out of scope",
|
|
762
|
+
10,
|
|
763
|
+
)
|
|
764
|
+
except engine.RobotsDenied as exc:
|
|
765
|
+
return _fail(
|
|
766
|
+
args,
|
|
767
|
+
"robots_denied",
|
|
768
|
+
f"robots.txt denied this request: {exc}",
|
|
769
|
+
"robots.txt denied this request. Use --robots-policy warn only when "
|
|
770
|
+
"you are authorised to read the path (the record will mark the override), "
|
|
771
|
+
"or choose another source; do not silently switch to a tool that ignores robots",
|
|
772
|
+
3,
|
|
773
|
+
)
|
|
774
|
+
except engine.UnsafeAddress as exc:
|
|
775
|
+
return _fail(
|
|
776
|
+
args,
|
|
777
|
+
"unsafe_address",
|
|
778
|
+
str(exc),
|
|
779
|
+
"loopback, private, link-local and cloud-metadata addresses are refused",
|
|
780
|
+
2,
|
|
781
|
+
)
|
|
782
|
+
except engine.InvalidURLError as exc:
|
|
783
|
+
return _fail(args, "invalid_url", f"invalid URL: {exc}", "pass an absolute http(s) URL", 2)
|
|
784
|
+
except engine.PdfUnavailable as exc:
|
|
785
|
+
return _fail(args, "pdf_unavailable", str(exc), "pip install pypdf", 7)
|
|
786
|
+
except engine.PdfExtractionError as exc:
|
|
787
|
+
return _fail(args, "pdf_extraction_failed", str(exc), "check that the file is a valid PDF", 5)
|
|
788
|
+
except engine.VerificationRequired as exc:
|
|
789
|
+
return _fail(
|
|
790
|
+
args,
|
|
791
|
+
"verification_required",
|
|
792
|
+
f"verification required: {exc}",
|
|
793
|
+
"open it in browser mode with the wechat-dig playbook "
|
|
794
|
+
"(node browser/run.js --playbook wechat-dig --url <URL> --backend cdp) "
|
|
795
|
+
"after signing in once with browser/launch_chrome.py; "
|
|
796
|
+
"web-forager does not solve challenges",
|
|
797
|
+
9,
|
|
798
|
+
)
|
|
799
|
+
except engine.FetchError as exc:
|
|
800
|
+
return _fail(args, "fetch_failed", f"fetch failed: {exc}", "check the URL and network", 4)
|
|
801
|
+
except engine.ExtractionError as exc:
|
|
802
|
+
return _fail(
|
|
803
|
+
args,
|
|
804
|
+
"extraction_failed",
|
|
805
|
+
f"extraction failed: {exc}",
|
|
806
|
+
"try --render always (needs Playwright) for JavaScript-heavy pages",
|
|
807
|
+
5,
|
|
808
|
+
)
|
|
809
|
+
except engine.RenderUnavailable as exc:
|
|
810
|
+
return _fail(
|
|
811
|
+
args,
|
|
812
|
+
"render_unavailable",
|
|
813
|
+
f"dynamic rendering unavailable: {exc}",
|
|
814
|
+
"pip install playwright && playwright install chromium",
|
|
815
|
+
6,
|
|
816
|
+
)
|
|
817
|
+
except ValueError as exc:
|
|
818
|
+
return _fail(args, "invalid_argument", str(exc), "", 2)
|
|
819
|
+
except KeyboardInterrupt:
|
|
820
|
+
return _fail(args, "interrupted", "interrupted", "", 130)
|
|
821
|
+
except Exception as exc: # never surface a raw traceback to the user
|
|
822
|
+
if getattr(args, "verbose", False):
|
|
823
|
+
import traceback
|
|
824
|
+
|
|
825
|
+
traceback.print_exc()
|
|
826
|
+
return _fail(args, "unexpected_error", f"unexpected failure: {exc}", "run with --verbose for the traceback", 1)
|
|
827
|
+
|
|
828
|
+
if getattr(args, "core", False):
|
|
829
|
+
document = engine.apply_core(document)
|
|
830
|
+
|
|
831
|
+
json_text = json.dumps(document, ensure_ascii=False, indent=2)
|
|
832
|
+
markdown_text = document_to_markdown(document)
|
|
833
|
+
if args.agent:
|
|
834
|
+
output = json.dumps({"ok": True, "data": document}, ensure_ascii=False, indent=2)
|
|
835
|
+
elif args.format == "markdown":
|
|
836
|
+
output = markdown_text
|
|
837
|
+
else:
|
|
838
|
+
output = json_text
|
|
839
|
+
|
|
840
|
+
if args.root and not args.out:
|
|
841
|
+
try:
|
|
842
|
+
write_workspace(args, document, json_text, markdown_text)
|
|
843
|
+
except (OSError, ValueError) as exc:
|
|
844
|
+
print(f"error: cannot write Chelate workspace: {exc}", file=sys.stderr)
|
|
845
|
+
return 2
|
|
846
|
+
|
|
847
|
+
emit(output, args.out)
|
|
848
|
+
return 0
|
|
849
|
+
|
|
850
|
+
|
|
851
|
+
if __name__ == "__main__":
|
|
852
|
+
raise SystemExit(main())
|