web-forager 0.0.0-stage → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +119 -0
  18. package/CHANGELOG.md +508 -0
  19. package/LICENSE +21 -0
  20. package/README.md +498 -2
  21. package/README_zh.md +498 -0
  22. package/SKILL.md +451 -0
  23. package/THIRD_PARTY_LICENSES.txt +40 -0
  24. package/browser/bridges/playwright-bridge.js +382 -0
  25. package/browser/launch_chrome.py +193 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +90 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +141 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +130 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +215 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +252 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +107 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +441 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/research.md +35 -0
  58. package/commands/scholar.md +28 -0
  59. package/commands/surf.md +28 -0
  60. package/cordis.patch.yml +19 -0
  61. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  62. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  63. package/docs/DEPENDENCIES.md +40 -0
  64. package/docs/INSTALL_COMMANDS.md +113 -0
  65. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  66. package/docs/REFERENCE.md +349 -0
  67. package/docs/REFERENCE_zh.md +342 -0
  68. package/index.js +7 -0
  69. package/install.py +254 -0
  70. package/manifest.json +98 -0
  71. package/mcp_server.py +850 -0
  72. package/package.json +82 -6
  73. package/requirements.txt +15 -0
  74. package/run.py +908 -0
  75. package/scripts/__init__.py +1 -0
  76. package/scripts/bilibili.py +882 -0
  77. package/scripts/documents.py +437 -0
  78. package/scripts/engine.py +5141 -0
  79. package/scripts/feeds.py +180 -0
  80. package/scripts/pdf.py +156 -0
  81. package/scripts/research.py +2087 -0
  82. package/scripts/scholar.py +381 -0
  83. package/scripts/sitemap.py +195 -0
  84. package/scripts/untrusted.py +312 -0
@@ -0,0 +1,437 @@
1
+ """Office-document and ebook extraction without extra dependencies.
2
+
3
+ DOCX, XLSX and EPUB are ZIP containers with XML inside, so the standard
4
+ library is enough. XML is parsed through :func:`engine.parse_xml` (defused
5
+ when available, DOCTYPE/ENTITY rejected otherwise), and archive expansion is
6
+ capped to avoid zip bombs.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import csv
12
+ import io
13
+ import re
14
+ import zipfile
15
+ from typing import Any
16
+
17
+ from . import engine
18
+
19
+ ZIP_MAGIC = b"PK\x03\x04"
20
+ _MAX_ENTRIES = 5_000
21
+ _MAX_UNCOMPRESSED = 64 * 1024 * 1024
22
+ _MAX_ENTRY = 24 * 1024 * 1024
23
+
24
+ _W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
25
+ _S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
26
+ _R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
27
+ _PKG_REL = "{http://schemas.openxmlformats.org/package/2006/relationships}"
28
+ _DC = "{http://purl.org/dc/elements/1.1/}"
29
+ _CONTAINER = "{urn:oasis:names:tc:opendocument:xmlns:container}"
30
+ _OPF = "{http://www.idpf.org/2007/opf}"
31
+
32
+ CONTENT_TYPES = {
33
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
34
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
35
+ "application/epub+zip": "epub",
36
+ "text/csv": "csv",
37
+ "application/csv": "csv",
38
+ }
39
+
40
+
41
+ def _clean(value: Any) -> str:
42
+ return re.sub(r"\s+", " ", str(value or "")).strip()
43
+
44
+
45
+ def _open_zip(data: bytes) -> zipfile.ZipFile:
46
+ archive = zipfile.ZipFile(io.BytesIO(data))
47
+ infos = archive.infolist()
48
+ if len(infos) > _MAX_ENTRIES:
49
+ raise engine.ExtractionError("archive contains too many entries")
50
+ if sum(info.file_size for info in infos) > _MAX_UNCOMPRESSED:
51
+ raise engine.ExtractionError("archive expands beyond the safety limit")
52
+ return archive
53
+
54
+
55
+ def _read_member(archive: zipfile.ZipFile, name: str) -> bytes:
56
+ info = archive.getinfo(name)
57
+ if info.file_size > _MAX_ENTRY:
58
+ raise engine.ExtractionError(f"archive member too large: {name}")
59
+ return archive.read(name)
60
+
61
+
62
+ def detect(data: bytes, content_type: str = "", url: str = "") -> str:
63
+ """Return docx/xlsx/epub/csv, or '' when the payload is not one of them."""
64
+
65
+ ctype = (content_type or "").split(";", 1)[0].strip().lower()
66
+ if ctype in CONTENT_TYPES:
67
+ return CONTENT_TYPES[ctype]
68
+ lowered = (url or "").lower().split("?", 1)[0]
69
+ if lowered.endswith(".csv"):
70
+ return "csv"
71
+ if data[:4] != ZIP_MAGIC:
72
+ return ""
73
+ try:
74
+ with _open_zip(data) as archive:
75
+ names = set(archive.namelist())
76
+ if "word/document.xml" in names:
77
+ return "docx"
78
+ if "xl/workbook.xml" in names:
79
+ return "xlsx"
80
+ if "META-INF/container.xml" in names:
81
+ return "epub"
82
+ except (zipfile.BadZipFile, engine.ExtractionError, KeyError):
83
+ return ""
84
+ return ""
85
+
86
+
87
+ # ---------------------------------------------------------------------------
88
+ # DOCX
89
+ # ---------------------------------------------------------------------------
90
+
91
+
92
+ def _docx_paragraph(node: Any) -> str:
93
+ pieces: list[str] = []
94
+ for text in node.iter(f"{_W}t"):
95
+ pieces.append(text.text or "")
96
+ for tab in node.iter(f"{_W}tab"):
97
+ pieces.append("\t")
98
+ for br in node.iter(f"{_W}br"):
99
+ pieces.append("\n")
100
+ return "".join(pieces).strip()
101
+
102
+
103
+ def _docx_style(node: Any) -> str:
104
+ style = node.find(f"{_W}pPr/{_W}pStyle")
105
+ if style is not None:
106
+ return style.get(f"{_W}val", "") or ""
107
+ return ""
108
+
109
+
110
+ def _docx_table(node: Any) -> dict[str, Any]:
111
+ rows: list[list[str]] = []
112
+ for row in node.findall(f"{_W}tr"):
113
+ cells = []
114
+ for cell in row.findall(f"{_W}tc"):
115
+ text = " ".join(
116
+ _docx_paragraph(paragraph)
117
+ for paragraph in cell.findall(f"{_W}p")
118
+ if _docx_paragraph(paragraph)
119
+ )
120
+ cells.append(text)
121
+ if cells:
122
+ rows.append(cells)
123
+ return {"caption": "", "headers": rows[0] if rows else [], "rows": rows[1:] if rows else []}
124
+
125
+
126
+ def _table_markdown(table: dict[str, Any]) -> str:
127
+ headers = [str(value) for value in table.get("headers") or []]
128
+ rows = [[str(value) for value in row] for row in table.get("rows") or []]
129
+ width = max([len(headers)] + [len(row) for row in rows]) if headers or rows else 0
130
+ if not width:
131
+ return ""
132
+ headers += [""] * (width - len(headers))
133
+ lines = ["| " + " | ".join(headers) + " |", "| " + " | ".join(["---"] * width) + " |"]
134
+ for row in rows:
135
+ row += [""] * (width - len(row))
136
+ lines.append("| " + " | ".join(cell.replace("|", "\\|") for cell in row) + " |")
137
+ return "\n".join(lines)
138
+
139
+
140
+ def extract_docx(data: bytes) -> dict[str, Any]:
141
+ with _open_zip(data) as archive:
142
+ document = engine.parse_xml(_read_member(archive, "word/document.xml").decode("utf-8", "replace"))
143
+ metadata: dict[str, str] = {}
144
+ if "docProps/core.xml" in archive.namelist():
145
+ core = engine.parse_xml(_read_member(archive, "docProps/core.xml").decode("utf-8", "replace"))
146
+ metadata = {
147
+ "title": _clean((core.findtext(f"{_DC}title") or "")),
148
+ "creator": _clean((core.findtext(f"{_DC}creator") or "")),
149
+ "subject": _clean((core.findtext(f"{_DC}subject") or "")),
150
+ "created": _clean((core.findtext(f"{_DC}created") or "")),
151
+ }
152
+
153
+ body = document.find(f"{_W}body")
154
+ blocks: list[str] = []
155
+ tables: list[dict[str, Any]] = []
156
+ if body is not None:
157
+ for child in body:
158
+ if child.tag == f"{_W}p":
159
+ text = _docx_paragraph(child)
160
+ if not text:
161
+ continue
162
+ style = _docx_style(child)
163
+ match = re.search(r"Heading\s*(\d)", style, re.I)
164
+ if match:
165
+ blocks.append("#" * min(int(match.group(1)), 6) + " " + text)
166
+ elif style.lower() in ("title",):
167
+ blocks.append("# " + text)
168
+ elif style.lower() in ("subtitle",):
169
+ blocks.append("## " + text)
170
+ elif child.find(f".//{_W}numPr") is not None:
171
+ blocks.append("- " + text)
172
+ else:
173
+ blocks.append(text)
174
+ elif child.tag == f"{_W}tbl":
175
+ table = _docx_table(child)
176
+ if table["headers"] or table["rows"]:
177
+ tables.append(table)
178
+ blocks.append(_table_markdown(table))
179
+
180
+ markdown = "\n\n".join(blocks).strip()
181
+ if not markdown:
182
+ raise engine.ExtractionError("DOCX contains no readable text")
183
+ title = metadata.get("title") or (blocks[0].lstrip("# ").strip() if blocks else "")
184
+ return {
185
+ "kind": "docx",
186
+ "title": title,
187
+ "markdown": markdown,
188
+ "tables": tables,
189
+ "metadata": {
190
+ "title": title,
191
+ "authors": [metadata["creator"]] if metadata.get("creator") else [],
192
+ "published": metadata.get("created", ""),
193
+ "abstract": metadata.get("subject", ""),
194
+ "site_name": "DOCX",
195
+ },
196
+ }
197
+
198
+
199
+ # ---------------------------------------------------------------------------
200
+ # XLSX
201
+ # ---------------------------------------------------------------------------
202
+
203
+
204
+ def _column_index(reference: str) -> int:
205
+ letters = re.match(r"([A-Z]+)", reference.upper())
206
+ if not letters:
207
+ return 0
208
+ index = 0
209
+ for letter in letters.group(1):
210
+ index = index * 26 + (ord(letter) - 64)
211
+ return max(0, index - 1)
212
+
213
+
214
+ def extract_xlsx(data: bytes, max_rows: int = 500, max_columns: int = 40) -> dict[str, Any]:
215
+ with _open_zip(data) as archive:
216
+ names = set(archive.namelist())
217
+ shared: list[str] = []
218
+ if "xl/sharedStrings.xml" in names:
219
+ root = engine.parse_xml(_read_member(archive, "xl/sharedStrings.xml").decode("utf-8", "replace"))
220
+ for item in root.findall(f"{_S}si"):
221
+ shared.append("".join(node.text or "" for node in item.iter(f"{_S}t")))
222
+
223
+ workbook = engine.parse_xml(_read_member(archive, "xl/workbook.xml").decode("utf-8", "replace"))
224
+ rels: dict[str, str] = {}
225
+ if "xl/_rels/workbook.xml.rels" in names:
226
+ rel_root = engine.parse_xml(
227
+ _read_member(archive, "xl/_rels/workbook.xml.rels").decode("utf-8", "replace")
228
+ )
229
+ for rel in rel_root.findall(f"{_PKG_REL}Relationship"):
230
+ rels[rel.get("Id", "")] = rel.get("Target", "")
231
+
232
+ sheets: list[dict[str, Any]] = []
233
+ for sheet in workbook.findall(f"{_S}sheets/{_S}sheet"):
234
+ name = sheet.get("name", "Sheet")
235
+ target = rels.get(sheet.get(f"{_R}id", ""), "")
236
+ target = target.lstrip("/")
237
+ if not target.startswith("xl/"):
238
+ target = "xl/" + target
239
+ if target not in names:
240
+ continue
241
+ root = engine.parse_xml(_read_member(archive, target).decode("utf-8", "replace"))
242
+
243
+ grid: list[list[str]] = []
244
+ for row_index, row in enumerate(root.findall(f".//{_S}sheetData/{_S}row")):
245
+ if row_index >= max_rows:
246
+ break
247
+ cells: dict[int, str] = {}
248
+ for cell in row.findall(f"{_S}c"):
249
+ reference = cell.get("r", "")
250
+ col = _column_index(reference)
251
+ if col >= max_columns:
252
+ continue
253
+ kind = cell.get("t", "")
254
+ if kind == "inlineStr":
255
+ value = "".join(node.text or "" for node in cell.iter(f"{_S}t"))
256
+ else:
257
+ raw = cell.findtext(f"{_S}v") or ""
258
+ if kind == "s":
259
+ try:
260
+ value = shared[int(raw)]
261
+ except (ValueError, IndexError):
262
+ value = ""
263
+ elif kind == "b":
264
+ value = "TRUE" if raw == "1" else "FALSE"
265
+ else:
266
+ value = raw
267
+ cells[col] = _clean(value)
268
+ if cells:
269
+ width = max(cells) + 1
270
+ grid.append([cells.get(index, "") for index in range(width)])
271
+
272
+ if not grid:
273
+ continue
274
+ width = max(len(row) for row in grid)
275
+ grid = [row + [""] * (width - len(row)) for row in grid]
276
+ table = {"caption": name, "headers": grid[0], "rows": grid[1:]}
277
+ sheets.append({"name": name, "table": table})
278
+
279
+ if not sheets:
280
+ raise engine.ExtractionError("XLSX contains no readable cells")
281
+
282
+ blocks: list[str] = []
283
+ tables: list[dict[str, Any]] = []
284
+ for sheet in sheets:
285
+ table = sheet["table"]
286
+ tables.append(table)
287
+ blocks.append(f"## {sheet['name']}\n\n{_table_markdown(table)}")
288
+ return {
289
+ "kind": "xlsx",
290
+ "title": sheets[0]["name"],
291
+ "markdown": "\n\n".join(blocks).strip(),
292
+ "tables": tables,
293
+ "metadata": {"title": sheets[0]["name"], "site_name": "XLSX"},
294
+ }
295
+
296
+
297
+ # ---------------------------------------------------------------------------
298
+ # EPUB
299
+ # ---------------------------------------------------------------------------
300
+
301
+
302
+ def extract_epub(data: bytes) -> dict[str, Any]:
303
+ with _open_zip(data) as archive:
304
+ names = set(archive.namelist())
305
+ container = engine.parse_xml(
306
+ _read_member(archive, "META-INF/container.xml").decode("utf-8", "replace")
307
+ )
308
+ rootfile = container.find(f".//{_CONTAINER}rootfile")
309
+ opf_path = rootfile.get("full-path", "") if rootfile is not None else ""
310
+ if not opf_path or opf_path not in names:
311
+ raise engine.ExtractionError("EPUB container does not point to a readable OPF file")
312
+ opf = engine.parse_xml(_read_member(archive, opf_path).decode("utf-8", "replace"))
313
+ base = opf_path.rsplit("/", 1)[0] + "/" if "/" in opf_path else ""
314
+
315
+ metadata_node = opf.find(f"{_OPF}metadata")
316
+ title = ""
317
+ creators: list[str] = []
318
+ if metadata_node is not None:
319
+ title = _clean(metadata_node.findtext(f"{_DC}title") or "")
320
+ creators = [
321
+ _clean(node.text)
322
+ for node in metadata_node.findall(f"{_DC}creator")
323
+ if _clean(node.text)
324
+ ]
325
+
326
+ manifest: dict[str, str] = {}
327
+ for item in opf.findall(f"{_OPF}manifest/{_OPF}item"):
328
+ manifest[item.get("id", "")] = item.get("href", "")
329
+ spine = [
330
+ manifest.get(item.get("idref", ""), "")
331
+ for item in opf.findall(f"{_OPF}spine/{_OPF}itemref")
332
+ ]
333
+
334
+ from . import engine as engine_mod # lazy: avoids an import cycle at load time
335
+
336
+ chapters: list[str] = []
337
+ for href in spine:
338
+ if not href:
339
+ continue
340
+ member = base + href
341
+ if member not in names:
342
+ continue
343
+ raw = _read_member(archive, member)
344
+ html = engine_mod.decode_bytes(raw, "application/xhtml+xml")
345
+ soup = engine_mod.make_soup(html)
346
+ heading_node = soup.find(["h1", "h2"])
347
+ heading = _clean(heading_node.get_text(" ", strip=True)) if heading_node else ""
348
+ if heading_node is not None:
349
+ heading_node.decompose()
350
+ chapter = engine_mod.html_to_markdown(
351
+ str(soup), url=member, include_tables=True, keep_links=False, include_images=False
352
+ ).strip()
353
+ if heading:
354
+ chapter = f"# {heading}\n\n{chapter}" if chapter else f"# {heading}"
355
+ if chapter:
356
+ chapters.append(chapter)
357
+
358
+ if not chapters:
359
+ raise engine.ExtractionError("EPUB contains no readable chapters")
360
+ markdown = "\n\n---\n\n".join(chapters)
361
+ return {
362
+ "kind": "epub",
363
+ "title": title or (chapters[0].splitlines()[0].lstrip("# ").strip() if chapters else ""),
364
+ "markdown": markdown,
365
+ "tables": [],
366
+ "metadata": {
367
+ "title": title,
368
+ "authors": creators,
369
+ "site_name": "EPUB",
370
+ },
371
+ }
372
+
373
+
374
+ # ---------------------------------------------------------------------------
375
+ # CSV
376
+ # ---------------------------------------------------------------------------
377
+
378
+
379
+ def extract_csv(data: bytes, content_type: str = "", max_rows: int = 1000) -> dict[str, Any]:
380
+ text = engine.decode_bytes(data, content_type or "text/csv")
381
+ sample = text[:8192]
382
+ try:
383
+ dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
384
+ except csv.Error:
385
+ dialect = csv.excel
386
+ rows = [row for row in csv.reader(io.StringIO(text), dialect)]
387
+ rows = [[_clean(cell) for cell in row] for row in rows[: max_rows + 1] if row]
388
+ if not rows:
389
+ raise engine.ExtractionError("CSV contains no rows")
390
+ width = max(len(row) for row in rows)
391
+ rows = [row + [""] * (width - len(row)) for row in rows]
392
+ table = {"caption": "", "headers": rows[0], "rows": rows[1:]}
393
+ return {
394
+ "kind": "csv",
395
+ "title": "CSV table",
396
+ "markdown": _table_markdown(table),
397
+ "tables": [table],
398
+ "metadata": {"title": "CSV table", "site_name": "CSV", "rows": len(rows) - 1},
399
+ }
400
+
401
+
402
+ def extract(data: bytes, kind: str = "", content_type: str = "", url: str = "") -> dict[str, Any]:
403
+ kind = kind or detect(data, content_type, url)
404
+ if kind == "docx":
405
+ return extract_docx(data)
406
+ if kind == "xlsx":
407
+ return extract_xlsx(data)
408
+ if kind == "epub":
409
+ return extract_epub(data)
410
+ if kind == "csv":
411
+ return extract_csv(data, content_type)
412
+ raise engine.ExtractionError(f"unsupported document kind: {kind or 'unknown'}")
413
+
414
+
415
+ def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
416
+ """Return document metadata in the same shape as article metadata."""
417
+
418
+ source: dict[str, Any] = document.get("metadata") or {}
419
+ return {
420
+ "title": source.get("title", ""),
421
+ "authors": source.get("authors") or [],
422
+ "published": source.get("published", ""),
423
+ "modified": "",
424
+ "journal": "",
425
+ "volume": "",
426
+ "issue": "",
427
+ "first_page": "",
428
+ "last_page": "",
429
+ "doi": "",
430
+ "arxiv_id": "",
431
+ "abstract": source.get("abstract", ""),
432
+ "keywords": [],
433
+ "language": "",
434
+ "site_name": source.get("site_name", document.get("kind", "")),
435
+ "canonical_url": url,
436
+ "og_type": document.get("kind", ""),
437
+ }