web-forager 0.0.0-stage → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +112 -0
- package/CHANGELOG.md +454 -0
- package/LICENSE +21 -0
- package/README.md +477 -3
- package/README_zh.md +477 -0
- package/SKILL.md +351 -0
- package/THIRD_PARTY_LICENSES.txt +35 -0
- package/browser/bridges/playwright-bridge.js +375 -0
- package/browser/launch_chrome.py +144 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +79 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +115 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +116 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +56 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +272 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +105 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +432 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +308 -0
- package/docs/REFERENCE_zh.md +309 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +97 -0
- package/mcp_server.py +597 -0
- package/package.json +80 -6
- package/requirements.txt +12 -0
- package/run.py +852 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +874 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +4220 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +135 -0
- package/scripts/scholar.py +354 -0
- package/scripts/sitemap.py +180 -0
- package/scripts/untrusted.py +95 -0
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
"""Office-document and ebook extraction without extra dependencies.
|
|
2
|
+
|
|
3
|
+
DOCX, XLSX and EPUB are ZIP containers with XML inside, so the standard
|
|
4
|
+
library is enough. XML is parsed through :func:`engine.parse_xml` (defused
|
|
5
|
+
when available, DOCTYPE/ENTITY rejected otherwise), and archive expansion is
|
|
6
|
+
capped to avoid zip bombs.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import csv
|
|
12
|
+
import io
|
|
13
|
+
import re
|
|
14
|
+
import zipfile
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from . import engine
|
|
18
|
+
|
|
19
|
+
ZIP_MAGIC = b"PK\x03\x04"
|
|
20
|
+
_MAX_ENTRIES = 5_000
|
|
21
|
+
_MAX_UNCOMPRESSED = 64 * 1024 * 1024
|
|
22
|
+
_MAX_ENTRY = 24 * 1024 * 1024
|
|
23
|
+
|
|
24
|
+
_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
|
|
25
|
+
_S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
|
26
|
+
_R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
|
|
27
|
+
_PKG_REL = "{http://schemas.openxmlformats.org/package/2006/relationships}"
|
|
28
|
+
_DC = "{http://purl.org/dc/elements/1.1/}"
|
|
29
|
+
_CONTAINER = "{urn:oasis:names:tc:opendocument:xmlns:container}"
|
|
30
|
+
_OPF = "{http://www.idpf.org/2007/opf}"
|
|
31
|
+
|
|
32
|
+
CONTENT_TYPES = {
|
|
33
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
|
|
34
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
|
|
35
|
+
"application/epub+zip": "epub",
|
|
36
|
+
"text/csv": "csv",
|
|
37
|
+
"application/csv": "csv",
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _clean(value: Any) -> str:
|
|
42
|
+
return re.sub(r"\s+", " ", str(value or "")).strip()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _open_zip(data: bytes) -> zipfile.ZipFile:
|
|
46
|
+
archive = zipfile.ZipFile(io.BytesIO(data))
|
|
47
|
+
infos = archive.infolist()
|
|
48
|
+
if len(infos) > _MAX_ENTRIES:
|
|
49
|
+
raise engine.ExtractionError("archive contains too many entries")
|
|
50
|
+
if sum(info.file_size for info in infos) > _MAX_UNCOMPRESSED:
|
|
51
|
+
raise engine.ExtractionError("archive expands beyond the safety limit")
|
|
52
|
+
return archive
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _read_member(archive: zipfile.ZipFile, name: str) -> bytes:
|
|
56
|
+
info = archive.getinfo(name)
|
|
57
|
+
if info.file_size > _MAX_ENTRY:
|
|
58
|
+
raise engine.ExtractionError(f"archive member too large: {name}")
|
|
59
|
+
return archive.read(name)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def detect(data: bytes, content_type: str = "", url: str = "") -> str:
|
|
63
|
+
"""Return docx/xlsx/epub/csv, or '' when the payload is not one of them."""
|
|
64
|
+
|
|
65
|
+
ctype = (content_type or "").split(";", 1)[0].strip().lower()
|
|
66
|
+
if ctype in CONTENT_TYPES:
|
|
67
|
+
return CONTENT_TYPES[ctype]
|
|
68
|
+
lowered = (url or "").lower().split("?", 1)[0]
|
|
69
|
+
if lowered.endswith(".csv"):
|
|
70
|
+
return "csv"
|
|
71
|
+
if data[:4] != ZIP_MAGIC:
|
|
72
|
+
return ""
|
|
73
|
+
try:
|
|
74
|
+
with _open_zip(data) as archive:
|
|
75
|
+
names = set(archive.namelist())
|
|
76
|
+
if "word/document.xml" in names:
|
|
77
|
+
return "docx"
|
|
78
|
+
if "xl/workbook.xml" in names:
|
|
79
|
+
return "xlsx"
|
|
80
|
+
if "META-INF/container.xml" in names:
|
|
81
|
+
return "epub"
|
|
82
|
+
except (zipfile.BadZipFile, engine.ExtractionError, KeyError):
|
|
83
|
+
return ""
|
|
84
|
+
return ""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# ---------------------------------------------------------------------------
|
|
88
|
+
# DOCX
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _docx_paragraph(node: Any) -> str:
|
|
93
|
+
pieces: list[str] = []
|
|
94
|
+
for text in node.iter(f"{_W}t"):
|
|
95
|
+
pieces.append(text.text or "")
|
|
96
|
+
for tab in node.iter(f"{_W}tab"):
|
|
97
|
+
pieces.append("\t")
|
|
98
|
+
for br in node.iter(f"{_W}br"):
|
|
99
|
+
pieces.append("\n")
|
|
100
|
+
return "".join(pieces).strip()
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _docx_style(node: Any) -> str:
|
|
104
|
+
style = node.find(f"{_W}pPr/{_W}pStyle")
|
|
105
|
+
if style is not None:
|
|
106
|
+
return style.get(f"{_W}val", "") or ""
|
|
107
|
+
return ""
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _docx_table(node: Any) -> dict[str, Any]:
|
|
111
|
+
rows: list[list[str]] = []
|
|
112
|
+
for row in node.findall(f"{_W}tr"):
|
|
113
|
+
cells = []
|
|
114
|
+
for cell in row.findall(f"{_W}tc"):
|
|
115
|
+
text = " ".join(
|
|
116
|
+
_docx_paragraph(paragraph)
|
|
117
|
+
for paragraph in cell.findall(f"{_W}p")
|
|
118
|
+
if _docx_paragraph(paragraph)
|
|
119
|
+
)
|
|
120
|
+
cells.append(text)
|
|
121
|
+
if cells:
|
|
122
|
+
rows.append(cells)
|
|
123
|
+
return {"caption": "", "headers": rows[0] if rows else [], "rows": rows[1:] if rows else []}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _table_markdown(table: dict[str, Any]) -> str:
|
|
127
|
+
headers = [str(value) for value in table.get("headers") or []]
|
|
128
|
+
rows = [[str(value) for value in row] for row in table.get("rows") or []]
|
|
129
|
+
width = max([len(headers)] + [len(row) for row in rows]) if headers or rows else 0
|
|
130
|
+
if not width:
|
|
131
|
+
return ""
|
|
132
|
+
headers += [""] * (width - len(headers))
|
|
133
|
+
lines = ["| " + " | ".join(headers) + " |", "| " + " | ".join(["---"] * width) + " |"]
|
|
134
|
+
for row in rows:
|
|
135
|
+
row += [""] * (width - len(row))
|
|
136
|
+
lines.append("| " + " | ".join(cell.replace("|", "\\|") for cell in row) + " |")
|
|
137
|
+
return "\n".join(lines)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def extract_docx(data: bytes) -> dict[str, Any]:
|
|
141
|
+
with _open_zip(data) as archive:
|
|
142
|
+
document = engine.parse_xml(_read_member(archive, "word/document.xml").decode("utf-8", "replace"))
|
|
143
|
+
metadata: dict[str, str] = {}
|
|
144
|
+
if "docProps/core.xml" in archive.namelist():
|
|
145
|
+
core = engine.parse_xml(_read_member(archive, "docProps/core.xml").decode("utf-8", "replace"))
|
|
146
|
+
metadata = {
|
|
147
|
+
"title": _clean((core.findtext(f"{_DC}title") or "")),
|
|
148
|
+
"creator": _clean((core.findtext(f"{_DC}creator") or "")),
|
|
149
|
+
"subject": _clean((core.findtext(f"{_DC}subject") or "")),
|
|
150
|
+
"created": _clean((core.findtext(f"{_DC}created") or "")),
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
body = document.find(f"{_W}body")
|
|
154
|
+
blocks: list[str] = []
|
|
155
|
+
tables: list[dict[str, Any]] = []
|
|
156
|
+
if body is not None:
|
|
157
|
+
for child in body:
|
|
158
|
+
if child.tag == f"{_W}p":
|
|
159
|
+
text = _docx_paragraph(child)
|
|
160
|
+
if not text:
|
|
161
|
+
continue
|
|
162
|
+
style = _docx_style(child)
|
|
163
|
+
match = re.search(r"Heading\s*(\d)", style, re.I)
|
|
164
|
+
if match:
|
|
165
|
+
blocks.append("#" * min(int(match.group(1)), 6) + " " + text)
|
|
166
|
+
elif style.lower() in ("title",):
|
|
167
|
+
blocks.append("# " + text)
|
|
168
|
+
elif style.lower() in ("subtitle",):
|
|
169
|
+
blocks.append("## " + text)
|
|
170
|
+
elif child.find(f".//{_W}numPr") is not None:
|
|
171
|
+
blocks.append("- " + text)
|
|
172
|
+
else:
|
|
173
|
+
blocks.append(text)
|
|
174
|
+
elif child.tag == f"{_W}tbl":
|
|
175
|
+
table = _docx_table(child)
|
|
176
|
+
if table["headers"] or table["rows"]:
|
|
177
|
+
tables.append(table)
|
|
178
|
+
blocks.append(_table_markdown(table))
|
|
179
|
+
|
|
180
|
+
markdown = "\n\n".join(blocks).strip()
|
|
181
|
+
if not markdown:
|
|
182
|
+
raise engine.ExtractionError("DOCX contains no readable text")
|
|
183
|
+
title = metadata.get("title") or (blocks[0].lstrip("# ").strip() if blocks else "")
|
|
184
|
+
return {
|
|
185
|
+
"kind": "docx",
|
|
186
|
+
"title": title,
|
|
187
|
+
"markdown": markdown,
|
|
188
|
+
"tables": tables,
|
|
189
|
+
"metadata": {
|
|
190
|
+
"title": title,
|
|
191
|
+
"authors": [metadata["creator"]] if metadata.get("creator") else [],
|
|
192
|
+
"published": metadata.get("created", ""),
|
|
193
|
+
"abstract": metadata.get("subject", ""),
|
|
194
|
+
"site_name": "DOCX",
|
|
195
|
+
},
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# ---------------------------------------------------------------------------
|
|
200
|
+
# XLSX
|
|
201
|
+
# ---------------------------------------------------------------------------
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _column_index(reference: str) -> int:
|
|
205
|
+
letters = re.match(r"([A-Z]+)", reference.upper())
|
|
206
|
+
if not letters:
|
|
207
|
+
return 0
|
|
208
|
+
index = 0
|
|
209
|
+
for letter in letters.group(1):
|
|
210
|
+
index = index * 26 + (ord(letter) - 64)
|
|
211
|
+
return max(0, index - 1)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def extract_xlsx(data: bytes, max_rows: int = 500, max_columns: int = 40) -> dict[str, Any]:
|
|
215
|
+
with _open_zip(data) as archive:
|
|
216
|
+
names = set(archive.namelist())
|
|
217
|
+
shared: list[str] = []
|
|
218
|
+
if "xl/sharedStrings.xml" in names:
|
|
219
|
+
root = engine.parse_xml(_read_member(archive, "xl/sharedStrings.xml").decode("utf-8", "replace"))
|
|
220
|
+
for item in root.findall(f"{_S}si"):
|
|
221
|
+
shared.append("".join(node.text or "" for node in item.iter(f"{_S}t")))
|
|
222
|
+
|
|
223
|
+
workbook = engine.parse_xml(_read_member(archive, "xl/workbook.xml").decode("utf-8", "replace"))
|
|
224
|
+
rels: dict[str, str] = {}
|
|
225
|
+
if "xl/_rels/workbook.xml.rels" in names:
|
|
226
|
+
rel_root = engine.parse_xml(
|
|
227
|
+
_read_member(archive, "xl/_rels/workbook.xml.rels").decode("utf-8", "replace")
|
|
228
|
+
)
|
|
229
|
+
for rel in rel_root.findall(f"{_PKG_REL}Relationship"):
|
|
230
|
+
rels[rel.get("Id", "")] = rel.get("Target", "")
|
|
231
|
+
|
|
232
|
+
sheets: list[dict[str, Any]] = []
|
|
233
|
+
for sheet in workbook.findall(f"{_S}sheets/{_S}sheet"):
|
|
234
|
+
name = sheet.get("name", "Sheet")
|
|
235
|
+
target = rels.get(sheet.get(f"{_R}id", ""), "")
|
|
236
|
+
target = target.lstrip("/")
|
|
237
|
+
if not target.startswith("xl/"):
|
|
238
|
+
target = "xl/" + target
|
|
239
|
+
if target not in names:
|
|
240
|
+
continue
|
|
241
|
+
root = engine.parse_xml(_read_member(archive, target).decode("utf-8", "replace"))
|
|
242
|
+
|
|
243
|
+
grid: list[list[str]] = []
|
|
244
|
+
for row_index, row in enumerate(root.findall(f".//{_S}sheetData/{_S}row")):
|
|
245
|
+
if row_index >= max_rows:
|
|
246
|
+
break
|
|
247
|
+
cells: dict[int, str] = {}
|
|
248
|
+
for cell in row.findall(f"{_S}c"):
|
|
249
|
+
reference = cell.get("r", "")
|
|
250
|
+
col = _column_index(reference)
|
|
251
|
+
if col >= max_columns:
|
|
252
|
+
continue
|
|
253
|
+
kind = cell.get("t", "")
|
|
254
|
+
if kind == "inlineStr":
|
|
255
|
+
value = "".join(node.text or "" for node in cell.iter(f"{_S}t"))
|
|
256
|
+
else:
|
|
257
|
+
raw = cell.findtext(f"{_S}v") or ""
|
|
258
|
+
if kind == "s":
|
|
259
|
+
try:
|
|
260
|
+
value = shared[int(raw)]
|
|
261
|
+
except (ValueError, IndexError):
|
|
262
|
+
value = ""
|
|
263
|
+
elif kind == "b":
|
|
264
|
+
value = "TRUE" if raw == "1" else "FALSE"
|
|
265
|
+
else:
|
|
266
|
+
value = raw
|
|
267
|
+
cells[col] = _clean(value)
|
|
268
|
+
if cells:
|
|
269
|
+
width = max(cells) + 1
|
|
270
|
+
grid.append([cells.get(index, "") for index in range(width)])
|
|
271
|
+
|
|
272
|
+
if not grid:
|
|
273
|
+
continue
|
|
274
|
+
width = max(len(row) for row in grid)
|
|
275
|
+
grid = [row + [""] * (width - len(row)) for row in grid]
|
|
276
|
+
table = {"caption": name, "headers": grid[0], "rows": grid[1:]}
|
|
277
|
+
sheets.append({"name": name, "table": table})
|
|
278
|
+
|
|
279
|
+
if not sheets:
|
|
280
|
+
raise engine.ExtractionError("XLSX contains no readable cells")
|
|
281
|
+
|
|
282
|
+
blocks: list[str] = []
|
|
283
|
+
tables: list[dict[str, Any]] = []
|
|
284
|
+
for sheet in sheets:
|
|
285
|
+
table = sheet["table"]
|
|
286
|
+
tables.append(table)
|
|
287
|
+
blocks.append(f"## {sheet['name']}\n\n{_table_markdown(table)}")
|
|
288
|
+
return {
|
|
289
|
+
"kind": "xlsx",
|
|
290
|
+
"title": sheets[0]["name"],
|
|
291
|
+
"markdown": "\n\n".join(blocks).strip(),
|
|
292
|
+
"tables": tables,
|
|
293
|
+
"metadata": {"title": sheets[0]["name"], "site_name": "XLSX"},
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
# ---------------------------------------------------------------------------
|
|
298
|
+
# EPUB
|
|
299
|
+
# ---------------------------------------------------------------------------
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def extract_epub(data: bytes) -> dict[str, Any]:
|
|
303
|
+
with _open_zip(data) as archive:
|
|
304
|
+
names = set(archive.namelist())
|
|
305
|
+
container = engine.parse_xml(
|
|
306
|
+
_read_member(archive, "META-INF/container.xml").decode("utf-8", "replace")
|
|
307
|
+
)
|
|
308
|
+
rootfile = container.find(f".//{_CONTAINER}rootfile")
|
|
309
|
+
opf_path = rootfile.get("full-path", "") if rootfile is not None else ""
|
|
310
|
+
if not opf_path or opf_path not in names:
|
|
311
|
+
raise engine.ExtractionError("EPUB container does not point to a readable OPF file")
|
|
312
|
+
opf = engine.parse_xml(_read_member(archive, opf_path).decode("utf-8", "replace"))
|
|
313
|
+
base = opf_path.rsplit("/", 1)[0] + "/" if "/" in opf_path else ""
|
|
314
|
+
|
|
315
|
+
metadata_node = opf.find(f"{_OPF}metadata")
|
|
316
|
+
title = ""
|
|
317
|
+
creators: list[str] = []
|
|
318
|
+
if metadata_node is not None:
|
|
319
|
+
title = _clean(metadata_node.findtext(f"{_DC}title") or "")
|
|
320
|
+
creators = [
|
|
321
|
+
_clean(node.text)
|
|
322
|
+
for node in metadata_node.findall(f"{_DC}creator")
|
|
323
|
+
if _clean(node.text)
|
|
324
|
+
]
|
|
325
|
+
|
|
326
|
+
manifest: dict[str, str] = {}
|
|
327
|
+
for item in opf.findall(f"{_OPF}manifest/{_OPF}item"):
|
|
328
|
+
manifest[item.get("id", "")] = item.get("href", "")
|
|
329
|
+
spine = [
|
|
330
|
+
manifest.get(item.get("idref", ""), "")
|
|
331
|
+
for item in opf.findall(f"{_OPF}spine/{_OPF}itemref")
|
|
332
|
+
]
|
|
333
|
+
|
|
334
|
+
from . import engine as engine_mod # lazy: avoids an import cycle at load time
|
|
335
|
+
|
|
336
|
+
chapters: list[str] = []
|
|
337
|
+
for href in spine:
|
|
338
|
+
if not href:
|
|
339
|
+
continue
|
|
340
|
+
member = base + href
|
|
341
|
+
if member not in names:
|
|
342
|
+
continue
|
|
343
|
+
raw = _read_member(archive, member)
|
|
344
|
+
html = engine_mod.decode_bytes(raw, "application/xhtml+xml")
|
|
345
|
+
soup = engine_mod.make_soup(html)
|
|
346
|
+
heading_node = soup.find(["h1", "h2"])
|
|
347
|
+
heading = _clean(heading_node.get_text(" ", strip=True)) if heading_node else ""
|
|
348
|
+
if heading_node is not None:
|
|
349
|
+
heading_node.decompose()
|
|
350
|
+
chapter = engine_mod.html_to_markdown(
|
|
351
|
+
str(soup), url=member, include_tables=True, keep_links=False, include_images=False
|
|
352
|
+
).strip()
|
|
353
|
+
if heading:
|
|
354
|
+
chapter = f"# {heading}\n\n{chapter}" if chapter else f"# {heading}"
|
|
355
|
+
if chapter:
|
|
356
|
+
chapters.append(chapter)
|
|
357
|
+
|
|
358
|
+
if not chapters:
|
|
359
|
+
raise engine.ExtractionError("EPUB contains no readable chapters")
|
|
360
|
+
markdown = "\n\n---\n\n".join(chapters)
|
|
361
|
+
return {
|
|
362
|
+
"kind": "epub",
|
|
363
|
+
"title": title or (chapters[0].splitlines()[0].lstrip("# ").strip() if chapters else ""),
|
|
364
|
+
"markdown": markdown,
|
|
365
|
+
"tables": [],
|
|
366
|
+
"metadata": {
|
|
367
|
+
"title": title,
|
|
368
|
+
"authors": creators,
|
|
369
|
+
"site_name": "EPUB",
|
|
370
|
+
},
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
# ---------------------------------------------------------------------------
|
|
375
|
+
# CSV
|
|
376
|
+
# ---------------------------------------------------------------------------
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def extract_csv(data: bytes, content_type: str = "", max_rows: int = 1000) -> dict[str, Any]:
|
|
380
|
+
text = engine.decode_bytes(data, content_type or "text/csv")
|
|
381
|
+
sample = text[:8192]
|
|
382
|
+
try:
|
|
383
|
+
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
|
384
|
+
except csv.Error:
|
|
385
|
+
dialect = csv.excel
|
|
386
|
+
rows = [row for row in csv.reader(io.StringIO(text), dialect)]
|
|
387
|
+
rows = [[_clean(cell) for cell in row] for row in rows[: max_rows + 1] if row]
|
|
388
|
+
if not rows:
|
|
389
|
+
raise engine.ExtractionError("CSV contains no rows")
|
|
390
|
+
width = max(len(row) for row in rows)
|
|
391
|
+
rows = [row + [""] * (width - len(row)) for row in rows]
|
|
392
|
+
table = {"caption": "", "headers": rows[0], "rows": rows[1:]}
|
|
393
|
+
return {
|
|
394
|
+
"kind": "csv",
|
|
395
|
+
"title": "CSV table",
|
|
396
|
+
"markdown": _table_markdown(table),
|
|
397
|
+
"tables": [table],
|
|
398
|
+
"metadata": {"title": "CSV table", "site_name": "CSV", "rows": len(rows) - 1},
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def extract(data: bytes, kind: str = "", content_type: str = "", url: str = "") -> dict[str, Any]:
|
|
403
|
+
kind = kind or detect(data, content_type, url)
|
|
404
|
+
if kind == "docx":
|
|
405
|
+
return extract_docx(data)
|
|
406
|
+
if kind == "xlsx":
|
|
407
|
+
return extract_xlsx(data)
|
|
408
|
+
if kind == "epub":
|
|
409
|
+
return extract_epub(data)
|
|
410
|
+
if kind == "csv":
|
|
411
|
+
return extract_csv(data, content_type)
|
|
412
|
+
raise engine.ExtractionError(f"unsupported document kind: {kind or 'unknown'}")
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
|
|
416
|
+
"""Return document metadata in the same shape as article metadata."""
|
|
417
|
+
|
|
418
|
+
source: dict[str, Any] = document.get("metadata") or {}
|
|
419
|
+
return {
|
|
420
|
+
"title": source.get("title", ""),
|
|
421
|
+
"authors": source.get("authors") or [],
|
|
422
|
+
"published": source.get("published", ""),
|
|
423
|
+
"modified": "",
|
|
424
|
+
"journal": "",
|
|
425
|
+
"volume": "",
|
|
426
|
+
"issue": "",
|
|
427
|
+
"first_page": "",
|
|
428
|
+
"last_page": "",
|
|
429
|
+
"doi": "",
|
|
430
|
+
"arxiv_id": "",
|
|
431
|
+
"abstract": source.get("abstract", ""),
|
|
432
|
+
"keywords": [],
|
|
433
|
+
"language": "",
|
|
434
|
+
"site_name": source.get("site_name", document.get("kind", "")),
|
|
435
|
+
"canonical_url": url,
|
|
436
|
+
"og_type": document.get("kind", ""),
|
|
437
|
+
}
|