tablecheck 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tablecheck/__init__.py +7 -0
- tablecheck/cjk.py +128 -0
- tablecheck/cli.py +252 -0
- tablecheck/detect.py +81 -0
- tablecheck/schema.py +131 -0
- tablecheck/struct.py +277 -0
- tablecheck-0.1.0.dist-info/METADATA +239 -0
- tablecheck-0.1.0.dist-info/RECORD +12 -0
- tablecheck-0.1.0.dist-info/WHEEL +5 -0
- tablecheck-0.1.0.dist-info/entry_points.txt +2 -0
- tablecheck-0.1.0.dist-info/licenses/LICENSE +21 -0
- tablecheck-0.1.0.dist-info/top_level.txt +1 -0
tablecheck/__init__.py
ADDED
tablecheck/cjk.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""CJK 识别:中文/日文/韩文字符占比、是否混排、是否疑似竖排。
|
|
2
|
+
|
|
3
|
+
这三个信号是给中文/繁体 PDF 用的前置判断:
|
|
4
|
+
- ``script_ratio`` 低 → 大概率是英文文档,表头判定的规则要另说;
|
|
5
|
+
- ``mixed`` 为真 → 中英混排,单元格换行会破坏表格对齐;
|
|
6
|
+
- ``vertical_suspect`` 为真 → 可能是直排(竖排)文本,几何表格检测会失准。
|
|
7
|
+
|
|
8
|
+
判定全部基于 PyMuPDF 的文本块,纯本地、确定性,同一文件多次跑结果一致。
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
CJK_RANGES = (
|
|
14
|
+
(0x2E80, 0x2EFF), # CJK 部首补充
|
|
15
|
+
(0x3000, 0x303F), # CJK 符号与标点
|
|
16
|
+
(0x3040, 0x30FF), # 平假名、片假名
|
|
17
|
+
(0x3400, 0x4DBF), # CJK 扩展 A
|
|
18
|
+
(0x4E00, 0x9FFF), # CJK 统一汉字
|
|
19
|
+
(0xF900, 0xFAFF), # 兼容汉字
|
|
20
|
+
(0xAC00, 0xD7AF), # 韩文音节
|
|
21
|
+
(0x1100, 0x11FF), # 韩文字母
|
|
22
|
+
(0x20000, 0x2FA1F), # CJK 扩展 B 及以后
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
CJK_FONT_HINTS = (
|
|
26
|
+
"song", "sun", "ming", "kai", "hei", "gothic", "mincho", "fangsong",
|
|
27
|
+
"noto sans cjk", "notosanscjk", "noto serif cjk", "source han", "sourcehan",
|
|
28
|
+
"msung", "mingliu", "dfkai", "pmingliu", "simsun", "simhei", "kaiti",
|
|
29
|
+
"yuanti", "droidsansfallback", "wqy", "arphic", "uming", "ukai",
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
MIN_BLOCKS_FOR_VERTICAL = 8
|
|
33
|
+
VERTICAL_RATIO = 0.05
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _is_cjk(ch: str) -> bool:
|
|
37
|
+
o = ord(ch)
|
|
38
|
+
for lo, hi in CJK_RANGES:
|
|
39
|
+
if lo <= o <= hi:
|
|
40
|
+
return True
|
|
41
|
+
return False
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _is_latin(ch: str) -> bool:
|
|
45
|
+
o = ord(ch)
|
|
46
|
+
return (0x41 <= o <= 0x5A) or (0x61 <= o <= 0x7A) or (0xC0 <= o <= 0x24F)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _font_is_cjk(name: str) -> bool:
|
|
50
|
+
n = (name or "").lower()
|
|
51
|
+
return any(h in n for h in CJK_FONT_HINTS)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def scan(pdf_path: str, max_pages: int | None = None) -> dict:
|
|
55
|
+
"""扫全文(或前 max_pages 页),返回 cjk 段的字段。"""
|
|
56
|
+
import pymupdf
|
|
57
|
+
|
|
58
|
+
doc = pymupdf.open(pdf_path)
|
|
59
|
+
try:
|
|
60
|
+
cjk_chars = latin_chars = 0
|
|
61
|
+
fonts: list[str] = []
|
|
62
|
+
seen_fonts: set[str] = set()
|
|
63
|
+
vertical_blocks = 0
|
|
64
|
+
total_blocks = 0
|
|
65
|
+
vertical_wmode_spans = 0
|
|
66
|
+
pages_scanned = 0
|
|
67
|
+
|
|
68
|
+
limit = doc.page_count if max_pages is None else min(max_pages, doc.page_count)
|
|
69
|
+
|
|
70
|
+
for pno in range(limit):
|
|
71
|
+
page = doc[pno]
|
|
72
|
+
pages_scanned += 1
|
|
73
|
+
data = page.get_text("dict")
|
|
74
|
+
for block in data.get("blocks", []):
|
|
75
|
+
if block.get("type") != 0: # 0 = 文本块
|
|
76
|
+
continue
|
|
77
|
+
lines = block.get("lines", [])
|
|
78
|
+
if not lines:
|
|
79
|
+
continue
|
|
80
|
+
total_blocks += 1
|
|
81
|
+
text_chars = 0
|
|
82
|
+
for line in lines:
|
|
83
|
+
for span in line.get("spans", []):
|
|
84
|
+
txt = span.get("text", "")
|
|
85
|
+
text_chars += len(txt)
|
|
86
|
+
for ch in txt:
|
|
87
|
+
if _is_cjk(ch):
|
|
88
|
+
cjk_chars += 1
|
|
89
|
+
elif _is_latin(ch):
|
|
90
|
+
latin_chars += 1
|
|
91
|
+
fname = span.get("font", "")
|
|
92
|
+
if _font_is_cjk(fname) and fname not in seen_fonts:
|
|
93
|
+
seen_fonts.add(fname)
|
|
94
|
+
fonts.append(fname)
|
|
95
|
+
# 竖排写的字:wmode == 1
|
|
96
|
+
if span.get("wmode", 0) == 1:
|
|
97
|
+
vertical_wmode_spans += 1
|
|
98
|
+
x0, y0, x1, y1 = block["bbox"]
|
|
99
|
+
w, h = x1 - x0, y1 - y0
|
|
100
|
+
# 疑似竖排:块窄高、且每行平均不到 3 个字、至少 3 行
|
|
101
|
+
if h > w and len(lines) >= 3 and (text_chars / len(lines)) < 3:
|
|
102
|
+
vertical_blocks += 1
|
|
103
|
+
|
|
104
|
+
letters = cjk_chars + latin_chars
|
|
105
|
+
ratio = (cjk_chars / letters) if letters else 0.0
|
|
106
|
+
latin_ratio = (latin_chars / letters) if letters else 0.0
|
|
107
|
+
vertical_suspect = bool(
|
|
108
|
+
vertical_wmode_spans > 0
|
|
109
|
+
or (
|
|
110
|
+
vertical_blocks >= MIN_BLOCKS_FOR_VERTICAL
|
|
111
|
+
and total_blocks > 0
|
|
112
|
+
and vertical_blocks / total_blocks >= VERTICAL_RATIO
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
return {
|
|
117
|
+
"script_ratio": round(ratio, 4),
|
|
118
|
+
"mixed": bool(ratio >= 0.05 and latin_ratio >= 0.05),
|
|
119
|
+
"vertical_suspect": vertical_suspect,
|
|
120
|
+
"cjk_chars": cjk_chars,
|
|
121
|
+
"latin_chars": latin_chars,
|
|
122
|
+
"fonts": fonts[:20],
|
|
123
|
+
"vertical_blocks": vertical_blocks,
|
|
124
|
+
"total_blocks": total_blocks,
|
|
125
|
+
"pages_scanned": pages_scanned,
|
|
126
|
+
}
|
|
127
|
+
finally:
|
|
128
|
+
doc.close()
|
tablecheck/cli.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"""命令行入口:``tablecheck <pdf...> [--json out.json] [--md out.md] [--strict]``。
|
|
2
|
+
|
|
3
|
+
退出码:
|
|
4
|
+
0 跑完,没问题(或在非 strict 模式下发现问题)
|
|
5
|
+
1 跑不起来(文件不存在 / 打不开 / schema 自检失败)
|
|
6
|
+
2 --strict 下发现结构问题(用来当 CI 门禁)
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from . import __version__
|
|
18
|
+
from .cjk import scan as cjk_scan
|
|
19
|
+
from .detect import detect_pages, shape_of
|
|
20
|
+
from .schema import BAD_STATUSES, SCHEMA_VERSION, validate
|
|
21
|
+
from .struct import header_quality, read_structure
|
|
22
|
+
|
|
23
|
+
TOOL_NAME = "tablecheck"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def sha256_of(path: str) -> str:
|
|
27
|
+
h = hashlib.sha256()
|
|
28
|
+
with open(path, "rb") as fh:
|
|
29
|
+
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
30
|
+
h.update(chunk)
|
|
31
|
+
return h.hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _status(detected: int, tagged: int, th: int) -> str:
|
|
35
|
+
"""沿用 table_audit.py 的枚举与判定顺序,不改名、不重排。"""
|
|
36
|
+
if detected and tagged == 0:
|
|
37
|
+
return "NOT TAGGED"
|
|
38
|
+
if detected and th == 0:
|
|
39
|
+
return "NO HEADER"
|
|
40
|
+
if detected and th < tagged:
|
|
41
|
+
return "PARTIAL HEADER"
|
|
42
|
+
if detected:
|
|
43
|
+
return "OK"
|
|
44
|
+
return "TAGGED-ONLY"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def audit_file(path: str) -> dict:
|
|
48
|
+
"""跑单个文件,返回 schema v1 的 file 段(不含 schema_version/tool)。"""
|
|
49
|
+
p = Path(path)
|
|
50
|
+
if not p.is_file():
|
|
51
|
+
raise FileNotFoundError(path)
|
|
52
|
+
|
|
53
|
+
detected, page_count = detect_pages(path)
|
|
54
|
+
struct = read_structure(path)
|
|
55
|
+
|
|
56
|
+
tables_by_page: dict[int, list[dict]] = {}
|
|
57
|
+
orphan_tables = 0
|
|
58
|
+
for t in struct["tables"]:
|
|
59
|
+
if t["page"] is None:
|
|
60
|
+
orphan_tables += 1
|
|
61
|
+
continue
|
|
62
|
+
tables_by_page.setdefault(t["page"], []).append(t)
|
|
63
|
+
|
|
64
|
+
rows = []
|
|
65
|
+
pages = sorted(set(detected) | set(struct["per_page"]))
|
|
66
|
+
for pno in pages:
|
|
67
|
+
det = detected.get(pno, [])
|
|
68
|
+
mk = struct["per_page"].get(pno, {"tables": 0, "th": 0, "td": 0, "tr": 0})
|
|
69
|
+
n_det = len(det)
|
|
70
|
+
hq = header_quality(tables_by_page.get(pno, []))
|
|
71
|
+
rows.append(
|
|
72
|
+
{
|
|
73
|
+
"page": pno,
|
|
74
|
+
"detected": n_det,
|
|
75
|
+
"shape": shape_of(det),
|
|
76
|
+
"tagged_tables": mk["tables"],
|
|
77
|
+
"th": mk["th"],
|
|
78
|
+
"td": mk["td"],
|
|
79
|
+
"header_quality": hq,
|
|
80
|
+
"status": _status(n_det, mk["tables"], mk["th"]),
|
|
81
|
+
}
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
totals = struct["totals"]
|
|
85
|
+
summary = {
|
|
86
|
+
"pages": page_count,
|
|
87
|
+
"pages_with_tables": len(rows),
|
|
88
|
+
"tables_detected": sum(r["detected"] for r in rows),
|
|
89
|
+
"tables_tagged": totals["Table"],
|
|
90
|
+
"tr_total": totals["TR"],
|
|
91
|
+
"th_total": totals["TH"],
|
|
92
|
+
"td_total": totals["TD"],
|
|
93
|
+
"pages_not_tagged": sum(1 for r in rows if r["status"] == "NOT TAGGED"),
|
|
94
|
+
"pages_no_header": sum(1 for r in rows if r["status"] == "NO HEADER"),
|
|
95
|
+
"pages_partial_header": sum(1 for r in rows if r["status"] == "PARTIAL HEADER"),
|
|
96
|
+
"pages_ok": sum(1 for r in rows if r["status"] == "OK"),
|
|
97
|
+
"pages_tagged_only": sum(1 for r in rows if r["status"] == "TAGGED-ONLY"),
|
|
98
|
+
"pages_with_structparents": struct["pages_with_structparents"],
|
|
99
|
+
"has_struct_tree": struct["has_struct_tree"],
|
|
100
|
+
"tables_without_page": orphan_tables,
|
|
101
|
+
"th_missing_scope": sum(r["header_quality"]["th_missing_scope"] for r in rows),
|
|
102
|
+
"col_count_mismatch": sum(r["header_quality"]["col_count_mismatch"] for r in rows),
|
|
103
|
+
"empty_header_cells": sum(r["header_quality"]["empty_header_cells"] for r in rows),
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return {
|
|
107
|
+
"file": {"name": p.name, "pages": page_count, "sha256": sha256_of(path)},
|
|
108
|
+
"cjk": cjk_scan(path),
|
|
109
|
+
"summary": summary,
|
|
110
|
+
"pages": rows,
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _envelope(results: list[dict]) -> dict:
|
|
115
|
+
head = {"schema_version": SCHEMA_VERSION, "tool": {"name": TOOL_NAME, "version": __version__}}
|
|
116
|
+
if len(results) == 1:
|
|
117
|
+
head.update(results[0])
|
|
118
|
+
return head
|
|
119
|
+
head["files"] = results
|
|
120
|
+
return head
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def has_problems(doc: dict) -> bool:
|
|
124
|
+
"""strict 模式的判据:结构状态不合格,或表头质量有硬伤。
|
|
125
|
+
|
|
126
|
+
注意 ``col_count_mismatch`` **不参与**门禁:合并单元格(ColSpan/RowSpan)会让它必然非零,
|
|
127
|
+
拿它当门禁会在真实文档上大面积误报。它在 JSON 里作为信息项保留,交给用户自己看。
|
|
128
|
+
"""
|
|
129
|
+
files = doc["files"] if "files" in doc else [doc]
|
|
130
|
+
for f in files:
|
|
131
|
+
for r in f["pages"]:
|
|
132
|
+
if r["status"] in BAD_STATUSES:
|
|
133
|
+
return True
|
|
134
|
+
hq = r["header_quality"]
|
|
135
|
+
if hq["th_missing_scope"] or hq["empty_header_cells"]:
|
|
136
|
+
return True
|
|
137
|
+
s = f["summary"]
|
|
138
|
+
if s["tables_detected"] and s["tables_tagged"] == 0:
|
|
139
|
+
return True
|
|
140
|
+
return False
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def to_md(doc: dict) -> str:
|
|
144
|
+
files = doc["files"] if "files" in doc else [doc]
|
|
145
|
+
out = []
|
|
146
|
+
for f in files:
|
|
147
|
+
s = f["summary"]
|
|
148
|
+
out.append(f"## {f['file']['name']}({f['file']['pages']} 页)\n\n")
|
|
149
|
+
out.append(
|
|
150
|
+
f"- sha256 `{f['file']['sha256']}`\n"
|
|
151
|
+
f"- 检出表格 **{s['tables_detected']}** 张 · 结构树 Table **{s['tables_tagged']}** 个 · "
|
|
152
|
+
f"TR {s['tr_total']} · TH {s['th_total']} · TD {s['td_total']}\n"
|
|
153
|
+
f"- 页面判定:未打标 {s['pages_not_tagged']} · 无表头 {s['pages_no_header']} · "
|
|
154
|
+
f"表头不全 {s['pages_partial_header']} · 正常 {s['pages_ok']} · "
|
|
155
|
+
f"仅标注 {s['pages_tagged_only']}\n"
|
|
156
|
+
f"- 表头质量:缺 Scope 的 TH {s['th_missing_scope']} 个 · 列数不一致 {s['col_count_mismatch']} 表 · "
|
|
157
|
+
f"空表头单元格 {s['empty_header_cells']} 个\n"
|
|
158
|
+
f"- CJK:script_ratio {f['cjk']['script_ratio']} · mixed {f['cjk']['mixed']} · "
|
|
159
|
+
f"vertical_suspect {f['cjk']['vertical_suspect']}\n\n"
|
|
160
|
+
)
|
|
161
|
+
bad = [r for r in f["pages"] if r["status"] in BAD_STATUSES]
|
|
162
|
+
if bad:
|
|
163
|
+
out.append("| 页 / page | 检出 det | 形状 shape | 已标 Table | TH | TD | 判定 status |\n")
|
|
164
|
+
out.append("|---|---|---|---|---|---|---|\n")
|
|
165
|
+
for r in bad:
|
|
166
|
+
out.append(
|
|
167
|
+
f"| {r['page']} | {r['detected']} | {r['shape']} | "
|
|
168
|
+
f"{r['tagged_tables']} | {r['th']} | {r['td']} | {r['status']} |\n"
|
|
169
|
+
)
|
|
170
|
+
out.append("\n")
|
|
171
|
+
return "".join(out)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _text_summary(doc: dict) -> str:
|
|
175
|
+
files = doc["files"] if "files" in doc else [doc]
|
|
176
|
+
lines = []
|
|
177
|
+
for f in files:
|
|
178
|
+
s = f["summary"]
|
|
179
|
+
lines.append(
|
|
180
|
+
f"{f['file']['name']}: {f['file']['pages']}p "
|
|
181
|
+
f"detected={s['tables_detected']} tagged={s['tables_tagged']} "
|
|
182
|
+
f"TH={s['th_total']} TD={s['td_total']} "
|
|
183
|
+
f"NOT TAGGED={s['pages_not_tagged']} NO HEADER={s['pages_no_header']} "
|
|
184
|
+
f"PARTIAL={s['pages_partial_header']} OK={s['pages_ok']} "
|
|
185
|
+
f"TAGGED-ONLY={s['pages_tagged_only']}"
|
|
186
|
+
)
|
|
187
|
+
c = f["cjk"]
|
|
188
|
+
lines.append(
|
|
189
|
+
f" cjk: script_ratio={c['script_ratio']} mixed={c['mixed']} "
|
|
190
|
+
f"vertical_suspect={c['vertical_suspect']}"
|
|
191
|
+
)
|
|
192
|
+
return "\n".join(lines) + "\n"
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def main(argv: list[str] | None = None) -> int:
|
|
196
|
+
ap = argparse.ArgumentParser(
|
|
197
|
+
prog=TOOL_NAME,
|
|
198
|
+
description="检查 PDF 里几何存在的表格与结构树标注的差距(中文/繁体友好,离线确定性)。",
|
|
199
|
+
)
|
|
200
|
+
ap.add_argument("pdf", nargs="+", help="待检查的 PDF(可多个)")
|
|
201
|
+
ap.add_argument("--json", metavar="OUT", help="把结果写成 JSON(schema v1)")
|
|
202
|
+
ap.add_argument("--md", metavar="OUT", help="把结果写成 Markdown 报告")
|
|
203
|
+
ap.add_argument(
|
|
204
|
+
"--strict",
|
|
205
|
+
action="store_true",
|
|
206
|
+
help="发现结构问题(未打标/缺表头/缺 Scope/列数不一致/空表头)时返回退出码 2",
|
|
207
|
+
)
|
|
208
|
+
ap.add_argument("--quiet", action="store_true", help="不在 stdout 打印摘要")
|
|
209
|
+
ap.add_argument("--version", action="version", version=f"{TOOL_NAME} {__version__}")
|
|
210
|
+
a = ap.parse_args(argv)
|
|
211
|
+
|
|
212
|
+
results = []
|
|
213
|
+
try:
|
|
214
|
+
for pdf in a.pdf:
|
|
215
|
+
results.append(audit_file(pdf))
|
|
216
|
+
except FileNotFoundError as e:
|
|
217
|
+
print(f"tablecheck: 文件不存在: {e}", file=sys.stderr)
|
|
218
|
+
return 1
|
|
219
|
+
except Exception as e: # 打不开/损坏的 PDF
|
|
220
|
+
print(f"tablecheck: 读取失败: {type(e).__name__}: {e}", file=sys.stderr)
|
|
221
|
+
return 1
|
|
222
|
+
|
|
223
|
+
doc = _envelope(results)
|
|
224
|
+
|
|
225
|
+
errors = validate(doc)
|
|
226
|
+
if errors:
|
|
227
|
+
print("tablecheck: 输出不符合 schema v1(内部错误):", file=sys.stderr)
|
|
228
|
+
for e in errors[:20]:
|
|
229
|
+
print(f" - {e}", file=sys.stderr)
|
|
230
|
+
return 1
|
|
231
|
+
|
|
232
|
+
if not a.quiet:
|
|
233
|
+
sys.stdout.write(_text_summary(doc))
|
|
234
|
+
|
|
235
|
+
if a.json:
|
|
236
|
+
Path(a.json).parent.mkdir(parents=True, exist_ok=True)
|
|
237
|
+
Path(a.json).write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
238
|
+
if not a.quiet:
|
|
239
|
+
print(f"JSON -> {a.json}")
|
|
240
|
+
if a.md:
|
|
241
|
+
Path(a.md).parent.mkdir(parents=True, exist_ok=True)
|
|
242
|
+
Path(a.md).write_text(to_md(doc), encoding="utf-8")
|
|
243
|
+
if not a.quiet:
|
|
244
|
+
print(f"MD -> {a.md}")
|
|
245
|
+
|
|
246
|
+
if a.strict and has_problems(doc):
|
|
247
|
+
return 2
|
|
248
|
+
return 0
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
if __name__ == "__main__":
|
|
252
|
+
sys.exit(main())
|
tablecheck/detect.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""几何表格检测(纯 PyMuPDF,不依赖结构树)。
|
|
2
|
+
|
|
3
|
+
检测逻辑沿用既有基线工具的做法,保持数字可比:
|
|
4
|
+
|
|
5
|
+
1. ``page.find_tables()`` 拿几何表;
|
|
6
|
+
2. ``clean_cols`` 去掉只含货币符号/空白/破折号一类的假列;
|
|
7
|
+
3. 过滤假表:有效列 < 2、行数 < 3 一律不算表。
|
|
8
|
+
|
|
9
|
+
这两条过滤规则是历史基线的来源,改动会让 374 / 11 这些数字漂移,所以保持原样。
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
FAKE_CELL_VALUES = ("$", "€", "£", "%", "-", "—")
|
|
15
|
+
|
|
16
|
+
MIN_COLS = 2
|
|
17
|
+
MIN_ROWS = 3
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def clean_cols(table) -> int:
|
|
21
|
+
"""返回去掉假列后的有效列数。
|
|
22
|
+
|
|
23
|
+
``table`` 是 pymupdf 的 Table 对象;用 ``extract()`` 的文本矩阵判断,
|
|
24
|
+
某列只要有一个非空、且不属于货币符号/破折号的单元格就算有效列。
|
|
25
|
+
"""
|
|
26
|
+
data = table.extract()
|
|
27
|
+
if not data:
|
|
28
|
+
return 0
|
|
29
|
+
ncols = max(len(row) for row in data)
|
|
30
|
+
keep = 0
|
|
31
|
+
for c in range(ncols):
|
|
32
|
+
vals = []
|
|
33
|
+
for row in data:
|
|
34
|
+
if c < len(row) and row[c]:
|
|
35
|
+
vals.append(str(row[c]).strip())
|
|
36
|
+
meaningful = [v for v in vals if v and v not in FAKE_CELL_VALUES]
|
|
37
|
+
if meaningful:
|
|
38
|
+
keep += 1
|
|
39
|
+
return keep
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def detect_pages(pdf_path: str) -> tuple[dict[int, list[dict]], int]:
|
|
43
|
+
"""逐页检测几何表格。
|
|
44
|
+
|
|
45
|
+
返回 ``({页码1-based: [table_dict, ...]}, 总页数)``。
|
|
46
|
+
table_dict: ``{"rows", "cols_raw", "cols", "bbox"}``。
|
|
47
|
+
"""
|
|
48
|
+
import pymupdf
|
|
49
|
+
|
|
50
|
+
doc = pymupdf.open(pdf_path)
|
|
51
|
+
try:
|
|
52
|
+
found: dict[int, list[dict]] = {}
|
|
53
|
+
for pno, page in enumerate(doc, 1):
|
|
54
|
+
try:
|
|
55
|
+
tabs = page.find_tables()
|
|
56
|
+
except Exception:
|
|
57
|
+
# 单页解析失败不影响整份文件,跳过该页(与历史实现一致)
|
|
58
|
+
continue
|
|
59
|
+
items = []
|
|
60
|
+
for t in tabs.tables:
|
|
61
|
+
cols = clean_cols(t) or t.col_count
|
|
62
|
+
if cols < MIN_COLS or t.row_count < MIN_ROWS:
|
|
63
|
+
continue
|
|
64
|
+
items.append(
|
|
65
|
+
{
|
|
66
|
+
"rows": t.row_count,
|
|
67
|
+
"cols_raw": t.col_count,
|
|
68
|
+
"cols": cols,
|
|
69
|
+
"bbox": [round(float(v), 1) for v in t.bbox],
|
|
70
|
+
}
|
|
71
|
+
)
|
|
72
|
+
if items:
|
|
73
|
+
found[pno] = items
|
|
74
|
+
return found, doc.page_count
|
|
75
|
+
finally:
|
|
76
|
+
doc.close()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def shape_of(items: list[dict]) -> str:
|
|
80
|
+
"""把一页里的表形状压成 ``3r x 4c; 2r x 2c`` 这种字符串。"""
|
|
81
|
+
return "; ".join(f"{d['rows']}r x {d['cols']}c" for d in items) if items else ""
|
tablecheck/schema.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""输出 JSON 的 schema 版本与校验。
|
|
2
|
+
|
|
3
|
+
字段名是合同:外部脚本会按这里的名字取值,改名字 = 破坏性变更 = 升大版本。
|
|
4
|
+
|
|
5
|
+
v1.0.0 单文件形态::
|
|
6
|
+
|
|
7
|
+
{
|
|
8
|
+
"schema_version": "1.0.0",
|
|
9
|
+
"tool": {"name": "tablecheck", "version": "0.1.0"},
|
|
10
|
+
"file": {"name": "...", "pages": 20, "sha256": "..."},
|
|
11
|
+
"cjk": {"script_ratio": 0.0, "mixed": false, "vertical_suspect": false, ...},
|
|
12
|
+
"summary": {...},
|
|
13
|
+
"pages": [
|
|
14
|
+
{"page": 1, "detected": 2, "shape": "3r x 4c", "tagged_tables": 2,
|
|
15
|
+
"th": 4, "td": 12, "header_quality": {...}, "status": "OK"}
|
|
16
|
+
]
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
多文件形态只是把上面的 ``file``/``cjk``/``summary``/``pages`` 四项装进 ``files[]``::
|
|
20
|
+
|
|
21
|
+
{"schema_version": "1.0.0", "tool": {...}, "files": [ {...}, {...} ]}
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
SCHEMA_VERSION = "1.0.0"
|
|
27
|
+
|
|
28
|
+
STATUSES = ("NOT TAGGED", "NO HEADER", "PARTIAL HEADER", "OK", "TAGGED-ONLY")
|
|
29
|
+
BAD_STATUSES = ("NOT TAGGED", "NO HEADER", "PARTIAL HEADER")
|
|
30
|
+
|
|
31
|
+
FILE_KEYS = ("file", "cjk", "summary", "pages")
|
|
32
|
+
CJK_KEYS = ("script_ratio", "mixed", "vertical_suspect")
|
|
33
|
+
PAGE_KEYS = ("page", "detected", "shape", "tagged_tables", "th", "td",
|
|
34
|
+
"header_quality", "status")
|
|
35
|
+
HQ_KEYS = ("tables", "th_in_first_row", "th_in_first_col", "th_total",
|
|
36
|
+
"th_missing_scope", "col_count_mismatch", "empty_header_cells")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _type_errors(obj, key, types, where, errors):
|
|
40
|
+
if key not in obj:
|
|
41
|
+
errors.append(f"{where}: 缺字段 {key}")
|
|
42
|
+
return
|
|
43
|
+
if not isinstance(obj[key], types):
|
|
44
|
+
errors.append(f"{where}.{key}: 类型应为 {types}, 实际 {type(obj[key]).__name__}")
|
|
45
|
+
elif isinstance(obj[key], bool) and bool not in (types,):
|
|
46
|
+
errors.append(f"{where}.{key}: 不该是 bool")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def validate(doc: dict) -> list[str]:
|
|
50
|
+
"""校验输出的 JSON 结构,返回错误列表(空列表 = 合规)。"""
|
|
51
|
+
errors: list[str] = []
|
|
52
|
+
if not isinstance(doc, dict):
|
|
53
|
+
return ["顶层必须是对象"]
|
|
54
|
+
|
|
55
|
+
if doc.get("schema_version") != SCHEMA_VERSION:
|
|
56
|
+
errors.append(f"schema_version 应为 {SCHEMA_VERSION},实际 {doc.get('schema_version')!r}")
|
|
57
|
+
|
|
58
|
+
tool = doc.get("tool")
|
|
59
|
+
if not isinstance(tool, dict):
|
|
60
|
+
errors.append("缺 tool 对象")
|
|
61
|
+
else:
|
|
62
|
+
for k in ("name", "version"):
|
|
63
|
+
_type_errors(tool, k, (str,), "tool", errors)
|
|
64
|
+
|
|
65
|
+
if "files" in doc:
|
|
66
|
+
files = doc["files"]
|
|
67
|
+
if not isinstance(files, list) or not files:
|
|
68
|
+
errors.append("files 必须是非空数组")
|
|
69
|
+
return errors
|
|
70
|
+
for i, f in enumerate(files):
|
|
71
|
+
errors.extend(_validate_file(f, f"files[{i}]"))
|
|
72
|
+
return errors
|
|
73
|
+
|
|
74
|
+
errors.extend(_validate_file(doc, ""))
|
|
75
|
+
return errors
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _validate_file(doc: dict, where: str) -> list[str]:
|
|
79
|
+
errors: list[str] = []
|
|
80
|
+
prefix = f"{where}." if where else ""
|
|
81
|
+
|
|
82
|
+
f = doc.get("file")
|
|
83
|
+
if not isinstance(f, dict):
|
|
84
|
+
errors.append(f"{prefix}缺 file 对象")
|
|
85
|
+
else:
|
|
86
|
+
_type_errors(f, "name", (str,), prefix + "file", errors)
|
|
87
|
+
_type_errors(f, "pages", (int,), prefix + "file", errors)
|
|
88
|
+
_type_errors(f, "sha256", (str,), prefix + "file", errors)
|
|
89
|
+
if isinstance(f.get("sha256"), str) and len(f["sha256"]) != 64:
|
|
90
|
+
errors.append(f"{prefix}file.sha256 长度应为 64")
|
|
91
|
+
|
|
92
|
+
cjk = doc.get("cjk")
|
|
93
|
+
if not isinstance(cjk, dict):
|
|
94
|
+
errors.append(f"{prefix}缺 cjk 对象")
|
|
95
|
+
else:
|
|
96
|
+
for k in CJK_KEYS:
|
|
97
|
+
if k not in cjk:
|
|
98
|
+
errors.append(f"{prefix}cjk: 缺字段 {k}")
|
|
99
|
+
if isinstance(cjk.get("script_ratio"), (int, float)) and not isinstance(
|
|
100
|
+
cjk["script_ratio"], bool
|
|
101
|
+
):
|
|
102
|
+
if not 0.0 <= float(cjk["script_ratio"]) <= 1.0:
|
|
103
|
+
errors.append(f"{prefix}cjk.script_ratio 应在 0..1 之间")
|
|
104
|
+
for k in ("mixed", "vertical_suspect"):
|
|
105
|
+
if k in cjk and not isinstance(cjk[k], bool):
|
|
106
|
+
errors.append(f"{prefix}cjk.{k} 应为布尔")
|
|
107
|
+
|
|
108
|
+
if not isinstance(doc.get("summary"), dict):
|
|
109
|
+
errors.append(f"{prefix}缺 summary 对象")
|
|
110
|
+
|
|
111
|
+
pages = doc.get("pages")
|
|
112
|
+
if not isinstance(pages, list):
|
|
113
|
+
errors.append(f"{prefix}pages 必须是数组")
|
|
114
|
+
return errors
|
|
115
|
+
for i, p in enumerate(pages):
|
|
116
|
+
w = f"{prefix}pages[{i}]"
|
|
117
|
+
if not isinstance(p, dict):
|
|
118
|
+
errors.append(f"{w} 必须是对象")
|
|
119
|
+
continue
|
|
120
|
+
for k in PAGE_KEYS:
|
|
121
|
+
if k not in p:
|
|
122
|
+
errors.append(f"{w}: 缺字段 {k}")
|
|
123
|
+
if p.get("status") not in STATUSES:
|
|
124
|
+
errors.append(f"{w}.status 非法值 {p.get('status')!r}")
|
|
125
|
+
if not isinstance(p.get("header_quality"), dict):
|
|
126
|
+
errors.append(f"{w}.header_quality 必须是对象")
|
|
127
|
+
else:
|
|
128
|
+
for k in HQ_KEYS:
|
|
129
|
+
if k not in p["header_quality"]:
|
|
130
|
+
errors.append(f"{w}.header_quality: 缺字段 {k}")
|
|
131
|
+
return errors
|
tablecheck/struct.py
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""结构树读取:/Table /TR /TH /TD、/Scope、/StructParents。
|
|
2
|
+
|
|
3
|
+
用 pikepdf 走 /StructTreeRoot,产出一棵「表 → 行 → 单元格」的朴素树,
|
|
4
|
+
供上层做表头质量判定。只读,绝不改文件。
|
|
5
|
+
|
|
6
|
+
页归属规则(依次回退):
|
|
7
|
+
1. 元素自带的 ``/Pg``;
|
|
8
|
+
2. 最近的祖先元素的 ``/Pg``(很多生成器只在 /Table 上写 /Pg);
|
|
9
|
+
3. 都没有 → ``None``(记进 ``table["page"] = None``,只进全局总数,不算到任何页)。
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
TABLE_TAGS = ("/Table", "/TR", "/TH", "/TD")
|
|
15
|
+
CONTENT_TAGS = ("/TH", "/TD")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _as_int(obj):
|
|
19
|
+
"""pikepdf 的整数 Object 转 python int;不是整数返回 None。"""
|
|
20
|
+
if obj is None:
|
|
21
|
+
return None
|
|
22
|
+
try:
|
|
23
|
+
return int(obj)
|
|
24
|
+
except (TypeError, ValueError):
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _kids(node):
|
|
29
|
+
"""把 /K 归一化成 list(可能是单 dict / 单 ref / 整数 MCID / Array)。"""
|
|
30
|
+
import pikepdf
|
|
31
|
+
|
|
32
|
+
if not isinstance(node, pikepdf.Dictionary):
|
|
33
|
+
return []
|
|
34
|
+
kids = node.get("/K")
|
|
35
|
+
if kids is None:
|
|
36
|
+
return []
|
|
37
|
+
if isinstance(kids, pikepdf.Dictionary):
|
|
38
|
+
return [kids]
|
|
39
|
+
if isinstance(kids, pikepdf.Array):
|
|
40
|
+
return list(kids)
|
|
41
|
+
return []
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _scope_of(elem):
|
|
45
|
+
"""读 /Scope:既认元素上的直写,也认 /A 属性字典(Word/InDesign 走的是 /A)。"""
|
|
46
|
+
import pikepdf
|
|
47
|
+
|
|
48
|
+
direct = elem.get("/Scope")
|
|
49
|
+
if direct is not None:
|
|
50
|
+
return str(direct).lstrip("/").lower()
|
|
51
|
+
# pikepdf 的 Array/Dictionary 都是 Object;/A 还可能是单个字典而非数组
|
|
52
|
+
attrs = elem.get("/A")
|
|
53
|
+
if isinstance(attrs, pikepdf.Array):
|
|
54
|
+
candidates = list(attrs)
|
|
55
|
+
elif attrs is not None:
|
|
56
|
+
candidates = [attrs]
|
|
57
|
+
else:
|
|
58
|
+
candidates = []
|
|
59
|
+
for a in candidates:
|
|
60
|
+
if not isinstance(a, pikepdf.Dictionary):
|
|
61
|
+
continue
|
|
62
|
+
s = a.get("/Scope")
|
|
63
|
+
if s is not None:
|
|
64
|
+
return str(s).lstrip("/").lower()
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _span_of(elem) -> int:
|
|
69
|
+
"""读 /ColSpan(直写或 /A 里的),缺省 1。合并单元格靠它算实际列数。"""
|
|
70
|
+
import pikepdf
|
|
71
|
+
|
|
72
|
+
def pick(obj):
|
|
73
|
+
if obj is None:
|
|
74
|
+
return None
|
|
75
|
+
v = obj.get("/ColSpan") if isinstance(obj, pikepdf.Dictionary) else None
|
|
76
|
+
return _as_int(v)
|
|
77
|
+
|
|
78
|
+
span = pick(elem)
|
|
79
|
+
if span is None:
|
|
80
|
+
attrs = elem.get("/A")
|
|
81
|
+
if isinstance(attrs, pikepdf.Array):
|
|
82
|
+
for a in attrs:
|
|
83
|
+
span = pick(a)
|
|
84
|
+
if span:
|
|
85
|
+
break
|
|
86
|
+
else:
|
|
87
|
+
span = pick(attrs)
|
|
88
|
+
return span if span and span > 0 else 1
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _content_refs(elem, depth=0) -> int:
|
|
92
|
+
"""单元格里有多少个内容引用(MCID 整数 / OBJR 对象引用)。
|
|
93
|
+
|
|
94
|
+
MCID 常常不是直接挂在 TH/TD 上,而是挂在一层嵌套的 /P 之类 StructElem 里,
|
|
95
|
+
所以这里要往下钻几层,否则所有单元格都会被误判成「空表头」。
|
|
96
|
+
"""
|
|
97
|
+
if depth > 6:
|
|
98
|
+
return 0
|
|
99
|
+
n = 0
|
|
100
|
+
for k in _kids(elem):
|
|
101
|
+
if _as_int(k) is not None:
|
|
102
|
+
n += 1
|
|
103
|
+
continue
|
|
104
|
+
try:
|
|
105
|
+
if str(k.get("/Type", "")) == "/OBJR":
|
|
106
|
+
n += 1
|
|
107
|
+
continue
|
|
108
|
+
except AttributeError:
|
|
109
|
+
continue
|
|
110
|
+
n += _content_refs(k, depth + 1)
|
|
111
|
+
return n
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def read_structure(pdf_path: str) -> dict:
|
|
115
|
+
"""返回 ``{"has_struct_tree", "totals", "per_page", "tables", "pages_with_structparents"}``。"""
|
|
116
|
+
import pikepdf
|
|
117
|
+
|
|
118
|
+
result = {
|
|
119
|
+
"has_struct_tree": False,
|
|
120
|
+
"totals": {"Table": 0, "TR": 0, "TH": 0, "TD": 0},
|
|
121
|
+
"per_page": {},
|
|
122
|
+
"tables": [],
|
|
123
|
+
"pages_with_structparents": 0,
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
pdf = pikepdf.open(pdf_path)
|
|
127
|
+
try:
|
|
128
|
+
# /StructParents 只做统计:有多少页挂了 /StructParents(结构树的锚点页)
|
|
129
|
+
for pg in pdf.pages:
|
|
130
|
+
if pg.obj.get("/StructParents") is not None:
|
|
131
|
+
result["pages_with_structparents"] += 1
|
|
132
|
+
|
|
133
|
+
root = pdf.Root.get("/StructTreeRoot")
|
|
134
|
+
if root is None:
|
|
135
|
+
return result
|
|
136
|
+
result["has_struct_tree"] = True
|
|
137
|
+
|
|
138
|
+
page_index = {pg.obj.objgen: i for i, pg in enumerate(pdf.pages, 1)}
|
|
139
|
+
|
|
140
|
+
def page_of(obj, inherited):
|
|
141
|
+
ref = obj.get("/Pg") if isinstance(obj, pikepdf.Dictionary) else None
|
|
142
|
+
if ref is None:
|
|
143
|
+
return inherited
|
|
144
|
+
try:
|
|
145
|
+
return page_index.get(ref.objgen, inherited)
|
|
146
|
+
except AttributeError:
|
|
147
|
+
return inherited
|
|
148
|
+
|
|
149
|
+
seen = set()
|
|
150
|
+
|
|
151
|
+
def walk(node, inherited_page, depth=0):
|
|
152
|
+
if depth > 120:
|
|
153
|
+
return
|
|
154
|
+
for k in _kids(node):
|
|
155
|
+
if not isinstance(k, pikepdf.Dictionary):
|
|
156
|
+
continue
|
|
157
|
+
try:
|
|
158
|
+
ident = k.objgen
|
|
159
|
+
except AttributeError:
|
|
160
|
+
ident = None
|
|
161
|
+
# 结构树里出现环是真实存在的毛病,按对象号去重兜住
|
|
162
|
+
if ident is not None:
|
|
163
|
+
if ident in seen:
|
|
164
|
+
continue
|
|
165
|
+
seen.add(ident)
|
|
166
|
+
|
|
167
|
+
s = str(k.get("/S", ""))
|
|
168
|
+
pno = page_of(k, inherited_page)
|
|
169
|
+
if s in TABLE_TAGS:
|
|
170
|
+
result["totals"][s.lstrip("/")] += 1
|
|
171
|
+
if pno is not None:
|
|
172
|
+
d = result["per_page"].setdefault(
|
|
173
|
+
pno, {"tables": 0, "th": 0, "td": 0, "tr": 0}
|
|
174
|
+
)
|
|
175
|
+
if s == "/Table":
|
|
176
|
+
d["tables"] += 1
|
|
177
|
+
elif s == "/TH":
|
|
178
|
+
d["th"] += 1
|
|
179
|
+
elif s == "/TD":
|
|
180
|
+
d["td"] += 1
|
|
181
|
+
elif s == "/TR":
|
|
182
|
+
d["tr"] += 1
|
|
183
|
+
|
|
184
|
+
if s == "/Table":
|
|
185
|
+
result["tables"].append(_read_table(k, pno))
|
|
186
|
+
|
|
187
|
+
# 继续下钻:TH/TD/TR 的计数要靠这一趟走完(seen 已按对象号去重,不会重复计)
|
|
188
|
+
walk(k, pno, depth + 1)
|
|
189
|
+
|
|
190
|
+
walk(root, None)
|
|
191
|
+
return result
|
|
192
|
+
finally:
|
|
193
|
+
pdf.close()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _read_table(table_elem, page_no):
|
|
197
|
+
"""把一棵 /Table 子树读成 ``{page, n_rows, rows:[[cell,...]], th, td}``。"""
|
|
198
|
+
import pikepdf
|
|
199
|
+
|
|
200
|
+
rows = []
|
|
201
|
+
th = td = 0
|
|
202
|
+
# /Table 的直接子元素应该是 /TR;有的生成器会插一层 /THead /TBody
|
|
203
|
+
trs = []
|
|
204
|
+
|
|
205
|
+
def collect_tr(node, depth=0):
|
|
206
|
+
if depth > 12:
|
|
207
|
+
return
|
|
208
|
+
for k in _kids(node):
|
|
209
|
+
if not isinstance(k, pikepdf.Dictionary):
|
|
210
|
+
continue
|
|
211
|
+
s = str(k.get("/S", ""))
|
|
212
|
+
if s == "/TR":
|
|
213
|
+
trs.append(k)
|
|
214
|
+
elif s in ("/THead", "/TBody", "/TFoot"):
|
|
215
|
+
collect_tr(k, depth + 1)
|
|
216
|
+
|
|
217
|
+
collect_tr(table_elem)
|
|
218
|
+
|
|
219
|
+
for tr in trs:
|
|
220
|
+
row = []
|
|
221
|
+
for k in _kids(tr):
|
|
222
|
+
if not isinstance(k, pikepdf.Dictionary):
|
|
223
|
+
continue
|
|
224
|
+
s = str(k.get("/S", ""))
|
|
225
|
+
if s not in CONTENT_TAGS:
|
|
226
|
+
continue
|
|
227
|
+
is_th = s == "/TH"
|
|
228
|
+
if is_th:
|
|
229
|
+
th += 1
|
|
230
|
+
else:
|
|
231
|
+
td += 1
|
|
232
|
+
row.append(
|
|
233
|
+
{
|
|
234
|
+
"type": "TH" if is_th else "TD",
|
|
235
|
+
"scope": _scope_of(k) if is_th else None,
|
|
236
|
+
"colspan": _span_of(k),
|
|
237
|
+
"empty": _content_refs(k) == 0,
|
|
238
|
+
}
|
|
239
|
+
)
|
|
240
|
+
if row:
|
|
241
|
+
rows.append(row)
|
|
242
|
+
|
|
243
|
+
return {"page": page_no, "n_rows": len(rows), "rows": rows, "th": th, "td": td}
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def header_quality(tables: list[dict]) -> dict:
|
|
247
|
+
"""对一组已标注的表给表头质量指标(聚合计数,不是布尔)。"""
|
|
248
|
+
q = {
|
|
249
|
+
"tables": 0,
|
|
250
|
+
"th_in_first_row": 0,
|
|
251
|
+
"th_in_first_col": 0,
|
|
252
|
+
"th_total": 0,
|
|
253
|
+
"th_missing_scope": 0,
|
|
254
|
+
"col_count_mismatch": 0,
|
|
255
|
+
"empty_header_cells": 0,
|
|
256
|
+
}
|
|
257
|
+
for t in tables:
|
|
258
|
+
rows = t["rows"]
|
|
259
|
+
if not rows:
|
|
260
|
+
continue
|
|
261
|
+
q["tables"] += 1
|
|
262
|
+
q["th_total"] += t["th"]
|
|
263
|
+
if any(c["type"] == "TH" for c in rows[0]):
|
|
264
|
+
q["th_in_first_row"] += 1
|
|
265
|
+
if any(r and r[0]["type"] == "TH" for r in rows):
|
|
266
|
+
q["th_in_first_col"] += 1
|
|
267
|
+
for r in rows:
|
|
268
|
+
for c in r:
|
|
269
|
+
if c["type"] == "TH":
|
|
270
|
+
if c["scope"] is None:
|
|
271
|
+
q["th_missing_scope"] += 1
|
|
272
|
+
if c["empty"]:
|
|
273
|
+
q["empty_header_cells"] += 1
|
|
274
|
+
width = sum(c.get("colspan", 1) for c in rows[0])
|
|
275
|
+
if any(sum(c.get("colspan", 1) for c in r) != width for r in rows):
|
|
276
|
+
q["col_count_mismatch"] += 1
|
|
277
|
+
return q
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tablecheck
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Check PDF table structure: geometry-detected tables vs tagged /Table /TH /TD, CJK-aware (中文/繁體).
|
|
5
|
+
Author: tablecheck contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/xeology13/tablecheck
|
|
8
|
+
Keywords: pdf,pdfua,accessibility,table,cjk,chinese
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Topic :: Text Processing
|
|
12
|
+
Requires-Python: <3.15,>=3.11
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: pymupdf>=2.0
|
|
16
|
+
Requires-Dist: pikepdf>=8.0
|
|
17
|
+
Provides-Extra: dev
|
|
18
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# tablecheck
|
|
22
|
+
|
|
23
|
+
PDF 表格结构检查器 / PDF table-structure checker — 专治中文、繁体 PDF 里「看得见表格、结构树里没有」的问题。
|
|
24
|
+
|
|
25
|
+
离线、确定性、无模型、无网络。同输入同输出,`sha256` 写在结果里,方便多方对账。
|
|
26
|
+
|
|
27
|
+
An offline, deterministic CLI that compares **geometrically detected tables** against the
|
|
28
|
+
**tagged structure tree** (`/Table` `/TR` `/TH` `/TD`, `/Scope`), with CJK-specific signals
|
|
29
|
+
for Chinese/Traditional-Chinese/Japanese/Korean documents.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## 它查什么 / What it checks
|
|
34
|
+
|
|
35
|
+
1. **几何表格 vs 结构标注** — 每页给出「检出几张表、结构树里标了几个 `/Table`」,再判状态:
|
|
36
|
+
`NOT TAGGED`(有表没标)、`NO HEADER`(标了但一个 `/TH` 都没有)、
|
|
37
|
+
`PARTIAL HEADER`(TH 少于表数)、`OK`、`TAGGED-ONLY`(结构树里有表,几何上没检出)。
|
|
38
|
+
2. **表头质量** — 每张已标注的表:TH 是否出现在首行/首列、缺 `/Scope` 的 TH 有多少、
|
|
39
|
+
同一张表里各 `TR` 的单元格数与表头行是否一致、空表头单元格(无任何内容引用的 TH)有几个。
|
|
40
|
+
3. **CJK 识别** — 中日韩字符占比 `script_ratio`、是否中英混排 `mixed`、
|
|
41
|
+
是否疑似竖排 `vertical_suspect`(直排文本会让几何检测失准,先给出信号)。
|
|
42
|
+
4. **可复跑** — 每个结果带文件 `sha256` 与页数;同一文件重复跑,JSON 逐字节相同。
|
|
43
|
+
|
|
44
|
+
## 它不查什么 / What it does NOT check
|
|
45
|
+
|
|
46
|
+
- 不判 PDF/UA 或 WCAG 整体合规(只判表格这一段;元数据、语言、替代文本不在此列)。
|
|
47
|
+
- 不判阅读顺序、不做 OCR:扫描件没有文字层时,几何检测只会看到图片。
|
|
48
|
+
- 不修改文件,一个字节都不写进输入 PDF(只读)。
|
|
49
|
+
- 不联网、不调用任何模型、不下载任何东西。
|
|
50
|
+
- 合并单元格(rowspan/colspan)会被如实记成「列数不一致」,这是**报告**不是**修复**。
|
|
51
|
+
- 竖排文本只给「疑似」信号,不做竖排表格的几何修正。
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## 安装 / Install
|
|
56
|
+
|
|
57
|
+
要求 Python **3.11 – 3.14**。下界是 pymupdf/pikepdf 2.x 世代 wheel 的共同起点;
|
|
58
|
+
上界卡在 3.14 是因为传递依赖 onnxruntime 尚未发布 cp315 wheel(实测 3.15 下
|
|
59
|
+
`pip install` 直接 ResolutionImpossible,与 tablecheck 无关)。
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python3.14 -m venv .venv
|
|
63
|
+
.venv/bin/pip install . # 发布后可改为: pip install tablecheck
|
|
64
|
+
.venv/bin/tablecheck --version
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
English: requires Python 3.11–3.14. Direct dependencies are just `pymupdf>=2.0` and
|
|
68
|
+
`pikepdf>=8.0`; pymupdf 2.x pulls onnxruntime/numpy etc. transitively (~200 MB installed) —
|
|
69
|
+
that is the price of the v1 detection baseline. pymupdf 1.28 installs much smaller but
|
|
70
|
+
its `find_tables()` drifts (measured 380 vs 374 on a 264-page sample), so it is not
|
|
71
|
+
recommended. No extras to choose from: the layout engine ships inside pymupdf 2.x.
|
|
72
|
+
Dev install: `pip install -e ".[dev]"` (pytest).
|
|
73
|
+
|
|
74
|
+
## 用法 / Usage
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
# 人看的摘要
|
|
78
|
+
tablecheck 文件.pdf
|
|
79
|
+
|
|
80
|
+
# 机器读的 JSON(schema v1.0.0)
|
|
81
|
+
tablecheck 文件.pdf --json out.json
|
|
82
|
+
|
|
83
|
+
# Markdown 报告(只列问题页)
|
|
84
|
+
tablecheck 文件.pdf --md out.md
|
|
85
|
+
|
|
86
|
+
# 当 CI 门禁:发现问题就返回退出码 2
|
|
87
|
+
tablecheck 文件.pdf --strict
|
|
88
|
+
|
|
89
|
+
# 多份文件一起跑(JSON 变成 {"schema_version","tool","files":[...]})
|
|
90
|
+
tablecheck a.pdf b.pdf --json batch.json
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
退出码 / exit codes:
|
|
94
|
+
|
|
95
|
+
| code | 含义 |
|
|
96
|
+
|---|---|
|
|
97
|
+
| 0 | 跑完,未发现问题(或未加 `--strict`,发现问题也返回 0) |
|
|
98
|
+
| 1 | 跑不起来:文件不存在、PDF 打不开、内部 schema 自检失败 |
|
|
99
|
+
| 2 | `--strict` 模式下发现结构问题 |
|
|
100
|
+
|
|
101
|
+
`--strict` 判定为问题的条件:任一页 `status ∈ {NOT TAGGED, NO HEADER, PARTIAL HEADER}`,
|
|
102
|
+
或任一页存在缺 `/Scope` 的 TH、空表头单元格,或「有检出表但全篇零标注」。
|
|
103
|
+
|
|
104
|
+
`col_count_mismatch`(列数不一致)**不进** strict 门禁:合并单元格会让它必然非零,
|
|
105
|
+
拿它当 CI 门禁在真实文档上一定误报;它只在 JSON 里作为信息项给出。
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## 输出 JSON(schema v1.0.0,字段名固定)
|
|
110
|
+
|
|
111
|
+
单文件:
|
|
112
|
+
|
|
113
|
+
```json
|
|
114
|
+
{
|
|
115
|
+
"schema_version": "1.0.0",
|
|
116
|
+
"tool": {"name": "tablecheck", "version": "0.1.0"},
|
|
117
|
+
"file": {"name": "样本.pdf", "pages": 20, "sha256": "…64 hex…"},
|
|
118
|
+
"cjk": {
|
|
119
|
+
"script_ratio": 0.9929,
|
|
120
|
+
"mixed": false,
|
|
121
|
+
"vertical_suspect": false,
|
|
122
|
+
"cjk_chars": 9392, "latin_chars": 67,
|
|
123
|
+
"fonts": ["SimSun", "SimHei"], "vertical_blocks": 0,
|
|
124
|
+
"total_blocks": 641, "pages_scanned": 20
|
|
125
|
+
},
|
|
126
|
+
"summary": {
|
|
127
|
+
"pages": 20, "pages_with_tables": 8,
|
|
128
|
+
"tables_detected": 11, "tables_tagged": 13,
|
|
129
|
+
"tr_total": 51, "th_total": 36, "td_total": 134,
|
|
130
|
+
"pages_not_tagged": 0, "pages_no_header": 0, "pages_partial_header": 0,
|
|
131
|
+
"pages_ok": 8, "pages_tagged_only": 0,
|
|
132
|
+
"pages_with_structparents": 20, "has_struct_tree": true,
|
|
133
|
+
"tables_without_page": 0,
|
|
134
|
+
"th_missing_scope": 0, "col_count_mismatch": 2, "empty_header_cells": 0
|
|
135
|
+
},
|
|
136
|
+
"pages": [
|
|
137
|
+
{
|
|
138
|
+
"page": 13, "detected": 2, "shape": "4r x 4c; 4r x 4c",
|
|
139
|
+
"tagged_tables": 2, "th": 4, "td": 24,
|
|
140
|
+
"header_quality": {
|
|
141
|
+
"tables": 2, "th_in_first_row": 2, "th_in_first_col": 2,
|
|
142
|
+
"th_total": 4, "th_missing_scope": 0,
|
|
143
|
+
"col_count_mismatch": 0, "empty_header_cells": 0
|
|
144
|
+
},
|
|
145
|
+
"status": "OK"
|
|
146
|
+
}
|
|
147
|
+
]
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
多文件时,`file` / `cjk` / `summary` / `pages` 这四个键装进 `files[]`,其余不变:
|
|
152
|
+
|
|
153
|
+
```json
|
|
154
|
+
{"schema_version": "1.0.0", "tool": {...}, "files": [ {...}, {...} ]}
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
字段语义速查:
|
|
158
|
+
|
|
159
|
+
| 字段 | 含义 |
|
|
160
|
+
|---|---|
|
|
161
|
+
| `file.sha256` | 输入文件的 sha256(小写 hex),与 `shasum -a 256` 一致 |
|
|
162
|
+
| `cjk.script_ratio` | 中日韩字符 / (中日韩字符 + 拉丁字母),0..1 |
|
|
163
|
+
| `cjk.mixed` | `script_ratio >= 0.05` 且拉丁字母占比 `>= 0.05` |
|
|
164
|
+
| `cjk.vertical_suspect` | 存在 wmode=1 的文字,或「窄高块 + 单字换行」块 ≥ 8 且占比 ≥ 5% |
|
|
165
|
+
| `pages[].detected` | 几何检出的表数(有效列 ≥ 2 且行数 ≥ 3) |
|
|
166
|
+
| `pages[].shape` | 每张表的 `行r x 列c`,多张用 `; ` 分隔 |
|
|
167
|
+
| `pages[].tagged_tables` | 该页结构树里 `/Table` 的个数 |
|
|
168
|
+
| `pages[].status` | `NOT TAGGED` / `NO HEADER` / `PARTIAL HEADER` / `OK` / `TAGGED-ONLY` |
|
|
169
|
+
| `header_quality.th_missing_scope` | 既没有 `/Scope` 也没有 `/A /Scope` 的 TH 个数 |
|
|
170
|
+
| `header_quality.col_count_mismatch` | 各 TR 单元格数与首行不一致的**表**数 |
|
|
171
|
+
| `header_quality.empty_header_cells` | 没有任何内容引用(MCID/OBJR)的 TH 个数 |
|
|
172
|
+
| `summary.tables_without_page` | 结构树里没有 `/Pg` 且祖先也没写的表(只进总数,不归任何页) |
|
|
173
|
+
|
|
174
|
+
判定顺序(沿用既有实现,不重排):
|
|
175
|
+
`detected>0 且 tagged==0 → NOT TAGGED`;`detected>0 且 th==0 → NO HEADER`;
|
|
176
|
+
`detected>0 且 th<tagged → PARTIAL HEADER`;`detected>0 → OK`;`否则 TAGGED-ONLY`。
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## 实测基线 / Measured baseline
|
|
181
|
+
|
|
182
|
+
三份真实样本(私有医疗/公文 PDF,**不进仓库**),pymupdf 2.x 下逐条对齐,2026-11-14 复测仍 0 差:
|
|
183
|
+
|
|
184
|
+
| 指标 / metric | AR-1 (264p, 简中) | 回归-20页-FINAL-FINAL (20p, 中文打标产物) | reg-ar01-OUT (274p, 繁体未标) |
|
|
185
|
+
|---|---|---|---|
|
|
186
|
+
| 检出 / 已标 Table | 374 / 378 | 11 / 13 | 109 / 0 |
|
|
187
|
+
| TR / TH / TD | 1510 / 866 / 4330 | 51 / 36 / 134 | 0 / 0 / 0 |
|
|
188
|
+
| 缺 `/Scope` 的 TH | 0 | 0 | — |
|
|
189
|
+
| 列数不一致的表 | 22 | 2 | — |
|
|
190
|
+
| 空表头单元格 | 4 | 0 | — |
|
|
191
|
+
| 页面判定 | 未标 3 · 正常 237 · 仅标注 7 | 正常 8 | 未标 99 |
|
|
192
|
+
| `cjk.script_ratio` | 0.9926 | 0.9929 | 0.2352 |
|
|
193
|
+
| `cjk.mixed` / `vertical_suspect` | false / false | false / false | **true** / false |
|
|
194
|
+
|
|
195
|
+
读法:两份中文样本 `/Scope` 全齐说明生成路径规范;AR-1 的 22 张「列数不一致」几乎全是
|
|
196
|
+
合并单元格;繁体样本 `mixed=true` 是拉丁字符占 76% 的真实混排,274 页里 99 页检出表但零标注,
|
|
197
|
+
是「全篇表格没标」的典型形态。
|
|
198
|
+
|
|
199
|
+
## 检测口径 / Detection rules
|
|
200
|
+
|
|
201
|
+
几何检测(`detect.py`):
|
|
202
|
+
|
|
203
|
+
- 用 PyMuPDF `page.find_tables()`(基于页面上的格线);
|
|
204
|
+
- 去掉「只含 `$ € £ % - —` 或空白」的假列后,有效列 < 2 或行数 < 3 的表不算表;
|
|
205
|
+
- 单页解析失败跳过该页,不影响整份文件。
|
|
206
|
+
|
|
207
|
+
这两条过滤是历史基线的来源(264 页样本检出 374 张表),改口径会让数字漂移,
|
|
208
|
+
所以对外承诺:**v1 期间口径不变**。
|
|
209
|
+
|
|
210
|
+
结构树(`struct.py`):`/StructTreeRoot` 深度优先,`/Pg` 逐级继承;
|
|
211
|
+
遇到对象号重复(结构树有环)直接跳过,不死循环。
|
|
212
|
+
|
|
213
|
+
## 复跑与对账 / Reproduce
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
shasum -a 256 文件.pdf
|
|
217
|
+
tablecheck 文件.pdf --json a.json
|
|
218
|
+
tablecheck 文件.pdf --json b.json
|
|
219
|
+
diff a.json b.json # 应该完全相同
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
`tests/test_gold.py` 固化了四个基线用例(上表样本)。样本目录用环境变量
|
|
223
|
+
`TABLE_CHECK_SAMPLES` 指定(默认 `~/HermesWorkspace/tablecheck/samples`);
|
|
224
|
+
样本是私有 PDF,不在仓库里,没有样本时这些用例自动 skip。
|
|
225
|
+
`pytest tests/test_smoke.py` 用现造 PDF 冒烟,无外部依赖。
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
## 已知短板 / Known limits
|
|
230
|
+
|
|
231
|
+
- 无框线(只靠留白)的表格 `find_tables()` 抓不到,会低估 `detected`。
|
|
232
|
+
- 扫描件、图片型表格不检测(无文字层)。
|
|
233
|
+
- 合并单元格一定会在 `col_count_mismatch` 里出现,属于预期噪音。
|
|
234
|
+
- 只判「结构树里有没有表、表头长什么样」,不判单元格内容对不对。
|
|
235
|
+
- 严格来说这是**体检工具**,不是修复工具:它告诉你差在哪一页哪一张表。
|
|
236
|
+
|
|
237
|
+
## License
|
|
238
|
+
|
|
239
|
+
MIT,见 `LICENSE`。
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
tablecheck/__init__.py,sha256=UIKVEHrrxOEhPbZruSawwg8QrOD4UFwOBbBcui92J7Y,245
|
|
2
|
+
tablecheck/cjk.py,sha256=rcmwZgDoVampUcoTfKs3e2V67Ja2A2A9hHZb65GO5xw,4676
|
|
3
|
+
tablecheck/cli.py,sha256=yxWaIj3tRSBAFPcN9tBWFaDvXwY783zDB9WZ_29a_Ho,9493
|
|
4
|
+
tablecheck/detect.py,sha256=w8xyCu3nqistE1E11wnP7R1LaON3nDwAbxTDxrfVOKE,2674
|
|
5
|
+
tablecheck/schema.py,sha256=2gPt5UZicJ-VkSrr1grhVAXaqE8A-_2ySF6jBpuHo3c,4936
|
|
6
|
+
tablecheck/struct.py,sha256=m6K4Ht-kEdS3Xdmlwaf_Ar6SGxa1b6rcgWCsMyHaoRU,8810
|
|
7
|
+
tablecheck-0.1.0.dist-info/licenses/LICENSE,sha256=2_dBN6YgLicWy13Tob7uuKB-n8nd1yQz7tmuut28lZM,1080
|
|
8
|
+
tablecheck-0.1.0.dist-info/METADATA,sha256=LFnIz3oNhicgRZNG2XzXGzjLFvyG8tdm0nuOL3V5C7Q,10333
|
|
9
|
+
tablecheck-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
10
|
+
tablecheck-0.1.0.dist-info/entry_points.txt,sha256=42Tf7-mmXkXzB_ngOLpQKPKFhkA1mJUjmAm-l_8qXc8,51
|
|
11
|
+
tablecheck-0.1.0.dist-info/top_level.txt,sha256=Ixo-C8B9nv6BD_2YC7I7_EV7Nkx-6Wat1sZprsKZ2b8,11
|
|
12
|
+
tablecheck-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 tablecheck contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tablecheck
|