kba-cli 0.3.0rc4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kba/__init__.py +3 -0
- kba/__main__.py +6 -0
- kba/analyze.py +908 -0
- kba/cli.py +552 -0
- kba/doctor.py +73 -0
- kba/eval.py +332 -0
- kba/extract.py +511 -0
- kba/fingerprint.py +333 -0
- kba/jobs.py +69 -0
- kba/links.py +38 -0
- kba/pii.py +37 -0
- kba/report.py +565 -0
- kba/summary.py +55 -0
- kba/tags.py +108 -0
- kba/telemetry.py +78 -0
- kba/util.py +204 -0
- kba_cli-0.3.0rc4.dist-info/METADATA +135 -0
- kba_cli-0.3.0rc4.dist-info/RECORD +22 -0
- kba_cli-0.3.0rc4.dist-info/WHEEL +5 -0
- kba_cli-0.3.0rc4.dist-info/entry_points.txt +2 -0
- kba_cli-0.3.0rc4.dist-info/licenses/LICENSE +21 -0
- kba_cli-0.3.0rc4.dist-info/top_level.txt +1 -0
kba/cli.py
ADDED
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
"""KBA command line interface."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
import time
|
|
11
|
+
import traceback
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from . import (__version__, analyze, doctor, eval as eval_mod, extract,
|
|
15
|
+
fingerprint, jobs, report, summary as summary_mod, telemetry, util)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _print_json(obj) -> None:
|
|
19
|
+
print(json.dumps(obj, ensure_ascii=False, indent=2))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _brand(path: str | None) -> dict:
|
|
23
|
+
if not path:
|
|
24
|
+
return {}
|
|
25
|
+
return util.read_json(Path(path), {}) or {}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# ---------- audit ----------
|
|
29
|
+
|
|
30
|
+
def _write_params(job: Path, args) -> dict:
|
|
31
|
+
params = {
|
|
32
|
+
"root": str(Path(args.root).resolve()),
|
|
33
|
+
"exclude": args.exclude or sorted(util.DEFAULT_EXCLUDE),
|
|
34
|
+
"max_file_mb": args.max_file_mb,
|
|
35
|
+
"extract_dir": str(Path(args.extract_dir).resolve()) if args.extract_dir else None,
|
|
36
|
+
"redact": bool(args.redact_paths),
|
|
37
|
+
}
|
|
38
|
+
util.write_json(job / "params.json", params)
|
|
39
|
+
return params
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _run_scan(job: Path, params: dict) -> dict:
|
|
43
|
+
try:
|
|
44
|
+
fp = fingerprint.scan(
|
|
45
|
+
root=Path(params["root"]),
|
|
46
|
+
job=job,
|
|
47
|
+
exclude=params.get("exclude"),
|
|
48
|
+
max_file_mb=params.get("max_file_mb", 200),
|
|
49
|
+
extract_dir=Path(params["extract_dir"]) if params.get("extract_dir") else None,
|
|
50
|
+
redact=params.get("redact", False),
|
|
51
|
+
)
|
|
52
|
+
return fp
|
|
53
|
+
except Exception as exc: # noqa: BLE001
|
|
54
|
+
jobs.write_state(job, status="error", error=str(exc))
|
|
55
|
+
raise
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _worker_main(job_dir: str) -> int:
|
|
59
|
+
job = Path(job_dir)
|
|
60
|
+
params = util.read_json(job / "params.json", {}) or {}
|
|
61
|
+
if not params:
|
|
62
|
+
jobs.write_state(job, status="error", error="params.json missing")
|
|
63
|
+
return 1
|
|
64
|
+
try:
|
|
65
|
+
fp = _run_scan(job, params)
|
|
66
|
+
s = fp["summary"]
|
|
67
|
+
print(f"[ok] {s['files']} files, {util.human_bytes(s['bytes'])}, "
|
|
68
|
+
f"text {s['with_text']}, unsupported {s.get('unsupported', 0)}")
|
|
69
|
+
return 0
|
|
70
|
+
except Exception: # noqa: BLE001
|
|
71
|
+
(job / "worker.log").write_text(traceback.format_exc(), encoding="utf-8")
|
|
72
|
+
return 1
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def cmd_scan_summary(args) -> int:
|
|
76
|
+
root = Path(args.root).resolve()
|
|
77
|
+
if not root.is_dir():
|
|
78
|
+
print(f"[error] 目录不存在:{root}")
|
|
79
|
+
return 2
|
|
80
|
+
out = Path(args.out)
|
|
81
|
+
extract_dir = Path(args.extract_dir).resolve() if args.extract_dir else None
|
|
82
|
+
try:
|
|
83
|
+
with tempfile.TemporaryDirectory(prefix="kba-scan-") as td:
|
|
84
|
+
job = Path(td)
|
|
85
|
+
fp = fingerprint.scan(
|
|
86
|
+
root, job, exclude=args.exclude, max_file_mb=args.max_file_mb,
|
|
87
|
+
extract_dir=extract_dir,
|
|
88
|
+
)
|
|
89
|
+
result = analyze.analyze(job)
|
|
90
|
+
data = summary_mod.build_summary(fp, result)
|
|
91
|
+
util.write_json(out, data)
|
|
92
|
+
except (OSError, ValueError) as exc:
|
|
93
|
+
print(f"[error] {exc}")
|
|
94
|
+
return 2
|
|
95
|
+
if args.json:
|
|
96
|
+
_print_json(data)
|
|
97
|
+
else:
|
|
98
|
+
print(f"[ok] 摘要:{out}({data['scan']['files']} 文件,置信度 {data['confidence']})")
|
|
99
|
+
print("下一步:把摘要交给付费专家解读;不要上传原始文档。")
|
|
100
|
+
return 0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def cmd_start(args) -> int:
|
|
104
|
+
root = Path(args.root).resolve()
|
|
105
|
+
if not root.is_dir():
|
|
106
|
+
print(f"[error] 目录不存在:{root}")
|
|
107
|
+
return 2
|
|
108
|
+
job_id = args.job or jobs.new_job_id()
|
|
109
|
+
job = jobs.job_dir(job_id)
|
|
110
|
+
job.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
params = _write_params(job, args)
|
|
112
|
+
jobs.write_state(job, status="queued", stage="fingerprint", root=str(root),
|
|
113
|
+
total=0, done=0, created_at=util.now_iso(), job_id=job_id)
|
|
114
|
+
|
|
115
|
+
swept = fingerprint.sweep_expired_text(skip_job=job_id)
|
|
116
|
+
if swept.get("files"):
|
|
117
|
+
print(f"[info] 已清理 {swept['files']} 个过期正文缓存(TTL {swept['ttl_days']} 天)")
|
|
118
|
+
|
|
119
|
+
if args.foreground:
|
|
120
|
+
fp = _run_scan(job, params)
|
|
121
|
+
s = fp["summary"]
|
|
122
|
+
print(f"[ok] 扫描完成:{s['files']} 个文件 / {util.human_bytes(s['bytes'])} / "
|
|
123
|
+
f"有文本 {s['with_text']} / 未支持格式 {s.get('unsupported', 0)}")
|
|
124
|
+
print(f"job: {job_id}")
|
|
125
|
+
print(f"下一步:kba audit analyze --job {job_id}")
|
|
126
|
+
return 0
|
|
127
|
+
|
|
128
|
+
log = open(job / "worker.log", "a", encoding="utf-8")
|
|
129
|
+
flags = 0
|
|
130
|
+
if os.name == "nt":
|
|
131
|
+
flags = 0x00000008 | 0x08000000 # DETACHED_PROCESS | CREATE_NO_WINDOW
|
|
132
|
+
subprocess.Popen([sys.executable, "-m", "kba", "_worker", str(job)],
|
|
133
|
+
stdout=log, stderr=log, stdin=subprocess.DEVNULL,
|
|
134
|
+
creationflags=flags, close_fds=True)
|
|
135
|
+
print(f"已启动:{job_id}")
|
|
136
|
+
print(f"进度:kba audit status --job {job_id}")
|
|
137
|
+
return 0
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def cmd_extract(args) -> int:
|
|
141
|
+
root = Path(args.root).resolve()
|
|
142
|
+
if not root.is_dir():
|
|
143
|
+
print(f"[error] 目录不存在:{root}")
|
|
144
|
+
return 2
|
|
145
|
+
out = Path(args.out).resolve()
|
|
146
|
+
ttl = args.ttl_days if args.ttl_days is not None and args.ttl_days >= 0 else None
|
|
147
|
+
try:
|
|
148
|
+
manifest = extract.extract_root(
|
|
149
|
+
root, out, exclude=args.exclude, overwrite=args.overwrite,
|
|
150
|
+
max_file_mb=args.max_file_mb, workers=args.workers, ttl_days=ttl,
|
|
151
|
+
)
|
|
152
|
+
except (ValueError, OSError) as exc:
|
|
153
|
+
print(f"[error] {exc}")
|
|
154
|
+
return 2
|
|
155
|
+
s = manifest["summary"]
|
|
156
|
+
print(f"[ok] 抽取完成:{s['files']} 个待处理文件 / 已抽取 {s['extracted']} / "
|
|
157
|
+
f"无文本 {s['no_text']} / 失败 {s['failed']} / 未支持 {s['unsupported']} / "
|
|
158
|
+
f"跳过已存在 {s['skipped_existing']}(workers={args.workers})")
|
|
159
|
+
if s.get("missing_dependencies"):
|
|
160
|
+
deps = "、".join(f"{k}({v})" for k, v in s["missing_dependencies"].items())
|
|
161
|
+
print(f"[warn] 缺少抽取依赖:{deps};安装:pip install -e .[extract]")
|
|
162
|
+
print(f"输出:{out}")
|
|
163
|
+
print(f"下一步:kba audit start --root \"{root}\" --extract-dir \"{out}\" --foreground")
|
|
164
|
+
return 0
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _latest_with(name: str) -> str | None:
|
|
168
|
+
dirs = sorted([p for p in jobs.jobs_root().iterdir() if p.is_dir()],
|
|
169
|
+
key=lambda p: p.stat().st_mtime, reverse=True)
|
|
170
|
+
for j in dirs:
|
|
171
|
+
if (j / name).exists():
|
|
172
|
+
return j.name
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _job_from_args(args, need: str | None = None) -> Path:
|
|
177
|
+
job_id = getattr(args, "job", None)
|
|
178
|
+
if not job_id and need:
|
|
179
|
+
job_id = _latest_with(need)
|
|
180
|
+
if not job_id:
|
|
181
|
+
job_id = jobs.latest_job()
|
|
182
|
+
if not job_id:
|
|
183
|
+
raise SystemExit("没有可用任务,先运行 kba audit start")
|
|
184
|
+
return jobs.job_dir(job_id)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def cmd_status(args) -> int:
|
|
188
|
+
job = _job_from_args(args)
|
|
189
|
+
st = jobs.read_state(job)
|
|
190
|
+
if not st:
|
|
191
|
+
print(f"[error] 任务不存在:{job}")
|
|
192
|
+
return 2
|
|
193
|
+
if args.json:
|
|
194
|
+
_print_json(st)
|
|
195
|
+
return 0
|
|
196
|
+
total = st.get("total") or 0
|
|
197
|
+
done = st.get("done") or 0
|
|
198
|
+
pct = f"{done / total * 100:.0f}%" if total else "—"
|
|
199
|
+
print(f"任务 {job.name}:{st.get('status')} / {st.get('stage')} {done}/{total} ({pct})")
|
|
200
|
+
if st.get("error"):
|
|
201
|
+
print(f"错误:{st['error']}")
|
|
202
|
+
if st.get("confidence"):
|
|
203
|
+
wc = st.get("weight_coverage")
|
|
204
|
+
wc_text = "—" if wc is None else f"{wc * 100:.0f}%"
|
|
205
|
+
print(f"数据置信度:{st['confidence']} | 权重覆盖度 {wc_text}")
|
|
206
|
+
if st.get("status") == "done" and st.get("stage") == "fingerprint":
|
|
207
|
+
print(f"下一步:kba audit analyze --job {job.name}")
|
|
208
|
+
if st.get("status") == "done" and st.get("stage") == "analyze":
|
|
209
|
+
print(f"下一步:kba audit report --job {job.name} --out report.html")
|
|
210
|
+
return 0
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def cmd_analyze(args) -> int:
|
|
214
|
+
job = _job_from_args(args, "fp.json")
|
|
215
|
+
result = analyze.analyze(job, pack=args.pack)
|
|
216
|
+
telemetry.record(job)
|
|
217
|
+
m = result["metrics"]
|
|
218
|
+
dc = result.get("data_coverage") or {}
|
|
219
|
+
grade = dc.get("confidence") or result.get("confidence") or "C"
|
|
220
|
+
cov = dc.get("text_coverage")
|
|
221
|
+
cov_text = "—(无二进制文件)" if cov is None else f"{cov * 100:.1f}%"
|
|
222
|
+
print(f"[ok] 数据置信度 {grade}(文本覆盖率 {cov_text})")
|
|
223
|
+
if grade == "C":
|
|
224
|
+
print(" 元数据盘点版:不展示综合健康分;覆盖范围与结论边界见报告第 2 节")
|
|
225
|
+
else:
|
|
226
|
+
wc = ((result.get("health") or {}).get("weight_coverage") or 0) * 100
|
|
227
|
+
print(f" 归一化健康分 {result['health_score']}/100(权重覆盖度 {wc:.0f}%)")
|
|
228
|
+
print(f" 正文:纯文本 {dc.get('text_files', 0)} | 可抽取文件已抽取 {dc.get('extracted', 0)}、"
|
|
229
|
+
f"无文本 {dc.get('no_text', 0)}、失败 {dc.get('failed', 0)}、未启用 {dc.get('not_enabled', 0)}"
|
|
230
|
+
f" | 未支持 {dc.get('unsupported', 0)}")
|
|
231
|
+
print(f" 重复 {m['dup_groups']} 组 / 冗余 {m['dup_redundant']} 个({m['dup_rate'] * 100:.1f}%)")
|
|
232
|
+
hit = "—" if m['top5_hit'] is None else f"{m['top5_hit'] * 100:.1f}%"
|
|
233
|
+
print(f" 过时 {m['stale_rate'] * 100:.1f}% | 有效标签覆盖 {m['valid_tag_coverage'] * 100:.1f}%"
|
|
234
|
+
f"(推断 {m['inferred_tag_coverage'] * 100:.1f}%)| 自检索命中 {hit}")
|
|
235
|
+
print(f" 负责人覆盖 {m['owner_coverage'] * 100:.1f}% | 断链 {m['broken_links']} 个"
|
|
236
|
+
f"({m['broken_link_files']} 个文件)")
|
|
237
|
+
print(f" 预计工作量 {result['plan']['effort_days']} 人日")
|
|
238
|
+
print(f"下一步:kba audit report --job {job.name} --out report.html")
|
|
239
|
+
return 0
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def cmd_report(args) -> int:
|
|
243
|
+
job = _job_from_args(args, "analysis.json")
|
|
244
|
+
out = Path(args.out)
|
|
245
|
+
report.write_report(job, out, brand=_brand(args.brand), fmt=args.format)
|
|
246
|
+
analysis = util.read_json(job / "analysis.json", {}) or {}
|
|
247
|
+
grade = (analysis.get("data_coverage") or {}).get("confidence")
|
|
248
|
+
print(f"[ok] 报告:{out.resolve()}" + (f"(数据置信度 {grade})" if grade else ""))
|
|
249
|
+
return 0
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def cmd_share(args) -> int:
|
|
253
|
+
job = _job_from_args(args, "analysis.json")
|
|
254
|
+
fp = util.read_json(job / "fp.json", {}) or {}
|
|
255
|
+
analysis = util.read_json(job / "analysis.json", {}) or {}
|
|
256
|
+
if not analysis:
|
|
257
|
+
print("[error] 先运行 kba audit analyze")
|
|
258
|
+
return 2
|
|
259
|
+
out = Path(args.out)
|
|
260
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
261
|
+
out.write_text(report.render_share_md(analysis, fp), encoding="utf-8")
|
|
262
|
+
print(f"[ok] 摘要:{out.resolve()}")
|
|
263
|
+
return 0
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def cmd_summary(args) -> int:
|
|
267
|
+
job = _job_from_args(args, "analysis.json")
|
|
268
|
+
analysis = util.read_json(job / "analysis.json", {}) or {}
|
|
269
|
+
if not analysis:
|
|
270
|
+
print("[error] 先运行 kba audit analyze")
|
|
271
|
+
return 2
|
|
272
|
+
dc = analysis.get("data_coverage") or {}
|
|
273
|
+
grade = dc.get("confidence") or analysis.get("confidence")
|
|
274
|
+
out = {
|
|
275
|
+
"job_id": analysis.get("job_id"),
|
|
276
|
+
"confidence": grade,
|
|
277
|
+
# C-level metadata reports do not publish a composite score.
|
|
278
|
+
"health_score": None if grade == "C" else analysis.get("health_score"),
|
|
279
|
+
"weight_coverage": (analysis.get("health") or {}).get("weight_coverage"),
|
|
280
|
+
"data_coverage": dc,
|
|
281
|
+
"metrics": analysis.get("metrics"),
|
|
282
|
+
"findings": [{"type": f.get("type"), "title": f.get("title"), "count": f.get("count")}
|
|
283
|
+
for f in analysis.get("findings", [])],
|
|
284
|
+
}
|
|
285
|
+
_print_json(out) if args.json else print(json.dumps(out, ensure_ascii=False, indent=2))
|
|
286
|
+
return 0
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def cmd_clusters(args) -> int:
|
|
290
|
+
job = _job_from_args(args, "analysis.json")
|
|
291
|
+
analysis = util.read_json(job / "analysis.json", {}) or {}
|
|
292
|
+
clusters = analysis.get("clusters", [])
|
|
293
|
+
if args.json:
|
|
294
|
+
_print_json(clusters[: args.top])
|
|
295
|
+
return 0
|
|
296
|
+
if not clusters:
|
|
297
|
+
print("未发现重复或同名组。")
|
|
298
|
+
return 0
|
|
299
|
+
print(f"{'组':<8}{'类型':<11}{'成员':<5}{'建议保留':<40}{'处置'}")
|
|
300
|
+
for c in clusters[: args.top]:
|
|
301
|
+
rec = c.get("recommend", {})
|
|
302
|
+
keep = next((x["name"] for x in c["members"] if x["p"] == rec.get("keep")), "—")
|
|
303
|
+
act = f"折叠 {len(rec.get('fold') or [])} 个" if rec.get("fold") else ("人工合并" if rec.get("merge") else "全部保留")
|
|
304
|
+
print(f"{c['id']:<8}{c['type']:<11}{len(c['members']):<5}{keep[:38]:<40}{act}")
|
|
305
|
+
if len(clusters) > args.top:
|
|
306
|
+
print(f"…还有 {len(clusters) - args.top} 组,用 --top 调整")
|
|
307
|
+
return 0
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
# ---------- misc ----------
|
|
311
|
+
|
|
312
|
+
def cmd_doctor(args) -> int:
|
|
313
|
+
result = doctor.run()
|
|
314
|
+
if args.json:
|
|
315
|
+
_print_json(result)
|
|
316
|
+
else:
|
|
317
|
+
doctor.print_report(result)
|
|
318
|
+
return 0 if result["ok"] else 2
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def cmd_version(args) -> int:
|
|
322
|
+
if args.json:
|
|
323
|
+
_print_json({"name": "kba", "version": __version__, "schema": "kba.version/1"})
|
|
324
|
+
else:
|
|
325
|
+
print(f"kba {__version__}")
|
|
326
|
+
return 0
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def cmd_eval(args) -> int:
|
|
330
|
+
root = Path(args.root).resolve()
|
|
331
|
+
if not root.is_dir():
|
|
332
|
+
print(f"[error] 目录不存在:{root}")
|
|
333
|
+
return 2
|
|
334
|
+
labels = {}
|
|
335
|
+
if args.labels:
|
|
336
|
+
labels = util.read_json(Path(args.labels), {}) or {}
|
|
337
|
+
if not isinstance(labels, dict):
|
|
338
|
+
print("[error] labels 必须是 JSON 对象")
|
|
339
|
+
return 2
|
|
340
|
+
try:
|
|
341
|
+
result = eval_mod.eval_corpus(
|
|
342
|
+
root, labels=labels,
|
|
343
|
+
extract_dir=Path(args.extract_dir).resolve() if args.extract_dir else None,
|
|
344
|
+
)
|
|
345
|
+
except (OSError, ValueError) as exc:
|
|
346
|
+
print(f"[error] {exc}")
|
|
347
|
+
return 2
|
|
348
|
+
if args.out:
|
|
349
|
+
util.write_json(Path(args.out), result)
|
|
350
|
+
if args.json:
|
|
351
|
+
_print_json(result)
|
|
352
|
+
else:
|
|
353
|
+
print(eval_mod.format_text(result))
|
|
354
|
+
return 0
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def cmd_telemetry(args) -> int:
|
|
358
|
+
if args.action == "on":
|
|
359
|
+
telemetry.set_enabled(True)
|
|
360
|
+
print("已开启匿名遥测(仅聚合指标,不含文件名/路径/正文)。")
|
|
361
|
+
elif args.action == "off":
|
|
362
|
+
telemetry.set_enabled(False)
|
|
363
|
+
q = jobs.kba_home() / "telemetry_queue.jsonl"
|
|
364
|
+
if q.exists():
|
|
365
|
+
q.unlink()
|
|
366
|
+
print("已关闭遥测,并清空未发送队列。")
|
|
367
|
+
elif args.action == "preview":
|
|
368
|
+
job = _job_from_args(args, "analysis.json")
|
|
369
|
+
_print_json(telemetry.preview(job))
|
|
370
|
+
else:
|
|
371
|
+
print(f"遥测:{'开启' if telemetry.enabled() else '关闭'}")
|
|
372
|
+
print(f"配置:{jobs.kba_home() / 'config.json'}")
|
|
373
|
+
return 0
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def cmd_clean(args) -> int:
|
|
377
|
+
if args.purge_extract:
|
|
378
|
+
if not args.dir:
|
|
379
|
+
print("[error] --purge-extract 需要 --dir <kba extract --out 目录>")
|
|
380
|
+
return 2
|
|
381
|
+
if args.purge_text:
|
|
382
|
+
print("[error] --purge-text 与 --purge-extract 不能同时使用")
|
|
383
|
+
return 2
|
|
384
|
+
try:
|
|
385
|
+
result = extract.purge_sidecars(
|
|
386
|
+
Path(args.dir), older_than_days=args.older_than_days
|
|
387
|
+
)
|
|
388
|
+
except OSError as exc:
|
|
389
|
+
print(f"[error] {exc}")
|
|
390
|
+
return 2
|
|
391
|
+
print(f"已清理 {result['files']} 个抽取 sidecar;剩余 {result['remaining']} 个,"
|
|
392
|
+
f"manifest {'保留' if result['manifest'] else '已删除'}")
|
|
393
|
+
return 0
|
|
394
|
+
if args.dir:
|
|
395
|
+
print("[error] --dir 只能与 --purge-extract 一起使用")
|
|
396
|
+
return 2
|
|
397
|
+
if args.purge_text:
|
|
398
|
+
if args.job:
|
|
399
|
+
targets = [jobs.job_dir(args.job)]
|
|
400
|
+
else:
|
|
401
|
+
targets = [p for p in jobs.jobs_root().iterdir() if p.is_dir()]
|
|
402
|
+
removed = 0
|
|
403
|
+
hit = 0
|
|
404
|
+
for j in targets:
|
|
405
|
+
n = fingerprint.purge_text(j)
|
|
406
|
+
if n:
|
|
407
|
+
removed += n
|
|
408
|
+
hit += 1
|
|
409
|
+
try:
|
|
410
|
+
jobs.write_state(j, text_cache_purged_at=util.now_iso(),
|
|
411
|
+
text_cache_purged_files=n)
|
|
412
|
+
except OSError:
|
|
413
|
+
pass
|
|
414
|
+
print(f"已清理 {removed} 个缓存正文文件({hit} 个任务)")
|
|
415
|
+
return 0
|
|
416
|
+
if args.job:
|
|
417
|
+
print("[error] --job 只能与 --purge-text 一起使用")
|
|
418
|
+
return 2
|
|
419
|
+
cutoff = time.time() - args.days * 86400
|
|
420
|
+
removed = 0
|
|
421
|
+
for p in jobs.jobs_root().iterdir():
|
|
422
|
+
if p.is_dir() and p.stat().st_mtime < cutoff:
|
|
423
|
+
import shutil
|
|
424
|
+
shutil.rmtree(p, ignore_errors=True)
|
|
425
|
+
removed += 1
|
|
426
|
+
print(f"已清理 {removed} 个超过 {args.days} 天的任务")
|
|
427
|
+
return 0
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
431
|
+
ap = argparse.ArgumentParser(prog="kba", description="只读知识库体检工具")
|
|
432
|
+
ap.add_argument("--version", action="version", version=f"kba {__version__}")
|
|
433
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
434
|
+
|
|
435
|
+
p = sub.add_parser("doctor", help="环境检查")
|
|
436
|
+
p.add_argument("--json", action="store_true")
|
|
437
|
+
p.set_defaults(func=cmd_doctor)
|
|
438
|
+
|
|
439
|
+
p = sub.add_parser("version", help="版本")
|
|
440
|
+
p.add_argument("--json", action="store_true")
|
|
441
|
+
p.set_defaults(func=cmd_version)
|
|
442
|
+
|
|
443
|
+
p = sub.add_parser("extract", help="从 PDF/Office 抽取正文到 sidecar 目录")
|
|
444
|
+
p.add_argument("--root", required=True)
|
|
445
|
+
p.add_argument("--out", required=True, help="抽取文本目录,供 audit start --extract-dir 使用")
|
|
446
|
+
p.add_argument("--exclude", nargs="*")
|
|
447
|
+
p.add_argument("--overwrite", action="store_true", help="覆盖已存在的 sidecar")
|
|
448
|
+
p.add_argument("--max-file-mb", type=int, default=200)
|
|
449
|
+
p.add_argument("--workers", type=int, default=1,
|
|
450
|
+
help="并行抽取线程数(1–16,默认 1)")
|
|
451
|
+
p.add_argument("--ttl-days", type=int, default=7,
|
|
452
|
+
help="抽取前清理超过 N 天的 sidecar(-1 关闭,默认 7)")
|
|
453
|
+
p.set_defaults(func=cmd_extract)
|
|
454
|
+
|
|
455
|
+
p = sub.add_parser("eval", help="用标注语料评测抽取/分析指标")
|
|
456
|
+
p.add_argument("--root", required=True)
|
|
457
|
+
p.add_argument("--labels", help="标注 JSON;省略则只输出抽取/覆盖统计")
|
|
458
|
+
p.add_argument("--extract-dir", help="kba extract --out 目录")
|
|
459
|
+
p.add_argument("--out", help="把评测结果写入 JSON")
|
|
460
|
+
p.add_argument("--json", action="store_true")
|
|
461
|
+
p.set_defaults(func=cmd_eval)
|
|
462
|
+
|
|
463
|
+
p = sub.add_parser("scan-summary", help="一次性扫描+分析,输出脱敏摘要")
|
|
464
|
+
p.add_argument("--root", required=True)
|
|
465
|
+
p.add_argument("--out", required=True)
|
|
466
|
+
p.add_argument("--extract-dir")
|
|
467
|
+
p.add_argument("--exclude", nargs="*")
|
|
468
|
+
p.add_argument("--max-file-mb", type=int, default=200)
|
|
469
|
+
p.add_argument("--json", action="store_true", help="把摘要 JSON 打到 stdout")
|
|
470
|
+
p.set_defaults(func=cmd_scan_summary)
|
|
471
|
+
|
|
472
|
+
audit = sub.add_parser("audit", help="体检流程")
|
|
473
|
+
asub = audit.add_subparsers(dest="audit_cmd", required=True)
|
|
474
|
+
|
|
475
|
+
s = asub.add_parser("start", help="启动只读扫描")
|
|
476
|
+
s.add_argument("--root", required=True)
|
|
477
|
+
s.add_argument("--job")
|
|
478
|
+
s.add_argument("--exclude", nargs="*")
|
|
479
|
+
s.add_argument("--max-file-mb", type=int, default=200)
|
|
480
|
+
s.add_argument("--extract-dir", help="已有的抽取文本目录(可选)")
|
|
481
|
+
s.add_argument("--redact-paths", action="store_true", help="指纹中不保留原始路径/文件名")
|
|
482
|
+
s.add_argument("--foreground", action="store_true", help="前台执行(调试用)")
|
|
483
|
+
s.set_defaults(func=cmd_start)
|
|
484
|
+
|
|
485
|
+
s = asub.add_parser("status", help="查看进度")
|
|
486
|
+
s.add_argument("--job")
|
|
487
|
+
s.add_argument("--json", action="store_true")
|
|
488
|
+
s.set_defaults(func=cmd_status)
|
|
489
|
+
|
|
490
|
+
s = asub.add_parser("analyze", help="分析")
|
|
491
|
+
s.add_argument("--job")
|
|
492
|
+
s.add_argument("--pack", default="generic")
|
|
493
|
+
s.set_defaults(func=cmd_analyze)
|
|
494
|
+
|
|
495
|
+
s = asub.add_parser("report", help="生成报告")
|
|
496
|
+
s.add_argument("--job")
|
|
497
|
+
s.add_argument("--out", required=True)
|
|
498
|
+
s.add_argument("--format", choices=["html", "md"], default="html")
|
|
499
|
+
s.add_argument("--brand", help="品牌配置 JSON")
|
|
500
|
+
s.set_defaults(func=cmd_report)
|
|
501
|
+
|
|
502
|
+
s = asub.add_parser("share", help="生成一页纸摘要")
|
|
503
|
+
s.add_argument("--job")
|
|
504
|
+
s.add_argument("--out", required=True)
|
|
505
|
+
s.set_defaults(func=cmd_share)
|
|
506
|
+
|
|
507
|
+
s = asub.add_parser("summary", help="输出指标与问题清单(JSON)")
|
|
508
|
+
s.add_argument("--job")
|
|
509
|
+
s.add_argument("--json", action="store_true")
|
|
510
|
+
s.set_defaults(func=cmd_summary)
|
|
511
|
+
|
|
512
|
+
cl = sub.add_parser("clusters", help="重复簇")
|
|
513
|
+
clsub = cl.add_subparsers(dest="cluster_cmd", required=True)
|
|
514
|
+
s = clsub.add_parser("list", help="列出重复簇")
|
|
515
|
+
s.add_argument("--job")
|
|
516
|
+
s.add_argument("--top", type=int, default=20)
|
|
517
|
+
s.add_argument("--json", action="store_true")
|
|
518
|
+
s.set_defaults(func=cmd_clusters)
|
|
519
|
+
|
|
520
|
+
t = sub.add_parser("telemetry", help="匿名遥测")
|
|
521
|
+
t.add_argument("action", choices=["status", "on", "off", "preview"])
|
|
522
|
+
t.add_argument("--job")
|
|
523
|
+
t.set_defaults(func=cmd_telemetry)
|
|
524
|
+
|
|
525
|
+
c = sub.add_parser("clean", help="清理旧任务或正文缓存")
|
|
526
|
+
c.add_argument("--days", type=int, default=30, help="删除超过 N 天的整个任务目录")
|
|
527
|
+
c.add_argument("--purge-text", action="store_true",
|
|
528
|
+
help="只清理 job/text/ 正文缓存,不删除任务(默认 TTL 7 天,可随时手动清理)")
|
|
529
|
+
c.add_argument("--purge-extract", action="store_true",
|
|
530
|
+
help="按 extract.json 清理 kba extract --out 目录的 sidecar")
|
|
531
|
+
c.add_argument("--dir", help="配合 --purge-extract,指定抽取输出目录")
|
|
532
|
+
c.add_argument("--older-than-days", type=int,
|
|
533
|
+
help="配合 --purge-extract,只清理超过 N 天的 sidecar;省略则全部清理")
|
|
534
|
+
c.add_argument("--job", help="配合 --purge-text,只清理指定任务")
|
|
535
|
+
c.set_defaults(func=cmd_clean)
|
|
536
|
+
|
|
537
|
+
return ap
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def main(argv=None) -> int:
|
|
541
|
+
try:
|
|
542
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
543
|
+
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
|
544
|
+
except (AttributeError, ValueError):
|
|
545
|
+
pass
|
|
546
|
+
if argv is None:
|
|
547
|
+
argv = sys.argv[1:]
|
|
548
|
+
if argv and argv[0] == "_worker":
|
|
549
|
+
return _worker_main(argv[1])
|
|
550
|
+
parser = build_parser()
|
|
551
|
+
args = parser.parse_args(argv)
|
|
552
|
+
return args.func(args)
|
kba/doctor.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Environment self-check."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import importlib.util
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from . import util
|
|
12
|
+
from .jobs import kba_home
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def run() -> dict:
|
|
16
|
+
checks = []
|
|
17
|
+
|
|
18
|
+
def add(name, ok, detail, fix=""):
|
|
19
|
+
checks.append({"name": name, "status": "ok" if ok else "warn", "detail": detail, "fix": fix})
|
|
20
|
+
|
|
21
|
+
ver = sys.version_info
|
|
22
|
+
add("Python", ver >= (3, 10), f"{ver.major}.{ver.minor}.{ver.micro}", "安装 Python 3.10+")
|
|
23
|
+
add("平台", True, f"{sys.platform} ({os.name})")
|
|
24
|
+
home = kba_home()
|
|
25
|
+
try:
|
|
26
|
+
home.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
probe = home / ".probe"
|
|
28
|
+
probe.write_text("ok", encoding="utf-8")
|
|
29
|
+
probe.unlink()
|
|
30
|
+
add("数据目录可写", True, str(home))
|
|
31
|
+
except OSError as exc:
|
|
32
|
+
add("数据目录可写", False, f"{home}: {exc}", "检查目录权限或设置 KBA_HOME")
|
|
33
|
+
|
|
34
|
+
try:
|
|
35
|
+
free = shutil.disk_usage(home).free
|
|
36
|
+
add("磁盘空间", free > 200 * 1024 * 1024, util.human_bytes(free), "清理磁盘")
|
|
37
|
+
except OSError:
|
|
38
|
+
add("磁盘空间", True, "未知")
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
d = Path(tempfile.mkdtemp(prefix="kba-中文路径-"))
|
|
42
|
+
p = d / "测试 文件.md"
|
|
43
|
+
p.write_text("ok", encoding="utf-8")
|
|
44
|
+
p.unlink()
|
|
45
|
+
d.rmdir()
|
|
46
|
+
add("中文/空格路径", True, "通过")
|
|
47
|
+
except OSError as exc:
|
|
48
|
+
add("中文/空格路径", False, str(exc), "检查系统编码设置")
|
|
49
|
+
|
|
50
|
+
for mod, label, fix in (
|
|
51
|
+
("pypdf", "pypdf(pdf 文本)", "pip install -e .[extract]"),
|
|
52
|
+
("docx", "python-docx(docx 文本)", "pip install -e .[extract]"),
|
|
53
|
+
("pptx", "python-pptx(pptx 文本)", "pip install -e .[extract]"),
|
|
54
|
+
("openpyxl", "openpyxl(xlsx 文本/结构指纹)", "pip install -e .[extract]"),
|
|
55
|
+
):
|
|
56
|
+
ok = importlib.util.find_spec(mod) is not None
|
|
57
|
+
add(label, ok, "已安装" if ok else "未安装(相应格式只做盘点,不做文本分析)", fix)
|
|
58
|
+
|
|
59
|
+
return {
|
|
60
|
+
"ok": all(c["status"] == "ok" for c in checks),
|
|
61
|
+
"version": __import__("kba").__version__,
|
|
62
|
+
"checks": checks,
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def print_report(result: dict) -> None:
|
|
67
|
+
print(f"KBA {result['version']} 环境检查")
|
|
68
|
+
for c in result["checks"]:
|
|
69
|
+
mark = "OK " if c["status"] == "ok" else "WARN"
|
|
70
|
+
print(f"[{mark}] {c['name']}: {c['detail']}")
|
|
71
|
+
if c["status"] != "ok" and c["fix"]:
|
|
72
|
+
print(f" 修复:{c['fix']}")
|
|
73
|
+
print("\n结论:", "可用" if result["ok"] else "有警告,部分能力受限")
|