kba-cli 0.3.0rc4__py3-none-any.whl

kba/analyze.py

@@ -0,0 +1,908 @@
+"""Analysis: duplicate clustering, canonical recommendation, quality metrics."""
+from __future__ import annotations
+
+import difflib
+import math
+import os
+import re
+import zlib
+from collections import Counter, defaultdict
+from pathlib import Path
+
+from . import fingerprint, pii, util
+from .jobs import read_state, write_state
+
+SHINGLE_K = 5
+SHINGLE_CAP = 60_000
+JACCARD_NEAR = 0.85
+CONTAIN_SUPERSET = 0.90
+STALE_DAYS = 730
+
+# Data-confidence grades are defined on text coverage of binary files.
+CONFIDENCE_A = 0.90
+CONFIDENCE_B = 0.60
+
+# Fixed weights (percent). Only components with usable data are scored;
+# the remaining weight is reported as "weight coverage", never faked with 0/0.5.
+HEALTH_COMPONENTS = (
+    ("duplicate", "重复", 30),
+    ("stale", "过时", 20),
+    ("tagging", "标签", 15),
+    ("structure", "结构", 15),
+    ("retrieval", "自检索", 20),
+)
+HEALTH_WEIGHT_TOTAL = sum(w for _, _, w in HEALTH_COMPONENTS)
+
+
+def _confidence(text_coverage: float | None) -> str:
+    if text_coverage is None:
+        return "A"
+    if text_coverage >= CONFIDENCE_A:
+        return "A"
+    if text_coverage >= CONFIDENCE_B:
+        return "B"
+    return "C"
+
+
+def _shingles(text: str) -> frozenset[int]:
+    t = "".join((text or "")[:SHINGLE_CAP].split())
+    if len(t) < SHINGLE_K:
+        return frozenset()
+    return frozenset(zlib.crc32(t[i:i + SHINGLE_K].encode("utf-8")) for i in range(len(t) - SHINGLE_K + 1))
+
+
+class _UF:
+    def __init__(self, n: int):
+        self.p = list(range(n))
+
+    def find(self, x: int) -> int:
+        while self.p[x] != x:
+            self.p[x] = self.p[self.p[x]]
+            x = self.p[x]
+        return x
+
+    def union(self, a: int, b: int) -> None:
+        ra, rb = self.find(a), self.find(b)
+        if ra != rb:
+            self.p[rb] = ra
+
+
+def _near_pairs(records: list[dict]) -> list[tuple[int, int, float, float]]:
+    """Candidate pairs via shared rare shingles; return (i, j, jaccard, containment)."""
+    sets: dict[int, frozenset[int]] = {}
+    for i, r in enumerate(records):
+        s = _shingles(r.get("_text", ""))
+        if s:
+            sets[i] = s
+    if len(sets) < 2:
+        return []
+
+    index: dict[int, list[int]] = defaultdict(list)
+    for i, s in sets.items():
+        for sh in s:
+            index[sh].append(i)
+
+    counts: dict[tuple[int, int], int] = defaultdict(int)
+    for sh, docs in index.items():
+        if 2 <= len(docs) <= 50:
+            for a in range(len(docs)):
+                for b in range(a + 1, len(docs)):
+                    x, y = docs[a], docs[b]
+                    counts[(x, y) if x < y else (y, x)] += 1
+
+    pairs = []
+    for (i, j), shared in counts.items():
+        if shared < 5:
+            continue
+        si, sj = sets[i], sets[j]
+        if not si or not sj:
+            continue
+        ratio = len(si) / len(sj)
+        # Draft/final versions can differ a lot in length; containment below still decides.
+        if ratio < 0.2 or ratio > 5.0:
+            continue
+        inter = len(si & sj)
+        union = len(si | sj)
+        if not union:
+            continue
+        jac = inter / union
+        contain = inter / min(len(si), len(sj))
+        if jac >= JACCARD_NEAR or contain >= CONTAIN_SUPERSET:
+            pairs.append((i, j, jac, contain))
+    return pairs
+
+
+PAGE_MIN_SHARED = 8
+PAGE_MAX_PAGES = 200
+PAGE_MAX_CANDIDATES = 20000
+
+
+def _page_sets(records: list[dict]) -> dict[int, list[frozenset[int]]]:
+    pages: dict[int, list[frozenset[int]]] = {}
+    for i, r in enumerate(records):
+        text = r.get("_text", "")
+        if "\f" not in text:
+            continue
+        sets = [_shingles(p) for p in text.split("\f") if p.strip()]
+        sets = [s for s in sets if s][:PAGE_MAX_PAGES]
+        if len(sets) >= 2:
+            pages[i] = sets
+    return pages
+
+
+def _page_overlaps(records: list[dict], clusters: list[dict]) -> tuple[list[dict], int]:
+    """Find cross-document page-level overlaps; never auto-merges whole documents."""
+    pages = _page_sets(records)
+    if len(pages) < 2:
+        return [], 0
+    index: dict[int, list[tuple[int, int]]] = defaultdict(list)
+    for i, sets in pages.items():
+        for pi, s in enumerate(sets):
+            for sh in s:
+                index[sh].append((i, pi))
+
+    counts: dict[tuple[int, int], int] = defaultdict(int)
+    full = False
+    for postings in index.values():
+        if not (2 <= len(postings) <= 50):
+            continue
+        for a in range(len(postings)):
+            i, _ = postings[a]
+            for b in range(a + 1, len(postings)):
+                j, _ = postings[b]
+                if i == j:
+                    continue
+                counts[(i, j) if i < j else (j, i)] += 1
+                if len(counts) > PAGE_MAX_CANDIDATES:
+                    full = True
+                    break
+            if full:
+                break
+        if full:
+            break
+
+    cluster_of: dict[str, str] = {}
+    for c in clusters:
+        for m in c["members"]:
+            cluster_of[m["p"]] = c["id"]
+
+    overlaps = []
+    for (i, j), shared in counts.items():
+        if shared < PAGE_MIN_SHARED:
+            continue
+        if records[i].get("sha256") and records[i].get("sha256") == records[j].get("sha256"):
+            continue
+        cid = cluster_of.get(records[i]["p"])
+        if cid and cid == cluster_of.get(records[j]["p"]):
+            continue
+        best = None
+        for pi, si in enumerate(pages[i]):
+            for pj, sj in enumerate(pages[j]):
+                inter = len(si & sj)
+                if not inter:
+                    continue
+                union = len(si | sj)
+                jac = inter / union if union else 0.0
+                contain = inter / min(len(si), len(sj))
+                if jac >= JACCARD_NEAR or contain >= CONTAIN_SUPERSET:
+                    cand = (jac, contain, pi, pj)
+                    if best is None or (cand[0], cand[1]) > (best[0], best[1]):
+                        best = cand
+        if best:
+            jac, contain, pi, pj = best
+            overlaps.append({
+                "a": records[i]["p"], "a_name": records[i].get("name", ""),
+                "a_page": pi + 1, "pages_a": len(pages[i]),
+                "b": records[j]["p"], "b_name": records[j].get("name", ""),
+                "b_page": pj + 1, "pages_b": len(pages[j]),
+                "page_jaccard": round(jac, 3),
+                "page_containment": round(contain, 3),
+            })
+    overlaps.sort(key=lambda o: (-o["page_jaccard"], -o["page_containment"]))
+    return overlaps, len(pages)
+
+
+DIFF_CAP = 20_000
+
+
+def _content_delta(keep: dict, other: dict) -> dict:
+    """Summary-only text diff; no excerpts are stored."""
+    keep_text = keep.get("_text", "") or ""
+    other_text = other.get("_text", "") or ""
+    a = re.sub(r"\s+", " ", keep_text[:DIFF_CAP]).strip()
+    b = re.sub(r"\s+", " ", other_text[:DIFF_CAP]).strip()
+    added = removed = 0
+    similarity = None
+    if a and b:
+        matcher = difflib.SequenceMatcher(None, a, b, autojunk=False)
+        similarity = round(matcher.ratio(), 4)
+        for tag, i1, i2, j1, j2 in matcher.get_opcodes():
+            if tag == "insert":
+                added += j2 - j1
+            elif tag == "delete":
+                removed += i2 - i1
+            elif tag == "replace":
+                added += j2 - j1
+                removed += i2 - i1
+    return {
+        "p": other.get("p", ""),
+        "name": other.get("name", ""),
+        "version_stage": other.get("version_stage", "normal"),
+        "chars": len(other_text),
+        "delta_chars": len(other_text) - len(keep_text),
+        "added_chars": added,
+        "removed_chars": removed,
+        "similarity": similarity,
+    }
+
+
+def _attach_content_diff(cluster: dict, records_by_p: dict[str, dict]) -> None:
+    members = [records_by_p.get(m["p"]) for m in cluster.get("members", [])]
+    members = [m for m in members if m and m.get("_text")]
+    if len(members) < 2:
+        return
+    keep_p = cluster.get("recommend", {}).get("keep") or ""
+    keep = records_by_p.get(keep_p) if keep_p else None
+    if not keep or not keep.get("_text"):
+        keep = max(members, key=lambda m: len(m.get("_text", "")))
+    others = [m for m in members if m is not keep]
+    cluster["content_diff"] = {
+        "keep": {
+            "p": keep.get("p", ""), "name": keep.get("name", ""),
+            "chars": len(keep.get("_text", "")),
+            "version_stage": keep.get("version_stage", "normal"),
+        },
+        "compared": [_content_delta(keep, m) for m in others[:5]],
+        "truncated": len(others) > 5,
+    }
+
+
+STAGE_RANK = {"final": 0, "normal": 1, "draft": 2}
+
+
+def _score_keep(r: dict) -> tuple:
+    stage = STAGE_RANK.get(r.get("version_stage"), 1)
+    return (stage, r.get("depth", 0), -float(r.get("mtime", 0) or 0),
+            len(r.get("name", "")), r.get("p", ""))
+
+
+def _keep_near(members: list[dict]) -> dict:
+    finals = [m for m in members if m.get("version_stage") == "final"]
+    pool = finals or [m for m in members if m.get("version_stage") != "draft"] or members
+    return max(pool, key=lambda m: (len(m.get("_text", "")), -m.get("depth", 0),
+                                    float(m.get("mtime", 0) or 0), m.get("p", "")))
+
+
+def _length_evidence(members: list[dict]) -> list[str]:
+    lengths = [len(m.get("_text", "")) for m in members if m.get("_text")]
+    if len(lengths) > 1 and max(lengths) != min(lengths):
+        return [f"正文字符 {min(lengths)}–{max(lengths)}（差 {max(lengths) - min(lengths)}）"]
+    return []
+
+
+def _stage_evidence(members: list[dict], keep: dict) -> list[str]:
+    evidence = []
+    if keep.get("version_stage") == "final":
+        evidence.append("文件名/路径含终稿标记")
+    if any(m.get("version_stage") == "draft" for m in members):
+        evidence.append("组内含底稿/草稿标记，优先保留终稿")
+    return evidence
+
+
+def _build_cluster(cid: str, idxs: list[int], records: list[dict], jac: float | None,
+                   contain: float | None = None) -> dict:
+    members = [records[i] for i in idxs]
+    hashes = {m.get("sha256") for m in members if m.get("sha256")}
+    ctype = "exact" if len(hashes) == 1 and hashes else "near"
+    if ctype == "exact":
+        keep = min(members, key=_score_keep)
+        folds = [m for m in members if m is not keep]
+        rec = {"keep": keep["p"], "fold": [m["p"] for m in folds], "merge": [],
+               "evidence": ["内容完全一致（sha256）", "路径更短/更新时间更近"]
+                           + _stage_evidence(members, keep), "confidence": 1.0}
+    else:
+        keep = _keep_near(members)
+        others = [m for m in members if m is not keep]
+        stage_ev = _stage_evidence(members, keep)
+        if contain is not None and contain >= CONTAIN_SUPERSET:
+            rec = {"keep": keep["p"], "fold": [m["p"] for m in others], "merge": [],
+                   "evidence": [f"内容为包含关系（覆盖率≈{contain:.2f}）", "保留内容更完整的一版"]
+                               + stage_ev,
+                   "confidence": round(max(contain, jac or 0), 2)}
+        elif jac is not None and jac >= 0.95:
+            rec = {"keep": keep["p"], "fold": [m["p"] for m in others], "merge": [],
+                   "evidence": [f"内容高度重合（Jaccard≈{jac:.2f}）"] + stage_ev,
+                   "confidence": round(jac, 2)}
+        else:
+            rec = {"keep": keep["p"], "fold": [], "merge": [m["p"] for m in others],
+                   "evidence": [f"内容部分重合（Jaccard≈{(jac or 0):.2f}）", "建议人工合并"]
+                               + stage_ev,
+                   "confidence": round(jac or 0.8, 2)}
+    rec["evidence"] = rec.get("evidence", []) + _length_evidence(members)
+    return {
+        "id": cid, "type": ctype,
+        "members": [{"p": m["p"], "name": m.get("name", ""), "size": m.get("size", 0),
+                     "mtime": m.get("mtime", 0),
+                     "depth": m.get("depth", 0),
+                     "version_stage": m.get("version_stage", "normal"),
+                     "version_number": m.get("version_number", "")} for m in members],
+        "similarity": round(jac, 3) if jac is not None else 1.0,
+        "recommend": rec,
+    }
+
+
+def _self_retrieval(records: list[dict], max_docs: int = 50) -> tuple[float | None, int, str]:
+    docs = [r for r in records if r.get("has_text") and len(r.get("title", "")) >= 4
+            and r.get("_text") and r.get("title_source") in ("frontmatter", "heading")]
+    mode = "title"
+    if len(docs) < 5:
+        docs = [r for r in records if r.get("has_text") and len(r.get("title", "")) >= 4 and r.get("_text")]
+        mode = "fallback"
+    if len(docs) < 5:
+        return None, 0, mode
+    index: dict[int, list[tuple[int, int]]] = defaultdict(list)
+    for i, r in enumerate(docs):
+        text = "".join((r["_text"] or "")[:30_000].split())
+        tf = Counter(zlib.crc32(text[j:j + 2].encode("utf-8")) for j in range(len(text) - 1))
+        for bg, n in tf.items():
+            index[bg].append((i, n))
+    n = len(docs)
+    step = max(1, len(docs) // max_docs)
+    sample = list(enumerate(docs))[::step][:max_docs]
+    hits = 0
+    for src_i, src in sample:
+        q = "".join(src["title"].split())[:60]
+        if len(q) < 2:
+            continue
+        scores: dict[int, float] = defaultdict(float)
+        for j in range(len(q) - 1):
+            bg = zlib.crc32(q[j:j + 2].encode("utf-8"))
+            post = index.get(bg)
+            if not post:
+                continue
+            idf = math.log(1 + n / len(post))
+            for doc_i, tf in post:
+                scores[doc_i] += idf * tf
+        top = sorted(scores.items(), key=lambda kv: -kv[1])[:5]
+        if src_i in [i for i, _ in top]:
+            hits += 1
+    return (hits / len(sample) if sample else None), len(sample), mode
+
+
+def analyze(job: Path, pack: str = "generic") -> dict:
+    job = Path(job)
+    fp = util.read_json(job / "fp.json")
+    if not fp:
+        raise FileNotFoundError("fp.json not found; run `kba audit start` first")
+    records = fp["files"]
+    for r in records:
+        r["_text"] = ""
+
+    params = fp.get("params") or {}
+    extraction_enabled = bool(params.get("extract_enabled", params.get("extract_dir")))
+    if "extract_enabled" not in params and not extraction_enabled:
+        # Older fingerprints have no extract params; infer from recorded statuses.
+        extraction_enabled = any(r.get("text_status") in ("extracted", "no_text", "failed")
+                                 for r in records)
+    for r in records:
+        r["_status"] = util.text_status_of(r, extraction_enabled)
+
+    # attach locally cached text (job/text, never uploaded)
+    root_path = (read_state(job).get("root") or "").strip()
+    extract_dir = Path(params["extract_dir"]) if params.get("extract_dir") else None
+    text_dir = job / "text"
+    text_recovered = 0
+    for r in records:
+        tp = text_dir / (util.safe_name(r["p"]) + ".txt")
+        if text_dir.exists() and tp.exists():
+            r["_text"] = util.read_text(tp)
+            continue
+        # Cache expired/absent: read the source once more in memory, never re-cache.
+        if not r.get("has_text"):
+            continue
+        rel = r.get("rel") or ""
+        if not rel:
+            continue
+        try:
+            if util.ext_kind(r.get("ext")) == "text" and root_path:
+                src = Path(root_path) / rel
+                if src.is_file():
+                    text = util.read_text(src)
+                    if text.strip():
+                        r["_text"] = text
+                        text_recovered += 1
+            elif util.ext_kind(r.get("ext")) == "extractable" and extract_dir:
+                ep = Path(str(Path(extract_dir) / rel) + ".txt")
+                if ep.is_file():
+                    text = util.read_text(ep)
+                    if text.strip():
+                        r["_text"] = text
+                        text_recovered += 1
+        except OSError:
+            continue
+
+    # Retention: analysis uses body text in memory, then expired cache is removed.
+    ttl = params.get("text_ttl_days")
+    text_cache_purged = fingerprint.purge_text(
+        job, older_than_days=int(ttl) if ttl is not None else util.text_ttl_days()
+    )
+    content_analyzed = sum(1 for r in records if r.get("has_text") and r.get("_text"))
+    text_missing = sum(1 for r in records if r.get("has_text") and not r.get("_text"))
+
+    total = len(records)
+    uf = _UF(total)
+    exact: dict[str, list[int]] = defaultdict(list)
+    for i, r in enumerate(records):
+        h = r.get("sha256")
+        if h:
+            exact[h].append(i)
+    for idxs in exact.values():
+        for i in idxs[1:]:
+            uf.union(idxs[0], i)
+
+    near = _near_pairs(records)
+    for i, j, _, _ in near:
+        uf.union(i, j)
+
+    comps: dict[int, list[int]] = defaultdict(list)
+    for i in range(total):
+        comps[uf.find(i)].append(i)
+
+    clusters = []
+    cid = 0
+    for idxs in comps.values():
+        if len(idxs) < 2:
+            continue
+        cid += 1
+        jac = None
+        contain = None
+        if len({records[i].get("sha256") for i in idxs}) > 1:
+            cand = [p for p in near if p[0] in idxs and p[1] in idxs]
+            jac = max((p[2] for p in cand), default=0.86)
+            contain = max((p[3] for p in cand), default=0.0)
+        clusters.append(_build_cluster(f"c{cid:03d}", idxs, records, jac, contain))
+
+    in_cluster = {m["p"] for c in clusters for m in c["members"]}
+    by_name: dict[str, list[int]] = defaultdict(list)
+    for i, r in enumerate(records):
+        by_name[util.normalize_name(r.get("name", ""))].append(i)
+    for name, idxs in by_name.items():
+        if len(idxs) < 2:
+            continue
+        if any(records[i]["p"] in in_cluster for i in idxs):
+            continue
+        if len({records[i].get("sha256") for i in idxs}) == 1:
+            continue
+        cid += 1
+        clusters.append(_build_cluster(f"c{cid:03d}", idxs, records, None, None))
+        clusters[-1]["type"] = "same-name"
+        clusters[-1]["similarity"] = 0.0
+        clusters[-1]["recommend"] = {
+            "keep": "", "fold": [], "merge": [],
+            "evidence": ["文件名相同但内容不同（同名异内容）", "建议人工判定，全部保留并标记 dis"],
+            "confidence": 0.6,
+        }
+
+    records_by_p = {r["p"]: r for r in records}
+    for c in clusters:
+        _attach_content_diff(c, records_by_p)
+
+    redundant = sum(max(0, len(c["members"]) - 1) for c in clusters if c["type"] in ("exact", "near"))
+    dup_rate = redundant / total if total else 0.0
+    same_name_clusters = [c for c in clusters if c["type"] == "same-name"]
+
+    final_files = sum(1 for r in records if r.get("version_stage") == "final")
+    draft_files = sum(1 for r in records if r.get("version_stage") == "draft")
+    version_conflicts = []
+    for c in clusters:
+        finals = [m["name"] for m in c["members"] if m.get("version_stage") == "final"]
+        drafts = [m["name"] for m in c["members"] if m.get("version_stage") == "draft"]
+        if finals and drafts:
+            keep = next((m["name"] for m in c["members"] if m["p"] == c["recommend"].get("keep")), "")
+            version_conflicts.append({"id": c["id"], "keep": keep, "final": finals, "draft": drafts})
+
+    page_overlaps, page_docs = _page_overlaps(records, clusters)
+
+    clusters_by_id = {c["id"]: c for c in clusters}
+    manual_queue = []
+    for c in same_name_clusters:
+        manual_queue.append({
+            "type": "same-name", "id": c["id"],
+            "files": [m["name"] for m in c["members"]],
+            "action": "人工判定，全部保留并标记 dis",
+            "diff": "",
+        })
+    for v in version_conflicts:
+        c = clusters_by_id.get(v["id"]) or {}
+        diff = c.get("content_diff") or {}
+        compared = (diff.get("compared") or [{}])[0]
+        keep_name = (v.get("keep") or (diff.get("keep") or {}).get("name")
+                     or (v["final"][0] if v.get("final") else ""))
+        note = ""
+        if compared and keep_name:
+            sim = compared.get("similarity")
+            note = (f"{keep_name} vs {compared.get('name', '')}："
+                    f"{int(compared.get('delta_chars') or 0):+d} 字"
+                    + (f"，相似度 {sim:.2f}" if isinstance(sim, (int, float)) else ""))
+        manual_queue.append({
+            "type": "version", "id": v["id"],
+            "files": list(v["final"]) + list(v["draft"]),
+            "action": "保留终稿；确认正文差异后再处置",
+            "diff": note,
+        })
+    for o in page_overlaps:
+        manual_queue.append({
+            "type": "page-overlap", "id": "",
+            "files": [o["a_name"], o["b_name"]],
+            "action": f"人工确认共享页（页 Jaccard {o['page_jaccard']:.2f}）",
+            "diff": "",
+        })
+
+    pii_counts: Counter = Counter()
+    pii_affected = 0
+    pii_examples = []
+    for r in records:
+        hits = pii.scan_text(r.get("_text", ""))
+        if not hits:
+            continue
+        pii_affected += 1
+        for kind in hits:
+            pii_counts[kind] += 1
+        if len(pii_examples) < 50:
+            pii_examples.append({"p": r.get("p", ""), "name": r.get("name", ""),
+                                 "types": sorted(hits)})
+
+    # Excel models are versioned by structure (sheets/formulas/defined names), not text alone.
+    xlsx_with_fp = [r for r in records
+                    if isinstance(r.get("xlsx_fp"), dict) and r["xlsx_fp"].get("sha256")]
+    xlsx_groups: dict[str, list[dict]] = defaultdict(list)
+    for r in xlsx_with_fp:
+        xlsx_groups[r["xlsx_fp"]["sha256"]].append(r)
+    xlsx_model_groups = []
+    for fp_hash, members in xlsx_groups.items():
+        distinct = {m.get("sha256") for m in members if m.get("sha256")}
+        if len(members) < 2 or len(distinct) < 2:
+            continue
+        first_fp = members[0]["xlsx_fp"]
+        # Plain value workbooks with identical sheet names are not "model versions".
+        if not (first_fp.get("formula_cells") or first_fp.get("defined_names")):
+            continue
+        xlsx_model_groups.append({
+            "fingerprint": fp_hash,
+            "sheets": first_fp.get("sheets") or [],
+            "formula_cells": int(first_fp.get("formula_cells") or 0),
+            "defined_names": first_fp.get("defined_names") or [],
+            "versions": len(distinct),
+            "members": [{"p": m["p"], "name": m.get("name", ""), "size": m.get("size", 0),
+                         "mtime": m.get("mtime", 0), "sha256": m.get("sha256", "")}
+                        for m in members],
+        })
+    xlsx_model_groups.sort(key=lambda g: (-g["versions"], -len(g["members"]), g["fingerprint"]))
+
+    stale = [r for r in records if util.mtime_days_ago(r.get("mtime", 0)) > STALE_DAYS]
+    stale_rate = len(stale) / total if total else 0.0
+    recent = sum(1 for r in records if util.mtime_days_ago(r.get("mtime", 0)) <= 30)
+    stale_reliable = not (total and recent / total > 0.9)
+
+    tagged = [r for r in records if r.get("tags")]
+    valid = [r for r in records if len(r.get("tags") or []) >= 3]
+    fm_tagged = [r for r in records if r.get("tags_frontmatter")]
+    inferred_tagged = [r for r in records if r.get("tags_inferred")]
+    tag_coverage = len(tagged) / total if total else 0.0
+    valid_tag_coverage = len(valid) / total if total else 0.0
+    frontmatter_tag_coverage = len(fm_tagged) / total if total else 0.0
+    inferred_tag_coverage = len(inferred_tagged) / total if total else 0.0
+    owner_files = sum(1 for r in records if r.get("owner"))
+    broken_link_files = sum(1 for r in records if r.get("broken_links"))
+    broken_links_total = sum(int(r.get("broken_links") or 0) for r in records)
+    text_files = [r for r in records if util.ext_kind(r.get("ext")) == "text"]
+    text_content_analyzed = sum(1 for r in text_files if r.get("has_text") and r.get("_text"))
+    text_tagged = [r for r in text_files if r.get("tags")]
+    text_valid = [r for r in text_files if len(r.get("tags") or []) >= 3]
+    text_tag_coverage = len(text_tagged) / len(text_files) if text_files else 0.0
+    text_valid_tag_coverage = len(text_valid) / len(text_files) if text_files else 0.0
+    text_unreadable = sum(1 for r in text_files
+                          if not r.get("has_text") and (r.get("size") or 0) > 0)
+    dom_c = Counter(r.get("domain") for r in records if r.get("domain"))
+
+    extractable = [r for r in records if util.ext_kind(r.get("ext")) == "extractable"]
+    unsupported_files = [r for r in records if util.ext_kind(r.get("ext")) == "unsupported"]
+    binaries = extractable + unsupported_files
+    # Coverage must reflect text this run can actually read, not just the fingerprint.
+    binary_with_text = [r for r in binaries if r.get("has_text") and r.get("_text")]
+    text_coverage = (len(binary_with_text) / len(binaries)) if binaries else None
+    confidence = "C" if total == 0 else _confidence(text_coverage)
+    # Fingerprint coverage cannot exceed what this run could actually read.
+    if content_analyzed == 0 and total:
+        confidence = "C"
+
+    status_counts = Counter(r["_status"] for r in records)
+    unsupported_by_ext = Counter(r.get("ext") for r in unsupported_files)
+    extract_ok = sum(1 for r in extractable if r["_status"] == "extracted")
+    extract_no_text = sum(1 for r in extractable if r["_status"] == "no_text")
+    extract_failed = sum(1 for r in extractable if r["_status"] == "failed")
+    extract_not_enabled = sum(1 for r in extractable if r["_status"] == "not_enabled")
+    extract_attempted = extract_ok + extract_no_text + extract_failed
+    extract_fail = round(extract_failed / extract_attempted, 4) if extract_attempted else None
+    extract_no_text_rate = round(extract_no_text / extract_attempted, 4) if extract_attempted else None
+
+    depth_bad = [r for r in records if r.get("depth", 0) > 4]
+    root_files = [r for r in records if r.get("depth", 0) == 0]
+    naming_bad = [r for r in records if re.search(r"无标题|untitled|^\d{4}[-_]?\d{2}|__dup\d+", r.get("name", ""), re.I)]
+    empty_dirs = []
+    if root_path and Path(root_path).exists():
+        for dirpath, dirnames, filenames in os.walk(root_path):
+            if not dirnames and not filenames:
+                empty_dirs.append(dirpath)
+    structure_files = {r["p"] for r in depth_bad + naming_bad}
+    structure_rate = (len(structure_files) + len(root_files)) / total if total else 0.0
+    structure_rate = min(1.0, structure_rate)
+
+    top5_hit, top5_n, top5_mode = _self_retrieval(records)
+
+    def _clamp(v) -> float:
+        return max(0.0, min(1.0, float(v)))
+
+    health_components: list[dict] = []
+
+    def _component(key: str, label: str, weight: int, value, badness, available: bool, note: str):
+        health_components.append({
+            "key": key, "label": label, "weight": weight,
+            "available": bool(available),
+            "value": round(value, 4) if value is not None else None,
+            "badness": round(_clamp(badness), 4) if (available and badness is not None) else None,
+            "note": note,
+        })
+
+    _component("duplicate", "重复", 30, dup_rate, dup_rate, True,
+               "SHA256 精确分组 + 文本近重复")
+    if stale_reliable:
+        _component("stale", "过时", 20, stale_rate, stale_rate, True, "24 个月未更新占比")
+    else:
+        _component("stale", "过时", 20, None, None, False,
+                   "mtime 可能被复制/迁移重置，不计分")
+    if total:
+        _component("tagging", "标签", 15, valid_tag_coverage,
+                   1 - valid_tag_coverage, True,
+                   f"frontmatter+路径/内容推断（推断 {inferred_tag_coverage:.0%}）")
+    else:
+        _component("tagging", "标签", 15, None, None, False, "无文件，标签不可评估")
+    _component("structure", "结构", 15, structure_rate, structure_rate, True,
+               "层级/根目录散文件/命名")
+    if top5_hit is not None:
+        _component("retrieval", "自检索", 20, top5_hit, 1 - top5_hit, True,
+                   "标题反查代理指标")
+    else:
+        _component("retrieval", "自检索", 20, None, None, False,
+                   "文本样本不足，不以 0.5 顶替")
+
+    available_components = [c for c in health_components if c["available"]]
+    weight_sum = sum(c["weight"] for c in available_components)
+    weight_coverage = round(weight_sum / HEALTH_WEIGHT_TOTAL, 4) if HEALTH_WEIGHT_TOTAL else 0.0
+    if weight_sum:
+        penalty = sum(c["badness"] * c["weight"] for c in available_components) / weight_sum * 100
+        health = max(0, min(100, round(100 - penalty)))
+    else:
+        health = None
+    # C-level reports must never publish a composite score; keep the raw value
+    # for internal benchmarking only (reports/summaries do not read it).
+    published_score = None if confidence == "C" else health
+    health_info = {
+        "score": published_score,
+        "score_raw": health,
+        "weight_coverage": weight_coverage,
+        "weights": {c["key"]: c["weight"] for c in health_components},
+        "components": health_components,
+        "excluded": [c["key"] for c in health_components if not c["available"]],
+    }
+
+    data_coverage = {
+        "confidence": confidence,
+        "confidence_thresholds": {"A": CONFIDENCE_A, "B": CONFIDENCE_B},
+        "confidence_reason": ("empty" if total == 0
+                              else "no_content" if content_analyzed == 0
+                              else "text_coverage"),
+        "empty_corpus": total == 0,
+        "text_coverage": round(text_coverage, 4) if text_coverage is not None else None,
+        "extraction_enabled": extraction_enabled,
+        "binary_files": len(binaries),
+        "binary_with_text": len(binary_with_text),
+        "content_analyzed": content_analyzed,
+        "text_missing": text_missing,
+        "text_files": len(text_files),
+        "text_content_analyzed": text_content_analyzed,
+        "text_unreadable": text_unreadable,
+        "extractable_files": len(extractable),
+        "extracted": extract_ok,
+        "no_text": extract_no_text,
+        "failed": extract_failed,
+        "not_enabled": extract_not_enabled,
+        "extract_attempted": extract_attempted,
+        "unsupported": len(unsupported_files),
+        "unsupported_by_ext": dict(unsupported_by_ext.most_common()),
+        "text_status": dict(status_counts),
+        "text_ttl_days": params.get("text_ttl_days", util.text_ttl_days()),
+        "text_cache_purged": text_cache_purged,
+        "text_recovered": text_recovered,
+    }
+
+    findings = []
+    if redundant:
+        findings.append({"type": "duplicate", "count": redundant, "severity": "high",
+                         "title": f"{len([c for c in clusters if c['type'] in ('exact','near')])} 组重复/近重复，冗余 {redundant} 个",
+                         "examples": [c["members"][0]["name"] for c in clusters[:5]]})
+    if same_name_clusters:
+        findings.append({"type": "same-name", "count": len(same_name_clusters), "severity": "medium",
+                         "title": f"{len(same_name_clusters)} 组同名异内容，需人工判定（标记 dis，不自动处置）",
+                         "examples": [c["members"][0]["name"] for c in same_name_clusters[:5]]})
+    if stale:
+        findings.append({"type": "stale", "count": len(stale), "severity": "medium",
+                         "title": f"{len(stale)} 个文件超过 24 个月未更新",
+                         "examples": [r["name"] for r in stale[:5]]})
+    if total and valid_tag_coverage < 0.9:
+        findings.append({"type": "tagging", "count": total - len(valid), "severity": "medium",
+                         "title": f"有效标签覆盖率 {valid_tag_coverage:.0%}（frontmatter {frontmatter_tag_coverage:.0%}，推断 {inferred_tag_coverage:.0%}）",
+                         "examples": [r["name"] for r in records if len(r.get("tags") or []) < 3][:5]})
+    if broken_link_files:
+        findings.append({"type": "broken-link", "count": broken_links_total, "severity": "medium",
+                         "title": f"{broken_link_files} 个文件有 {broken_links_total} 个断链（本地相对链接目标不存在）",
+                         "examples": [r["name"] for r in records if r.get("broken_links")][:5]})
+    if text_unreadable:
+        findings.append({"type": "text-unreadable", "count": text_unreadable, "severity": "medium",
+                         "title": f"{text_unreadable} 个文本文件无法解码/读取，未参与内容分析",
+                         "examples": [r["name"] for r in text_files
+                                      if not r.get("has_text") and (r.get("size") or 0) > 0][:5]})
+    if text_missing:
+        findings.append({"type": "text-missing", "count": text_missing, "severity": "medium",
+                         "title": f"{text_missing} 个文件的正文在分析时不可读（缓存过期且源文件/抽取结果不可用）",
+                         "examples": [r["name"] for r in records
+                                      if r.get("has_text") and not r.get("_text")][:5]})
+    if extract_failed:
+        findings.append({"type": "extract", "count": extract_failed, "severity": "medium",
+                         "title": f"{extract_failed} 个 pdf/docx 抽取失败（已尝试 {extract_attempted} 个）",
+                         "examples": [r["name"] for r in extractable if r["_status"] == "failed"][:5]})
+    if extract_no_text:
+        findings.append({"type": "extract-no-text", "count": extract_no_text, "severity": "low",
+                         "title": f"{extract_no_text} 个 pdf/docx 无可用文本（扫描件候选，需 OCR）",
+                         "examples": [r["name"] for r in extractable if r["_status"] == "no_text"][:5]})
+    if unsupported_files:
+        exts = "、".join(ext for ext, _ in unsupported_by_ext.most_common())
+        findings.append({"type": "unsupported", "count": len(unsupported_files), "severity": "medium",
+                         "title": f"{len(unsupported_files)} 个未支持格式（{exts}）仅计入盘点，未做内容分析",
+                         "examples": [r["name"] for r in unsupported_files][:5]})
+    if xlsx_model_groups:
+        members = sum(len(g["members"]) for g in xlsx_model_groups)
+        findings.append({"type": "xlsx-model", "count": len(xlsx_model_groups), "severity": "medium",
+                         "title": f"{len(xlsx_model_groups)} 组 Excel 同结构不同内容（模型版本候选，共 {members} 个文件）",
+                         "examples": [g["members"][0]["name"] for g in xlsx_model_groups[:5]]})
+    if version_conflicts:
+        findings.append({"type": "version", "count": len(version_conflicts), "severity": "medium",
+                         "title": f"{len(version_conflicts)} 组重复/近重复同时含终稿与底稿，建议保留终稿并人工确认",
+                         "examples": [v["keep"] for v in version_conflicts[:5]]})
+    if page_overlaps:
+        findings.append({"type": "page-overlap", "count": len(page_overlaps), "severity": "medium",
+                         "title": f"{len(page_overlaps)} 对文档存在页级高重合（可能共享页/转载），建议人工确认，不自动合并",
+                         "examples": [f"{o['a_name']} ↔ {o['b_name']}" for o in page_overlaps[:3]]})
+    if pii_affected:
+        kinds = "、".join(pii.LABELS.get(k, k) for k in sorted(pii_counts))
+        findings.append({"type": "pii", "count": pii_affected, "severity": "high",
+                         "title": f"{pii_affected} 个文件命中敏感信息候选（{kinds}）；报告仅显示计数，建议人工核查",
+                         "examples": [p["name"] for p in pii_examples[:5]]})
+    if structure_files or root_files or empty_dirs:
+        findings.append({"type": "structure", "count": len(structure_files) + len(root_files) + len(empty_dirs), "severity": "low",
+                         "title": f"结构问题：{len(depth_bad)} 个层级过深、{len(root_files)} 个根目录散文件、{len(naming_bad)} 个命名问题、{len(empty_dirs)} 个空目录",
+                         "examples": [r["name"] for r in (depth_bad + root_files)[:5]]})
+    if top5_hit is not None and top5_hit < 0.85:
+        findings.append({"type": "retrieval", "count": top5_n, "severity": "high",
+                         "title": f"自检索 Top-5 命中率 {top5_hit:.0%}（{top5_n} 个查询抽样）",
+                         "examples": []})
+    if confidence == "C":
+        if content_analyzed == 0 and total:
+            coverage_title = "数据置信度 C：本次分析没有可读取的正文，本报告为元数据盘点版，不展示综合健康分"
+        else:
+            cov_pct = f"{text_coverage:.0%}" if text_coverage is not None else "—"
+            coverage_title = f"数据置信度 C：二进制文本覆盖率 {cov_pct}，本报告为元数据盘点版，不展示综合健康分"
+        findings.insert(0, {
+            "type": "coverage", "count": len(binaries) - len(binary_with_text), "severity": "high",
+            "title": coverage_title,
+            "examples": [],
+        })
+
+    effort = min(60.0, round(
+        redundant * 0.05 + len([c for c in clusters if c["type"] == "near"]) * 0.1
+        + len(stale) * 0.005 + (total - len(tagged)) * 0.002
+        + (len(structure_files) + len(root_files)) * 0.01, 1)) or 1.0
+
+    plan = {
+        "effort_days": effort,
+        "weeks": [
+            {"week": 1, "tasks": ["确认分类标准与裁决规则", "处理完全重复（折叠，不删除）"], "owner": "客户+顾问"},
+            {"week": 2, "tasks": ["处理近重复与同名异内容", "修正目录层级与命名"], "owner": "客户"},
+            {"week": 3, "tasks": ["补齐标签与负责人", "处理扫描件/抽取失败"], "owner": "客户"},
+            {"week": 4, "tasks": ["重建索引", "抽检检索命中率", "出验收报告"], "owner": "顾问"},
+        ],
+    }
+
+    result = {
+        "schema": "kba.analysis/1",
+        "job_id": fp.get("job_id"),
+        "pack": pack,
+        "confidence": confidence,
+        "health_score": published_score,
+        "health": health_info,
+        "data_coverage": data_coverage,
+        "metrics": {
+            "files": total,
+            "dup_groups": len([c for c in clusters if c["type"] in ("exact", "near")]),
+            "dup_redundant": redundant,
+            "dup_rate": round(dup_rate, 4),
+            "same_name_groups": len(same_name_clusters),
+            "same_name_members": sum(len(c["members"]) for c in same_name_clusters),
+            "final_files": final_files,
+            "draft_files": draft_files,
+            "version_conflicts": len(version_conflicts),
+            "manual_queue": len(manual_queue),
+            "page_overlap_pairs": len(page_overlaps),
+            "page_overlap_files": len({o["a"] for o in page_overlaps} | {o["b"] for o in page_overlaps}),
+            "page_aware_docs": page_docs,
+            "pii_files": pii_affected,
+            "pii_types": len(pii_counts),
+            "stale_rate": round(stale_rate, 4),
+            "stale_reliable": stale_reliable,
+            "tag_coverage": round(tag_coverage, 4),
+            "valid_tag_coverage": round(valid_tag_coverage, 4),
+            "frontmatter_tag_coverage": round(frontmatter_tag_coverage, 4),
+            "inferred_tag_coverage": round(inferred_tag_coverage, 4),
+            "text_tag_coverage": round(text_tag_coverage, 4),
+            "text_valid_tag_coverage": round(text_valid_tag_coverage, 4),
+            "owner_coverage": round(owner_files / total, 4) if total else 0.0,
+            "broken_link_files": broken_link_files,
+            "broken_links": broken_links_total,
+            "by_domain": dict(dom_c.most_common(20)),
+            "confidence": confidence,
+            "text_coverage": round(text_coverage, 4) if text_coverage is not None else None,
+            "binary_files": len(binaries),
+            "text_files": len(text_files),
+            "text_unreadable": text_unreadable,
+            "content_analyzed": content_analyzed,
+            "text_missing": text_missing,
+            "text_content_analyzed": text_content_analyzed,
+            "extractable_files": len(extractable),
+            "unsupported_files": len(unsupported_files),
+            "unsupported_by_ext": dict(unsupported_by_ext.most_common()),
+            "xlsx_with_fp": len(xlsx_with_fp),
+            "xlsx_model_groups": len(xlsx_model_groups),
+            "xlsx_model_members": sum(len(g["members"]) for g in xlsx_model_groups),
+            "extract_status": {
+                "extracted": extract_ok,
+                "no_text": extract_no_text,
+                "failed": extract_failed,
+                "not_enabled": extract_not_enabled,
+                "unsupported": len(unsupported_files),
+            },
+            "extract_fail": extract_fail,
+            "extract_no_text": extract_no_text_rate,
+            "weight_coverage": weight_coverage,
+            "top5_hit": round(top5_hit, 4) if top5_hit is not None else None,
+            "top5_samples": top5_n,
+            "top5_mode": top5_mode,
+            "structure_rate": round(structure_rate, 4),
+        },
+        "clusters": clusters,
+        "xlsx_model_groups": xlsx_model_groups,
+        "version_conflicts": version_conflicts,
+        "manual_queue": manual_queue,
+        "page_overlaps": page_overlaps,
+        "pii": {
+            "files": pii_affected,
+            "types": dict(sorted(pii_counts.items())),
+            "examples": pii_examples,
+        },
+        "findings": findings,
+        "plan": plan,
+        "meta": {"created_at": util.now_iso(), "engine": "kba-analyze/1"},
+    }
+    util.write_json(job / "analysis.json", result)
+    write_state(job, status="done", stage="analyze", health_score=published_score,
+                confidence=confidence, weight_coverage=weight_coverage)
+    return result