kba-cli 0.3.0rc4__py3-none-any.whl

kba/fingerprint.py

@@ -0,0 +1,333 @@
+"""Read-only fingerprint: metadata + hash, no content leaves the machine."""
+from __future__ import annotations
+
+import json
+import time
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from . import links as link_rules
+from . import tags as tag_rules
+from . import util
+from .extract import read_manifest
+from .jobs import jobs_root, write_state
+
+TEXT_CACHE_DIR = "text"
+
+
+def _extract_text(p: Path, rel: str, ext: str, extract_dir: Path | None) -> tuple[str, str]:
+    """Return (text, text_status).
+
+    text_status is one of: text | extracted | no_text | failed | not_enabled | unsupported.
+    The tri-state matters for honesty: "not extracted" is not an extraction failure.
+    """
+    if ext in util.TEXT_EXTS:
+        try:
+            return util.read_text(p), "text"
+        except OSError:
+            return "", "failed"
+    if ext in util.UNSUPPORTED_TEXT_EXTS:
+        return "", "unsupported"
+    if extract_dir is None:
+        return "", "not_enabled"
+    ep = Path(extract_dir) / rel
+    ep = Path(str(ep) + ".txt")
+    if not ep.exists():
+        return "", "failed"
+    try:
+        text = util.read_text(ep)
+    except OSError:
+        return "", "failed"
+    if not text.strip():
+        return text, "no_text"
+    return text, "extracted"
+
+
+def purge_text(job: Path, older_than_days: int | None = None) -> int:
+    """Delete cached body text under job/text/.
+
+    older_than_days=None purges everything; otherwise only files older than N days.
+    Returns the number of removed cache files.
+    """
+    text_dir = Path(job) / TEXT_CACHE_DIR
+    if not text_dir.is_dir():
+        return 0
+    cutoff = None if older_than_days is None else time.time() - max(0, int(older_than_days)) * 86400
+    removed = 0
+    for p in text_dir.rglob("*"):
+        if not p.is_file():
+            continue
+        try:
+            if cutoff is not None and p.stat().st_mtime >= cutoff:
+                continue
+            p.unlink()
+            removed += 1
+        except OSError:
+            continue
+    for d in sorted((d for d in text_dir.rglob("*") if d.is_dir()),
+                    key=lambda d: len(d.parts), reverse=True):
+        try:
+            d.rmdir()
+        except OSError:
+            pass
+    try:
+        if not any(text_dir.iterdir()):
+            text_dir.rmdir()
+    except OSError:
+        pass
+    return removed
+
+
+def sweep_expired_text(ttl_days: int | None = None, skip_job: str | None = None) -> dict:
+    """Apply the text-cache TTL to all jobs (except skip_job)."""
+    ttl = util.text_ttl_days() if ttl_days is None else max(0, int(ttl_days))
+    removed = 0
+    hit_jobs = 0
+    for p in jobs_root().iterdir():
+        if not p.is_dir() or (skip_job and p.name == skip_job):
+            continue
+        n = purge_text(p, older_than_days=ttl)
+        if n:
+            removed += n
+            hit_jobs += 1
+            try:
+                write_state(p, text_cache_purged_at=util.now_iso(),
+                            text_cache_purged_files=n, text_ttl_days=ttl)
+            except OSError:
+                pass
+    return {"files": removed, "jobs": hit_jobs, "ttl_days": ttl}
+
+
+def _describe(p: Path, root: Path, extract_dir: Path | None, redact: bool,
+              extract_manifest: dict | None = None):
+    rel = util.rel_posix(p, root)
+    st = p.stat()
+    ext = p.suffix.lower()
+    text, text_status = _extract_text(p, rel, ext, extract_dir)
+    meta = (extract_manifest or {}).get(rel) or {}
+    if text_status == "failed" and meta.get("status") in ("no_text", "failed", "unsupported"):
+        text_status = meta["status"]
+    fm, body = util.parse_frontmatter(text) if ext in util.TEXT_EXTS else ({}, text)
+    title = fm.get("title") if isinstance(fm.get("title"), str) else ""
+    title_src = "frontmatter" if title else ""
+    if not title:
+        for line in (body or "").splitlines():
+            if line.startswith("# "):
+                title = line[2:].strip()
+                title_src = "heading"
+                break
+    if not title:
+        title = p.stem
+        title_src = "stem"
+    fm_tags = fm.get("topicTags", []) or []
+    if isinstance(fm_tags, str):
+        fm_tags = [t.strip().strip("'\"") for t in fm_tags.strip("[]").split(",") if t.strip()]
+    fm_tags = [str(t) for t in fm_tags] if isinstance(fm_tags, list) else []
+    fm_domain = fm.get("domain", "") if isinstance(fm.get("domain", ""), str) else ""
+    tags, tag_sources, inferred_tags, domain, domain_source = tag_rules.infer_metadata(
+        rel, p.name, body, fm_tags, fm_domain
+    )
+    owner = fm.get("owner") or fm.get("负责人") or fm.get("Owner") or ""
+    owner = str(owner).strip()[:120] if isinstance(owner, (str, int)) else ""
+    links_total, broken = ((link_rules.scan_links(body or "", p.parent)
+                            if ext in util.TEXT_EXTS else (0, []))
+                           if body else (0, []))
+    key = util.safe_name(rel) if redact else rel
+    name = util.safe_name(rel) + ext if redact else p.name
+    return {
+        "p": key,
+        "name": name,
+        "rel": "" if redact else rel,
+        "ext": ext,
+        "size": st.st_size,
+        "sha256": "",
+        "mtime": st.st_mtime,
+        "depth": rel.count("/"),
+        "chars": len(body) if body else 0,
+        "has_text": bool((body or "").strip()),
+        "text_status": text_status,
+        "text_error": str(meta.get("error") or "")[:300],
+        "xlsx_fp": meta.get("xlsx_fp") if isinstance(meta.get("xlsx_fp"), dict) else None,
+        "version_stage": util.version_stage(p.name, rel),
+        "version_number": util.version_number(p.name, rel),
+        "title": title[:200],
+        "title_source": title_src,
+        "tags": tags,
+        "tags_frontmatter": fm_tags,
+        "tags_inferred": inferred_tags,
+        "tag_sources": tag_sources,
+        "domain": domain,
+        "domain_source": domain_source,
+        "owner": owner,
+        "links_total": links_total,
+        "broken_links": len(broken),
+        "broken_examples": broken[:3],
+        "reviewed": bool(fm.get("reviewed")) if fm else False,
+    }, (body or "")
+
+
+def scan(
+    root: Path,
+    job: Path,
+    exclude=None,
+    max_file_mb: int = 200,
+    extract_dir: Path | None = None,
+    redact: bool = False,
+    progress=None,
+    text_ttl_days: int | None = None,
+) -> dict:
+    root = Path(root).resolve()
+    job = Path(job)
+    job.mkdir(parents=True, exist_ok=True)
+    extract_dir = Path(extract_dir).resolve() if extract_dir else None
+    extract_manifest = read_manifest(extract_dir) if extract_dir else {}
+    if extract_manifest:
+        # Redacted fingerprints use safe_name(rel) as p; index both.
+        aliased = {}
+        for rel, entry in extract_manifest.items():
+            aliased[rel] = entry
+            aliased[util.safe_name(rel)] = entry
+        extract_manifest = aliased
+    ttl = util.text_ttl_days() if text_ttl_days is None else max(0, int(text_ttl_days))
+
+    files = list(util.iter_files(root, exclude=exclude))
+    write_state(job, status="running", stage="fingerprint", root=str(root),
+                total=len(files), done=0, started_at=util.now_iso())
+
+    jsonl = job / "files.jsonl"
+    done_keys: set[str] = set()
+    done_records: list[dict] = []
+    if jsonl.exists():
+        for line in jsonl.read_text(encoding="utf-8").splitlines():
+            try:
+                rec = json.loads(line)
+                done_keys.add(rec["p"])
+                done_records.append(rec)
+            except (ValueError, KeyError):
+                continue
+
+    max_bytes = max_file_mb * 1024 * 1024
+    text_dir = job / "text"
+    text_dir.mkdir(parents=True, exist_ok=True)
+
+    # If the body cache was purged, resuming would silently lose text. Force a clean rescan.
+    cache_missing = any(
+        rec.get("has_text") and not (text_dir / (util.safe_name(rec["p"]) + ".txt")).exists()
+        for rec in done_records
+    )
+    # A job scanned before `kba extract` must not silently keep "not extracted" records.
+    needs_extract_rescan = any(
+        (extract_manifest.get(rec.get("p")) or {}).get("status") in ("extracted", "no_text")
+        and rec.get("text_status") not in ("extracted", "no_text")
+        for rec in done_records
+    )
+    if cache_missing or needs_extract_rescan:
+        try:
+            jsonl.unlink()
+        except OSError:
+            pass
+        else:
+            done_keys.clear()
+
+    def work(p: Path):
+        rec, text = _describe(p, root, extract_dir, redact, extract_manifest)
+        if rec["size"] > max_bytes:
+            rec["skipped"] = "too_large"
+            rec["sha256"] = ""
+        else:
+            rec["sha256"] = util.sha256_file(p)
+        return rec, text
+
+    todo = []
+    for p in files:
+        rel_key = util.safe_name(util.rel_posix(p, root)) if redact else util.rel_posix(p, root)
+        if rel_key not in done_keys:
+            todo.append(p)
+
+    done = len(done_keys)
+    errors = 0
+    with open(jsonl, "a", encoding="utf-8") as f, ThreadPoolExecutor(max_workers=8) as ex:
+        futures = {ex.submit(work, p): p for p in todo}
+        for fut in as_completed(futures):
+            p = futures[fut]
+            try:
+                rec, text = fut.result()
+            except Exception as exc:  # noqa: BLE001
+                errors += 1
+                rec, text = {"p": util.rel_posix(p, root), "name": p.name, "ext": p.suffix.lower(),
+                             "size": 0, "sha256": "", "mtime": 0, "depth": 0, "chars": 0,
+                             "has_text": False, "text_status": "failed",
+                             "text_error": str(exc)[:300], "xlsx_fp": None,
+                             "version_stage": util.version_stage(p.name, util.rel_posix(p, root)),
+                             "version_number": "", "title": p.stem, "title_source": "stem",
+                             "tags": [], "tags_frontmatter": [], "tags_inferred": [],
+                             "tag_sources": [], "domain": "", "domain_source": "",
+                             "owner": "", "links_total": 0, "broken_links": 0,
+                             "broken_examples": [], "reviewed": False, "error": str(exc)}, ""
+            if text and text.strip():
+                try:
+                    (text_dir / (util.safe_name(rec["p"]) + ".txt")).write_text(text, encoding="utf-8")
+                except OSError:
+                    pass
+            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+            f.flush()
+            done += 1
+            if progress and done % 20 == 0:
+                progress(done, len(files))
+            if done % 20 == 0:
+                write_state(job, done=done)
+
+    records = []
+    for line in jsonl.read_text(encoding="utf-8").splitlines():
+        try:
+            records.append(json.loads(line))
+        except ValueError:
+            continue
+
+    by_ext: dict[str, int] = {}
+    unsupported_by_ext: dict[str, int] = {}
+    status_counts: Counter = Counter()
+    total_bytes = 0
+    max_depth = 0
+    for r in records:
+        by_ext[r["ext"]] = by_ext.get(r["ext"], 0) + 1
+        if r.get("ext") in util.UNSUPPORTED_TEXT_EXTS:
+            unsupported_by_ext[r["ext"]] = unsupported_by_ext.get(r["ext"], 0) + 1
+        status_counts[util.text_status_of(r, extract_dir is not None)] += 1
+        total_bytes += r.get("size", 0)
+        max_depth = max(max_depth, r.get("depth", 0))
+
+    fp = {
+        "schema": "kba.fingerprint/1",
+        "job_id": job.name,
+        "created_at": util.now_iso(),
+        "tool": {"name": "kba", "version": __import__("kba").__version__},
+        "root": {"redacted": redact, "name": root.name, "files": len(records)},
+        "params": {
+            "max_file_mb": max_file_mb,
+            "exclude": sorted(exclude or util.DEFAULT_EXCLUDE),
+            "extract_enabled": extract_dir is not None,
+            "extract_dir": str(extract_dir) if (extract_dir and not redact) else None,
+            "text_ttl_days": ttl,
+        },
+        "summary": {
+            "files": len(records),
+            "bytes": total_bytes,
+            "by_ext": dict(sorted(by_ext.items(), key=lambda kv: -kv[1])),
+            "max_depth": max_depth,
+            "errors": errors,
+            "skipped": sum(1 for r in records if r.get("skipped")),
+            "with_text": sum(1 for r in records if r.get("has_text")),
+            "unsupported": sum(unsupported_by_ext.values()),
+            "unsupported_by_ext": dict(sorted(unsupported_by_ext.items(), key=lambda kv: -kv[1])),
+            "text_status": dict(sorted(status_counts.items())),
+            "xlsx_with_fp": sum(1 for r in records if isinstance(r.get("xlsx_fp"), dict)),
+            "text_ttl_days": ttl,
+        },
+        "files": records,
+    }
+    util.write_json(job / "fp.json", fp)
+    write_state(job, status="done", stage="fingerprint", done=len(records), total=len(files),
+                finished_at=util.now_iso())
+    return fp

kba/jobs.py

@@ -0,0 +1,69 @@
+"""Job storage under ~/.kba/jobs/<job_id>/."""
+from __future__ import annotations
+
+import json
+import os
+import secrets
+import time
+from datetime import datetime
+from pathlib import Path
+
+
+def kba_home() -> Path:
+    return Path(os.environ.get("KBA_HOME", Path.home() / ".kba"))
+
+
+def jobs_root() -> Path:
+    d = kba_home() / "jobs"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def new_job_id() -> str:
+    return datetime.now().strftime("%Y%m%d-%H%M%S-") + secrets.token_hex(2)
+
+
+def job_dir(job_id: str) -> Path:
+    return jobs_root() / job_id
+
+
+def state_path(j: Path) -> Path:
+    return Path(j) / "state.json"
+
+
+def write_state(j: Path, **fields) -> dict:
+    j = Path(j)
+    j.mkdir(parents=True, exist_ok=True)
+    p = state_path(j)
+    try:
+        st = json.loads(p.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        st = {}
+    st.update(fields)
+    st["updated_at"] = datetime.now().astimezone().isoformat(timespec="seconds")
+    tmp = p.with_name(f".{p.name}.{os.getpid()}.{secrets.token_hex(4)}.tmp")
+    tmp.write_text(json.dumps(st, ensure_ascii=False, indent=2), encoding="utf-8")
+    # Windows can briefly lock state.json (AV/indexer); retry the atomic replace.
+    for attempt in range(8):
+        try:
+            tmp.replace(p)
+            break
+        except PermissionError:
+            if attempt == 7:
+                raise
+            time.sleep(0.02)
+    return st
+
+
+def read_state(j: Path) -> dict:
+    try:
+        return json.loads(state_path(j).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def latest_job() -> str | None:
+    jobs = [p for p in jobs_root().iterdir() if p.is_dir()]
+    if not jobs:
+        return None
+    return max(jobs, key=lambda p: p.stat().st_mtime).name

kba/links.py

@@ -0,0 +1,38 @@
+"""Local Markdown link checks (never fetches external URLs)."""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from urllib.parse import unquote
+
+LINK_RE = re.compile(r"\[[^\]]*\]\(([^)\s]+)(?:\s+[\"'][^\"']*[\"'])?\)")
+SKIP_PREFIXES = ("#", "http://", "https://", "mailto:", "tel:", "data:", "javascript:", "//")
+SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
+
+
+def scan_links(text: str, base_dir: Path, cap: int = 200) -> tuple[int, list[str]]:
+    """Return (local_link_count, broken_targets)."""
+    total = 0
+    broken: list[str] = []
+    for match in LINK_RE.finditer(text or ""):
+        target = match.group(1).strip()
+        if not target or target.startswith(SKIP_PREFIXES) or target.startswith("/"):
+            continue
+        # Skip URI schemes (file:, obsidian:, javascript\:void(...)) and @placeholders.
+        if target.startswith("@") or SCHEME_RE.match(target.replace("\\:", ":")):
+            continue
+        if "{{" in target or "}}" in target:
+            continue
+        total += 1
+        clean = unquote(target.split("#", 1)[0].split("?", 1)[0]).strip()
+        if not clean:
+            continue
+        try:
+            candidate = Path(base_dir) / clean
+            if not candidate.exists():
+                broken.append(clean[:200])
+        except OSError:
+            broken.append(clean[:200])
+        if len(broken) >= cap:
+            break
+    return total, broken

kba/pii.py

@@ -0,0 +1,37 @@
+"""PII candidate detection: counts only, never returns or stores matched text."""
+from __future__ import annotations
+
+import re
+
+PATTERNS = (
+    ("email", re.compile(r"(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}(?![A-Za-z0-9.-])")),
+    ("id_cn", re.compile(r"(?<!\d)\d{17}[\dXx](?!\d)")),
+    ("phone_cn", re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)")),
+    ("bank_card", re.compile(r"(?<!\d)(?:\d[ -]?){15,18}\d(?!\d)")),
+    ("secret", re.compile(r"(?:AKIA[0-9A-Z]{16}|sk-[A-Za-z0-9]{16,}|gh[pousr]_[A-Za-z0-9]{20,})")),
+    ("contract_no", re.compile(
+        r"(?:合同|协议|订单|contract)\s*(?:编号|号|no\.?)?\s*[:：]?\s*([A-Z]{2,}[A-Z0-9-]{4,})",
+        re.I,
+    )),
+)
+
+LABELS = {
+    "email": "邮箱",
+    "id_cn": "身份证号",
+    "phone_cn": "手机号",
+    "bank_card": "银行卡号",
+    "secret": "密钥/Token",
+    "contract_no": "合同/协议编号",
+}
+
+
+def scan_text(text: str, cap: int = 200_000) -> dict[str, int]:
+    """Return {type: occurrences}; matched spans are blanked to avoid double counting."""
+    text = (text or "")[:cap]
+    counts: dict[str, int] = {}
+    for name, pattern in PATTERNS:
+        found = list(pattern.finditer(text))
+        if found:
+            counts[name] = len(found)
+            text = pattern.sub(lambda m: " " * len(m.group(0)), text)
+    return counts