repowiki-cli 0.5.0__py3-none-any.whl

repowiki/__init__.py

@@ -0,0 +1,18 @@
+"""repowiki: deterministic repo-wiki build system driven by coding agents.
+
+It plans tasks, validates agent-produced output, and assembles metadata;
+intelligence is supplied by whatever agent drives the worker loop:
+
+    loop: task = repowiki next --claim --json
+          if empty and busy > 0: wait and retry   # others are mid-flight
+          if empty and busy == 0: exit            # all done
+          execute the task spec (read source, write output)
+          repowiki check --task <id> --json       # auto-fix + status flip
+"""
+
+from importlib.metadata import PackageNotFoundError, version as _package_version
+
+try:
+    __version__ = _package_version("repowiki")
+except PackageNotFoundError:  # running from an uninstalled source tree
+    __version__ = "0.0.0"

repowiki/catalog.py

@@ -0,0 +1,193 @@
+"""Catalog schema validation, flattening and output-path derivation.
+
+The catalog is the chapter tree produced by the planning task
+(``state/catalog.json``). This module is the single source of truth for
+what a valid catalog looks like and where each page lands under
+``<locale>/content/``.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from .paths import sanitize_component, unique_name, nfc
+from .validate import PLACEHOLDER_RE
+
+SLUG_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$")
+MAX_DEPTH = 4  # root chapters are depth 1
+ARCHETYPES = ("module", "flow")  # page templates; default "module"
+
+
+@dataclass
+class FlatNode:
+    id: str
+    title: str
+    slug: str
+    summary: str
+    kind: str  # "chapter" | "page"
+    dependent_files: list[str]
+    page_brief: str
+    parent_id: str | None
+    depth: int
+    output: str  # relative to .repowiki/, e.g. zh/content/Overview/Overview.md
+    archetype: str = "module"  # page template shape: "module" | "flow"
+
+    def chapter_path(self, by_id: dict[str, "FlatNode"]) -> str:
+        parts = [self.title]
+        p = self.parent_id
+        while p and p in by_id:
+            parts.append(by_id[p].title)
+            p = by_id[p].parent_id
+        return " > ".join(reversed(parts))
+
+
+def validate_catalog(data, known_paths: set[str]) -> tuple[list[str], list[str]]:
+    """Return (errors, warnings). Unknown dependent_files are dropped with a warning."""
+    errors: list[str] = []
+    warnings: list[str] = []
+    if not isinstance(data, dict):
+        return ["catalog 必须是 JSON 对象"], warnings
+    if not isinstance(data.get("repo_name"), str) or not data.get("repo_name", "").strip():
+        errors.append("缺少 repo_name（非空字符串）")
+    chapters = data.get("chapters")
+    if not isinstance(chapters, list) or not chapters:
+        errors.append("缺少 chapters（非空数组）")
+        return errors, warnings
+
+    seen_ids: dict[str, str] = {}
+    seen_titles: dict[str, str] = {}
+    seen_slugs: dict[str, str] = {}
+    dropped_files: list[str] = []
+
+    def walk(nodes, depth: int, path_desc: str) -> None:
+        if nodes is None:
+            nodes = []
+        if not isinstance(nodes, list):
+            errors.append(f"{path_desc}: children 必须是数组")
+            return
+        if depth > MAX_DEPTH:
+            errors.append(f"{path_desc}: 树深度超过 {MAX_DEPTH} 层")
+            return
+        for i, node in enumerate(nodes):
+            where = f"{path_desc}[{i}]"
+            if not isinstance(node, dict):
+                errors.append(f"{where}: 节点必须是对象")
+                continue
+            nid = node.get("id")
+            title = nfc(node.get("title", "")).strip()
+            slug = node.get("slug", "")
+            kind = node.get("kind")
+            if not isinstance(nid, str) or not nid.strip():
+                errors.append(f"{where}: 缺少 id")
+            elif nid in seen_ids:
+                errors.append(f"{where}: id 重复 `{nid}`（首次出现于 {seen_ids[nid]}）")
+            else:
+                seen_ids[nid] = where
+            if not title:
+                errors.append(f"{where}: 缺少 title")
+            elif title in seen_titles:
+                errors.append(f"{where}: title 重复 `{title}`（首次出现于 {seen_titles[title]}）")
+            elif PLACEHOLDER_RE.search(title):
+                errors.append(
+                    f"{where}: title 含模板占位符形态 `{title}`"
+                    "（标题会写进页面 H1 并触发「未替换占位符」误判，请改用普通措辞）"
+                )
+            else:
+                seen_titles[title] = where
+            if not isinstance(slug, str) or not SLUG_RE.match(slug or ""):
+                errors.append(f"{where}: slug 非法 `{slug}`（应为小写英文与连字符，如 project-overview）")
+            elif slug in seen_slugs:
+                errors.append(f"{where}: slug 重复 `{slug}`（首次出现于 {seen_slugs[slug]}）")
+            else:
+                seen_slugs[slug] = where
+            if kind not in ("chapter", "page"):
+                errors.append(f"{where}: kind 必须是 chapter 或 page，得到 {kind!r}")
+                kind = "page"
+            archetype = node.get("archetype", "module")
+            if archetype not in ARCHETYPES:
+                errors.append(
+                    f"{where}（{title or nid}）: archetype 必须是 module 或 flow（可省略，默认 module），得到 {archetype!r}"
+                )
+                archetype = "module"
+            brief = node.get("page_brief")
+            if not isinstance(brief, str) or not brief.strip():
+                errors.append(f"{where}（{title or nid}）: 缺少 page_brief（页面要点提示词）")
+            deps = node.get("dependent_files")
+            if deps is None:
+                deps = []
+            if not isinstance(deps, list):
+                errors.append(f"{where}（{title or nid}）: dependent_files 必须是数组")
+                deps = []
+            else:
+                bad = [d for d in deps if not isinstance(d, str) or d not in known_paths]
+                for d in bad:
+                    dropped_files.append(f"{title or nid}: {d}")
+                deps = [d for d in deps if isinstance(d, str) and d in known_paths]
+            node["dependent_files"] = deps
+            if kind == "page" and node.get("children"):
+                errors.append(f"{where}（{title or nid}）: kind=page 不能有 children")
+            walk(node.get("children") if kind == "chapter" else None, depth + 1, f"{where}/children")
+
+    walk(chapters, 1, "chapters")
+    if dropped_files:
+        warnings.append("以下 dependent_files 不在仓库清单中，已剔除: " + "; ".join(dropped_files))
+    return errors, warnings
+
+
+def flatten(data: dict, locale: str = "zh") -> list[FlatNode]:
+    """Flatten validated catalog into nodes with derived output paths.
+
+    Path rules: every chapter gets a directory named after its title plus an
+    index page with the same name; root-level pages are standalone files at
+    ``<locale>/content/``. Collisions among siblings get ``__2`` suffixes.
+    """
+    content_root = f"{locale}/content"
+    by_id: dict[str, FlatNode] = {}
+    out: list[FlatNode] = []
+
+    def walk(nodes, parent: FlatNode | None, dir_chain: list[str], used_here: set[str]) -> None:
+        for node in nodes:
+            kind = node.get("kind", "page")
+            comp = unique_name(sanitize_component(node["title"]), used_here)
+            if kind == "chapter":
+                chain = dir_chain + [comp]
+                rel_dir = "/".join(chain)
+                output = f"{content_root}/{rel_dir}/{comp}.md"
+            else:
+                rel_dir = "/".join(dir_chain)
+                output = f"{content_root}/{rel_dir}/{comp}.md" if rel_dir else f"{content_root}/{comp}.md"
+            flat = FlatNode(
+                id=node["id"],
+                title=nfc(node["title"]).strip(),
+                slug=node.get("slug", ""),
+                summary=node.get("summary", ""),
+                kind=kind,
+                dependent_files=list(node.get("dependent_files", [])),
+                page_brief=node.get("page_brief", ""),
+                parent_id=parent.id if parent else None,
+                depth=(parent.depth + 1) if parent else 1,
+                output=output,
+                archetype=node.get("archetype", "module") if node.get("archetype") in ARCHETYPES else "module",
+            )
+            by_id[flat.id] = flat
+            out.append(flat)
+            if kind == "chapter":
+                walk(node.get("children") or [], flat, chain, set())
+
+    walk(data.get("chapters") or [], None, [], set())
+    return out
+
+
+def catalog_tree_text(flat: list[FlatNode]) -> str:
+    """Render the flattened catalog as an indented tree with page_briefs."""
+    by_id = {n.id: n for n in flat}
+    lines: list[str] = []
+    for n in flat:
+        indent = "  " * (n.depth - 1)
+        marker = "章" if n.kind == "chapter" else "页"
+        lines.append(f"{indent}- [{marker}] {n.title}  ({n.output})")
+        if n.page_brief:
+            brief = n.page_brief.replace("\n", " ")
+            lines.append(f"{indent}    · {brief}")
+    return "\n".join(lines)

repowiki/cli.py

@@ -0,0 +1,164 @@
+"""Command-line interface.
+
+Exit codes: 0 = ok, 1 = validation failure / usage error / corrupted state,
+2 = state conflict (e.g. task already claimed by a live worker).
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __version__
+from .dispatch import run_check, run_next, run_release, run_status, run_touch, run_watch
+from .errors import ConflictError, StateError, UsageError  # noqa: F401 (re-exported)
+from .coverage import run_coverage
+from .knowledge import run_knowledge
+from .metadata import run_finalize
+from .output import emit_error
+from .paths import WikiPaths
+from .plan import run_plan
+from .site import run_site
+from .state import run_clean
+from .updater import run_stale, run_update
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="repowiki",
+        description="Deterministic repo-wiki build system driven by coding agents.",
+    )
+    parser.add_argument("--version", action="version", version=f"repowiki {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("plan", help="scan repo and create the task manifest (phase 1: catalog task)")
+    p.add_argument("repo", help="path to the repository")
+    p.add_argument("--replan", action="store_true", help="discard existing catalog and plan again")
+    p.add_argument("--force", action="store_true", help="with --replan: proceed even if tasks are in flight")
+    p.add_argument("--max-pages", type=int, default=None, help="cap number of page tasks (for cheap trial runs)")
+    p.add_argument("--knowledge", action="store_true", help="also append the knowledge-card task set")
+    p.add_argument("--locale", default="auto", choices=["auto", "zh", "en"],
+                   help="output language: auto-detect from the repo (README-weighted) or force zh/en")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_plan(
+        paths, replan=a.replan, max_pages=a.max_pages, knowledge=a.knowledge,
+        force=a.force, as_json=a.json, locale=a.locale))
+
+    p = sub.add_parser("next", help="list (and optionally claim) ready tasks")
+    p.add_argument("repo")
+    p.add_argument("--claim", action="store_true", help="atomically claim the returned tasks")
+    p.add_argument("--worker", default=None, help="worker identifier recorded on claim")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_next(paths, claim=a.claim, worker=a.worker, as_json=a.json))
+
+    p = sub.add_parser("check", help="validate task output, auto-fix deterministic defects, update status")
+    p.add_argument("repo")
+    p.add_argument("--task", default=None, help="check a single task id")
+    p.add_argument("--all", dest="select_all", action="store_true",
+                   help="check all in_progress/failed tasks (crash recovery / main agent)")
+    p.add_argument("--worker", default=None, help="caller identity; in_progress tasks held by others are refused")
+    p.add_argument("--force", action="store_true", help="check even if claimed by another worker")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_check(
+        paths, task_id=a.task, as_json=a.json, select_all=a.select_all,
+        worker=a.worker, force=a.force))
+
+    p = sub.add_parser("touch", help="refresh a task's claim while executing (heartbeat)")
+    p.add_argument("repo")
+    p.add_argument("--task", required=True)
+    p.add_argument("--worker", default=None)
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_touch(paths, task_id=a.task, worker=a.worker, as_json=a.json))
+
+    p = sub.add_parser("watch", help="block until all tasks are done (or stalled/timeout); exit 0=completed, 1=stalled/timeout")
+    p.add_argument("repo")
+    p.add_argument("--interval", type=float, default=10.0, help="poll interval seconds (default 10)")
+    p.add_argument("--timeout", type=float, default=3600.0, help="give up after this many seconds (default 3600)")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_watch(
+        paths, interval=a.interval, timeout=a.timeout, as_json=a.json))
+
+    p = sub.add_parser("release", help="return an in_progress task to pending")
+    p.add_argument("repo")
+    p.add_argument("--task", required=True)
+    p.add_argument("--force", action="store_true", help="release even if claimed by another worker")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_release(
+        paths, task_id=a.task, force=a.force, as_json=a.json))
+
+    p = sub.add_parser("finalize", help="assemble zh/meta/repowiki-metadata.json (requires all tasks done)")
+    p.add_argument("repo")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_finalize(paths, as_json=a.json))
+
+    p = sub.add_parser("site", help="render the finished wiki into one offline HTML file (.repowiki/<locale>/wiki.html) plus llms.txt / llms-full.txt agent indexes")
+    p.add_argument("repo")
+    p.add_argument("--open", dest="open_browser", action="store_true",
+                   help="open the generated file in the default browser")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_site(
+        paths, open_browser=a.open_browser, as_json=a.json))
+
+    p = sub.add_parser("update", help="map git changes to page_update tasks (incremental regeneration)")
+    p.add_argument("repo")
+    p.add_argument("--since", default=None, help="commit sha to diff from (default: last_commit_id in metadata)")
+    p.add_argument("--dirty", action="store_true",
+                   help="also see uncommitted (staged+unstaged) and untracked changes, not just commits")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_update(paths, since=a.since, as_json=a.json, dirty=a.dirty))
+
+    p = sub.add_parser("stale", help="read-only: report which pages `update` would affect for since..HEAD (CI staleness gate)")
+    p.add_argument("repo")
+    p.add_argument("--since", default=None, help="git ref to diff from (default: last_commit_id in metadata)")
+    p.add_argument("--dirty", action="store_true",
+                   help="also see uncommitted (staged+unstaged) and untracked changes, not just commits")
+    p.add_argument("--fail-if-stale", dest="fail_if_stale", action="store_true",
+                   help="exit 1 when any page/card/module is affected (for CI gates)")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_stale(
+        paths, since=a.since, fail_if_stale=a.fail_if_stale, as_json=a.json, dirty=a.dirty))
+
+    p = sub.add_parser("knowledge", help="append the knowledge-card task set (planning + cards)")
+    p.add_argument("repo")
+    p.add_argument("--categories", default=None, metavar="FILE",
+                   help="YAML/JSON file with a custom mechanism-card category list "
+                        "(replaces the built-in six; persisted in state/knowledge_categories.json)")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_knowledge(paths, as_json=a.json, categories=a.categories))
+
+    p = sub.add_parser("status", help="show task statistics, failures and stale claims")
+    p.add_argument("repo")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_status(paths, as_json=a.json))
+
+    p = sub.add_parser("coverage", help="read-only: which repo files has the wiki never cited (coverage report)")
+    p.add_argument("repo")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_coverage(paths, as_json=a.json))
+
+    p = sub.add_parser("clean", help="remove .repowiki/state entirely (wiki output is kept)")
+    p.add_argument("repo")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=lambda a, paths: run_clean(paths, as_json=a.json))
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    paths = WikiPaths(args.repo)
+    try:
+        return args.func(args, paths)
+    except ConflictError as e:
+        emit_error("conflict", str(e), args.json)
+        return 2
+    except StateError as e:
+        emit_error("state_corrupt", str(e), args.json)
+        return 1
+    except UsageError as e:
+        emit_error("usage", str(e), args.json)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

repowiki/coverage.py

@@ -0,0 +1,96 @@
+"""``repowiki coverage``: which repo files has the wiki never cited?
+
+A read-only quality report in repowiki's deterministic spirit: compare the
+repository inventory against the union of ``file://`` citations across all
+wiki pages and the overview, plus knowledge-card source files. No agent, no
+LLM — the numbers are computable facts, useful as a writing guide ("these
+modules have no page yet") and as a provable quality metric.
+"""
+
+from __future__ import annotations
+
+import json
+
+from .catalog import flatten
+from .errors import UsageError
+from .output import emit
+from .paths import WikiPaths
+from .scanner import scan
+from .validate import extract_refs
+
+MAX_LISTING = 50  # human output caps the uncited listing; JSON is complete
+
+
+def run_coverage(paths: WikiPaths, as_json: bool) -> int:
+    if not paths.catalog_file.exists():
+        raise UsageError("state/catalog.json 不存在，请先完成首次生成")
+    try:
+        catalog = json.loads(paths.catalog_file.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as e:
+        raise UsageError(
+            f"state/catalog.json 损坏（{e}）：可手工修复该文件，或 `repowiki plan --replan` 重新规划"
+        ) from e
+
+    inv = scan(paths.repo_root)
+    known = {f.path for f in inv.files}
+
+    cited: set[str] = set()
+    pages: list[dict] = []
+    for n in flatten(catalog, paths.locale):
+        f = paths.root / n.output
+        refs = extract_refs(f.read_text(encoding="utf-8")) if f.is_file() else []
+        page_cited = {path for path, _s, _e in refs if path in known}
+        cited |= page_cited
+        pages.append({"id": n.id, "title": n.title, "cited_files": len(page_cited)})
+
+    if paths.overview_file.is_file():
+        for path, _s, _e in extract_refs(paths.overview_file.read_text(encoding="utf-8")):
+            if path in known:
+                cited.add(path)
+
+    knowledge_files: set[str] = set()
+    if paths.knowledge_plan_file.exists():
+        try:
+            plan = json.loads(paths.knowledge_plan_file.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            plan = {}
+        if isinstance(plan, dict):
+            for card in plan.get("cards") or []:
+                if isinstance(card, dict):
+                    knowledge_files |= {p for p in card.get("source_files") or [] if p in known}
+    cited |= knowledge_files
+
+    uncited = sorted(known - cited)
+    total = len(known)
+    covered = total - len(uncited)
+    result = {
+        "ok": True,
+        "repo_files": total,
+        "cited_files": covered,
+        "coverage": round(covered / total, 4) if total else 1.0,
+        "uncited_files": uncited,
+        "knowledge_files": sorted(knowledge_files),
+        "pages": pages,
+    }
+    emit(result, _coverage_human, as_json)
+    return 0
+
+
+def _coverage_human(r: dict) -> str:
+    lines = [
+        f"覆盖率 {r['cited_files']}/{r['repo_files']}（{r['coverage'] * 100:.1f}%）"
+        "——被 wiki 页面/总览/知识卡片引用过的仓库文件占比"
+    ]
+    uncited = r["uncited_files"]
+    if uncited:
+        lines.append(f"未被引用 {len(uncited)} 个（至多列出 {MAX_LISTING} 个，JSON 输出含全量）:")
+        lines += [f"  → {p}" for p in uncited[:MAX_LISTING]]
+    else:
+        lines.append("仓库全部文件都被引用 ✓")
+    zero = [p for p in r["pages"] if p["cited_files"] == 0]
+    if zero:
+        lines.append(
+            f"⚠ {len(zero)} 个页面没有任何 file:// 引用: "
+            + ", ".join(f"{p['id']}({p['title']})" for p in zero[:5])
+        )
+    return "\n".join(lines)