repowiki-cli 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- repowiki/__init__.py +18 -0
- repowiki/catalog.py +193 -0
- repowiki/cli.py +164 -0
- repowiki/coverage.py +96 -0
- repowiki/dispatch.py +419 -0
- repowiki/errors.py +21 -0
- repowiki/gitutil.py +19 -0
- repowiki/i18n.py +161 -0
- repowiki/knowledge.py +226 -0
- repowiki/llms.py +52 -0
- repowiki/metadata.py +176 -0
- repowiki/output.py +34 -0
- repowiki/paths.py +176 -0
- repowiki/plan.py +140 -0
- repowiki/scanner.py +170 -0
- repowiki/site.py +384 -0
- repowiki/state.py +437 -0
- repowiki/tasks.py +354 -0
- repowiki/templates/en/STYLE.md +49 -0
- repowiki/templates/en/catalog_task.md +69 -0
- repowiki/templates/en/flow_template.md +87 -0
- repowiki/templates/en/knowledge_card_task.md +65 -0
- repowiki/templates/en/knowledge_card_update_task.md +49 -0
- repowiki/templates/en/knowledge_module_task.md +33 -0
- repowiki/templates/en/knowledge_module_update_task.md +36 -0
- repowiki/templates/en/knowledge_task.md +65 -0
- repowiki/templates/en/overview_task.md +35 -0
- repowiki/templates/en/overview_update_task.md +50 -0
- repowiki/templates/en/page_task.md +45 -0
- repowiki/templates/en/page_template.md +113 -0
- repowiki/templates/en/update_task.md +49 -0
- repowiki/templates/site/app.js +378 -0
- repowiki/templates/site.html +373 -0
- repowiki/templates/zh/STYLE.md +50 -0
- repowiki/templates/zh/catalog_task.md +66 -0
- repowiki/templates/zh/flow_template.md +87 -0
- repowiki/templates/zh/knowledge_card_task.md +65 -0
- repowiki/templates/zh/knowledge_card_update_task.md +46 -0
- repowiki/templates/zh/knowledge_module_task.md +33 -0
- repowiki/templates/zh/knowledge_module_update_task.md +35 -0
- repowiki/templates/zh/knowledge_task.md +62 -0
- repowiki/templates/zh/overview_task.md +35 -0
- repowiki/templates/zh/overview_update_task.md +44 -0
- repowiki/templates/zh/page_task.md +45 -0
- repowiki/templates/zh/page_template.md +113 -0
- repowiki/templates/zh/update_task.md +49 -0
- repowiki/templates.py +38 -0
- repowiki/updater.py +322 -0
- repowiki/validate.py +356 -0
- repowiki/vendor/marked.min.js +69 -0
- repowiki/vendor/mermaid.min.js +3636 -0
- repowiki_cli-0.5.0.dist-info/METADATA +349 -0
- repowiki_cli-0.5.0.dist-info/RECORD +57 -0
- repowiki_cli-0.5.0.dist-info/WHEEL +5 -0
- repowiki_cli-0.5.0.dist-info/entry_points.txt +2 -0
- repowiki_cli-0.5.0.dist-info/licenses/LICENSE +21 -0
- repowiki_cli-0.5.0.dist-info/top_level.txt +1 -0
repowiki/plan.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""``repowiki plan``: scan the repo and lay down the task manifest."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import shutil
|
|
7
|
+
|
|
8
|
+
from .errors import UsageError
|
|
9
|
+
from .i18n import SUPPORTED, detect_locale, strings
|
|
10
|
+
from .output import emit
|
|
11
|
+
from .paths import WikiPaths
|
|
12
|
+
from . import tasks
|
|
13
|
+
from .catalog import validate_catalog, flatten
|
|
14
|
+
from .scanner import scan
|
|
15
|
+
from .state import TaskStore
|
|
16
|
+
|
|
17
|
+
MIN_CODE_FILES = 10
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _load_catalog(paths: WikiPaths) -> dict | None:
|
|
21
|
+
if not paths.catalog_file.exists():
|
|
22
|
+
return None
|
|
23
|
+
try:
|
|
24
|
+
return json.loads(paths.catalog_file.read_text(encoding="utf-8"))
|
|
25
|
+
except json.JSONDecodeError:
|
|
26
|
+
return None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _resolve_locale(paths: WikiPaths, locale_flag: str, inv) -> str:
|
|
30
|
+
"""Explicit --locale wins; else a persisted state/locale; else detect."""
|
|
31
|
+
if locale_flag != "auto":
|
|
32
|
+
return locale_flag
|
|
33
|
+
if (paths.state_dir / "locale").exists():
|
|
34
|
+
return paths.locale # lazy property reads the persisted value
|
|
35
|
+
return detect_locale(paths.repo_root, [f.path for f in inv.files if f.is_code])
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def run_plan(paths: WikiPaths, replan: bool = False, max_pages: int | None = None,
|
|
39
|
+
knowledge: bool = False, force: bool = False, as_json: bool = False,
|
|
40
|
+
locale: str = "auto") -> int:
|
|
41
|
+
if replan and paths.root.exists():
|
|
42
|
+
busy = _busy_tasks(paths)
|
|
43
|
+
if busy is None and not force:
|
|
44
|
+
raise UsageError(
|
|
45
|
+
"state/index.json 无法解析,无法确认是否有任务在执行;"
|
|
46
|
+
"replan 会删除现有状态与规格,确认放弃恢复请加 --force"
|
|
47
|
+
)
|
|
48
|
+
if busy and not force:
|
|
49
|
+
raise UsageError(
|
|
50
|
+
f"检测到 {len(busy)} 个 in_progress 任务({', '.join(busy[:5])}),"
|
|
51
|
+
"replan 会删除正在被写入的产物;确认请加 --force"
|
|
52
|
+
)
|
|
53
|
+
shutil.rmtree(paths.root)
|
|
54
|
+
store = TaskStore(paths)
|
|
55
|
+
inv = scan(paths.repo_root)
|
|
56
|
+
|
|
57
|
+
if inv.code_file_count < MIN_CODE_FILES:
|
|
58
|
+
raise UsageError(
|
|
59
|
+
f"代码文件数 {inv.code_file_count} < {MIN_CODE_FILES},仓库太小,不适合生成 RepoWiki"
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
# locale must be resolved before ensure(): the locale decides which
|
|
63
|
+
# <locale>/content and <locale>/meta directories get created
|
|
64
|
+
resolved = _resolve_locale(paths, locale, inv)
|
|
65
|
+
persisted = (paths.state_dir / "locale").exists()
|
|
66
|
+
if not persisted or locale != "auto":
|
|
67
|
+
paths.persist_locale(resolved)
|
|
68
|
+
paths.ensure()
|
|
69
|
+
|
|
70
|
+
warnings: list[str] = []
|
|
71
|
+
added: list[str] = []
|
|
72
|
+
catalog = _load_catalog(paths)
|
|
73
|
+
if catalog is not None:
|
|
74
|
+
errors, warns = validate_catalog(catalog, inv.known_paths())
|
|
75
|
+
warnings.extend(warns)
|
|
76
|
+
if errors:
|
|
77
|
+
warnings.append("已有 catalog.json 校验失败,将重新规划: " + "; ".join(errors[:5]))
|
|
78
|
+
catalog = None
|
|
79
|
+
|
|
80
|
+
if catalog is None:
|
|
81
|
+
added += store.add_tasks([tasks.build_catalog_task(paths, inv)])
|
|
82
|
+
else:
|
|
83
|
+
nodes = flatten(catalog, resolved)
|
|
84
|
+
added += store.add_tasks(tasks.build_page_tasks(paths, nodes, inv, max_pages))
|
|
85
|
+
warnings.extend(_warn_dangling_outputs(paths, nodes))
|
|
86
|
+
|
|
87
|
+
if knowledge:
|
|
88
|
+
added += store.add_tasks([tasks.build_knowledge_plan_task(paths, inv)])
|
|
89
|
+
|
|
90
|
+
stats = store.stats()
|
|
91
|
+
result = {
|
|
92
|
+
"repo": str(paths.repo_root),
|
|
93
|
+
"locale": resolved,
|
|
94
|
+
"code_file_count": inv.code_file_count,
|
|
95
|
+
"tasks_added": added,
|
|
96
|
+
"tasks_total": stats["total"],
|
|
97
|
+
"by_status": stats["by_status"],
|
|
98
|
+
"current_phase": stats["current_phase"],
|
|
99
|
+
"warnings": warnings,
|
|
100
|
+
"next": "执行 `repowiki next <repo> --claim` 领取任务",
|
|
101
|
+
}
|
|
102
|
+
emit(result, _plan_human, as_json)
|
|
103
|
+
return 0
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _plan_human(r: dict) -> str:
|
|
107
|
+
lines = [
|
|
108
|
+
f"仓库:{r['repo']}(代码文件 {r['code_file_count']} 个,产出语言:{strings(r['locale'])['name']})",
|
|
109
|
+
f"新增任务 {len(r['tasks_added'])} 个,总任务 {r['tasks_total']} 个,当前阶段 {r['current_phase']}",
|
|
110
|
+
*[f" ⚠ {w}" for w in r["warnings"]],
|
|
111
|
+
r["next"],
|
|
112
|
+
]
|
|
113
|
+
return "\n".join(lines)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _busy_tasks(paths: WikiPaths) -> list[str] | None:
|
|
117
|
+
"""Ids of in_progress tasks; None when the index exists but is unreadable."""
|
|
118
|
+
if not paths.index_file.exists():
|
|
119
|
+
return []
|
|
120
|
+
try:
|
|
121
|
+
data = json.loads(paths.index_file.read_text(encoding="utf-8"))
|
|
122
|
+
except json.JSONDecodeError:
|
|
123
|
+
return None
|
|
124
|
+
return [tid for tid, t in data.get("tasks", {}).items() if t.get("status") == "in_progress"]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _warn_dangling_outputs(paths: WikiPaths, nodes: list[dict]) -> list[str]:
|
|
128
|
+
"""Pages already on disk but no longer in the catalog (stale leftovers)."""
|
|
129
|
+
expected = {n.output for n in nodes}
|
|
130
|
+
dangling: list[str] = []
|
|
131
|
+
content = paths.content_dir
|
|
132
|
+
if not content.exists():
|
|
133
|
+
return dangling
|
|
134
|
+
for p in content.rglob("*.md"):
|
|
135
|
+
rel = f"{paths.locale}/content/" + p.relative_to(content).as_posix()
|
|
136
|
+
if rel not in expected:
|
|
137
|
+
dangling.append(rel)
|
|
138
|
+
if dangling:
|
|
139
|
+
return [f"{len(dangling)} 个已生成页面不在当前 catalog 中(可能为旧残留): " + ", ".join(dangling[:5])]
|
|
140
|
+
return []
|
repowiki/scanner.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""Repository scanner: deterministic file inventory for planning.
|
|
2
|
+
|
|
3
|
+
Uses ``git ls-files`` when available (accurate .gitignore handling), else
|
|
4
|
+
falls back to a manual walk. Skips binaries, VCS/build/vendor directories
|
|
5
|
+
and the tool's own output (``.repowiki``).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from .gitutil import run_git
|
|
15
|
+
|
|
16
|
+
CODE_EXTS = {
|
|
17
|
+
"py", "ts", "tsx", "js", "jsx", "mjs", "cjs", "go", "rs", "java", "kt",
|
|
18
|
+
"rb", "c", "h", "cc", "cpp", "hpp", "cs", "swift", "scala", "sh", "bash",
|
|
19
|
+
"sql", "php", "vue", "svelte", "dart", "lua", "pl",
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
LANG_BY_EXT = {
|
|
23
|
+
"py": "python", "ts": "typescript", "tsx": "typescript", "js": "javascript",
|
|
24
|
+
"jsx": "javascript", "mjs": "javascript", "cjs": "javascript", "go": "go",
|
|
25
|
+
"rs": "rust", "java": "java", "kt": "kotlin", "rb": "ruby", "c": "c",
|
|
26
|
+
"h": "c", "cc": "cpp", "cpp": "cpp", "hpp": "cpp", "cs": "csharp",
|
|
27
|
+
"swift": "swift", "scala": "scala", "sh": "shell", "bash": "shell",
|
|
28
|
+
"sql": "sql", "php": "php", "vue": "vue", "svelte": "svelte",
|
|
29
|
+
"dart": "dart", "lua": "lua", "pl": "perl",
|
|
30
|
+
"md": "markdown", "rst": "markdown", "txt": "text",
|
|
31
|
+
"yaml": "yaml", "yml": "yaml", "toml": "toml", "json": "json",
|
|
32
|
+
"ini": "ini", "cfg": "ini", "html": "html", "css": "css", "scss": "scss",
|
|
33
|
+
"xml": "xml",
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
IGNORE_DIRS = {
|
|
37
|
+
".git", "node_modules", "venv", ".venv", "env", "dist", "target", "build",
|
|
38
|
+
"out", "__pycache__", ".repowiki", ".idea", ".vscode", ".next", ".tox",
|
|
39
|
+
".eggs", "coverage", ".mypy_cache", ".pytest_cache", "site-packages",
|
|
40
|
+
".ruff_cache", ".hypothesis",
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
KEY_FILE_GLOBS = [
|
|
44
|
+
"README*", "pyproject.toml", "setup.py", "setup.cfg", "package.json",
|
|
45
|
+
"go.mod", "Cargo.toml", "docker-compose*.yml", "docker-compose*.yaml",
|
|
46
|
+
"Makefile", "requirements*.txt", "pom.xml", "build.gradle", "*.gemspec",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
MAX_TEXT_BYTES = 2 * 1024 * 1024 # larger files are listed but not counted by LOC
|
|
50
|
+
_PER_DIR_CAP = 20
|
|
51
|
+
_MAX_SUMMARY_LINES = 400
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class FileEntry:
|
|
56
|
+
path: str
|
|
57
|
+
loc: int
|
|
58
|
+
lang: str
|
|
59
|
+
is_code: bool
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class Inventory:
|
|
64
|
+
repo_root: str
|
|
65
|
+
files: list[FileEntry] = field(default_factory=list)
|
|
66
|
+
key_files: list[str] = field(default_factory=list)
|
|
67
|
+
tree_summary: str = ""
|
|
68
|
+
code_file_count: int = 0
|
|
69
|
+
|
|
70
|
+
def known_paths(self) -> set[str]:
|
|
71
|
+
return {f.path for f in self.files}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _git_ls_files(root: Path) -> list[str] | None:
|
|
75
|
+
if not (root / ".git").exists():
|
|
76
|
+
return None
|
|
77
|
+
out = run_git(root, "ls-files", "-z", timeout=120)
|
|
78
|
+
if out is None:
|
|
79
|
+
return None
|
|
80
|
+
return [p for p in out.split("\0") if p]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _walk_files(root: Path) -> list[str]:
|
|
84
|
+
found: list[str] = []
|
|
85
|
+
for dirpath, dirnames, filenames in os_walk_pruned(root):
|
|
86
|
+
for name in filenames:
|
|
87
|
+
rel = (Path(dirpath) / name).relative_to(root).as_posix()
|
|
88
|
+
found.append(rel)
|
|
89
|
+
return sorted(found)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def os_walk_pruned(root: Path):
|
|
93
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
94
|
+
dirnames[:] = [d for d in dirnames if d not in IGNORE_DIRS and not d.startswith(".git")]
|
|
95
|
+
yield dirpath, dirnames, filenames
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _is_binary(path: Path) -> bool:
|
|
99
|
+
try:
|
|
100
|
+
with open(path, "rb") as fh:
|
|
101
|
+
return b"\0" in fh.read(8192)
|
|
102
|
+
except OSError:
|
|
103
|
+
return True
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _count_loc(path: Path, size: int) -> int:
|
|
107
|
+
if size > MAX_TEXT_BYTES:
|
|
108
|
+
return 0
|
|
109
|
+
try:
|
|
110
|
+
with open(path, "rb") as fh:
|
|
111
|
+
return fh.read().count(b"\n")
|
|
112
|
+
except OSError:
|
|
113
|
+
return 0
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def scan(repo_root: str | Path) -> Inventory:
|
|
117
|
+
root = Path(repo_root).resolve()
|
|
118
|
+
rels = _git_ls_files(root)
|
|
119
|
+
if rels is None:
|
|
120
|
+
rels = _walk_files(root)
|
|
121
|
+
else:
|
|
122
|
+
rels = [r for r in rels if not any(part in IGNORE_DIRS for part in Path(r).parts[:-1])]
|
|
123
|
+
|
|
124
|
+
files: list[FileEntry] = []
|
|
125
|
+
for rel in sorted(rels):
|
|
126
|
+
p = root / rel
|
|
127
|
+
if not p.is_file() or _is_binary(p):
|
|
128
|
+
continue
|
|
129
|
+
size = p.stat().st_size
|
|
130
|
+
ext = p.suffix.lstrip(".").lower()
|
|
131
|
+
no_ext = p.name.lower()
|
|
132
|
+
lang = LANG_BY_EXT.get(ext, "" )
|
|
133
|
+
if not lang and no_ext in {"dockerfile", "makefile", "procfile"}:
|
|
134
|
+
lang = no_ext.lower()
|
|
135
|
+
is_code = ext in CODE_EXTS
|
|
136
|
+
files.append(FileEntry(path=rel, loc=_count_loc(p, size), lang=lang, is_code=is_code))
|
|
137
|
+
|
|
138
|
+
inv = Inventory(
|
|
139
|
+
repo_root=str(root),
|
|
140
|
+
files=files,
|
|
141
|
+
code_file_count=sum(1 for f in files if f.is_code),
|
|
142
|
+
)
|
|
143
|
+
inv.key_files = sorted(
|
|
144
|
+
{f.path for f in files for pat in KEY_FILE_GLOBS if Path(f.path).parent == Path(".") and Path(f.path).match(pat)}
|
|
145
|
+
)
|
|
146
|
+
inv.tree_summary = _render_tree(files)
|
|
147
|
+
return inv
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _render_tree(files: list[FileEntry]) -> str:
|
|
151
|
+
"""Compact repo tree, capped per directory and in total size."""
|
|
152
|
+
by_dir: dict[str, list[str]] = {}
|
|
153
|
+
for f in files:
|
|
154
|
+
by_dir.setdefault(str(Path(f.path).parent) if str(Path(f.path).parent) != "." else "", []).append(Path(f.path).name)
|
|
155
|
+
|
|
156
|
+
lines: list[str] = ["<repo>/"]
|
|
157
|
+
dirs = sorted(by_dir)
|
|
158
|
+
for i, d in enumerate(dirs):
|
|
159
|
+
names = by_dir[d]
|
|
160
|
+
shown = names[:_PER_DIR_CAP]
|
|
161
|
+
prefix = "" if d == "" else f"{d}/"
|
|
162
|
+
lines.append(f"{prefix} ({len(names)} files)")
|
|
163
|
+
for name in shown:
|
|
164
|
+
lines.append(f" {name}")
|
|
165
|
+
if len(names) > _PER_DIR_CAP:
|
|
166
|
+
lines.append(f" … (+{len(names) - _PER_DIR_CAP} more)")
|
|
167
|
+
if len(lines) > _MAX_SUMMARY_LINES:
|
|
168
|
+
lines.append(f"… (tree truncated at {_MAX_SUMMARY_LINES} lines, {len(dirs) - i - 1} dirs omitted)")
|
|
169
|
+
break
|
|
170
|
+
return "\n".join(lines)
|
repowiki/site.py
ADDED
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
"""``repowiki site``: render the finished wiki into one self-contained HTML file.
|
|
2
|
+
|
|
3
|
+
Reads every generated page plus the overview, embeds the source lines behind
|
|
4
|
+
each ``file://`` reference, and inlines the vendored markdown/mermaid JS.
|
|
5
|
+
Knowledge module docs and cards (``knowledge/<locale>/``) are included as
|
|
6
|
+
regular pages under a dedicated nav chapter.
|
|
7
|
+
The result (``<locale>/wiki.html``) opens in any browser with zero network
|
|
8
|
+
and zero server — double-click, or share the single file.
|
|
9
|
+
The same page collection is also exported as ``llms.txt`` /
|
|
10
|
+
``llms-full.txt`` (llmstxt.org convention) for direct agent consumption.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import webbrowser
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
|
|
23
|
+
from . import templates
|
|
24
|
+
from .catalog import FlatNode, flatten
|
|
25
|
+
from .errors import UsageError
|
|
26
|
+
from .i18n import strings
|
|
27
|
+
from .llms import write_llms
|
|
28
|
+
from .output import emit
|
|
29
|
+
from .paths import WikiPaths, sanitize_component
|
|
30
|
+
from .state import now_iso
|
|
31
|
+
from .validate import extract_refs
|
|
32
|
+
|
|
33
|
+
MAX_SNIPPET_LINES = 20_000 # larger spans are skipped rather than bloating the file
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def run_site(paths: WikiPaths, open_browser: bool, as_json: bool) -> int:
|
|
37
|
+
if not paths.metadata_file.is_file():
|
|
38
|
+
raise UsageError(
|
|
39
|
+
f"未找到 {paths.metadata_file}:请先完成全部任务并运行 `repowiki finalize <repo>`"
|
|
40
|
+
)
|
|
41
|
+
try:
|
|
42
|
+
metadata = json.loads(paths.metadata_file.read_text(encoding="utf-8"))
|
|
43
|
+
except json.JSONDecodeError as e:
|
|
44
|
+
raise UsageError(f"repowiki-metadata.json 损坏({e}):请重新运行 `repowiki finalize <repo>`") from e
|
|
45
|
+
|
|
46
|
+
nodes = _ordered_nodes(paths)
|
|
47
|
+
pages = _collect_pages(paths, metadata, nodes)
|
|
48
|
+
if not pages:
|
|
49
|
+
raise UsageError(f"未找到任何已生成的 wiki 页面({paths.content_dir} 为空)")
|
|
50
|
+
knowledge_pages, knowledge_nav = _collect_knowledge(paths, base=len(pages))
|
|
51
|
+
pages.extend(knowledge_pages)
|
|
52
|
+
nav = _build_nav(paths, pages, nodes)
|
|
53
|
+
if knowledge_nav:
|
|
54
|
+
nav.append(knowledge_nav)
|
|
55
|
+
snippets = _collect_snippets(paths.repo_root, pages)
|
|
56
|
+
|
|
57
|
+
payload = {
|
|
58
|
+
"repo": _repo_name(paths, metadata),
|
|
59
|
+
"locale": paths.locale,
|
|
60
|
+
"generatedAt": now_iso(),
|
|
61
|
+
"ui": strings(paths.locale)["site"],
|
|
62
|
+
"nav": nav,
|
|
63
|
+
"pages": pages,
|
|
64
|
+
"snippets": snippets,
|
|
65
|
+
}
|
|
66
|
+
html = _render_html(payload, paths.locale)
|
|
67
|
+
|
|
68
|
+
out = paths.site_file
|
|
69
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
70
|
+
tmp = out.with_name(f".{out.name}.{os.getpid()}.tmp")
|
|
71
|
+
tmp.write_text(html, encoding="utf-8")
|
|
72
|
+
os.replace(tmp, out)
|
|
73
|
+
write_llms(paths, payload["repo"], pages, nav)
|
|
74
|
+
if open_browser:
|
|
75
|
+
webbrowser.open(out.as_uri())
|
|
76
|
+
|
|
77
|
+
summary = {
|
|
78
|
+
"ok": True,
|
|
79
|
+
"site": str(out),
|
|
80
|
+
"pages": len(pages),
|
|
81
|
+
"knowledge_pages": len(knowledge_pages),
|
|
82
|
+
"snippets": len(snippets),
|
|
83
|
+
"size_mb": round(out.stat().st_size / 1024 / 1024, 2),
|
|
84
|
+
"llms": str(paths.llms_file),
|
|
85
|
+
"llms_full": str(paths.llms_full_file),
|
|
86
|
+
}
|
|
87
|
+
emit(summary, _site_human, as_json)
|
|
88
|
+
return 0
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _site_human(r: dict) -> str:
|
|
92
|
+
return (
|
|
93
|
+
f"✓ 站点已生成: {r['site']}\n"
|
|
94
|
+
f" 页面 {r['pages']}(含知识页 {r['knowledge_pages']})· 源码片段 {r['snippets']} · 体积 {r['size_mb']} MB\n"
|
|
95
|
+
f" agent 索引:{r['llms']} 与 {r['llms_full']}\n"
|
|
96
|
+
f" 单文件离线可用:浏览器直接打开即可(--open 自动打开)"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# --- content assembly ---
|
|
101
|
+
|
|
102
|
+
def _repo_name(paths: WikiPaths, metadata: dict) -> str:
|
|
103
|
+
return (metadata.get("wiki_repo") or {}).get("name") or paths.repo_root.name
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _collect_pages(paths: WikiPaths, metadata: dict, nodes: list[FlatNode]) -> list[dict]:
|
|
107
|
+
site = strings(paths.locale)["site"]
|
|
108
|
+
pages: list[dict] = []
|
|
109
|
+
|
|
110
|
+
overview = metadata.get("wiki_overview") or ""
|
|
111
|
+
if not overview and paths.overview_file.is_file():
|
|
112
|
+
overview = paths.overview_file.read_text(encoding="utf-8")
|
|
113
|
+
if overview and overview != "No overview yet.":
|
|
114
|
+
pages.append({
|
|
115
|
+
"id": "overview",
|
|
116
|
+
"title": site["overview_label"],
|
|
117
|
+
"path": paths.overview_file.relative_to(paths.root).as_posix(),
|
|
118
|
+
"md": overview,
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
for node in nodes:
|
|
122
|
+
f = paths.root / node.output
|
|
123
|
+
if not f.is_file():
|
|
124
|
+
continue
|
|
125
|
+
pages.append({
|
|
126
|
+
"id": node.id,
|
|
127
|
+
"title": node.title,
|
|
128
|
+
"path": node.output,
|
|
129
|
+
"md": f.read_text(encoding="utf-8"),
|
|
130
|
+
})
|
|
131
|
+
return pages
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _ordered_nodes(paths: WikiPaths) -> list[FlatNode]:
|
|
135
|
+
"""Plan order from state/catalog.json; after `repowiki clean`, degrade to
|
|
136
|
+
the on-disk directory layout so the site stays buildable."""
|
|
137
|
+
if paths.catalog_file.is_file():
|
|
138
|
+
try:
|
|
139
|
+
catalog = json.loads(paths.catalog_file.read_text(encoding="utf-8"))
|
|
140
|
+
except json.JSONDecodeError:
|
|
141
|
+
catalog = None
|
|
142
|
+
if catalog is not None:
|
|
143
|
+
return flatten(catalog, paths.locale)
|
|
144
|
+
return _nodes_from_disk(paths)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _nodes_from_disk(paths: WikiPaths) -> list[FlatNode]:
|
|
148
|
+
"""Chapter per sub-directory, page per .md file — plan order is lost,
|
|
149
|
+
but the site stays buildable after `repowiki clean`."""
|
|
150
|
+
nodes: list[FlatNode] = []
|
|
151
|
+
content = paths.content_dir
|
|
152
|
+
if not content.is_dir():
|
|
153
|
+
return nodes
|
|
154
|
+
|
|
155
|
+
def page_node(f: Path, parent_id: str | None) -> FlatNode:
|
|
156
|
+
return FlatNode(
|
|
157
|
+
id=f.stem, title=f.stem, slug=f.stem, summary="", kind="page",
|
|
158
|
+
dependent_files=[], page_brief="", parent_id=parent_id, depth=2,
|
|
159
|
+
output=f.relative_to(paths.root).as_posix(),
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
for entry in sorted(content.iterdir(), key=lambda p: p.name):
|
|
163
|
+
if entry.is_dir() and any(entry.glob("*.md")):
|
|
164
|
+
nodes.append(FlatNode(
|
|
165
|
+
id=entry.name, title=entry.name, slug=entry.name, summary="", kind="chapter",
|
|
166
|
+
dependent_files=[], page_brief="", parent_id=None, depth=1,
|
|
167
|
+
output=f"{entry.relative_to(paths.root).as_posix()}/__chapter__.md",
|
|
168
|
+
))
|
|
169
|
+
for f in sorted(entry.glob("*.md")):
|
|
170
|
+
nodes.append(page_node(f, entry.name))
|
|
171
|
+
elif entry.is_file() and entry.suffix == ".md":
|
|
172
|
+
nodes.append(page_node(entry, None))
|
|
173
|
+
return nodes
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _build_nav(paths: WikiPaths, pages: list[dict], nodes: list[FlatNode]) -> list[dict]:
|
|
177
|
+
site = strings(paths.locale)["site"]
|
|
178
|
+
by_output = {p["path"]: i for i, p in enumerate(pages)}
|
|
179
|
+
children_of: dict[str | None, list[FlatNode]] = {}
|
|
180
|
+
for n in nodes:
|
|
181
|
+
children_of.setdefault(n.parent_id, []).append(n)
|
|
182
|
+
|
|
183
|
+
def descendants(node_id: str) -> list[FlatNode]:
|
|
184
|
+
out: list[FlatNode] = []
|
|
185
|
+
for k in children_of.get(node_id, []):
|
|
186
|
+
out.append(k)
|
|
187
|
+
out.extend(descendants(k.id))
|
|
188
|
+
return out
|
|
189
|
+
|
|
190
|
+
entries: list[dict] = []
|
|
191
|
+
for i, p in enumerate(pages):
|
|
192
|
+
if p["id"] == "overview":
|
|
193
|
+
entries.append({"title": site["overview_label"], "page": i})
|
|
194
|
+
break
|
|
195
|
+
|
|
196
|
+
for top in children_of.get(None, []):
|
|
197
|
+
own = [{"title": top.title, "page": by_output[top.output]}] if top.output in by_output else []
|
|
198
|
+
kids = [
|
|
199
|
+
{"title": k.title, "page": by_output[k.output]}
|
|
200
|
+
for k in descendants(top.id) if k.output in by_output
|
|
201
|
+
]
|
|
202
|
+
if not kids:
|
|
203
|
+
if own: # standalone top-level page (with or without a chapter wrapper)
|
|
204
|
+
entries.append({"title": top.title, "page": own[0]["page"]})
|
|
205
|
+
continue
|
|
206
|
+
entries.append({"title": top.title, "children": own + kids})
|
|
207
|
+
return entries
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
# --- knowledge pages ---
|
|
211
|
+
|
|
212
|
+
_FM_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.S)
|
|
213
|
+
_H1_RE = re.compile(r"^#\s+(.+)$", re.M)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _collect_knowledge(paths: WikiPaths, base: int) -> tuple[list[dict], dict | None]:
|
|
217
|
+
"""Module docs + cards under knowledge/<locale>/ as site pages.
|
|
218
|
+
|
|
219
|
+
Returns (pages, nav_entry); nav children reference absolute page indexes
|
|
220
|
+
starting at ``base``. Order follows state/knowledge.json (modules then
|
|
221
|
+
cards, by sanitized title) and degrades to on-disk sorting.
|
|
222
|
+
"""
|
|
223
|
+
site = strings(paths.locale)["site"]
|
|
224
|
+
kdir = paths.knowledge_dir
|
|
225
|
+
if not kdir.is_dir():
|
|
226
|
+
return [], None
|
|
227
|
+
dirs = sorted(d for d in kdir.iterdir() if d.is_dir())
|
|
228
|
+
if not dirs:
|
|
229
|
+
return [], None
|
|
230
|
+
|
|
231
|
+
module_files = set(strings(paths.locale)["module_required_files"])
|
|
232
|
+
rank = _knowledge_plan_order(paths)
|
|
233
|
+
|
|
234
|
+
def sort_key(d: Path) -> tuple:
|
|
235
|
+
return (rank.get(d.name, len(rank)), d.name)
|
|
236
|
+
dirs.sort(key=sort_key)
|
|
237
|
+
|
|
238
|
+
module_title_of: dict[str, str] = {}
|
|
239
|
+
pages: list[dict] = []
|
|
240
|
+
children: list[dict] = []
|
|
241
|
+
for d in dirs:
|
|
242
|
+
docs = sorted(f for f in d.glob("*.md") if f.name in module_files)
|
|
243
|
+
if docs:
|
|
244
|
+
title = _module_title(paths, d)
|
|
245
|
+
module_title_of[d.name] = title
|
|
246
|
+
for f in docs:
|
|
247
|
+
pages.append({
|
|
248
|
+
"id": f"kb:{d.name}/{f.name}",
|
|
249
|
+
"title": f"{title} · {f.stem}",
|
|
250
|
+
"path": f.relative_to(paths.root).as_posix(),
|
|
251
|
+
"md": f.read_text(encoding="utf-8"),
|
|
252
|
+
})
|
|
253
|
+
children.append({"title": f"{title} · {f.stem}", "page": base + len(pages) - 1})
|
|
254
|
+
continue
|
|
255
|
+
card = d / f"{d.name}.md"
|
|
256
|
+
if card.is_file():
|
|
257
|
+
md, title = _card_body(card.read_text(encoding="utf-8"), d.name)
|
|
258
|
+
pages.append({
|
|
259
|
+
"id": f"kb:{d.name}/{d.name}.md",
|
|
260
|
+
"title": title,
|
|
261
|
+
"path": card.relative_to(paths.root).as_posix(),
|
|
262
|
+
"md": md,
|
|
263
|
+
})
|
|
264
|
+
children.append({"title": title, "page": base + len(pages) - 1})
|
|
265
|
+
if not pages:
|
|
266
|
+
return [], None
|
|
267
|
+
return pages, {"title": site["knowledge_label"], "children": children}
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _knowledge_plan_order(paths: WikiPaths) -> dict[str, int]:
|
|
271
|
+
"""Directory-name -> plan rank, from state/knowledge.json item titles."""
|
|
272
|
+
f = paths.knowledge_plan_file
|
|
273
|
+
if not f.is_file():
|
|
274
|
+
return {}
|
|
275
|
+
try:
|
|
276
|
+
plan = json.loads(f.read_text(encoding="utf-8"))
|
|
277
|
+
except json.JSONDecodeError:
|
|
278
|
+
return {}
|
|
279
|
+
rank: dict[str, int] = {}
|
|
280
|
+
if not isinstance(plan, dict):
|
|
281
|
+
return rank
|
|
282
|
+
i = 0
|
|
283
|
+
for kind in ("modules", "cards"):
|
|
284
|
+
for item in plan.get(kind) or []:
|
|
285
|
+
if isinstance(item, dict) and item.get("title"):
|
|
286
|
+
rank.setdefault(sanitize_component(item["title"]), i)
|
|
287
|
+
i += 1
|
|
288
|
+
return rank
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _module_title(paths: WikiPaths, d: Path) -> str:
|
|
292
|
+
f = d / "_module.yaml"
|
|
293
|
+
if f.is_file():
|
|
294
|
+
try:
|
|
295
|
+
meta = yaml.safe_load(f.read_text(encoding="utf-8")) or {}
|
|
296
|
+
if isinstance(meta, dict) and meta.get("title"):
|
|
297
|
+
return str(meta["title"])
|
|
298
|
+
except yaml.YAMLError:
|
|
299
|
+
pass
|
|
300
|
+
return d.name
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _card_body(raw: str, dirname: str) -> tuple[str, str]:
|
|
304
|
+
"""Strip YAML front matter for display; title = fm name / first H1 / dir."""
|
|
305
|
+
title = dirname
|
|
306
|
+
body = raw
|
|
307
|
+
fm = _FM_RE.match(raw)
|
|
308
|
+
meta = {}
|
|
309
|
+
if fm:
|
|
310
|
+
body = raw[fm.end():]
|
|
311
|
+
try:
|
|
312
|
+
meta = yaml.safe_load(fm.group(1)) or {}
|
|
313
|
+
except yaml.YAMLError:
|
|
314
|
+
meta = {}
|
|
315
|
+
if isinstance(meta, dict) and meta.get("name"):
|
|
316
|
+
title = str(meta["name"])
|
|
317
|
+
if title == dirname:
|
|
318
|
+
h1 = _H1_RE.search(body)
|
|
319
|
+
if h1:
|
|
320
|
+
title = h1.group(1).strip()
|
|
321
|
+
return body, title
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
# --- source snippet extraction ---
|
|
325
|
+
|
|
326
|
+
def _collect_snippets(repo_root: Path, pages: list[dict]) -> dict:
|
|
327
|
+
snippets: dict = {}
|
|
328
|
+
for p in pages:
|
|
329
|
+
for path, start, end in extract_refs(p["md"]):
|
|
330
|
+
if start is None:
|
|
331
|
+
key, s, e = path, 1, None
|
|
332
|
+
else:
|
|
333
|
+
key, s, e = f"{path}#L{start}-L{end}", start, end
|
|
334
|
+
if key in snippets:
|
|
335
|
+
continue
|
|
336
|
+
lines = _read_lines(repo_root / path)
|
|
337
|
+
if lines is None:
|
|
338
|
+
snippets[key] = {"path": path, "missing": True}
|
|
339
|
+
continue
|
|
340
|
+
e = min(e or len(lines), len(lines))
|
|
341
|
+
s = max(1, min(s, e))
|
|
342
|
+
if e - s + 1 > MAX_SNIPPET_LINES:
|
|
343
|
+
snippets[key] = {"path": path, "missing": True}
|
|
344
|
+
continue
|
|
345
|
+
snippets[key] = {"path": path, "start": s, "end": e, "lines": lines[s - 1:e]}
|
|
346
|
+
return snippets
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _read_lines(p: Path) -> list[str] | None:
|
|
350
|
+
try:
|
|
351
|
+
if not p.is_file():
|
|
352
|
+
return None
|
|
353
|
+
data = p.read_bytes()
|
|
354
|
+
if b"\0" in data[:8192]: # binary file
|
|
355
|
+
return None
|
|
356
|
+
return data.decode("utf-8", "replace").splitlines()
|
|
357
|
+
except OSError:
|
|
358
|
+
return None
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
# --- HTML assembly ---
|
|
362
|
+
|
|
363
|
+
def _render_html(payload: dict, locale: str) -> str:
|
|
364
|
+
payload_js = json.dumps(payload, ensure_ascii=False).replace("<", "\\u003c")
|
|
365
|
+
shell = (templates.TEMPLATE_DIR / "site.html").read_text(encoding="utf-8")
|
|
366
|
+
app_js = _script_safe((templates.TEMPLATE_DIR / "site" / "app.js").read_text(encoding="utf-8"))
|
|
367
|
+
return templates.render(
|
|
368
|
+
shell,
|
|
369
|
+
SITE_PAYLOAD=payload_js,
|
|
370
|
+
MARKED_JS=_vendor("marked.min.js"),
|
|
371
|
+
MERMAID_JS=_vendor("mermaid.min.js"),
|
|
372
|
+
APP_JS=app_js,
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _vendor(name: str) -> str:
|
|
377
|
+
js = (Path(__file__).parent / "vendor" / name).read_text(encoding="utf-8")
|
|
378
|
+
return _script_safe(js)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _script_safe(js: str) -> str:
|
|
382
|
+
# Inside an inline <script> the literal `</script` would close the tag;
|
|
383
|
+
# `<\/script` is an equivalent escape inside JS strings and regexes.
|
|
384
|
+
return js.replace("</script", "<\\/script")
|