pystdoc 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pystdoc/__init__.py +3 -0
- pystdoc/cache.py +38 -0
- pystdoc/call_graph.py +277 -0
- pystdoc/cli.py +184 -0
- pystdoc/compilation_db.py +112 -0
- pystdoc/db.py +231 -0
- pystdoc/design_engine.py +600 -0
- pystdoc/doc_writer.py +222 -0
- pystdoc/engine.py +362 -0
- pystdoc/hasher.py +77 -0
- pystdoc/llm_client.py +288 -0
- pystdoc/parser_clang.py +169 -0
- pystdoc/parser_generic.py +41 -0
- pystdoc/parser_python.py +120 -0
- pystdoc/report_engine.py +132 -0
- pystdoc/scanner.py +61 -0
- pystdoc/symbols.py +54 -0
- pystdoc-0.2.0.dist-info/METADATA +130 -0
- pystdoc-0.2.0.dist-info/RECORD +23 -0
- pystdoc-0.2.0.dist-info/WHEEL +5 -0
- pystdoc-0.2.0.dist-info/entry_points.txt +5 -0
- pystdoc-0.2.0.dist-info/licenses/LICENSE +21 -0
- pystdoc-0.2.0.dist-info/top_level.txt +1 -0
pystdoc/__init__.py
ADDED
pystdoc/cache.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Cache storage and atomic write operations with immediate disk sync."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Dict, Optional
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def write_flushed_text(file_path: Path, text: str) -> None:
|
|
11
|
+
"""Write text to file with immediate directory creation, flush, and fsync."""
|
|
12
|
+
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
13
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
14
|
+
f.write(text)
|
|
15
|
+
f.flush()
|
|
16
|
+
os.fsync(f.fileno())
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def save_symbol_cache(target_dir: Path, unique_id: str, data: Dict[str, Any]) -> Path:
|
|
20
|
+
"""Save symbol LLM analysis result to json cache file with immediate sync."""
|
|
21
|
+
safe_name = re.sub(r"[^\w\-.]", "_", unique_id)
|
|
22
|
+
cache_dir = target_dir / ".docgen" / "cache"
|
|
23
|
+
cache_file = cache_dir / f"{safe_name}.json"
|
|
24
|
+
content = json.dumps(data, ensure_ascii=False, indent=2)
|
|
25
|
+
write_flushed_text(cache_file, content)
|
|
26
|
+
return cache_file
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load_symbol_cache(target_dir: Path, unique_id: str) -> Optional[Dict[str, Any]]:
|
|
30
|
+
"""Load cached LLM analysis data for a symbol."""
|
|
31
|
+
safe_name = re.sub(r"[^\w\-.]", "_", unique_id)
|
|
32
|
+
cache_file = target_dir / ".docgen" / "cache" / f"{safe_name}.json"
|
|
33
|
+
if cache_file.exists():
|
|
34
|
+
try:
|
|
35
|
+
return json.loads(cache_file.read_text(encoding="utf-8"))
|
|
36
|
+
except Exception:
|
|
37
|
+
return None
|
|
38
|
+
return None
|
pystdoc/call_graph.py
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""Call graph construction, Tarjan SCC cycle condensation, FQDN matching, and Level-by-Level DAG parallel ordering."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections import defaultdict, deque
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Dict, List, Optional, Set, Tuple
|
|
8
|
+
|
|
9
|
+
from pystdoc.symbols import Symbol, get_kind_prefix
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class SymbolNode:
|
|
14
|
+
symbol: Symbol
|
|
15
|
+
rel_path: Path
|
|
16
|
+
full_path: Path
|
|
17
|
+
unique_id: str
|
|
18
|
+
fqdn: str = ""
|
|
19
|
+
direct_callee_ids: Set[str] = field(default_factory=set)
|
|
20
|
+
scc_group_ids: List[str] = field(default_factory=list)
|
|
21
|
+
dag_level: int = 0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def flatten_symbols(
|
|
25
|
+
symbols: List[Symbol],
|
|
26
|
+
rel_path: Path,
|
|
27
|
+
full_path: Path,
|
|
28
|
+
prefix: str = "",
|
|
29
|
+
) -> List[SymbolNode]:
|
|
30
|
+
"""Recursively collect all symbols into a flat list of SymbolNodes with FQDN and kind prefixes."""
|
|
31
|
+
nodes: List[SymbolNode] = []
|
|
32
|
+
for sym in symbols:
|
|
33
|
+
k_prefix = get_kind_prefix(sym.kind)
|
|
34
|
+
raw_name = f"{prefix}{sym.name}" if prefix else sym.name
|
|
35
|
+
identifier = f"{k_prefix}.{raw_name}"
|
|
36
|
+
uid = f"{rel_path.as_posix()}::{identifier}"
|
|
37
|
+
|
|
38
|
+
fqdn = sym.fqdn or f"{rel_path.with_suffix('').as_posix().replace('/', '.')}.{raw_name}"
|
|
39
|
+
|
|
40
|
+
node = SymbolNode(
|
|
41
|
+
symbol=sym,
|
|
42
|
+
rel_path=rel_path,
|
|
43
|
+
full_path=full_path,
|
|
44
|
+
unique_id=uid,
|
|
45
|
+
fqdn=fqdn,
|
|
46
|
+
direct_callee_ids=set(),
|
|
47
|
+
scc_group_ids=[],
|
|
48
|
+
dag_level=0,
|
|
49
|
+
)
|
|
50
|
+
nodes.append(node)
|
|
51
|
+
if sym.children:
|
|
52
|
+
child_nodes = flatten_symbols(
|
|
53
|
+
sym.children,
|
|
54
|
+
rel_path,
|
|
55
|
+
full_path,
|
|
56
|
+
prefix=f"{raw_name}.",
|
|
57
|
+
)
|
|
58
|
+
nodes.extend(child_nodes)
|
|
59
|
+
return nodes
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def link_variables_to_functions(nodes: List[SymbolNode]) -> None:
|
|
63
|
+
"""Detect which functions directly reference each variable using FQDN and scope matching."""
|
|
64
|
+
fn_nodes = [n for n in nodes if get_kind_prefix(n.symbol.kind) == "fn"]
|
|
65
|
+
var_nodes = [n for n in nodes if get_kind_prefix(n.symbol.kind) == "var"]
|
|
66
|
+
|
|
67
|
+
for v_node in var_nodes:
|
|
68
|
+
v_name = v_node.symbol.name
|
|
69
|
+
ref_funcs: Set[str] = set()
|
|
70
|
+
|
|
71
|
+
for f_node in fn_nodes:
|
|
72
|
+
if f_node.rel_path == v_node.rel_path:
|
|
73
|
+
if f_node.symbol.line_start <= v_node.symbol.line_start <= f_node.symbol.line_end:
|
|
74
|
+
ref_funcs.add(f"`{f_node.symbol.name}` (`{f_node.rel_path.name}`)")
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
try:
|
|
78
|
+
code_lines = f_node.full_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
79
|
+
start = max(0, f_node.symbol.line_start - 1)
|
|
80
|
+
end = min(len(code_lines), f_node.symbol.line_end)
|
|
81
|
+
func_code = "\n".join(code_lines[start:end])
|
|
82
|
+
|
|
83
|
+
if re.search(rf"\b{re.escape(v_name)}\b", func_code):
|
|
84
|
+
ref_funcs.add(f"`{f_node.symbol.name}` (`{f_node.rel_path.name}`)")
|
|
85
|
+
except Exception:
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
v_node.symbol.referencing_functions = sorted(list(ref_funcs))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def tarjan_scc(node_ids: List[str], adj: Dict[str, Set[str]]) -> List[List[str]]:
|
|
92
|
+
"""Tarjan's strongly connected components algorithm in O(V+E) time."""
|
|
93
|
+
index = 0
|
|
94
|
+
indices: Dict[str, int] = {}
|
|
95
|
+
lowlinks: Dict[str, int] = {}
|
|
96
|
+
on_stack: Set[str] = set()
|
|
97
|
+
stack: List[str] = []
|
|
98
|
+
sccs: List[List[str]] = []
|
|
99
|
+
|
|
100
|
+
def strongconnect(v: str) -> None:
|
|
101
|
+
nonlocal index
|
|
102
|
+
indices[v] = index
|
|
103
|
+
lowlinks[v] = index
|
|
104
|
+
index += 1
|
|
105
|
+
stack.append(v)
|
|
106
|
+
on_stack.add(v)
|
|
107
|
+
|
|
108
|
+
for w in adj.get(v, set()):
|
|
109
|
+
if w not in indices:
|
|
110
|
+
strongconnect(w)
|
|
111
|
+
lowlinks[v] = min(lowlinks[v], lowlinks[w])
|
|
112
|
+
elif w in on_stack:
|
|
113
|
+
lowlinks[v] = min(lowlinks[v], indices[w])
|
|
114
|
+
|
|
115
|
+
if lowlinks[v] == indices[v]:
|
|
116
|
+
scc: List[str] = []
|
|
117
|
+
while True:
|
|
118
|
+
w = stack.pop()
|
|
119
|
+
on_stack.remove(w)
|
|
120
|
+
scc.append(w)
|
|
121
|
+
if w == v:
|
|
122
|
+
break
|
|
123
|
+
sccs.append(scc)
|
|
124
|
+
|
|
125
|
+
for node_id in node_ids:
|
|
126
|
+
if node_id not in indices:
|
|
127
|
+
strongconnect(node_id)
|
|
128
|
+
|
|
129
|
+
return sccs
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def order_symbols_by_levels(nodes: List[SymbolNode]) -> List[List[SymbolNode]]:
|
|
133
|
+
"""Partition symbol nodes into strictly dependency-safe levels for parallel processing."""
|
|
134
|
+
id_to_node: Dict[str, SymbolNode] = {n.unique_id: n for n in nodes}
|
|
135
|
+
fqdn_to_node_ids: Dict[str, List[str]] = defaultdict(list)
|
|
136
|
+
short_name_to_ids: Dict[str, List[str]] = defaultdict(list)
|
|
137
|
+
|
|
138
|
+
for node in nodes:
|
|
139
|
+
short_name_to_ids[node.symbol.name].append(node.unique_id)
|
|
140
|
+
if node.fqdn:
|
|
141
|
+
fqdn_to_node_ids[node.fqdn].append(node.unique_id)
|
|
142
|
+
if "::" in node.unique_id:
|
|
143
|
+
sym_part = node.unique_id.split("::", 1)[1]
|
|
144
|
+
raw_without_prefix = sym_part.split(".", 1)[-1] if "." in sym_part else sym_part
|
|
145
|
+
short_name_to_ids[raw_without_prefix].append(node.unique_id)
|
|
146
|
+
|
|
147
|
+
caller_to_callees: Dict[str, Set[str]] = {n.unique_id: set() for n in nodes}
|
|
148
|
+
|
|
149
|
+
for node in nodes:
|
|
150
|
+
prefix_type = get_kind_prefix(node.symbol.kind)
|
|
151
|
+
if prefix_type == "fn":
|
|
152
|
+
for callee_name in node.symbol.callees:
|
|
153
|
+
matched_ids = fqdn_to_node_ids.get(callee_name, [])
|
|
154
|
+
if not matched_ids:
|
|
155
|
+
matched_ids = short_name_to_ids.get(callee_name, [])
|
|
156
|
+
|
|
157
|
+
for target_id in matched_ids:
|
|
158
|
+
if target_id != node.unique_id and get_kind_prefix(id_to_node[target_id].symbol.kind) == "fn":
|
|
159
|
+
caller_to_callees[node.unique_id].add(target_id)
|
|
160
|
+
node.direct_callee_ids.add(target_id)
|
|
161
|
+
|
|
162
|
+
base_types_and_vars = [n for n in nodes if get_kind_prefix(n.symbol.kind) in ("const", "type", "var")]
|
|
163
|
+
for n in base_types_and_vars:
|
|
164
|
+
n.dag_level = 0
|
|
165
|
+
|
|
166
|
+
fn_nodes = [n for n in nodes if get_kind_prefix(n.symbol.kind) == "fn"]
|
|
167
|
+
fn_ids = [n.unique_id for n in fn_nodes]
|
|
168
|
+
|
|
169
|
+
# Tarjan SCC
|
|
170
|
+
sccs = tarjan_scc(fn_ids, caller_to_callees)
|
|
171
|
+
|
|
172
|
+
node_to_scc_idx: Dict[str, int] = {}
|
|
173
|
+
for scc_idx, scc_members in enumerate(sccs):
|
|
174
|
+
for member_id in scc_members:
|
|
175
|
+
node_to_scc_idx[member_id] = scc_idx
|
|
176
|
+
if len(scc_members) > 1:
|
|
177
|
+
id_to_node[member_id].scc_group_ids = [m for m in scc_members if m != member_id]
|
|
178
|
+
|
|
179
|
+
# Condensation DAG construction
|
|
180
|
+
scc_callees: Dict[int, Set[int]] = defaultdict(set)
|
|
181
|
+
scc_callers: Dict[int, Set[int]] = defaultdict(set)
|
|
182
|
+
|
|
183
|
+
for caller_id, callee_ids in caller_to_callees.items():
|
|
184
|
+
if caller_id not in node_to_scc_idx:
|
|
185
|
+
continue
|
|
186
|
+
c_scc = node_to_scc_idx[caller_id]
|
|
187
|
+
for callee_id in callee_ids:
|
|
188
|
+
if callee_id not in node_to_scc_idx:
|
|
189
|
+
continue
|
|
190
|
+
target_scc = node_to_scc_idx[callee_id]
|
|
191
|
+
if target_scc != c_scc:
|
|
192
|
+
scc_callees[c_scc].add(target_scc)
|
|
193
|
+
scc_callers[target_scc].add(c_scc)
|
|
194
|
+
|
|
195
|
+
# Bottom-up topological ranking
|
|
196
|
+
scc_rank: Dict[int, int] = {}
|
|
197
|
+
queue = deque([i for i in range(len(sccs)) if len(scc_callees[i]) == 0])
|
|
198
|
+
for i in queue:
|
|
199
|
+
scc_rank[i] = 0
|
|
200
|
+
|
|
201
|
+
resolved_sccs = set(queue)
|
|
202
|
+
while queue:
|
|
203
|
+
curr_scc = queue.popleft()
|
|
204
|
+
for parent_scc in scc_callers[curr_scc]:
|
|
205
|
+
if scc_callees[parent_scc].issubset(resolved_sccs):
|
|
206
|
+
max_child_rank = max(scc_rank[c] for c in scc_callees[parent_scc])
|
|
207
|
+
scc_rank[parent_scc] = max_child_rank + 1
|
|
208
|
+
resolved_sccs.add(parent_scc)
|
|
209
|
+
queue.append(parent_scc)
|
|
210
|
+
|
|
211
|
+
for i in range(len(sccs)):
|
|
212
|
+
if i not in scc_rank:
|
|
213
|
+
scc_rank[i] = 0
|
|
214
|
+
|
|
215
|
+
# Assign DAG levels
|
|
216
|
+
level_groups: Dict[int, List[SymbolNode]] = defaultdict(list)
|
|
217
|
+
if base_types_and_vars:
|
|
218
|
+
level_groups[0].extend(base_types_and_vars)
|
|
219
|
+
|
|
220
|
+
for scc_idx, members in enumerate(sccs):
|
|
221
|
+
rank = scc_rank[scc_idx]
|
|
222
|
+
node_level = rank + (1 if base_types_and_vars else 0)
|
|
223
|
+
for member_id in members:
|
|
224
|
+
node = id_to_node[member_id]
|
|
225
|
+
node.dag_level = node_level
|
|
226
|
+
level_groups[node_level].append(node)
|
|
227
|
+
|
|
228
|
+
max_lvl = max(level_groups.keys()) if level_groups else 0
|
|
229
|
+
levels_list = [level_groups[i] for i in range(max_lvl + 1) if level_groups[i]]
|
|
230
|
+
return levels_list
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def order_symbols_bottom_up(nodes: List[SymbolNode]) -> List[SymbolNode]:
|
|
234
|
+
"""Flattened bottom-up order."""
|
|
235
|
+
levels = order_symbols_by_levels(nodes)
|
|
236
|
+
flat: List[SymbolNode] = []
|
|
237
|
+
for lvl in levels:
|
|
238
|
+
flat.extend(lvl)
|
|
239
|
+
return flat
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def build_callee_context_summary(
|
|
243
|
+
node: SymbolNode,
|
|
244
|
+
resolved_symbols: Dict[str, SymbolNode],
|
|
245
|
+
) -> str:
|
|
246
|
+
"""Build summary text of called low-level functions and mutual recursion peers for LLM prompt."""
|
|
247
|
+
lines: List[str] = []
|
|
248
|
+
|
|
249
|
+
if node.scc_group_ids:
|
|
250
|
+
peer_names = []
|
|
251
|
+
for peer_id in node.scc_group_ids:
|
|
252
|
+
peer_node = resolved_symbols.get(peer_id)
|
|
253
|
+
if peer_node:
|
|
254
|
+
peer_names.append(f"`{peer_node.symbol.name}` ({peer_node.rel_path.name})")
|
|
255
|
+
if peer_names:
|
|
256
|
+
lines.append("[Mutual Recursion Group]:")
|
|
257
|
+
lines.append(f"- Note: This function operates in mutual recursion with {', '.join(peer_names)}.")
|
|
258
|
+
lines.append("")
|
|
259
|
+
|
|
260
|
+
if node.direct_callee_ids:
|
|
261
|
+
lines.append("[Direct Callees Summary]:")
|
|
262
|
+
for callee_id in sorted(node.direct_callee_ids):
|
|
263
|
+
callee_node = resolved_symbols.get(callee_id)
|
|
264
|
+
if callee_node:
|
|
265
|
+
sym = callee_node.symbol
|
|
266
|
+
purpose = sym.purpose or "Executes operation"
|
|
267
|
+
inputs = sym.inputs_note or ", ".join(f"{p.name}: {p.type_hint}" for p in sym.parameters) or "None"
|
|
268
|
+
outputs = sym.outputs_note or sym.return_type or "None"
|
|
269
|
+
lines.append(f"- Function `{sym.name}` (FQDN: `{sym.fqdn or sym.name}`, File: `{callee_node.rel_path.name}`):")
|
|
270
|
+
lines.append(f" - Purpose: {purpose}")
|
|
271
|
+
lines.append(f" - Inputs: {inputs}")
|
|
272
|
+
lines.append(f" - Outputs: {outputs}")
|
|
273
|
+
if sym.overview:
|
|
274
|
+
first_lines = sym.overview.strip().splitlines()[:2]
|
|
275
|
+
lines.append(f" - Summary: {' / '.join(l.strip() for l in first_lines)}")
|
|
276
|
+
|
|
277
|
+
return "\n".join(lines)
|
pystdoc/cli.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Command-line interface entry points for docgen, designgen, and reportgen with multi-language support."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import pystdoc
|
|
8
|
+
from pystdoc.engine import run_docgen
|
|
9
|
+
from pystdoc.design_engine import run_design_generation
|
|
10
|
+
from pystdoc.report_engine import generate_readme_doc
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def docgen_main() -> None:
|
|
14
|
+
"""CLI entry point for `docgen` command."""
|
|
15
|
+
parser = argparse.ArgumentParser(description="Source code symbol document generator (docgen)")
|
|
16
|
+
parser.add_argument("--version", "-v", action="version", version=f"%(prog)s {pystdoc.__version__}")
|
|
17
|
+
parser.add_argument("--dir", default="./", help="Target project directory path (default: ./)")
|
|
18
|
+
parser.add_argument("--no-llm", action="store_true", help="Disable LLM explanation generation")
|
|
19
|
+
parser.add_argument("--host", "-H", default=None, help="LLM server host URL (default: http://127.0.0.1:11434)")
|
|
20
|
+
parser.add_argument("--base-url", "-b", default=None, help="Alias for --host")
|
|
21
|
+
parser.add_argument("--model", "-m", default="gemma4-26b-a4b", help="LLM model identifier (default: gemma4-26b-a4b)")
|
|
22
|
+
parser.add_argument("--token", default=None, help="API token / key (or env OPENAI_API_KEY / LLM_TOKEN)")
|
|
23
|
+
parser.add_argument("--api-key", default=None, help="Alias for --token")
|
|
24
|
+
parser.add_argument("--context-size", "--ctx-size", type=int, default=16384, help="LLM context window size (default: 16384)")
|
|
25
|
+
parser.add_argument("--language", "-l", default="English", help="Output documentation language (e.g. English, Japanese) (default: English)")
|
|
26
|
+
parser.add_argument("--force", "-f", action="store_true", help="Force regenerate all documents ignoring cache")
|
|
27
|
+
parser.add_argument("--allow-fallback", action="store_true", help="Allow fallback to static template on LLM failure")
|
|
28
|
+
parser.add_argument("--compile-commands", default=None, help="Path to compile_commands.json (auto-detected if omitted)")
|
|
29
|
+
parser.add_argument("--concurrency", "-j", type=int, default=4, help="Number of parallel LLM workers (default: 4)")
|
|
30
|
+
|
|
31
|
+
args = parser.parse_args()
|
|
32
|
+
sys.exit(
|
|
33
|
+
run_docgen(
|
|
34
|
+
target_dir=Path(args.dir),
|
|
35
|
+
use_llm=not args.no_llm,
|
|
36
|
+
host=args.host,
|
|
37
|
+
base_url=args.base_url,
|
|
38
|
+
model=args.model,
|
|
39
|
+
token=args.token,
|
|
40
|
+
api_key=args.api_key,
|
|
41
|
+
context_size=args.context_size,
|
|
42
|
+
language=args.language,
|
|
43
|
+
force=args.force,
|
|
44
|
+
allow_fallback=args.allow_fallback,
|
|
45
|
+
compile_commands_path=args.compile_commands,
|
|
46
|
+
concurrency=args.concurrency,
|
|
47
|
+
)
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def designgen_main() -> None:
|
|
52
|
+
"""CLI entry point for `designgen` command."""
|
|
53
|
+
parser = argparse.ArgumentParser(description="Architecture and system design generator (designgen)")
|
|
54
|
+
parser.add_argument("--version", "-v", action="version", version=f"%(prog)s {pystdoc.__version__}")
|
|
55
|
+
parser.add_argument("--dir", default="./", help="Target project directory path (default: ./)")
|
|
56
|
+
parser.add_argument("--no-llm", action="store_true", help="Disable LLM explanation generation")
|
|
57
|
+
parser.add_argument("--host", "-H", default=None, help="LLM server host URL (default: http://127.0.0.1:11434)")
|
|
58
|
+
parser.add_argument("--base-url", "-b", default=None, help="Alias for --host")
|
|
59
|
+
parser.add_argument("--model", "-m", default="gemma4-26b-a4b", help="LLM model identifier (default: gemma4-26b-a4b)")
|
|
60
|
+
parser.add_argument("--token", default=None, help="API token / key (or env OPENAI_API_KEY / LLM_TOKEN)")
|
|
61
|
+
parser.add_argument("--api-key", default=None, help="Alias for --token")
|
|
62
|
+
parser.add_argument("--context-size", "--ctx-size", type=int, default=16384, help="LLM context window size (default: 16384)")
|
|
63
|
+
parser.add_argument("--language", "-l", default="English", help="Output documentation language (e.g. English, Japanese) (default: English)")
|
|
64
|
+
parser.add_argument("--force", "-f", action="store_true", help="Force regenerate all design documents ignoring cache")
|
|
65
|
+
parser.add_argument("--allow-fallback", action="store_true", help="Allow fallback to static template on LLM failure")
|
|
66
|
+
|
|
67
|
+
args = parser.parse_args()
|
|
68
|
+
sys.exit(
|
|
69
|
+
run_design_generation(
|
|
70
|
+
target_dir=Path(args.dir),
|
|
71
|
+
use_llm=not args.no_llm,
|
|
72
|
+
host=args.host,
|
|
73
|
+
base_url=args.base_url,
|
|
74
|
+
model=args.model,
|
|
75
|
+
token=args.token,
|
|
76
|
+
api_key=args.api_key,
|
|
77
|
+
context_size=args.context_size,
|
|
78
|
+
language=args.language,
|
|
79
|
+
force=args.force,
|
|
80
|
+
allow_fallback=args.allow_fallback,
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def reportgen_main() -> None:
|
|
86
|
+
"""CLI entry point for `reportgen` and `pystdoc` commands."""
|
|
87
|
+
parser = argparse.ArgumentParser(description="Unified documentation orchestrator (reportgen / pystdoc)")
|
|
88
|
+
parser.add_argument("--version", "-v", action="version", version=f"%(prog)s {pystdoc.__version__}")
|
|
89
|
+
parser.add_argument("--dir", default="./", help="Target project directory path (default: ./)")
|
|
90
|
+
parser.add_argument("--no-llm", action="store_true", help="Disable LLM explanation generation")
|
|
91
|
+
parser.add_argument("--host", "-H", default=None, help="LLM server host URL (default: http://127.0.0.1:11434)")
|
|
92
|
+
parser.add_argument("--base-url", "-b", default=None, help="Alias for --host")
|
|
93
|
+
parser.add_argument("--model", "-m", default="gemma4-26b-a4b", help="LLM model identifier (default: gemma4-26b-a4b)")
|
|
94
|
+
parser.add_argument("--token", default=None, help="API token / key (or env OPENAI_API_KEY / LLM_TOKEN)")
|
|
95
|
+
parser.add_argument("--api-key", default=None, help="Alias for --token")
|
|
96
|
+
parser.add_argument("--context-size", "--ctx-size", type=int, default=16384, help="LLM context window size (default: 16384)")
|
|
97
|
+
parser.add_argument("--language", "-l", default="English", help="Output documentation language (e.g. English, Japanese) (default: English)")
|
|
98
|
+
parser.add_argument("--force", "-f", action="store_true", help="Force regenerate all documents ignoring cache")
|
|
99
|
+
parser.add_argument("--allow-fallback", action="store_true", help="Allow fallback to static template on LLM failure")
|
|
100
|
+
parser.add_argument("--compile-commands", default=None, help="Path to compile_commands.json (auto-detected if omitted)")
|
|
101
|
+
parser.add_argument("--concurrency", "-j", type=int, default=4, help="Number of parallel LLM workers (default: 4)")
|
|
102
|
+
parser.add_argument("--skip-docgen", action="store_true", help="Skip docgen step")
|
|
103
|
+
parser.add_argument("--skip-designgen", action="store_true", help="Skip designgen step")
|
|
104
|
+
|
|
105
|
+
args = parser.parse_args()
|
|
106
|
+
target_dir = Path(args.dir).resolve()
|
|
107
|
+
|
|
108
|
+
if not target_dir.exists() or not target_dir.is_dir():
|
|
109
|
+
print(f"Error: Specified directory does not exist: {target_dir}", file=sys.stderr)
|
|
110
|
+
sys.exit(1)
|
|
111
|
+
|
|
112
|
+
print("================================================================")
|
|
113
|
+
print(f"=== pystdoc Unified Pipeline v{pystdoc.__version__} (Lang: {args.language}): {target_dir} ===")
|
|
114
|
+
print("================================================================")
|
|
115
|
+
|
|
116
|
+
# 1. docgen
|
|
117
|
+
if not args.skip_docgen:
|
|
118
|
+
print("\n>>> [Step 1/3] docgen: Parsing source code and generating symbol docs...")
|
|
119
|
+
ret_docgen = run_docgen(
|
|
120
|
+
target_dir=target_dir,
|
|
121
|
+
use_llm=not args.no_llm,
|
|
122
|
+
host=args.host,
|
|
123
|
+
base_url=args.base_url,
|
|
124
|
+
model=args.model,
|
|
125
|
+
token=args.token,
|
|
126
|
+
api_key=args.api_key,
|
|
127
|
+
context_size=args.context_size,
|
|
128
|
+
language=args.language,
|
|
129
|
+
force=args.force,
|
|
130
|
+
allow_fallback=args.allow_fallback,
|
|
131
|
+
compile_commands_path=args.compile_commands,
|
|
132
|
+
concurrency=args.concurrency,
|
|
133
|
+
)
|
|
134
|
+
if ret_docgen != 0:
|
|
135
|
+
sys.exit(ret_docgen)
|
|
136
|
+
|
|
137
|
+
# 2. designgen
|
|
138
|
+
if not args.skip_designgen:
|
|
139
|
+
print("\n>>> [Step 2/3] designgen: Synthesizing architecture and data models...")
|
|
140
|
+
ret_design = run_design_generation(
|
|
141
|
+
target_dir=target_dir,
|
|
142
|
+
use_llm=not args.no_llm,
|
|
143
|
+
host=args.host,
|
|
144
|
+
base_url=args.base_url,
|
|
145
|
+
model=args.model,
|
|
146
|
+
token=args.token,
|
|
147
|
+
api_key=args.api_key,
|
|
148
|
+
context_size=args.context_size,
|
|
149
|
+
language=args.language,
|
|
150
|
+
force=args.force,
|
|
151
|
+
allow_fallback=args.allow_fallback,
|
|
152
|
+
)
|
|
153
|
+
if ret_design != 0:
|
|
154
|
+
sys.exit(ret_design)
|
|
155
|
+
|
|
156
|
+
# 3. reportgen
|
|
157
|
+
print("\n>>> [Step 3/3] reportgen: Generating project overview README (.docgen/README.md)...")
|
|
158
|
+
from pystdoc.llm_client import LLMClient
|
|
159
|
+
llm_client = None
|
|
160
|
+
if not args.no_llm:
|
|
161
|
+
client = LLMClient(
|
|
162
|
+
host=args.host or args.base_url,
|
|
163
|
+
model=args.model,
|
|
164
|
+
token=args.token or args.api_key,
|
|
165
|
+
context_size=args.context_size,
|
|
166
|
+
)
|
|
167
|
+
if client.check_availability():
|
|
168
|
+
llm_client = client
|
|
169
|
+
elif not args.allow_fallback:
|
|
170
|
+
print(f"Error: Failed to connect to LLM server ({client.base_url}).", file=sys.stderr)
|
|
171
|
+
sys.exit(1)
|
|
172
|
+
|
|
173
|
+
generate_readme_doc(
|
|
174
|
+
target_dir=target_dir,
|
|
175
|
+
llm_client=llm_client,
|
|
176
|
+
language=args.language,
|
|
177
|
+
allow_fallback=args.allow_fallback,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
out_readme = target_dir / ".docgen" / "README.md"
|
|
181
|
+
print("================================================================")
|
|
182
|
+
print(f"=== reportgen Finished: Created {out_readme} ===")
|
|
183
|
+
print("================================================================")
|
|
184
|
+
sys.exit(0)
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Compilation database (compile_commands.json) parser and compiler flags resolver."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import shlex
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Dict, List, Optional
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CompilationDatabase:
|
|
11
|
+
"""Compilation database resolver for C/C++ compilation flags."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, db_path: Optional[Path] = None, target_dir: Optional[Path] = None):
|
|
14
|
+
self.flags_map: Dict[str, List[str]] = {}
|
|
15
|
+
self.loaded_path: Optional[Path] = None
|
|
16
|
+
self.target_dir = target_dir
|
|
17
|
+
|
|
18
|
+
# Auto-discover compile_commands.json
|
|
19
|
+
candidates = []
|
|
20
|
+
if db_path:
|
|
21
|
+
candidates.append(Path(db_path))
|
|
22
|
+
if target_dir:
|
|
23
|
+
candidates.extend([
|
|
24
|
+
target_dir / "compile_commands.json",
|
|
25
|
+
target_dir / "build" / "compile_commands.json",
|
|
26
|
+
target_dir / "builddir" / "compile_commands.json",
|
|
27
|
+
])
|
|
28
|
+
candidates.extend([
|
|
29
|
+
Path("compile_commands.json"),
|
|
30
|
+
Path("build/compile_commands.json"),
|
|
31
|
+
Path("builddir/compile_commands.json"),
|
|
32
|
+
])
|
|
33
|
+
|
|
34
|
+
for c in candidates:
|
|
35
|
+
if c.exists() and c.is_file():
|
|
36
|
+
try:
|
|
37
|
+
self._load_db(c)
|
|
38
|
+
self.loaded_path = c
|
|
39
|
+
break
|
|
40
|
+
except Exception:
|
|
41
|
+
continue
|
|
42
|
+
|
|
43
|
+
def _load_db(self, db_file: Path) -> None:
|
|
44
|
+
"""Parse compile_commands.json and extract flags for each source file."""
|
|
45
|
+
data = json.loads(db_file.read_text(encoding="utf-8"))
|
|
46
|
+
for entry in data:
|
|
47
|
+
file_path = entry.get("file")
|
|
48
|
+
if not file_path:
|
|
49
|
+
continue
|
|
50
|
+
|
|
51
|
+
directory = entry.get("directory", "")
|
|
52
|
+
full_file = Path(directory) / file_path if directory else Path(file_path)
|
|
53
|
+
norm_key = full_file.resolve().as_posix()
|
|
54
|
+
|
|
55
|
+
# Extract arguments from command string or arguments list
|
|
56
|
+
raw_args = []
|
|
57
|
+
if "arguments" in entry:
|
|
58
|
+
raw_args = entry["arguments"]
|
|
59
|
+
elif "command" in entry:
|
|
60
|
+
raw_args = shlex.split(entry["command"])
|
|
61
|
+
|
|
62
|
+
# Filter valid flags for libclang
|
|
63
|
+
clang_args = []
|
|
64
|
+
skip_next = False
|
|
65
|
+
for i, arg in enumerate(raw_args[1:], start=1):
|
|
66
|
+
if skip_next:
|
|
67
|
+
skip_next = False
|
|
68
|
+
continue
|
|
69
|
+
|
|
70
|
+
if arg in ("-c", "-o"):
|
|
71
|
+
skip_next = True
|
|
72
|
+
continue
|
|
73
|
+
if arg.startswith("-o"):
|
|
74
|
+
continue
|
|
75
|
+
|
|
76
|
+
if arg.startswith(("-I", "-D", "-U", "-isystem", "-std=", "-m", "-f")):
|
|
77
|
+
if arg in ("-I", "-isystem") and i + 1 < len(raw_args):
|
|
78
|
+
inc_path = raw_args[i + 1]
|
|
79
|
+
if directory:
|
|
80
|
+
inc_path = (Path(directory) / inc_path).resolve().as_posix()
|
|
81
|
+
clang_args.append(f"{arg}{inc_path}")
|
|
82
|
+
skip_next = True
|
|
83
|
+
else:
|
|
84
|
+
if arg.startswith("-I") and len(arg) > 2:
|
|
85
|
+
inc_dir = arg[2:]
|
|
86
|
+
if directory and not inc_dir.startswith("/"):
|
|
87
|
+
inc_dir = (Path(directory) / inc_dir).resolve().as_posix()
|
|
88
|
+
clang_args.append(f"-I{inc_dir}")
|
|
89
|
+
else:
|
|
90
|
+
clang_args.append(arg)
|
|
91
|
+
|
|
92
|
+
self.flags_map[norm_key] = clang_args
|
|
93
|
+
self.flags_map[Path(file_path).name] = clang_args
|
|
94
|
+
|
|
95
|
+
def get_flags_for_file(self, file_path: Path) -> List[str]:
|
|
96
|
+
"""Get compilation flags for a given source file, or return smart defaults."""
|
|
97
|
+
norm_key = file_path.resolve().as_posix()
|
|
98
|
+
if norm_key in self.flags_map:
|
|
99
|
+
return self.flags_map[norm_key]
|
|
100
|
+
if file_path.name in self.flags_map:
|
|
101
|
+
return self.flags_map[file_path.name]
|
|
102
|
+
|
|
103
|
+
# Smart defaults with inferred include directories
|
|
104
|
+
base_dir = self.target_dir or file_path.parent
|
|
105
|
+
defaults = [
|
|
106
|
+
"-std=c11" if file_path.suffix in (".c", ".h") else "-std=c++17",
|
|
107
|
+
"-D_GNU_SOURCE",
|
|
108
|
+
f"-I{base_dir.resolve().as_posix()}",
|
|
109
|
+
f"-I{(base_dir / 'include').resolve().as_posix()}",
|
|
110
|
+
f"-I{(base_dir / 'src').resolve().as_posix()}",
|
|
111
|
+
]
|
|
112
|
+
return defaults
|