pystdoc 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pystdoc-0.2.0/LICENSE +21 -0
- pystdoc-0.2.0/PKG-INFO +130 -0
- pystdoc-0.2.0/README.md +107 -0
- pystdoc-0.2.0/pyproject.toml +39 -0
- pystdoc-0.2.0/setup.cfg +4 -0
- pystdoc-0.2.0/src/pystdoc/__init__.py +3 -0
- pystdoc-0.2.0/src/pystdoc/cache.py +38 -0
- pystdoc-0.2.0/src/pystdoc/call_graph.py +277 -0
- pystdoc-0.2.0/src/pystdoc/cli.py +184 -0
- pystdoc-0.2.0/src/pystdoc/compilation_db.py +112 -0
- pystdoc-0.2.0/src/pystdoc/db.py +231 -0
- pystdoc-0.2.0/src/pystdoc/design_engine.py +600 -0
- pystdoc-0.2.0/src/pystdoc/doc_writer.py +222 -0
- pystdoc-0.2.0/src/pystdoc/engine.py +362 -0
- pystdoc-0.2.0/src/pystdoc/hasher.py +77 -0
- pystdoc-0.2.0/src/pystdoc/llm_client.py +288 -0
- pystdoc-0.2.0/src/pystdoc/parser_clang.py +169 -0
- pystdoc-0.2.0/src/pystdoc/parser_generic.py +41 -0
- pystdoc-0.2.0/src/pystdoc/parser_python.py +120 -0
- pystdoc-0.2.0/src/pystdoc/report_engine.py +132 -0
- pystdoc-0.2.0/src/pystdoc/scanner.py +61 -0
- pystdoc-0.2.0/src/pystdoc/symbols.py +54 -0
- pystdoc-0.2.0/src/pystdoc.egg-info/PKG-INFO +130 -0
- pystdoc-0.2.0/src/pystdoc.egg-info/SOURCES.txt +29 -0
- pystdoc-0.2.0/src/pystdoc.egg-info/dependency_links.txt +1 -0
- pystdoc-0.2.0/src/pystdoc.egg-info/entry_points.txt +5 -0
- pystdoc-0.2.0/src/pystdoc.egg-info/requires.txt +1 -0
- pystdoc-0.2.0/src/pystdoc.egg-info/top_level.txt +1 -0
- pystdoc-0.2.0/tests/test_designgen.py +101 -0
- pystdoc-0.2.0/tests/test_docgen.py +170 -0
- pystdoc-0.2.0/tests/test_reportgen.py +60 -0
pystdoc-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 tab4moji
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pystdoc-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pystdoc
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Python Structural & Topological Documentation Engine (pystdoc): High-precision, LLM-powered hierarchical codebase and architecture documentation generator.
|
|
5
|
+
Author: tab4moji
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: documentation,llm,ast,clang,architecture,topological,dag,pystdoc,mermaid
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Software Development :: Documentation
|
|
18
|
+
Requires-Python: >=3.8
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: libclang>=16.0.0
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# pystdoc: Python Structural & Topological Documentation Engine
|
|
25
|
+
|
|
26
|
+
**pystdoc** (*Python Structural & Topological Documentation Engine*) is an enterprise-grade, high-precision codebase and architectural documentation generator powered by LLMs, AST parsing, and `libclang`.
|
|
27
|
+
|
|
28
|
+
It analyzes codebases in **bottom-up + top-down topological passes**, constructs hierarchical execution/data models, and synthesizes clean, human-centric Markdown and Mermaid diagrams within a **16K context window**.
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## 📁 Output Directory: `.docgen/`
|
|
33
|
+
|
|
34
|
+
> [!IMPORTANT]
|
|
35
|
+
> **All generated documentation, architecture designs, and caches are automatically centralized inside the `.docgen/` directory of your target project.**
|
|
36
|
+
> Your existing source code files are never modified.
|
|
37
|
+
|
|
38
|
+
When `pystdoc` finishes, you can explore the complete documentation suite starting from `.docgen/README.md`:
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
your_project/
|
|
42
|
+
├── .docgen/ # <-- Centralized output directory
|
|
43
|
+
│ ├── README.md # Executive summary: "What does this project actually do?"
|
|
44
|
+
│ ├── design/ # System architecture and design documentation
|
|
45
|
+
│ │ ├── overview.md # Architecture overview & inter-module Mermaid diagram
|
|
46
|
+
│ │ ├── data_models.md # Data structure design, models, lifecycle & integrity
|
|
47
|
+
│ │ ├── execution_model.md # Runtime execution model, paradigms, & control flow
|
|
48
|
+
│ │ └── modules/ # Module-by-module detailed design documents
|
|
49
|
+
│ │ ├── module_a.md
|
|
50
|
+
│ │ └── ...
|
|
51
|
+
│ ├── documents/ # Granular symbol & source code documentation
|
|
52
|
+
│ │ ├── src/main.c.md # File-level overview and symbol list
|
|
53
|
+
│ │ ├── src/main.c.fn.main.md # Individual symbol document (with call graph & context)
|
|
54
|
+
│ │ └── ...
|
|
55
|
+
│ ├── files.txt # List of scanned source files
|
|
56
|
+
│ └── index.db # SQLite WAL database for instantaneous incremental caching
|
|
57
|
+
├── src/
|
|
58
|
+
└── ...
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## 🌟 Key Features
|
|
64
|
+
|
|
65
|
+
1. **Topological & Structural Ordering (Tarjan SCC + Kahn DAG)**:
|
|
66
|
+
- Evaluates call graphs in $O(V+E)$ linear time.
|
|
67
|
+
- Automatically breaks cyclic mutual recursions and organizes code symbols into dependency-safe execution levels.
|
|
68
|
+
- Level-by-level parallel LLM execution guarantees context-rich bottom-up summaries without race conditions.
|
|
69
|
+
2. **3-in-1 Unified Documentation Pipeline**:
|
|
70
|
+
- `docgen`: Bottom-up & top-down symbol-level documentation with SHA-256 and SQLite caching (`.docgen/documents/`).
|
|
71
|
+
- `designgen`: Map-Reduce architectural synthesis (`.docgen/design/`).
|
|
72
|
+
- `reportgen` / `pystdoc`: Executive summary README (`.docgen/README.md`) answering *"What does this project actually do?"*
|
|
73
|
+
3. **C/C++, Python & Shell Deep Understanding**:
|
|
74
|
+
- **`compile_commands.json` Integration**: Full include path resolution and macro expansion via `libclang`.
|
|
75
|
+
- **Fully Qualified Domain Names (FQDN)**: Disambiguates identical symbol names across large monorepos.
|
|
76
|
+
4. **Standard LLM Options & Multi-Language Support**:
|
|
77
|
+
- Works with **Ollama, LiteRT-LM, vLLM, and OpenAI API**.
|
|
78
|
+
- Supports `--host`, `--model`, `--token` / `--api-key`, `--context-size`, and `--language` (e.g. `English`, `Japanese`, `日本語`).
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## 🚀 Quick Start
|
|
83
|
+
|
|
84
|
+
### Installation
|
|
85
|
+
```bash
|
|
86
|
+
pip install pystdoc
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### Basic Usage
|
|
90
|
+
|
|
91
|
+
#### 1. Generate Full Documentation & README (One Command)
|
|
92
|
+
```bash
|
|
93
|
+
pystdoc --dir ./my_project/
|
|
94
|
+
```
|
|
95
|
+
*Output will be created at `./my_project/.docgen/README.md`.*
|
|
96
|
+
|
|
97
|
+
#### 2. Generate in Japanese
|
|
98
|
+
```bash
|
|
99
|
+
pystdoc --dir ./my_project/ --language 日本語
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
#### 3. Run Individual Steps
|
|
103
|
+
```bash
|
|
104
|
+
# Generate symbol-level docs into .docgen/documents/
|
|
105
|
+
docgen --dir ./my_project/ -j 4
|
|
106
|
+
|
|
107
|
+
# Synthesize architecture design docs into .docgen/design/
|
|
108
|
+
designgen --dir ./my_project/
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## ⚙️ CLI Options
|
|
114
|
+
|
|
115
|
+
| Option | Alias / Env | Default | Description |
|
|
116
|
+
| :--- | :--- | :--- | :--- |
|
|
117
|
+
| `--dir` | | `./` | Target project directory path |
|
|
118
|
+
| `--language` | `-l` | `English` | Output documentation language (`English`, `Japanese`, `日本語`) |
|
|
119
|
+
| `--host` | `-H`, `--base-url` | `http://127.0.0.1:11434` | LLM server host endpoint URL |
|
|
120
|
+
| `--model` | `-m`, `LLM_MODEL` | `gemma4-26b-a4b` | LLM model identifier |
|
|
121
|
+
| `--token` | `--api-key`, `OPENAI_API_KEY` | `None` | API Bearer token |
|
|
122
|
+
| `--context-size` | `--ctx-size` | `16384` | Context window size |
|
|
123
|
+
| `--concurrency` | `-j` | `4` | Number of parallel LLM workers |
|
|
124
|
+
| `--force` | `-f` | `false` | Force regenerate all documents ignoring cache |
|
|
125
|
+
| `--compile-commands` | | `None` | Path to `compile_commands.json` |
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## 📄 License
|
|
130
|
+
MIT License. Author: **tab4moji**.
|
pystdoc-0.2.0/README.md
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# pystdoc: Python Structural & Topological Documentation Engine
|
|
2
|
+
|
|
3
|
+
**pystdoc** (*Python Structural & Topological Documentation Engine*) is an enterprise-grade, high-precision codebase and architectural documentation generator powered by LLMs, AST parsing, and `libclang`.
|
|
4
|
+
|
|
5
|
+
It analyzes codebases in **bottom-up + top-down topological passes**, constructs hierarchical execution/data models, and synthesizes clean, human-centric Markdown and Mermaid diagrams within a **16K context window**.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 📁 Output Directory: `.docgen/`
|
|
10
|
+
|
|
11
|
+
> [!IMPORTANT]
|
|
12
|
+
> **All generated documentation, architecture designs, and caches are automatically centralized inside the `.docgen/` directory of your target project.**
|
|
13
|
+
> Your existing source code files are never modified.
|
|
14
|
+
|
|
15
|
+
When `pystdoc` finishes, you can explore the complete documentation suite starting from `.docgen/README.md`:
|
|
16
|
+
|
|
17
|
+
```text
|
|
18
|
+
your_project/
|
|
19
|
+
├── .docgen/ # <-- Centralized output directory
|
|
20
|
+
│ ├── README.md # Executive summary: "What does this project actually do?"
|
|
21
|
+
│ ├── design/ # System architecture and design documentation
|
|
22
|
+
│ │ ├── overview.md # Architecture overview & inter-module Mermaid diagram
|
|
23
|
+
│ │ ├── data_models.md # Data structure design, models, lifecycle & integrity
|
|
24
|
+
│ │ ├── execution_model.md # Runtime execution model, paradigms, & control flow
|
|
25
|
+
│ │ └── modules/ # Module-by-module detailed design documents
|
|
26
|
+
│ │ ├── module_a.md
|
|
27
|
+
│ │ └── ...
|
|
28
|
+
│ ├── documents/ # Granular symbol & source code documentation
|
|
29
|
+
│ │ ├── src/main.c.md # File-level overview and symbol list
|
|
30
|
+
│ │ ├── src/main.c.fn.main.md # Individual symbol document (with call graph & context)
|
|
31
|
+
│ │ └── ...
|
|
32
|
+
│ ├── files.txt # List of scanned source files
|
|
33
|
+
│ └── index.db # SQLite WAL database for instantaneous incremental caching
|
|
34
|
+
├── src/
|
|
35
|
+
└── ...
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## 🌟 Key Features
|
|
41
|
+
|
|
42
|
+
1. **Topological & Structural Ordering (Tarjan SCC + Kahn DAG)**:
|
|
43
|
+
- Evaluates call graphs in $O(V+E)$ linear time.
|
|
44
|
+
- Automatically breaks cyclic mutual recursions and organizes code symbols into dependency-safe execution levels.
|
|
45
|
+
- Level-by-level parallel LLM execution guarantees context-rich bottom-up summaries without race conditions.
|
|
46
|
+
2. **3-in-1 Unified Documentation Pipeline**:
|
|
47
|
+
- `docgen`: Bottom-up & top-down symbol-level documentation with SHA-256 and SQLite caching (`.docgen/documents/`).
|
|
48
|
+
- `designgen`: Map-Reduce architectural synthesis (`.docgen/design/`).
|
|
49
|
+
- `reportgen` / `pystdoc`: Executive summary README (`.docgen/README.md`) answering *"What does this project actually do?"*
|
|
50
|
+
3. **C/C++, Python & Shell Deep Understanding**:
|
|
51
|
+
- **`compile_commands.json` Integration**: Full include path resolution and macro expansion via `libclang`.
|
|
52
|
+
- **Fully Qualified Domain Names (FQDN)**: Disambiguates identical symbol names across large monorepos.
|
|
53
|
+
4. **Standard LLM Options & Multi-Language Support**:
|
|
54
|
+
- Works with **Ollama, LiteRT-LM, vLLM, and OpenAI API**.
|
|
55
|
+
- Supports `--host`, `--model`, `--token` / `--api-key`, `--context-size`, and `--language` (e.g. `English`, `Japanese`, `日本語`).
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## 🚀 Quick Start
|
|
60
|
+
|
|
61
|
+
### Installation
|
|
62
|
+
```bash
|
|
63
|
+
pip install pystdoc
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### Basic Usage
|
|
67
|
+
|
|
68
|
+
#### 1. Generate Full Documentation & README (One Command)
|
|
69
|
+
```bash
|
|
70
|
+
pystdoc --dir ./my_project/
|
|
71
|
+
```
|
|
72
|
+
*Output will be created at `./my_project/.docgen/README.md`.*
|
|
73
|
+
|
|
74
|
+
#### 2. Generate in Japanese
|
|
75
|
+
```bash
|
|
76
|
+
pystdoc --dir ./my_project/ --language 日本語
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
#### 3. Run Individual Steps
|
|
80
|
+
```bash
|
|
81
|
+
# Generate symbol-level docs into .docgen/documents/
|
|
82
|
+
docgen --dir ./my_project/ -j 4
|
|
83
|
+
|
|
84
|
+
# Synthesize architecture design docs into .docgen/design/
|
|
85
|
+
designgen --dir ./my_project/
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## ⚙️ CLI Options
|
|
91
|
+
|
|
92
|
+
| Option | Alias / Env | Default | Description |
|
|
93
|
+
| :--- | :--- | :--- | :--- |
|
|
94
|
+
| `--dir` | | `./` | Target project directory path |
|
|
95
|
+
| `--language` | `-l` | `English` | Output documentation language (`English`, `Japanese`, `日本語`) |
|
|
96
|
+
| `--host` | `-H`, `--base-url` | `http://127.0.0.1:11434` | LLM server host endpoint URL |
|
|
97
|
+
| `--model` | `-m`, `LLM_MODEL` | `gemma4-26b-a4b` | LLM model identifier |
|
|
98
|
+
| `--token` | `--api-key`, `OPENAI_API_KEY` | `None` | API Bearer token |
|
|
99
|
+
| `--context-size` | `--ctx-size` | `16384` | Context window size |
|
|
100
|
+
| `--concurrency` | `-j` | `4` | Number of parallel LLM workers |
|
|
101
|
+
| `--force` | `-f` | `false` | Force regenerate all documents ignoring cache |
|
|
102
|
+
| `--compile-commands` | | `None` | Path to `compile_commands.json` |
|
|
103
|
+
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
## 📄 License
|
|
107
|
+
MIT License. Author: **tab4moji**.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pystdoc"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Python Structural & Topological Documentation Engine (pystdoc): High-precision, LLM-powered hierarchical codebase and architecture documentation generator."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{ name = "tab4moji" }
|
|
12
|
+
]
|
|
13
|
+
license = { text = "MIT" }
|
|
14
|
+
keywords = ["documentation", "llm", "ast", "clang", "architecture", "topological", "dag", "pystdoc", "mermaid"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.8",
|
|
21
|
+
"Programming Language :: Python :: 3.9",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Topic :: Software Development :: Documentation",
|
|
26
|
+
]
|
|
27
|
+
requires-python = ">=3.8"
|
|
28
|
+
dependencies = [
|
|
29
|
+
"libclang>=16.0.0",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
pystdoc = "pystdoc.cli:reportgen_main"
|
|
34
|
+
reportgen = "pystdoc.cli:reportgen_main"
|
|
35
|
+
docgen = "pystdoc.cli:docgen_main"
|
|
36
|
+
designgen = "pystdoc.cli:designgen_main"
|
|
37
|
+
|
|
38
|
+
[tool.setuptools.packages.find]
|
|
39
|
+
where = ["src"]
|
pystdoc-0.2.0/setup.cfg
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Cache storage and atomic write operations with immediate disk sync."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Dict, Optional
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def write_flushed_text(file_path: Path, text: str) -> None:
|
|
11
|
+
"""Write text to file with immediate directory creation, flush, and fsync."""
|
|
12
|
+
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
13
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
14
|
+
f.write(text)
|
|
15
|
+
f.flush()
|
|
16
|
+
os.fsync(f.fileno())
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def save_symbol_cache(target_dir: Path, unique_id: str, data: Dict[str, Any]) -> Path:
|
|
20
|
+
"""Save symbol LLM analysis result to json cache file with immediate sync."""
|
|
21
|
+
safe_name = re.sub(r"[^\w\-.]", "_", unique_id)
|
|
22
|
+
cache_dir = target_dir / ".docgen" / "cache"
|
|
23
|
+
cache_file = cache_dir / f"{safe_name}.json"
|
|
24
|
+
content = json.dumps(data, ensure_ascii=False, indent=2)
|
|
25
|
+
write_flushed_text(cache_file, content)
|
|
26
|
+
return cache_file
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load_symbol_cache(target_dir: Path, unique_id: str) -> Optional[Dict[str, Any]]:
|
|
30
|
+
"""Load cached LLM analysis data for a symbol."""
|
|
31
|
+
safe_name = re.sub(r"[^\w\-.]", "_", unique_id)
|
|
32
|
+
cache_file = target_dir / ".docgen" / "cache" / f"{safe_name}.json"
|
|
33
|
+
if cache_file.exists():
|
|
34
|
+
try:
|
|
35
|
+
return json.loads(cache_file.read_text(encoding="utf-8"))
|
|
36
|
+
except Exception:
|
|
37
|
+
return None
|
|
38
|
+
return None
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""Call graph construction, Tarjan SCC cycle condensation, FQDN matching, and Level-by-Level DAG parallel ordering."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections import defaultdict, deque
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Dict, List, Optional, Set, Tuple
|
|
8
|
+
|
|
9
|
+
from pystdoc.symbols import Symbol, get_kind_prefix
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class SymbolNode:
|
|
14
|
+
symbol: Symbol
|
|
15
|
+
rel_path: Path
|
|
16
|
+
full_path: Path
|
|
17
|
+
unique_id: str
|
|
18
|
+
fqdn: str = ""
|
|
19
|
+
direct_callee_ids: Set[str] = field(default_factory=set)
|
|
20
|
+
scc_group_ids: List[str] = field(default_factory=list)
|
|
21
|
+
dag_level: int = 0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def flatten_symbols(
|
|
25
|
+
symbols: List[Symbol],
|
|
26
|
+
rel_path: Path,
|
|
27
|
+
full_path: Path,
|
|
28
|
+
prefix: str = "",
|
|
29
|
+
) -> List[SymbolNode]:
|
|
30
|
+
"""Recursively collect all symbols into a flat list of SymbolNodes with FQDN and kind prefixes."""
|
|
31
|
+
nodes: List[SymbolNode] = []
|
|
32
|
+
for sym in symbols:
|
|
33
|
+
k_prefix = get_kind_prefix(sym.kind)
|
|
34
|
+
raw_name = f"{prefix}{sym.name}" if prefix else sym.name
|
|
35
|
+
identifier = f"{k_prefix}.{raw_name}"
|
|
36
|
+
uid = f"{rel_path.as_posix()}::{identifier}"
|
|
37
|
+
|
|
38
|
+
fqdn = sym.fqdn or f"{rel_path.with_suffix('').as_posix().replace('/', '.')}.{raw_name}"
|
|
39
|
+
|
|
40
|
+
node = SymbolNode(
|
|
41
|
+
symbol=sym,
|
|
42
|
+
rel_path=rel_path,
|
|
43
|
+
full_path=full_path,
|
|
44
|
+
unique_id=uid,
|
|
45
|
+
fqdn=fqdn,
|
|
46
|
+
direct_callee_ids=set(),
|
|
47
|
+
scc_group_ids=[],
|
|
48
|
+
dag_level=0,
|
|
49
|
+
)
|
|
50
|
+
nodes.append(node)
|
|
51
|
+
if sym.children:
|
|
52
|
+
child_nodes = flatten_symbols(
|
|
53
|
+
sym.children,
|
|
54
|
+
rel_path,
|
|
55
|
+
full_path,
|
|
56
|
+
prefix=f"{raw_name}.",
|
|
57
|
+
)
|
|
58
|
+
nodes.extend(child_nodes)
|
|
59
|
+
return nodes
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def link_variables_to_functions(nodes: List[SymbolNode]) -> None:
|
|
63
|
+
"""Detect which functions directly reference each variable using FQDN and scope matching."""
|
|
64
|
+
fn_nodes = [n for n in nodes if get_kind_prefix(n.symbol.kind) == "fn"]
|
|
65
|
+
var_nodes = [n for n in nodes if get_kind_prefix(n.symbol.kind) == "var"]
|
|
66
|
+
|
|
67
|
+
for v_node in var_nodes:
|
|
68
|
+
v_name = v_node.symbol.name
|
|
69
|
+
ref_funcs: Set[str] = set()
|
|
70
|
+
|
|
71
|
+
for f_node in fn_nodes:
|
|
72
|
+
if f_node.rel_path == v_node.rel_path:
|
|
73
|
+
if f_node.symbol.line_start <= v_node.symbol.line_start <= f_node.symbol.line_end:
|
|
74
|
+
ref_funcs.add(f"`{f_node.symbol.name}` (`{f_node.rel_path.name}`)")
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
try:
|
|
78
|
+
code_lines = f_node.full_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
79
|
+
start = max(0, f_node.symbol.line_start - 1)
|
|
80
|
+
end = min(len(code_lines), f_node.symbol.line_end)
|
|
81
|
+
func_code = "\n".join(code_lines[start:end])
|
|
82
|
+
|
|
83
|
+
if re.search(rf"\b{re.escape(v_name)}\b", func_code):
|
|
84
|
+
ref_funcs.add(f"`{f_node.symbol.name}` (`{f_node.rel_path.name}`)")
|
|
85
|
+
except Exception:
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
v_node.symbol.referencing_functions = sorted(list(ref_funcs))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def tarjan_scc(node_ids: List[str], adj: Dict[str, Set[str]]) -> List[List[str]]:
|
|
92
|
+
"""Tarjan's strongly connected components algorithm in O(V+E) time."""
|
|
93
|
+
index = 0
|
|
94
|
+
indices: Dict[str, int] = {}
|
|
95
|
+
lowlinks: Dict[str, int] = {}
|
|
96
|
+
on_stack: Set[str] = set()
|
|
97
|
+
stack: List[str] = []
|
|
98
|
+
sccs: List[List[str]] = []
|
|
99
|
+
|
|
100
|
+
def strongconnect(v: str) -> None:
|
|
101
|
+
nonlocal index
|
|
102
|
+
indices[v] = index
|
|
103
|
+
lowlinks[v] = index
|
|
104
|
+
index += 1
|
|
105
|
+
stack.append(v)
|
|
106
|
+
on_stack.add(v)
|
|
107
|
+
|
|
108
|
+
for w in adj.get(v, set()):
|
|
109
|
+
if w not in indices:
|
|
110
|
+
strongconnect(w)
|
|
111
|
+
lowlinks[v] = min(lowlinks[v], lowlinks[w])
|
|
112
|
+
elif w in on_stack:
|
|
113
|
+
lowlinks[v] = min(lowlinks[v], indices[w])
|
|
114
|
+
|
|
115
|
+
if lowlinks[v] == indices[v]:
|
|
116
|
+
scc: List[str] = []
|
|
117
|
+
while True:
|
|
118
|
+
w = stack.pop()
|
|
119
|
+
on_stack.remove(w)
|
|
120
|
+
scc.append(w)
|
|
121
|
+
if w == v:
|
|
122
|
+
break
|
|
123
|
+
sccs.append(scc)
|
|
124
|
+
|
|
125
|
+
for node_id in node_ids:
|
|
126
|
+
if node_id not in indices:
|
|
127
|
+
strongconnect(node_id)
|
|
128
|
+
|
|
129
|
+
return sccs
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def order_symbols_by_levels(nodes: List[SymbolNode]) -> List[List[SymbolNode]]:
|
|
133
|
+
"""Partition symbol nodes into strictly dependency-safe levels for parallel processing."""
|
|
134
|
+
id_to_node: Dict[str, SymbolNode] = {n.unique_id: n for n in nodes}
|
|
135
|
+
fqdn_to_node_ids: Dict[str, List[str]] = defaultdict(list)
|
|
136
|
+
short_name_to_ids: Dict[str, List[str]] = defaultdict(list)
|
|
137
|
+
|
|
138
|
+
for node in nodes:
|
|
139
|
+
short_name_to_ids[node.symbol.name].append(node.unique_id)
|
|
140
|
+
if node.fqdn:
|
|
141
|
+
fqdn_to_node_ids[node.fqdn].append(node.unique_id)
|
|
142
|
+
if "::" in node.unique_id:
|
|
143
|
+
sym_part = node.unique_id.split("::", 1)[1]
|
|
144
|
+
raw_without_prefix = sym_part.split(".", 1)[-1] if "." in sym_part else sym_part
|
|
145
|
+
short_name_to_ids[raw_without_prefix].append(node.unique_id)
|
|
146
|
+
|
|
147
|
+
caller_to_callees: Dict[str, Set[str]] = {n.unique_id: set() for n in nodes}
|
|
148
|
+
|
|
149
|
+
for node in nodes:
|
|
150
|
+
prefix_type = get_kind_prefix(node.symbol.kind)
|
|
151
|
+
if prefix_type == "fn":
|
|
152
|
+
for callee_name in node.symbol.callees:
|
|
153
|
+
matched_ids = fqdn_to_node_ids.get(callee_name, [])
|
|
154
|
+
if not matched_ids:
|
|
155
|
+
matched_ids = short_name_to_ids.get(callee_name, [])
|
|
156
|
+
|
|
157
|
+
for target_id in matched_ids:
|
|
158
|
+
if target_id != node.unique_id and get_kind_prefix(id_to_node[target_id].symbol.kind) == "fn":
|
|
159
|
+
caller_to_callees[node.unique_id].add(target_id)
|
|
160
|
+
node.direct_callee_ids.add(target_id)
|
|
161
|
+
|
|
162
|
+
base_types_and_vars = [n for n in nodes if get_kind_prefix(n.symbol.kind) in ("const", "type", "var")]
|
|
163
|
+
for n in base_types_and_vars:
|
|
164
|
+
n.dag_level = 0
|
|
165
|
+
|
|
166
|
+
fn_nodes = [n for n in nodes if get_kind_prefix(n.symbol.kind) == "fn"]
|
|
167
|
+
fn_ids = [n.unique_id for n in fn_nodes]
|
|
168
|
+
|
|
169
|
+
# Tarjan SCC
|
|
170
|
+
sccs = tarjan_scc(fn_ids, caller_to_callees)
|
|
171
|
+
|
|
172
|
+
node_to_scc_idx: Dict[str, int] = {}
|
|
173
|
+
for scc_idx, scc_members in enumerate(sccs):
|
|
174
|
+
for member_id in scc_members:
|
|
175
|
+
node_to_scc_idx[member_id] = scc_idx
|
|
176
|
+
if len(scc_members) > 1:
|
|
177
|
+
id_to_node[member_id].scc_group_ids = [m for m in scc_members if m != member_id]
|
|
178
|
+
|
|
179
|
+
# Condensation DAG construction
|
|
180
|
+
scc_callees: Dict[int, Set[int]] = defaultdict(set)
|
|
181
|
+
scc_callers: Dict[int, Set[int]] = defaultdict(set)
|
|
182
|
+
|
|
183
|
+
for caller_id, callee_ids in caller_to_callees.items():
|
|
184
|
+
if caller_id not in node_to_scc_idx:
|
|
185
|
+
continue
|
|
186
|
+
c_scc = node_to_scc_idx[caller_id]
|
|
187
|
+
for callee_id in callee_ids:
|
|
188
|
+
if callee_id not in node_to_scc_idx:
|
|
189
|
+
continue
|
|
190
|
+
target_scc = node_to_scc_idx[callee_id]
|
|
191
|
+
if target_scc != c_scc:
|
|
192
|
+
scc_callees[c_scc].add(target_scc)
|
|
193
|
+
scc_callers[target_scc].add(c_scc)
|
|
194
|
+
|
|
195
|
+
# Bottom-up topological ranking
|
|
196
|
+
scc_rank: Dict[int, int] = {}
|
|
197
|
+
queue = deque([i for i in range(len(sccs)) if len(scc_callees[i]) == 0])
|
|
198
|
+
for i in queue:
|
|
199
|
+
scc_rank[i] = 0
|
|
200
|
+
|
|
201
|
+
resolved_sccs = set(queue)
|
|
202
|
+
while queue:
|
|
203
|
+
curr_scc = queue.popleft()
|
|
204
|
+
for parent_scc in scc_callers[curr_scc]:
|
|
205
|
+
if scc_callees[parent_scc].issubset(resolved_sccs):
|
|
206
|
+
max_child_rank = max(scc_rank[c] for c in scc_callees[parent_scc])
|
|
207
|
+
scc_rank[parent_scc] = max_child_rank + 1
|
|
208
|
+
resolved_sccs.add(parent_scc)
|
|
209
|
+
queue.append(parent_scc)
|
|
210
|
+
|
|
211
|
+
for i in range(len(sccs)):
|
|
212
|
+
if i not in scc_rank:
|
|
213
|
+
scc_rank[i] = 0
|
|
214
|
+
|
|
215
|
+
# Assign DAG levels
|
|
216
|
+
level_groups: Dict[int, List[SymbolNode]] = defaultdict(list)
|
|
217
|
+
if base_types_and_vars:
|
|
218
|
+
level_groups[0].extend(base_types_and_vars)
|
|
219
|
+
|
|
220
|
+
for scc_idx, members in enumerate(sccs):
|
|
221
|
+
rank = scc_rank[scc_idx]
|
|
222
|
+
node_level = rank + (1 if base_types_and_vars else 0)
|
|
223
|
+
for member_id in members:
|
|
224
|
+
node = id_to_node[member_id]
|
|
225
|
+
node.dag_level = node_level
|
|
226
|
+
level_groups[node_level].append(node)
|
|
227
|
+
|
|
228
|
+
max_lvl = max(level_groups.keys()) if level_groups else 0
|
|
229
|
+
levels_list = [level_groups[i] for i in range(max_lvl + 1) if level_groups[i]]
|
|
230
|
+
return levels_list
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def order_symbols_bottom_up(nodes: List[SymbolNode]) -> List[SymbolNode]:
|
|
234
|
+
"""Flattened bottom-up order."""
|
|
235
|
+
levels = order_symbols_by_levels(nodes)
|
|
236
|
+
flat: List[SymbolNode] = []
|
|
237
|
+
for lvl in levels:
|
|
238
|
+
flat.extend(lvl)
|
|
239
|
+
return flat
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def build_callee_context_summary(
|
|
243
|
+
node: SymbolNode,
|
|
244
|
+
resolved_symbols: Dict[str, SymbolNode],
|
|
245
|
+
) -> str:
|
|
246
|
+
"""Build summary text of called low-level functions and mutual recursion peers for LLM prompt."""
|
|
247
|
+
lines: List[str] = []
|
|
248
|
+
|
|
249
|
+
if node.scc_group_ids:
|
|
250
|
+
peer_names = []
|
|
251
|
+
for peer_id in node.scc_group_ids:
|
|
252
|
+
peer_node = resolved_symbols.get(peer_id)
|
|
253
|
+
if peer_node:
|
|
254
|
+
peer_names.append(f"`{peer_node.symbol.name}` ({peer_node.rel_path.name})")
|
|
255
|
+
if peer_names:
|
|
256
|
+
lines.append("[Mutual Recursion Group]:")
|
|
257
|
+
lines.append(f"- Note: This function operates in mutual recursion with {', '.join(peer_names)}.")
|
|
258
|
+
lines.append("")
|
|
259
|
+
|
|
260
|
+
if node.direct_callee_ids:
|
|
261
|
+
lines.append("[Direct Callees Summary]:")
|
|
262
|
+
for callee_id in sorted(node.direct_callee_ids):
|
|
263
|
+
callee_node = resolved_symbols.get(callee_id)
|
|
264
|
+
if callee_node:
|
|
265
|
+
sym = callee_node.symbol
|
|
266
|
+
purpose = sym.purpose or "Executes operation"
|
|
267
|
+
inputs = sym.inputs_note or ", ".join(f"{p.name}: {p.type_hint}" for p in sym.parameters) or "None"
|
|
268
|
+
outputs = sym.outputs_note or sym.return_type or "None"
|
|
269
|
+
lines.append(f"- Function `{sym.name}` (FQDN: `{sym.fqdn or sym.name}`, File: `{callee_node.rel_path.name}`):")
|
|
270
|
+
lines.append(f" - Purpose: {purpose}")
|
|
271
|
+
lines.append(f" - Inputs: {inputs}")
|
|
272
|
+
lines.append(f" - Outputs: {outputs}")
|
|
273
|
+
if sym.overview:
|
|
274
|
+
first_lines = sym.overview.strip().splitlines()[:2]
|
|
275
|
+
lines.append(f" - Summary: {' / '.join(l.strip() for l in first_lines)}")
|
|
276
|
+
|
|
277
|
+
return "\n".join(lines)
|