pystdoc 0.8.0__tar.gz → 0.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {pystdoc-0.8.0/src/pystdoc.egg-info → pystdoc-0.8.2}/PKG-INFO +26 -8
  2. {pystdoc-0.8.0 → pystdoc-0.8.2}/README.md +25 -7
  3. {pystdoc-0.8.0 → pystdoc-0.8.2}/pyproject.toml +1 -1
  4. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/__init__.py +1 -1
  5. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/cli.py +178 -40
  6. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/config.py +1 -0
  7. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/db.py +50 -28
  8. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/design_engine.py +88 -22
  9. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/engine.py +150 -35
  10. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/llm_client.py +367 -80
  11. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/mcp_server.py +8 -0
  12. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/perf.py +108 -0
  13. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/progress.py +84 -12
  14. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/report_engine.py +44 -11
  15. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/ui_detector.py +88 -23
  16. {pystdoc-0.8.0 → pystdoc-0.8.2/src/pystdoc.egg-info}/PKG-INFO +26 -8
  17. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_cli.py +116 -1
  18. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_engine.py +116 -0
  19. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_interrupt.py +40 -0
  20. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_llm_client.py +146 -1
  21. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_perf.py +47 -0
  22. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_progress.py +104 -1
  23. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_ui_detector.py +60 -5
  24. {pystdoc-0.8.0 → pystdoc-0.8.2}/LICENSE +0 -0
  25. {pystdoc-0.8.0 → pystdoc-0.8.2}/setup.cfg +0 -0
  26. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/__main__.py +0 -0
  27. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/adapters/__init__.py +0 -0
  28. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/adapters/base.py +0 -0
  29. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/adapters/clang_adapter.py +0 -0
  30. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/adapters/generic_adapter.py +0 -0
  31. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/adapters/java_adapter.py +0 -0
  32. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/adapters/kotlin_adapter.py +0 -0
  33. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/adapters/python_adapter.py +0 -0
  34. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/cache.py +0 -0
  35. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/call_graph.py +0 -0
  36. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/compilation_db.py +0 -0
  37. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/doc_writer.py +0 -0
  38. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/hasher.py +0 -0
  39. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/interrupt.py +0 -0
  40. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/parser_clang.py +0 -0
  41. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/parser_generic.py +0 -0
  42. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/parser_java.py +0 -0
  43. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/parser_kotlin.py +0 -0
  44. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/parser_python.py +0 -0
  45. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/query.py +0 -0
  46. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/scanner.py +0 -0
  47. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/similarity.py +0 -0
  48. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/symbols.py +0 -0
  49. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc/watcher.py +0 -0
  50. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc.egg-info/SOURCES.txt +0 -0
  51. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc.egg-info/dependency_links.txt +0 -0
  52. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc.egg-info/entry_points.txt +0 -0
  53. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc.egg-info/requires.txt +0 -0
  54. {pystdoc-0.8.0 → pystdoc-0.8.2}/src/pystdoc.egg-info/top_level.txt +0 -0
  55. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_adapters.py +0 -0
  56. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_config.py +0 -0
  57. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_coverage_boost.py +0 -0
  58. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_design_engine_full.py +0 -0
  59. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_designgen.py +0 -0
  60. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_docgen.py +0 -0
  61. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_engine_full.py +0 -0
  62. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_final_100.py +0 -0
  63. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_live_server_100.py +0 -0
  64. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_mcp_server.py +0 -0
  65. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_parser_generic.py +0 -0
  66. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_parser_java.py +0 -0
  67. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_parser_kotlin.py +0 -0
  68. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_pep8.py +0 -0
  69. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_query.py +0 -0
  70. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_reduce_cache.py +0 -0
  71. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_reportgen.py +0 -0
  72. {pystdoc-0.8.0 → pystdoc-0.8.2}/tests/test_watcher.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pystdoc
3
- Version: 0.8.0
3
+ Version: 0.8.2
4
4
  Summary: Python Structural & Topological Documentation Engine (pystdoc): High-precision, LLM-powered hierarchical codebase and architecture documentation generator.
5
5
  Author: tab4moji
6
6
  License: MIT
@@ -78,25 +78,28 @@ your_project/
78
78
  - Evaluates call graphs in $O(V+E)$ linear time.
79
79
  - Automatically breaks cyclic mutual recursions and organizes code symbols into dependency-safe execution levels.
80
80
  - Level-by-level parallel LLM execution guarantees context-rich bottom-up summaries without race conditions.
81
- 2. **High-Speed Incremental Sync & Semantic Change Propagation (v0.8.0)**:
82
- - **Selective Bottom-Up Propagation**: Detects AST and logic-level code differences. If regenerated specifications remain semantically unchanged (evaluated by deterministic Jaccard / token-overlap metrics), change propagation terminates immediately without triggering cascading LLM re-analysis across upper modules.
81
+ 2. **High-Speed Incremental Sync & Semantic Change Propagation (v0.8.0 - v0.8.2)**:
82
+ - **Selective Bottom-Up Propagation & Immediate Snippet Flush**: Detects AST and logic-level code differences. If regenerated specifications remain semantically unchanged, cascading LLM re-analysis terminates early while Markdown documents (source code snippets, line ranges, caller/callee references) are immediately updated with latest code. Signature, kind, and callee alterations are guaranteed to propagate upstream.
83
83
  - **Content-Defined Map-Reduce Caching**: Intermediate summaries in architecture synthesis (`designgen`) are chunked using stable hash boundaries and cached per group in SQLite, ensuring that only groups containing edited functions are re-summarized.
84
84
  - **Architectural Entry-Point Filtering**: For system execution modeling (`execution_model.md`), key architectural functions (entry points, CLI routers, and core dispatchers) are automatically filtered and passed directly into the LLM context, bypassing heavy multi-step Map-Reduce loops and speeding up generation by up to 90%+.
85
- 3. **3-in-1 Unified Documentation Pipeline**:
85
+ 3. **Flexible Step Execution & High-Precision UI Inference (v0.8.1)**:
86
+ - **Phase Re-execution & Recovery (`--from-step`, `--step`)**: Resume or selectively re-run pipeline stages (e.g., `--from-step 3` to restart from architecture design, or `--step 4` for README synthesis only) without regenerating bottom-up symbol documents.
87
+ - **High-Precision UI & CLI Prioritization**: Restricts GUI classification to high-confidence UI frameworks (`PyQt`, `Tkinter`, `GTK`, `Jetpack Compose`, etc.) while eliminating False Positive hallucinations (e.g. JavaFX). Heavily prioritizes CLI classification for command pipelines and automatically overwrites existing symbol annotations on re-runs.
88
+ 4. **3-in-1 Unified Documentation Pipeline**:
86
89
  - `docgen`: Bottom-up & top-down symbol-level documentation with SHA-256 and SQLite caching (`.pystdoc/documents/`).
87
90
  - `designgen`: Map-Reduce architectural synthesis (`.pystdoc/design/`).
88
91
  - `reportgen` / `pystdoc`: Executive summary README (`.pystdoc/README.md`) answering *"What does this project actually do?"*
89
- 4. **Multi-Language Codebase Deep Understanding (C/C++, Python, Java, Kotlin, Shell)**:
92
+ 5. **Multi-Language Codebase Deep Understanding (C/C++, Python, Java, Kotlin, Shell)**:
90
93
  - **C/C++**: `compile_commands.json` integration, include path resolution, and macro expansion via `libclang`.
91
94
  - **Java / Kotlin**: AST parsing via `javalang` & `kopyt` with class hierarchy, companion objects, and data classes.
92
95
  - **Python & Shell**: AST visitor and generic regex parser with import/call extraction.
93
96
  - **Fully Qualified Domain Names (FQDN)**: Disambiguates identical symbol names across large monorepos.
94
- 5. **Feature Location & UI Component Pinpointing (`locate` / `pystdoc_locate_feature`)**:
97
+ 6. **Feature Location & UI Component Pinpointing (`locate` / `pystdoc_locate_feature`)**:
95
98
  - Pinpoint exact files, functions, UI elements, and line ranges (`Lines X-Y`) from high-level natural language queries (e.g., *"where is the debug notification button"*, *"add auth header"*) without running brute-force `Glob`/`Grep` across the entire project.
96
99
  - Built-in multi-lingual token scoring with automatic boosting for UI elements (`@Composable`, `Button`, `Dialog`, `Activity`, `Modifier`, etc.).
97
- 6. **Impact & Inbound/Outbound Dependency Analysis (`impact` / `pystdoc_trace_impact`)**:
100
+ 7. **Impact & Inbound/Outbound Dependency Analysis (`impact` / `pystdoc_trace_impact`)**:
98
101
  - Instant 360-degree dependency tracing: inspect where a symbol is called from (*Inbound Callers / References*) and what it uses (*Outbound Dependencies*) before making breaking code changes or deletions.
99
- 7. **Standard LLM Options & Multi-Language Support**:
102
+ 8. **Standard LLM Options & Multi-Language Support**:
100
103
  - Works with **Ollama, LiteRT-LM, vLLM, and OpenAI API**.
101
104
  - Supports `--host`, `--model`, `--token` / `--api-key`, `--context-size`, and `--language` (defaults to terminal language, e.g. `Japanese`, `English`, `日本語`).
102
105
 
@@ -116,6 +119,12 @@ pip install pystdoc
116
119
  # Full sync: complete 4-step pipeline (symbol docs, design docs, README overview)
117
120
  pystdoc --dir ./my_project/
118
121
 
122
+ # Partial / Step-level re-run: resume pipeline from Step 3 (designgen) to Step 4 (reportgen)
123
+ pystdoc --dir ./my_project/ --from-step 3
124
+
125
+ # Single-step re-run: regenerate project overview README (.pystdoc/README.md) only
126
+ pystdoc --dir ./my_project/ --step 4
127
+
119
128
  # Fast sync: bottom-up symbol analysis & indexing only (skips top-down design/report synthesis)
120
129
  pystdoc sync --fast --dir ./my_project/
121
130
 
@@ -186,7 +195,11 @@ pystdoc --watch --interval 0.5
186
195
 
187
196
  #### 4. Practical Real-World Example (Dedicated Remote LLM Server)
188
197
  ```bash
198
+ # Production execution with dedicated local/remote server
189
199
  pystdoc sync --dir ./target_project/ --host 192.168.0.11:11434 --model gemma4-26b-a4b --language 日本語
200
+
201
+ # Debugging LLM responses in realtime (tail -f tmp.debug.log in another terminal)
202
+ pystdoc sync --dir ./target_project/ --debug
190
203
  ```
191
204
 
192
205
  ---
@@ -276,7 +289,11 @@ pystdoc sync --dir ./target_project/ --host 192.168.0.11:11434 --model gemma4-26
276
289
  | Option | Alias / Env | Default | Description |
277
290
  | :--- | :--- | :--- | :--- |
278
291
  | `--dir` | | `./` | Target project directory path |
292
+ | `--from-step` | | `None` | Start pipeline from specific step (`1`..`4`, `docgen`, `data`, `design`, `report`) |
293
+ | `--step` | | `None` | Run only the specified step (`1`..`4`, `docgen`, `data`, `design`, `report`) |
279
294
  | `--fast` | | `false` | Fast bottom-up sync only (runs docgen symbol indexing, skips designgen and reportgen) |
295
+ | `--skip-docgen` | | `false` | Skip Step 1 & 2 (source parsing and symbol document generation) |
296
+ | `--skip-designgen` | | `false` | Skip Step 3 (architectural design and module synthesis) |
280
297
  | `--language` | `-l` | Terminal locale | Output documentation language (defaults to terminal language; e.g. `Japanese`, `English`, `日本語`) |
281
298
  | `--host` | `-H`, `--base-url` | `http://127.0.0.1:11434` | LLM server host endpoint URL |
282
299
  | `--model` | `-m`, `LLM_MODEL` | `gemma4-26b-a4b` | LLM model identifier |
@@ -284,6 +301,7 @@ pystdoc sync --dir ./target_project/ --host 192.168.0.11:11434 --model gemma4-26
284
301
  | `--context-size` | `--ctx-size` | `16384` | Context window size |
285
302
  | `--concurrency` | `-j` | `1` | Number of parallel LLM workers |
286
303
  | `--force` | `-f` | `false` | Force regenerate all documents ignoring cache |
304
+ | `--debug` | `PYSTDOC_DEBUG` | `false` | Enable realtime LLM request and streaming token logging to `tmp.debug.log` |
287
305
  | `--compile-commands` | | `None` | Path to `compile_commands.json` |
288
306
 
289
307
  ---
@@ -49,25 +49,28 @@ your_project/
49
49
  - Evaluates call graphs in $O(V+E)$ linear time.
50
50
  - Automatically breaks cyclic mutual recursions and organizes code symbols into dependency-safe execution levels.
51
51
  - Level-by-level parallel LLM execution guarantees context-rich bottom-up summaries without race conditions.
52
- 2. **High-Speed Incremental Sync & Semantic Change Propagation (v0.8.0)**:
53
- - **Selective Bottom-Up Propagation**: Detects AST and logic-level code differences. If regenerated specifications remain semantically unchanged (evaluated by deterministic Jaccard / token-overlap metrics), change propagation terminates immediately without triggering cascading LLM re-analysis across upper modules.
52
+ 2. **High-Speed Incremental Sync & Semantic Change Propagation (v0.8.0 - v0.8.2)**:
53
+ - **Selective Bottom-Up Propagation & Immediate Snippet Flush**: Detects AST and logic-level code differences. If regenerated specifications remain semantically unchanged, cascading LLM re-analysis terminates early while Markdown documents (source code snippets, line ranges, caller/callee references) are immediately updated with latest code. Signature, kind, and callee alterations are guaranteed to propagate upstream.
54
54
  - **Content-Defined Map-Reduce Caching**: Intermediate summaries in architecture synthesis (`designgen`) are chunked using stable hash boundaries and cached per group in SQLite, ensuring that only groups containing edited functions are re-summarized.
55
55
  - **Architectural Entry-Point Filtering**: For system execution modeling (`execution_model.md`), key architectural functions (entry points, CLI routers, and core dispatchers) are automatically filtered and passed directly into the LLM context, bypassing heavy multi-step Map-Reduce loops and speeding up generation by up to 90%+.
56
- 3. **3-in-1 Unified Documentation Pipeline**:
56
+ 3. **Flexible Step Execution & High-Precision UI Inference (v0.8.1)**:
57
+ - **Phase Re-execution & Recovery (`--from-step`, `--step`)**: Resume or selectively re-run pipeline stages (e.g., `--from-step 3` to restart from architecture design, or `--step 4` for README synthesis only) without regenerating bottom-up symbol documents.
58
+ - **High-Precision UI & CLI Prioritization**: Restricts GUI classification to high-confidence UI frameworks (`PyQt`, `Tkinter`, `GTK`, `Jetpack Compose`, etc.) while eliminating False Positive hallucinations (e.g. JavaFX). Heavily prioritizes CLI classification for command pipelines and automatically overwrites existing symbol annotations on re-runs.
59
+ 4. **3-in-1 Unified Documentation Pipeline**:
57
60
  - `docgen`: Bottom-up & top-down symbol-level documentation with SHA-256 and SQLite caching (`.pystdoc/documents/`).
58
61
  - `designgen`: Map-Reduce architectural synthesis (`.pystdoc/design/`).
59
62
  - `reportgen` / `pystdoc`: Executive summary README (`.pystdoc/README.md`) answering *"What does this project actually do?"*
60
- 4. **Multi-Language Codebase Deep Understanding (C/C++, Python, Java, Kotlin, Shell)**:
63
+ 5. **Multi-Language Codebase Deep Understanding (C/C++, Python, Java, Kotlin, Shell)**:
61
64
  - **C/C++**: `compile_commands.json` integration, include path resolution, and macro expansion via `libclang`.
62
65
  - **Java / Kotlin**: AST parsing via `javalang` & `kopyt` with class hierarchy, companion objects, and data classes.
63
66
  - **Python & Shell**: AST visitor and generic regex parser with import/call extraction.
64
67
  - **Fully Qualified Domain Names (FQDN)**: Disambiguates identical symbol names across large monorepos.
65
- 5. **Feature Location & UI Component Pinpointing (`locate` / `pystdoc_locate_feature`)**:
68
+ 6. **Feature Location & UI Component Pinpointing (`locate` / `pystdoc_locate_feature`)**:
66
69
  - Pinpoint exact files, functions, UI elements, and line ranges (`Lines X-Y`) from high-level natural language queries (e.g., *"where is the debug notification button"*, *"add auth header"*) without running brute-force `Glob`/`Grep` across the entire project.
67
70
  - Built-in multi-lingual token scoring with automatic boosting for UI elements (`@Composable`, `Button`, `Dialog`, `Activity`, `Modifier`, etc.).
68
- 6. **Impact & Inbound/Outbound Dependency Analysis (`impact` / `pystdoc_trace_impact`)**:
71
+ 7. **Impact & Inbound/Outbound Dependency Analysis (`impact` / `pystdoc_trace_impact`)**:
69
72
  - Instant 360-degree dependency tracing: inspect where a symbol is called from (*Inbound Callers / References*) and what it uses (*Outbound Dependencies*) before making breaking code changes or deletions.
70
- 7. **Standard LLM Options & Multi-Language Support**:
73
+ 8. **Standard LLM Options & Multi-Language Support**:
71
74
  - Works with **Ollama, LiteRT-LM, vLLM, and OpenAI API**.
72
75
  - Supports `--host`, `--model`, `--token` / `--api-key`, `--context-size`, and `--language` (defaults to terminal language, e.g. `Japanese`, `English`, `日本語`).
73
76
 
@@ -87,6 +90,12 @@ pip install pystdoc
87
90
  # Full sync: complete 4-step pipeline (symbol docs, design docs, README overview)
88
91
  pystdoc --dir ./my_project/
89
92
 
93
+ # Partial / Step-level re-run: resume pipeline from Step 3 (designgen) to Step 4 (reportgen)
94
+ pystdoc --dir ./my_project/ --from-step 3
95
+
96
+ # Single-step re-run: regenerate project overview README (.pystdoc/README.md) only
97
+ pystdoc --dir ./my_project/ --step 4
98
+
90
99
  # Fast sync: bottom-up symbol analysis & indexing only (skips top-down design/report synthesis)
91
100
  pystdoc sync --fast --dir ./my_project/
92
101
 
@@ -157,7 +166,11 @@ pystdoc --watch --interval 0.5
157
166
 
158
167
  #### 4. Practical Real-World Example (Dedicated Remote LLM Server)
159
168
  ```bash
169
+ # Production execution with dedicated local/remote server
160
170
  pystdoc sync --dir ./target_project/ --host 192.168.0.11:11434 --model gemma4-26b-a4b --language 日本語
171
+
172
+ # Debugging LLM responses in realtime (tail -f tmp.debug.log in another terminal)
173
+ pystdoc sync --dir ./target_project/ --debug
161
174
  ```
162
175
 
163
176
  ---
@@ -247,7 +260,11 @@ pystdoc sync --dir ./target_project/ --host 192.168.0.11:11434 --model gemma4-26
247
260
  | Option | Alias / Env | Default | Description |
248
261
  | :--- | :--- | :--- | :--- |
249
262
  | `--dir` | | `./` | Target project directory path |
263
+ | `--from-step` | | `None` | Start pipeline from specific step (`1`..`4`, `docgen`, `data`, `design`, `report`) |
264
+ | `--step` | | `None` | Run only the specified step (`1`..`4`, `docgen`, `data`, `design`, `report`) |
250
265
  | `--fast` | | `false` | Fast bottom-up sync only (runs docgen symbol indexing, skips designgen and reportgen) |
266
+ | `--skip-docgen` | | `false` | Skip Step 1 & 2 (source parsing and symbol document generation) |
267
+ | `--skip-designgen` | | `false` | Skip Step 3 (architectural design and module synthesis) |
251
268
  | `--language` | `-l` | Terminal locale | Output documentation language (defaults to terminal language; e.g. `Japanese`, `English`, `日本語`) |
252
269
  | `--host` | `-H`, `--base-url` | `http://127.0.0.1:11434` | LLM server host endpoint URL |
253
270
  | `--model` | `-m`, `LLM_MODEL` | `gemma4-26b-a4b` | LLM model identifier |
@@ -255,6 +272,7 @@ pystdoc sync --dir ./target_project/ --host 192.168.0.11:11434 --model gemma4-26
255
272
  | `--context-size` | `--ctx-size` | `16384` | Context window size |
256
273
  | `--concurrency` | `-j` | `1` | Number of parallel LLM workers |
257
274
  | `--force` | `-f` | `false` | Force regenerate all documents ignoring cache |
275
+ | `--debug` | `PYSTDOC_DEBUG` | `false` | Enable realtime LLM request and streaming token logging to `tmp.debug.log` |
258
276
  | `--compile-commands` | | `None` | Path to `compile_commands.json` |
259
277
 
260
278
  ---
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "pystdoc"
7
- version = "0.8.0"
7
+ version = "0.8.2"
8
8
  description = "Python Structural & Topological Documentation Engine (pystdoc): High-precision, LLM-powered hierarchical codebase and architecture documentation generator."
9
9
  readme = "README.md"
10
10
  authors = [
@@ -1,3 +1,3 @@
1
1
  """pystdoc: Python Structural & Topological Documentation Engine (pystdoc)."""
2
2
 
3
- __version__ = "0.8.0"
3
+ __version__ = "0.8.2"
@@ -10,7 +10,7 @@ from pystdoc.config import detect_terminal_language, load_config
10
10
  from pystdoc.db import DocgenDB
11
11
  from pystdoc.design_engine import run_design_generation
12
12
  from pystdoc.engine import run_docgen
13
- from pystdoc.llm_client import LLMClient
13
+ from pystdoc.llm_client import LLMClient, set_debug_mode
14
14
  from pystdoc.progress import is_terminal
15
15
  from pystdoc.query import (
16
16
  run_description,
@@ -25,6 +25,28 @@ from pystdoc.interrupt import KeyboardInterruptWatcher, safe_exit
25
25
  from pystdoc.report_engine import generate_readme_doc
26
26
 
27
27
 
28
+ def normalize_step_identifier(step_val: Optional[str]) -> Optional[int]:
29
+ """Normalize step number or name (1..4, docgen, designgen, reportgen)."""
30
+ if not step_val:
31
+ return None
32
+ s = str(step_val).strip().lower()
33
+ if s in ("1", "docgen", "symbol", "symbols"):
34
+ return 1
35
+ elif s in ("2", "data", "variable", "variables", "var"):
36
+ return 2
37
+ elif s in ("3", "designgen", "design", "architecture", "models"):
38
+ return 3
39
+ elif s in ("4", "reportgen", "report", "readme"):
40
+ return 4
41
+ try:
42
+ val = int(s)
43
+ if 1 <= val <= 4:
44
+ return val
45
+ except ValueError:
46
+ pass
47
+ return None
48
+
49
+
28
50
  def docgen_main(argv: Optional[List[str]] = None) -> None:
29
51
 
30
52
  """CLI entry point for `docgen` command."""
@@ -69,6 +91,12 @@ def docgen_main(argv: Optional[List[str]] = None) -> None:
69
91
  default=None,
70
92
  help="LLM context window size",
71
93
  )
94
+ parser.add_argument(
95
+ "--max-tokens",
96
+ type=int,
97
+ default=None,
98
+ help="Max output tokens for LLM generation",
99
+ )
72
100
  parser.add_argument(
73
101
  "--language",
74
102
  "-l",
@@ -99,8 +127,15 @@ def docgen_main(argv: Optional[List[str]] = None) -> None:
99
127
  default=None,
100
128
  help="Number of parallel LLM workers",
101
129
  )
130
+ parser.add_argument(
131
+ "--debug",
132
+ action="store_true",
133
+ help="Enable realtime LLM debug logging to tmp.debug.log",
134
+ )
102
135
 
103
136
  args = parser.parse_args(argv)
137
+ if getattr(args, "debug", False):
138
+ set_debug_mode(True)
104
139
  target = Path(args.dir)
105
140
  cfg = load_config(target)
106
141
 
@@ -108,6 +143,7 @@ def docgen_main(argv: Optional[List[str]] = None) -> None:
108
143
  model = args.model or cfg.get("model", "gemma4-26b-a4b")
109
144
  token = args.token or args.api_key or cfg.get("token")
110
145
  ctx_size = args.context_size or cfg.get("context_size", 16384)
146
+ max_tokens = args.max_tokens or cfg.get("max_tokens", 16384)
111
147
  lang = args.language or cfg.get("language") or detect_terminal_language()
112
148
  concurrency = args.concurrency or cfg.get("concurrency", 1)
113
149
  allow_fallback = (
@@ -126,6 +162,7 @@ def docgen_main(argv: Optional[List[str]] = None) -> None:
126
162
  model=model,
127
163
  token=token,
128
164
  context_size=ctx_size,
165
+ max_tokens=max_tokens,
129
166
  language=lang,
130
167
  force=args.force,
131
168
  allow_fallback=allow_fallback,
@@ -187,6 +224,12 @@ def designgen_main(argv: Optional[List[str]] = None) -> None:
187
224
  default=None,
188
225
  help="LLM context window size",
189
226
  )
227
+ parser.add_argument(
228
+ "--max-tokens",
229
+ type=int,
230
+ default=None,
231
+ help="Max output tokens for LLM generation",
232
+ )
190
233
  parser.add_argument(
191
234
  "--language",
192
235
  "-l",
@@ -205,8 +248,15 @@ def designgen_main(argv: Optional[List[str]] = None) -> None:
205
248
  default=None,
206
249
  help="Allow fallback to static template on LLM failure",
207
250
  )
251
+ parser.add_argument(
252
+ "--debug",
253
+ action="store_true",
254
+ help="Enable realtime LLM debug logging to tmp.debug.log",
255
+ )
208
256
 
209
257
  args = parser.parse_args(argv)
258
+ if getattr(args, "debug", False):
259
+ set_debug_mode(True)
210
260
  target = Path(args.dir)
211
261
  cfg = load_config(target)
212
262
 
@@ -214,6 +264,7 @@ def designgen_main(argv: Optional[List[str]] = None) -> None:
214
264
  model = args.model or cfg.get("model", "gemma4-26b-a4b")
215
265
  token = args.token or args.api_key or cfg.get("token")
216
266
  ctx_size = args.context_size or cfg.get("context_size", 16384)
267
+ max_tokens = args.max_tokens or cfg.get("max_tokens", 16384)
217
268
  lang = args.language or cfg.get("language") or detect_terminal_language()
218
269
  allow_fallback = (
219
270
  args.allow_fallback
@@ -231,6 +282,7 @@ def designgen_main(argv: Optional[List[str]] = None) -> None:
231
282
  model=model,
232
283
  token=token,
233
284
  context_size=ctx_size,
285
+ max_tokens=max_tokens,
234
286
  language=lang,
235
287
  force=args.force,
236
288
  allow_fallback=allow_fallback,
@@ -297,6 +349,7 @@ def run_watch(
297
349
  model: Optional[str] = None,
298
350
  token: Optional[str] = None,
299
351
  context_size: int = 16384,
352
+ max_tokens: int = 16384,
300
353
  concurrency: int = 1,
301
354
  language: Optional[str] = None,
302
355
  allow_fallback: bool = False,
@@ -318,6 +371,7 @@ def run_watch(
318
371
  model=model,
319
372
  token=token,
320
373
  context_size=context_size,
374
+ max_tokens=max_tokens,
321
375
  language=language,
322
376
  force=False,
323
377
  allow_fallback=allow_fallback,
@@ -334,6 +388,7 @@ def run_watch(
334
388
  model=model,
335
389
  token=token,
336
390
  context_size=context_size,
391
+ max_tokens=max_tokens,
337
392
  language=language,
338
393
  force=False,
339
394
  allow_fallback=allow_fallback,
@@ -348,6 +403,7 @@ def run_watch(
348
403
  model=model,
349
404
  token=token,
350
405
  context_size=context_size,
406
+ max_tokens=max_tokens,
351
407
  )
352
408
  if client.check_availability():
353
409
  llm_client = client
@@ -583,6 +639,12 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
583
639
  default=None,
584
640
  help="LLM context window size",
585
641
  )
642
+ parser.add_argument(
643
+ "--max-tokens",
644
+ type=int,
645
+ default=None,
646
+ help="Max output tokens for LLM generation",
647
+ )
586
648
  parser.add_argument(
587
649
  "--language",
588
650
  "-l",
@@ -629,14 +691,33 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
629
691
  action="store_true",
630
692
  help="Continuously watch for code changes (dnotify) and auto-sync",
631
693
  )
694
+ parser.add_argument(
695
+ "--from-step",
696
+ dest="from_step",
697
+ default=None,
698
+ help="Start from step (1..4, docgen, design, report)",
699
+ )
700
+ parser.add_argument(
701
+ "--step",
702
+ dest="single_step",
703
+ default=None,
704
+ help="Run step only (1..4, docgen, design, report)",
705
+ )
632
706
  parser.add_argument(
633
707
  "--interval",
634
708
  type=float,
635
709
  default=1.0,
636
710
  help="Debounce interval in seconds for watch mode (default: 1.0)",
637
711
  )
712
+ parser.add_argument(
713
+ "--debug",
714
+ action="store_true",
715
+ help="Enable realtime LLM debug logging to tmp.debug.log",
716
+ )
638
717
 
639
718
  args = parser.parse_args(sub_args)
719
+ if getattr(args, "debug", False):
720
+ set_debug_mode(True)
640
721
  target_dir = Path(args.dir).resolve()
641
722
 
642
723
  if not target_dir.exists() or not target_dir.is_dir():
@@ -652,6 +733,7 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
652
733
  model = args.model or cfg.get("model", "gemma4-26b-a4b")
653
734
  token = args.token or args.api_key or cfg.get("token")
654
735
  ctx_size = args.context_size or cfg.get("context_size", 16384)
736
+ max_tokens = args.max_tokens or cfg.get("max_tokens", 16384)
655
737
  lang = args.language or cfg.get("language") or detect_terminal_language()
656
738
  concurrency = args.concurrency or cfg.get("concurrency", 1)
657
739
  allow_fallback = (
@@ -660,6 +742,39 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
660
742
  else cfg.get("allow_fallback", False)
661
743
  )
662
744
 
745
+ start_step = 1
746
+ end_step = 4
747
+
748
+ if args.from_step:
749
+ norm_from = normalize_step_identifier(args.from_step)
750
+ if norm_from is None:
751
+ print(
752
+ f"Error: Invalid --from-step value: '{args.from_step}'. "
753
+ f"Valid steps: 1 (docgen), 2 (data), 3 (design), 4 (report)",
754
+ file=sys.stderr,
755
+ )
756
+ sys.exit(1)
757
+ start_step = norm_from
758
+ args.force = True
759
+
760
+ if args.single_step:
761
+ norm_step = normalize_step_identifier(args.single_step)
762
+ if norm_step is None:
763
+ print(
764
+ f"Error: Invalid --step value: '{args.single_step}'. "
765
+ f"Valid steps: 1 (docgen), 2 (data), 3 (design), 4 (report)",
766
+ file=sys.stderr,
767
+ )
768
+ sys.exit(1)
769
+ start_step = norm_step
770
+ end_step = norm_step
771
+ args.force = True
772
+
773
+ if start_step > 1:
774
+ args.skip_docgen = True
775
+ if start_step > 3 or end_step < 3:
776
+ args.skip_designgen = True
777
+
663
778
  if subcmd == "watch" or args.watch:
664
779
  sys.exit(
665
780
  run_watch(
@@ -669,6 +784,7 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
669
784
  model=model,
670
785
  token=token,
671
786
  context_size=ctx_size,
787
+ max_tokens=max_tokens,
672
788
  concurrency=concurrency,
673
789
  language=lang,
674
790
  allow_fallback=allow_fallback,
@@ -679,13 +795,21 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
679
795
  )
680
796
 
681
797
  is_tty = is_terminal(sys.stdout)
682
- pipeline_mode_label = " (Fast bottom-up)" if args.fast else ""
798
+ step_range_label = ""
799
+ if args.fast:
800
+ step_range_label = " (Fast bottom-up)"
801
+ elif start_step > 1 or end_step < 4:
802
+ if start_step == end_step:
803
+ step_range_label = f" (Step {start_step}/4 only)"
804
+ else:
805
+ step_range_label = f" (Steps {start_step}-{end_step}/4)"
806
+
683
807
  hint_msg = " [Ctrl-C or 'q' to abort safely]" if is_tty else ""
684
808
 
685
809
  print("=" * 64)
686
810
  hdr = (
687
811
  f"=== pystdoc Unified Pipeline v{pystdoc.__version__}"
688
- f"{pipeline_mode_label} (Lang: {lang}): {target_dir}{hint_msg} ==="
812
+ f"{step_range_label} (Lang: {lang}): {target_dir}{hint_msg} ==="
689
813
  )
690
814
  print(hdr)
691
815
  print("=" * 64)
@@ -710,6 +834,7 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
710
834
  model=model,
711
835
  token=token,
712
836
  context_size=ctx_size,
837
+ max_tokens=max_tokens,
713
838
  language=lang,
714
839
  force=args.force,
715
840
  allow_fallback=allow_fallback,
@@ -742,6 +867,7 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
742
867
  model=model,
743
868
  token=token,
744
869
  context_size=ctx_size,
870
+ max_tokens=max_tokens,
745
871
  language=lang,
746
872
  force=args.force,
747
873
  allow_fallback=allow_fallback,
@@ -751,47 +877,56 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
751
877
  sys.exit(ret_design)
752
878
 
753
879
  # 3. reportgen (Step 4/4)
754
- print(
755
- "\n>>> [Step 4/4] reportgen: "
756
- "Generating project overview README (.pystdoc/README.md)..."
757
- )
758
-
759
- llm_client = None
760
- if not args.no_llm:
761
- client = LLMClient(
762
- host=host,
763
- model=model,
764
- token=token,
765
- context_size=ctx_size,
880
+ if start_step <= 4 <= end_step:
881
+ print(
882
+ "\n>>> [Step 4/4] reportgen: Generating project overview "
883
+ "README (.pystdoc/README.md)..."
766
884
  )
767
- if client.check_availability():
768
- llm_client = client
769
- else:
770
- print(
771
- f"Error: Failed to connect to LLM server "
772
- f"({client.base_url}). Aborting.",
773
- file=sys.stderr,
774
- )
775
- sys.exit(1)
776
885
 
777
- db_path = target_dir / ".pystdoc" / "index.db"
778
- db = DocgenDB(db_path) if db_path.exists() else None
779
-
780
- generate_readme_doc(
781
- target_dir=target_dir,
782
- llm_client=llm_client,
783
- language=lang,
784
- force=args.force,
785
- allow_fallback=allow_fallback,
786
- db=db,
787
- is_tty=is_tty,
788
- )
886
+ llm_client = None
887
+ if not args.no_llm:
888
+ client = LLMClient(
889
+ host=host,
890
+ model=model,
891
+ token=token,
892
+ context_size=ctx_size,
893
+ max_tokens=max_tokens,
894
+ )
895
+ if client.check_availability():
896
+ llm_client = client
897
+ else:
898
+ print(
899
+ f"Error: Failed to connect to LLM server "
900
+ f"({client.base_url}). Aborting.",
901
+ file=sys.stderr,
902
+ )
903
+ sys.exit(1)
904
+
905
+ db_path = target_dir / ".pystdoc" / "index.db"
906
+ db = DocgenDB(db_path) if db_path.exists() else None
907
+
908
+ generate_readme_doc(
909
+ target_dir=target_dir,
910
+ llm_client=llm_client,
911
+ language=lang,
912
+ force=args.force,
913
+ allow_fallback=allow_fallback,
914
+ db=db,
915
+ is_tty=is_tty,
916
+ )
789
917
 
790
- out_readme = target_dir / ".pystdoc" / "README.md"
918
+ out_readme = target_dir / ".pystdoc" / "README.md"
791
919
 
792
- print("=" * 64)
793
- print(f"=== reportgen Finished: Created {out_readme} ===")
794
- print("=" * 64)
920
+ print("=" * 64)
921
+ print(f"=== reportgen Finished: Created {out_readme} ===")
922
+ print("=" * 64)
923
+ else:
924
+ print("=" * 64)
925
+ print(
926
+ f"=== pystdoc Pipeline Finished: "
927
+ f"Step {start_step}..{end_step} complete ==="
928
+ )
929
+ print("=" * 64)
795
930
  sys.exit(0)
796
931
  except KeyboardInterrupt:
797
932
  print(
@@ -801,3 +936,6 @@ def reportgen_main(argv: Optional[List[str]] = None) -> None:
801
936
  flush=True,
802
937
  )
803
938
  safe_exit(130)
939
+
940
+
941
+ pystdoc_main = reportgen_main
@@ -13,6 +13,7 @@ DEFAULT_CONFIG: Dict[str, Any] = {
13
13
  "token": None,
14
14
  "language": "English",
15
15
  "context_size": 16384,
16
+ "max_tokens": 16384,
16
17
  "concurrency": 1,
17
18
  "allow_fallback": False,
18
19
  }