pystdoc 0.2.0__tar.gz → 0.4.0__tar.gz

{pystdoc-0.2.0/src/pystdoc.egg-info → pystdoc-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pystdoc
-Version: 0.2.0
+Version: 0.4.0
 Summary: Python Structural & Topological Documentation Engine (pystdoc): High-precision, LLM-powered hierarchical codebase and architecture documentation generator.
 Author: tab4moji
 License: MIT
@@ -120,7 +120,7 @@ designgen --dir ./my_project/
 | `--model` | `-m`, `LLM_MODEL` | `gemma4-26b-a4b` | LLM model identifier |
 | `--token` | `--api-key`, `OPENAI_API_KEY` | `None` | API Bearer token |
 | `--context-size` | `--ctx-size` | `16384` | Context window size |
-| `--concurrency` | `-j` | `4` | Number of parallel LLM workers |
+| `--concurrency` | `-j` | `1` | Number of parallel LLM workers |
 | `--force` | `-f` | `false` | Force regenerate all documents ignoring cache |
 | `--compile-commands` | | `None` | Path to `compile_commands.json` |
 

{pystdoc-0.2.0 → pystdoc-0.4.0}/README.md

@@ -97,7 +97,7 @@ designgen --dir ./my_project/
 | `--model` | `-m`, `LLM_MODEL` | `gemma4-26b-a4b` | LLM model identifier |
 | `--token` | `--api-key`, `OPENAI_API_KEY` | `None` | API Bearer token |
 | `--context-size` | `--ctx-size` | `16384` | Context window size |
-| `--concurrency` | `-j` | `4` | Number of parallel LLM workers |
+| `--concurrency` | `-j` | `1` | Number of parallel LLM workers |
 | `--force` | `-f` | `false` | Force regenerate all documents ignoring cache |
 | `--compile-commands` | | `None` | Path to `compile_commands.json` |
 

{pystdoc-0.2.0 → pystdoc-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pystdoc"
-version = "0.2.0"
+version = "0.4.0"
 description = "Python Structural & Topological Documentation Engine (pystdoc): High-precision, LLM-powered hierarchical codebase and architecture documentation generator."
 readme = "README.md"
 authors = [

{pystdoc-0.2.0 → pystdoc-0.4.0}/src/pystdoc/__init__.py

@@ -1,3 +1,3 @@
 """pystdoc: Python Structural & Topological Documentation Engine."""
 
-__version__ = "0.2.0"
+__version__ = "0.4.0"

{pystdoc-0.2.0 → pystdoc-0.4.0}/src/pystdoc/cli.py

@@ -26,7 +26,7 @@ def docgen_main() -> None:
     parser.add_argument("--force", "-f", action="store_true", help="Force regenerate all documents ignoring cache")
     parser.add_argument("--allow-fallback", action="store_true", help="Allow fallback to static template on LLM failure")
     parser.add_argument("--compile-commands", default=None, help="Path to compile_commands.json (auto-detected if omitted)")
-    parser.add_argument("--concurrency", "-j", type=int, default=4, help="Number of parallel LLM workers (default: 4)")
+    parser.add_argument("--concurrency", "-j", type=int, default=1, help="Number of parallel LLM workers (default: 1)")
 
     args = parser.parse_args()
     sys.exit(
@@ -98,7 +98,7 @@ def reportgen_main() -> None:
     parser.add_argument("--force", "-f", action="store_true", help="Force regenerate all documents ignoring cache")
     parser.add_argument("--allow-fallback", action="store_true", help="Allow fallback to static template on LLM failure")
     parser.add_argument("--compile-commands", default=None, help="Path to compile_commands.json (auto-detected if omitted)")
-    parser.add_argument("--concurrency", "-j", type=int, default=4, help="Number of parallel LLM workers (default: 4)")
+    parser.add_argument("--concurrency", "-j", type=int, default=1, help="Number of parallel LLM workers (default: 1)")
     parser.add_argument("--skip-docgen", action="store_true", help="Skip docgen step")
     parser.add_argument("--skip-designgen", action="store_true", help="Skip designgen step")
 

{pystdoc-0.2.0 → pystdoc-0.4.0}/src/pystdoc/design_engine.py

@@ -4,6 +4,7 @@ import hashlib
 import json
 import re
 import sys
+import time
 from collections import defaultdict
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Set
@@ -197,6 +198,7 @@ def generate_data_models_doc(
             write_flushed_text(out_file, cached_content.strip() + "\n")
             return cached_content
 
+    start_t = time.time()
     reduced_types_summary = hierarchical_reduce_summaries(
         types_raw,
         "Type Definitions",
@@ -249,6 +251,9 @@ Structure the Markdown as follows:
             raise LLMError(f"Failed to generate data models document: {e}") from e
         content = "# Data Structure Design & Data Models\n\n## 1. Design Intent & Core Philosophy\nProvides core data structure definitions.\n"
 
+    elapsed = time.time() - start_t
+    print(f"       -> [Done in {elapsed:5.1f}s]: .docgen/design/data_models.md")
+
     write_flushed_text(out_file, content.strip() + "\n")
     if db:
         db.save_design_cache(cache_key, input_hash, content)
@@ -283,6 +288,7 @@ def generate_execution_model_doc(
             write_flushed_text(out_file, cached_content.strip() + "\n")
             return cached_content
 
+    start_t = time.time()
     reduced_funcs_summary = hierarchical_reduce_summaries(
         funcs_raw,
         "Function Call Structures",
@@ -325,6 +331,9 @@ Structure the Markdown as follows:
             raise LLMError(f"Failed to generate execution model document: {e}") from e
         content = "# System Execution Model & Runtime Architecture\n\n## 1. Design Intent & Execution Paradigm\nProvides system execution flow.\n"
 
+    elapsed = time.time() - start_t
+    print(f"       -> [Done in {elapsed:5.1f}s]: .docgen/design/execution_model.md")
+
     write_flushed_text(out_file, content.strip() + "\n")
     if db:
         db.save_design_cache(cache_key, input_hash, content)
@@ -359,6 +368,7 @@ def generate_module_docs(
                 module_summaries[mod_name] = cached_content
                 continue
 
+        start_t = time.time()
         reduced_module_elements = hierarchical_reduce_summaries(
             doc_lines,
             f"Module `{mod_name}` Elements",
@@ -402,6 +412,9 @@ Structure the Markdown as follows:
                 raise LLMError(f"Failed to generate module `{mod_name}` document: {e}") from e
             content = f"# Module Design: `{mod_name}`\n\n## 1. Design Intent & Responsibilities\nProvides capabilities for `{mod_name}`.\n"
 
+        elapsed = time.time() - start_t
+        print(f"         -> [Done in {elapsed:5.1f}s]: .docgen/design/modules/{mod_name}.md")
+
         write_flushed_text(out_file, content.strip() + "\n")
         if db:
             db.save_design_cache(cache_key, input_hash, content)
@@ -440,6 +453,7 @@ def generate_overview_doc(
             write_flushed_text(out_file, cached_content.strip() + "\n")
             return cached_content
 
+    start_t = time.time()
     reduced_modules_overview = hierarchical_reduce_summaries(
         mod_lines,
         "All Modules Overview",
@@ -478,6 +492,9 @@ Requirements:
             raise LLMError(f"Failed to generate overview document: {e}") from e
         content = "# System Architecture Overview\n\n## 1. Architectural Overview & Design Philosophy\nProvides overall system architectural design.\n"
 
+    elapsed = time.time() - start_t
+    print(f"       -> [Done in {elapsed:5.1f}s]: .docgen/design/overview.md")
+
     write_flushed_text(out_file, content.strip() + "\n")
     if db:
         db.save_design_cache(cache_key, input_hash, content)

{pystdoc-0.2.0 → pystdoc-0.4.0}/src/pystdoc/doc_writer.py

@@ -14,11 +14,11 @@ from pystdoc.symbols import Symbol, get_kind_prefix
 
 
 def normalize_language(lang: Optional[str]) -> str:
-    """Normalize language string into English representation (e.g. 'Japanese', 'English')."""
+    """Normalize language string into standard representation (e.g. 'Japanese', 'English')."""
     if not lang:
         return "English"
     l_lower = lang.strip().lower()
-    if l_lower in ("japanese", "ja", "jp"):
+    if l_lower in ("japanese", "ja", "jp", "日本語"):
         return "Japanese"
     elif l_lower in ("english", "en"):
         return "English"
@@ -78,6 +78,9 @@ def format_symbol_section(
     language: str = "English",
 ) -> str:
     """Format a symbol's documentation into standardized Markdown string."""
+    norm_lang = normalize_language(language)
+    is_ja = norm_lang == "Japanese"
+
     h_prefix = "#" * level
     kind_display = sym.kind
     if sym.fqdn:
@@ -86,11 +89,13 @@ def format_symbol_section(
     snippet = get_code_snippet(file_path, sym.line_start, sym.line_end)
     lang_id = get_code_language(file_path.suffix)
 
+    default_purpose = f"`{sym.name}` の処理を実行する。" if is_ja else f"Executes `{sym.name}` operations."
+
     lines = [
         f"{h_prefix} {sym.kind.capitalize()} Documentation: `{sym.name}`",
         "",
         "## 1. Design Intent & Purpose",
-        sym.purpose or f"Executes `{sym.name}` operations.",
+        sym.purpose or default_purpose,
         "",
         "## 2. Basic Information",
         f"- **Name**: `{sym.name}`",
@@ -132,7 +137,7 @@ def format_symbol_section(
     if sym.children:
         lines.append("")
         for child in sym.children:
-            lines.append(format_symbol_section(child, file_path, level=level + 1, language=language))
+            lines.append(format_symbol_section(child, file_path, level=level + 1, language=norm_lang))
 
     return "\n".join(lines)
 
@@ -145,6 +150,7 @@ def write_single_symbol_doc(
     language: str = "English",
 ) -> Path:
     """Write an individual symbol document with immediate flush."""
+    norm_lang = normalize_language(language)
     docgen_dir = target_dir / ".docgen" / "documents"
     docgen_dir.mkdir(parents=True, exist_ok=True)
 
@@ -153,7 +159,7 @@ def write_single_symbol_doc(
     out_file = docgen_dir / f"{rel_path.as_posix()}.{k_prefix}.{sym_id}.md"
 
     full_path = target_dir / rel_path
-    content = format_symbol_section(symbol, full_path, level=1, language=language)
+    content = format_symbol_section(symbol, full_path, level=1, language=norm_lang)
 
     write_flushed_text(out_file, content + "\n")
     return out_file
@@ -167,10 +173,11 @@ def write_individual_symbol_docs(
     language: str = "English",
 ) -> List[Path]:
     """Recursively write documentation files for each individual symbol with immediate flush."""
+    norm_lang = normalize_language(language)
     created_files: List[Path] = []
 
     for sym in symbols:
-        out_file = write_single_symbol_doc(target_dir, rel_path, sym, prefix_name=prefix_name, language=language)
+        out_file = write_single_symbol_doc(target_dir, rel_path, sym, prefix_name=prefix_name, language=norm_lang)
         created_files.append(out_file)
 
         if sym.children:
@@ -180,7 +187,7 @@ def write_individual_symbol_docs(
                 rel_path,
                 sym.children,
                 prefix_name=child_prefix,
-                language=language,
+                language=norm_lang,
             )
             created_files.extend(child_files)
 
@@ -195,6 +202,7 @@ def write_symbol_doc(
     language: str = "English",
 ) -> Path:
     """Write comprehensive file documentation with immediate flush."""
+    norm_lang = normalize_language(language)
     docgen_dir = target_dir / ".docgen" / "documents"
     docgen_dir.mkdir(parents=True, exist_ok=True)
 
@@ -215,7 +223,7 @@ def write_symbol_doc(
 
     lines.append("")
     for sym in symbols:
-        lines.append(format_symbol_section(sym, full_path, level=2, language=language))
+        lines.append(format_symbol_section(sym, full_path, level=2, language=norm_lang))
         lines.append("")
 
     write_flushed_text(out_file, "\n".join(lines).strip() + "\n")

{pystdoc-0.2.0 → pystdoc-0.4.0}/src/pystdoc/engine.py

@@ -1,10 +1,11 @@
-"""Core engine for docgen: level-by-level parallel LLM analysis and SQLite storage with standard LLM options and language support."""
+"""Core engine for docgen: level-by-level parallel LLM analysis and SQLite storage with static bypass for enums/constants and real-time progress."""
 
 import concurrent.futures
 import fcntl
 import os
 import sys
 import threading
+import time
 from pathlib import Path
 from typing import Dict, List, Optional
 
@@ -45,7 +46,7 @@ class FileLock:
         try:
             fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
         except BlockingIOError:
-            print(f"Error: Another docgen process is currently processing this directory: {self.lock_file}", file=sys.stderr)
+            print(f"Error: Another docgen process is currently processing this directory: {self.lock_file}", file=sys.stderr, flush=True)
             sys.exit(1)
         return self
 
@@ -58,6 +59,58 @@ class FileLock:
                 pass
 
 
+def generate_static_symbol_doc(sym, lang_norm: str) -> Dict[str, str]:
+    """Generate high-precision documentation for enums, constants, and basic types from static AST metadata."""
+    is_ja = lang_norm in ("Japanese", "日本語")
+    kind = sym.kind.lower()
+
+    if "enum" in kind or kind == "enum_constant":
+        if is_ja:
+            return {
+                "purpose": f"状態コードまたは識別子定数 `{sym.name}` を定義する列挙型要素。",
+                "inputs": "なし（定数定義）。",
+                "outputs": f"`{sym.signature or sym.name}` の定数識別値。",
+                "overview": f"システム全体で一貫したステータス管理や条件判定に使用される `{sym.name}` の定義。",
+            }
+        else:
+            return {
+                "purpose": f"Defines status code or identifier constant `{sym.name}`.",
+                "inputs": "None (constant definition).",
+                "outputs": f"Constant identifier value `{sym.signature or sym.name}`.",
+                "overview": f"Enumeration definition used for status handling and conditional flow across the system.",
+            }
+    elif "typedef" in kind or "type" in kind:
+        if is_ja:
+            return {
+                "purpose": f"データ型またはエイリアス `{sym.name}` の定義。",
+                "inputs": "なし（型定義）。",
+                "outputs": f"`{sym.signature or sym.name}` 型。",
+                "overview": f"システム全体で共通利用されるデータ構造または型エイリアス `{sym.name}` の仕様。",
+            }
+        else:
+            return {
+                "purpose": f"Defines data type or alias `{sym.name}`.",
+                "inputs": "None (type definition).",
+                "outputs": f"Type `{sym.signature or sym.name}`.",
+                "overview": f"Specification of data structure or type alias `{sym.name}` used across modules.",
+            }
+
+    if is_ja:
+        return {
+            "purpose": f"`{sym.name}` ({sym.kind}) の定義。",
+            "inputs": "パラメータ定義に従う。",
+            "outputs": "結果値または状態遷移。",
+            "overview": f"`{sym.name}` の基本仕様。",
+        }
+    else:
+        return {
+            "purpose": f"Defines `{sym.name}` ({sym.kind}).",
+            "inputs": "According to parameter definitions.",
+            "outputs": "Result value or state mutation.",
+            "overview": f"Basic specification for `{sym.name}`.",
+        }
+
+
 def run_docgen(
     target_dir: Path,
     use_llm: bool = True,
@@ -71,17 +124,17 @@ def run_docgen(
     force: bool = False,
     allow_fallback: bool = False,
     compile_commands_path: Optional[str] = None,
-    concurrency: int = 4,
+    concurrency: int = 1,
 ) -> int:
     target_dir = target_dir.resolve()
     norm_lang = normalize_language(language)
     if not target_dir.exists() or not target_dir.is_dir():
-        print(f"Error: Specified directory does not exist: {target_dir}", file=sys.stderr)
+        print(f"Error: Specified directory does not exist: {target_dir}", file=sys.stderr, flush=True)
         return 1
 
     lock_path = target_dir / ".docgen" / ".lock"
     with FileLock(lock_path):
-        print(f"=== docgen Started (Parallel: {concurrency} workers, Lang: {norm_lang}, SQLite index): {target_dir} ===")
+        print(f"=== docgen Started (Workers: {concurrency}, Lang: {norm_lang}, SQLite index): {target_dir} ===", flush=True)
 
         db_path = target_dir / ".docgen" / "index.db"
         db = DocgenDB(db_path)
@@ -91,7 +144,7 @@ def run_docgen(
             target_dir=target_dir,
         )
         if comp_db.loaded_path:
-            print(f"[CompDB] Detected compilation database: {comp_db.loaded_path}")
+            print(f"[CompDB] Detected compilation database: {comp_db.loaded_path}", flush=True)
 
         llm_client = None
         if use_llm:
@@ -104,23 +157,22 @@ def run_docgen(
             if client.check_availability():
                 llm_client = client
                 auth_info = " (authenticated)" if client.token else ""
-                print(f"[LLM] Connected to server: {client.base_url} (model: {client.model}, context: {client.context_size}{auth_info})")
+                print(f"[LLM] Connected to server: {client.base_url} (model: {client.model}, context: {client.context_size}{auth_info})", flush=True)
             else:
                 if not allow_fallback:
-                    print(f"Error: Failed to connect to LLM server ({client.base_url}). Aborting without --allow-fallback.", file=sys.stderr)
+                    print(f"Error: Failed to connect to LLM server ({client.base_url}). Aborting without --allow-fallback.", file=sys.stderr, flush=True)
                     db.close()
                     return 1
                 else:
-                    print(f"[LLM Warning] LLM server unreachable ({client.base_url}). Fallback to static templates.")
+                    print(f"[LLM Warning] LLM server unreachable ({client.base_url}). Fallback to static templates.", flush=True)
 
         # 1. Scan files
         matched_files = scan_files(target_dir)
         files_txt = write_files_list(target_dir, matched_files)
-        print(f"[1/5] Scanned files: {len(matched_files)} detected -> {files_txt}")
+        print(f"[1/5] Scanned files: {len(matched_files)} detected -> {files_txt}", flush=True)
 
         # 2. Compute hashes and enumerate all symbols
         file_hashes: Dict[Path, str] = {}
-        file_changed: Dict[Path, bool] = {}
         file_symbols: Dict[Path, list] = {}
         all_symbol_nodes: List[SymbolNode] = []
 
@@ -128,8 +180,7 @@ def run_docgen(
             full_path = target_dir / rel_path
             h = compute_file_hash(full_path)
             file_hashes[rel_path] = h
-            is_changed = db.update_file_hash(rel_path.as_posix(), h)
-            file_changed[rel_path] = is_changed
+            db.update_file_hash(rel_path.as_posix(), h)
 
             symbols = extract_symbols(full_path, comp_db=comp_db)
             file_symbols[rel_path] = symbols
@@ -138,7 +189,7 @@ def run_docgen(
             all_symbol_nodes.extend(nodes)
 
         total_symbols_count = len(all_symbol_nodes)
-        print(f"[2/5] Enumerated symbols (with FQDN): {total_symbols_count} symbols total")
+        print(f"[2/5] Enumerated symbols (with FQDN): {total_symbols_count} symbols total", flush=True)
 
         link_variables_to_functions(all_symbol_nodes)
 
@@ -148,9 +199,10 @@ def run_docgen(
         resolved_symbols_map: Dict[str, SymbolNode] = {node.unique_id: node for node in all_symbol_nodes}
 
         print_lock = threading.Lock()
-        processed_counter = [0]
+        started_counter = [0]
+        completed_counter = [0]
 
-        print(f"[3/5] Pass 1: Level-by-Level parallel analysis (Total {total_levels} levels, Workers: {concurrency})...")
+        print(f"[3/5] Pass 1: Level-by-Level analysis (Total {total_levels} levels, Workers: {concurrency})...", flush=True)
 
         def process_single_node(node: SymbolNode) -> None:
             sym = node.symbol
@@ -169,13 +221,19 @@ def run_docgen(
             is_sym_changed = db.update_symbol_hash(node.unique_id, rel_path.as_posix(), sym_hash)
             db.save_symbol_metadata(node.unique_id, sym, rel_path.as_posix())
 
-            cached_data = db.load_symbol_cache(node.unique_id) if not force and not is_sym_changed else None
+            # Cache check with language key
+            cache_key = f"{node.unique_id}::{norm_lang}"
+            cached_data = db.load_symbol_cache(cache_key) or db.load_symbol_cache(node.unique_id) if not force and not is_sym_changed else None
 
             with print_lock:
-                processed_counter[0] += 1
-                curr_count = processed_counter[0]
-                percent = (curr_count / total_symbols_count * 100.0) if total_symbols_count > 0 else 100.0
-                progress_str = f"[{curr_count}/{total_symbols_count} ({percent:5.1f}%)]"
+                started_counter[0] += 1
+                curr_idx = started_counter[0]
+                percent = (curr_idx / total_symbols_count * 100.0) if total_symbols_count > 0 else 100.0
+                progress_str = f"[{curr_idx}/{total_symbols_count} ({percent:5.1f}%)]"
+
+            # Check if this symbol should be statically resolved without LLM (enum, enum_constant, typedef)
+            sym_kind_lower = sym.kind.lower()
+            is_static_bypass = ("enum" in sym_kind_lower or sym_kind_lower in ("enum_constant", "typedef"))
 
             if cached_data:
                 sym.purpose = cached_data.get("purpose", sym.purpose)
@@ -184,16 +242,41 @@ def run_docgen(
                 sym.overview = cached_data.get("overview", sym.overview)
                 sym.top_down_context = cached_data.get("top_down_context", sym.top_down_context)
                 with print_lock:
-                    print(f"  {progress_str} [Cached]: {node.unique_id} (Lvl {node.dag_level})")
-            elif llm_client and (is_sym_changed or force or not cached_data):
+                    completed_counter[0] += 1
+                    print(f"  {progress_str} [Cached]: {node.unique_id} (Lvl {node.dag_level})", flush=True)
+            elif is_static_bypass:
+                # Fast AST-based static generation (0s, prevents LLM deadlock on enums)
+                static_doc = generate_static_symbol_doc(sym, norm_lang)
+                sym.purpose = static_doc["purpose"]
+                sym.inputs_note = static_doc["inputs"]
+                sym.outputs_note = static_doc["outputs"]
+                sym.overview = static_doc["overview"]
+
+                save_payload = {
+                    "purpose": sym.purpose,
+                    "inputs_note": sym.inputs_note,
+                    "outputs_note": sym.outputs_note,
+                    "overview": sym.overview,
+                    "top_down_context": sym.top_down_context,
+                }
+                db.save_symbol_cache(cache_key, save_payload)
+                db.save_symbol_cache(node.unique_id, save_payload)
+
+                with print_lock:
+                    completed_counter[0] += 1
+                    done_count = completed_counter[0]
+                    done_pct = (done_count / total_symbols_count * 100.0) if total_symbols_count > 0 else 100.0
+                    print(f"  {progress_str} [Static Spec]: {node.unique_id} (Lvl {node.dag_level})", flush=True)
+            elif llm_client:
                 callee_context = build_callee_context_summary(node, resolved_symbols_map)
                 dep_info = f" (Callees: {len(node.direct_callee_ids)})" if node.direct_callee_ids else ""
                 with print_lock:
-                    print(f"  {progress_str} [LLM Processing]: {node.unique_id} (Lvl {node.dag_level}){dep_info}")
+                    print(f"  {progress_str} [LLM Requesting...]: {node.unique_id} (Lvl {node.dag_level}){dep_info}", flush=True)
 
                 param_strs = [f"{p.name} ({p.type_hint})" if p.type_hint else p.name for p in sym.parameters]
                 lang = get_code_language(full_path.suffix)
 
+                start_sym_time = time.time()
                 explanation = llm_client.explain_symbol(
                     name=sym.name,
                     kind=sym.kind,
@@ -207,6 +290,7 @@ def run_docgen(
                     language=norm_lang,
                     allow_fallback=allow_fallback,
                 )
+                sym_elapsed = time.time() - start_sym_time
 
                 if explanation.get("purpose"):
                     sym.purpose = explanation["purpose"]
@@ -217,19 +301,25 @@ def run_docgen(
                 if explanation.get("overview"):
                     sym.overview = explanation["overview"]
 
-                db.save_symbol_cache(
-                    node.unique_id,
-                    {
-                        "purpose": sym.purpose,
-                        "inputs_note": sym.inputs_note,
-                        "outputs_note": sym.outputs_note,
-                        "overview": sym.overview,
-                        "top_down_context": sym.top_down_context,
-                    },
-                )
+                save_payload = {
+                    "purpose": sym.purpose,
+                    "inputs_note": sym.inputs_note,
+                    "outputs_note": sym.outputs_note,
+                    "overview": sym.overview,
+                    "top_down_context": sym.top_down_context,
+                }
+                db.save_symbol_cache(cache_key, save_payload)
+                db.save_symbol_cache(node.unique_id, save_payload)
+
+                with print_lock:
+                    completed_counter[0] += 1
+                    done_count = completed_counter[0]
+                    done_pct = (done_count / total_symbols_count * 100.0) if total_symbols_count > 0 else 100.0
+                    print(f"       -> [Done in {sym_elapsed:5.1f}s] ({done_count}/{total_symbols_count} - {done_pct:5.1f}%): {node.unique_id}", flush=True)
             else:
                 with print_lock:
-                    print(f"  {progress_str} [Static Info]: {node.unique_id}")
+                    completed_counter[0] += 1
+                    print(f"  {progress_str} [Static Info]: {node.unique_id}", flush=True)
 
             write_single_symbol_doc(target_dir, rel_path, sym, prefix_name=prefix_in_unique_id, language=norm_lang)
 
@@ -241,17 +331,18 @@ def run_docgen(
                     try:
                         f.result()
                     except Exception as e:
-                        print(f"\nError: {e}", file=sys.stderr)
+                        print(f"\nError: {e}", file=sys.stderr, flush=True)
                         db.close()
                         return 1
 
         # 4. Pass 2: Top-down variable refinement
         var_nodes = [n for n in all_symbol_nodes if get_kind_prefix(n.symbol.kind) == "var"]
         total_var_count = len(var_nodes)
-        print(f"[4/5] Pass 2: Top-down variable contextual refinement ({total_var_count} variables)...")
+        print(f"[4/5] Pass 2: Top-down variable contextual refinement ({total_var_count} variables)...", flush=True)
 
         fn_nodes = [n for n in all_symbol_nodes if get_kind_prefix(n.symbol.kind) == "fn"]
         var_counter = [0]
+        var_done_counter = [0]
 
         def process_var_node(v_node: SymbolNode) -> None:
             v_sym = v_node.symbol
@@ -287,11 +378,12 @@ def run_docgen(
             if llm_client and parent_funcs and (force or not v_sym.top_down_context):
                 funcs_names = ", ".join(f["name"] for f in parent_funcs[:2])
                 with print_lock:
-                    print(f"  {v_progress} [Top-down Variable Update]: {v_node.unique_id} (Referenced by: {funcs_names})")
+                    print(f"  {v_progress} [Top-down Variable Updating...]: {v_node.unique_id} (Referenced by: {funcs_names})", flush=True)
 
                 v_snippet = get_code_snippet(v_full, v_sym.line_start, v_sym.line_end)
                 lang = get_code_language(v_full.suffix)
 
+                start_var_time = time.time()
                 top_down_res = llm_client.refine_variable_top_down(
                     var_name=v_sym.name,
                     var_kind=v_sym.kind,
@@ -302,6 +394,7 @@ def run_docgen(
                     language=norm_lang,
                     allow_fallback=allow_fallback,
                 )
+                var_elapsed = time.time() - start_var_time
 
                 lines = []
                 if top_down_res.get("significance"):
@@ -313,16 +406,22 @@ def run_docgen(
 
                 v_sym.top_down_context = "\n".join(lines)
 
-                db.save_symbol_cache(
-                    v_node.unique_id,
-                    {
-                        "purpose": v_sym.purpose,
-                        "inputs_note": v_sym.inputs_note,
-                        "outputs_note": v_sym.outputs_note,
-                        "overview": v_sym.overview,
-                        "top_down_context": v_sym.top_down_context,
-                    },
-                )
+                save_payload = {
+                    "purpose": v_sym.purpose,
+                    "inputs_note": v_sym.inputs_note,
+                    "outputs_note": v_sym.outputs_note,
+                    "overview": v_sym.overview,
+                    "top_down_context": v_sym.top_down_context,
+                }
+                cache_key = f"{v_node.unique_id}::{norm_lang}"
+                db.save_symbol_cache(cache_key, save_payload)
+                db.save_symbol_cache(v_node.unique_id, save_payload)
+
+                with print_lock:
+                    var_done_counter[0] += 1
+                    vd = var_done_counter[0]
+                    v_pct = (vd / total_var_count * 100.0) if total_var_count > 0 else 100.0
+                    print(f"       -> [Done in {var_elapsed:5.1f}s] ({vd}/{total_var_count} - {v_pct:5.1f}%): {v_node.unique_id}", flush=True)
 
                 prefix_in_unique_id = ""
                 if "::" in v_node.unique_id:
@@ -333,7 +432,8 @@ def run_docgen(
                 write_single_symbol_doc(target_dir, v_rel, v_sym, prefix_name=prefix_in_unique_id, language=norm_lang)
             else:
                 with print_lock:
-                    print(f"  {v_progress} [Retained Variable Context]: {v_node.unique_id}")
+                    var_done_counter[0] += 1
+                    print(f"  {v_progress} [Retained Variable Context]: {v_node.unique_id}", flush=True)
 
         if var_nodes:
             with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
@@ -342,7 +442,7 @@ def run_docgen(
                     try:
                         f.result()
                     except Exception as e:
-                        print(f"\nError: {e}", file=sys.stderr)
+                        print(f"\nError: {e}", file=sys.stderr, flush=True)
                         db.close()
                         return 1
 
@@ -357,6 +457,6 @@ def run_docgen(
             total_individual_docs += len(ind_docs)
 
         db.close()
-        print(f"[5/5] Recorded documents: {len(matched_files)} files, {total_individual_docs} individual symbol docs")
-        print("=== docgen Finished ===")
+        print(f"[5/5] Recorded documents: {len(matched_files)} files, {total_individual_docs} individual symbol docs", flush=True)
+        print("=== docgen Finished ===", flush=True)
     return 0

{pystdoc-0.2.0 → pystdoc-0.4.0}/src/pystdoc/llm_client.py

@@ -1,4 +1,4 @@
-"""LLM Client for LiteRT-LM, Ollama, and OpenAI-compatible endpoints with standard host/token/context/language options."""
+"""LLM Client for LiteRT-LM, Ollama, and OpenAI-compatible endpoints with multi-language JSON normalization and retry resilience."""
 
 import json
 import os
@@ -31,6 +31,71 @@ def normalize_host_url(host: Optional[str]) -> str:
     return h
 
 
+def normalize_llm_json_dict(data: Dict[str, Any]) -> Dict[str, str]:
+    """Normalize JSON dictionary keys from any language (English, Japanese, etc.) into standardized keys."""
+    normalized: Dict[str, str] = {}
+
+    for k in ("purpose", "目的", "設計意図", "意図", "description", "target", "role", "機能"):
+        if k in data and data[k]:
+            normalized["purpose"] = str(data[k]).strip()
+            break
+
+    for k in ("inputs", "入力", "引数", "パラメータ", "input", "arguments", "parameters"):
+        if k in data and data[k]:
+            normalized["inputs"] = str(data[k]).strip()
+            break
+
+    for k in ("outputs", "出力", "戻り値", "返り値", "output", "returns", "result"):
+        if k in data and data[k]:
+            normalized["outputs"] = str(data[k]).strip()
+            break
+
+    for k in ("overview", "概要", "要約", "説明", "summary", "details"):
+        if k in data and data[k]:
+            normalized["overview"] = str(data[k]).strip()
+            break
+
+    for k in ("significance", "重要性", "役割", "意義", "role"):
+        if k in data and data[k]:
+            normalized["significance"] = str(data[k]).strip()
+            break
+
+    for k in ("usage_scenario", "利用シナリオ", "使用シナリオ", "データフロー", "データ受け渡し", "scenario"):
+        if k in data and data[k]:
+            normalized["usage_scenario"] = str(data[k]).strip()
+            break
+
+    for k in ("top_down_summary", "要約", "概要", "全体設計要約", "summary"):
+        if k in data and data[k]:
+            normalized["top_down_summary"] = str(data[k]).strip()
+            break
+
+    return normalized
+
+
+def extract_json_from_text(raw_text: str) -> Dict[str, Any]:
+    """Robustly extract and parse JSON object from LLM response text."""
+    clean = raw_text.strip()
+    # 1. Try markdown code block extraction
+    m_code = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", clean, re.IGNORECASE)
+    if m_code:
+        try:
+            return json.loads(m_code.group(1).strip())
+        except Exception:
+            pass
+
+    # 2. Try outermost curly braces
+    m_brace = re.search(r"\{[\s\S]*\}", clean)
+    if m_brace:
+        try:
+            return json.loads(m_brace.group(0).strip())
+        except Exception:
+            pass
+
+    # 3. Direct parse
+    return json.loads(clean)
+
+
 class LLMClient:
     """OpenAI-compatible client with standard host, token, context size, and model configuration."""
 
@@ -42,7 +107,7 @@ class LLMClient:
         token: Optional[str] = None,
         api_key: Optional[str] = None,
         context_size: int = 16384,
-        timeout: int = 180,
+        timeout: int = 60,
     ):
         raw_host = host or base_url or os.environ.get("LLM_HOST") or os.environ.get("OPENAI_BASE_URL")
         self.user_specified_host = raw_host is not None
@@ -50,7 +115,7 @@ class LLMClient:
         self.model = model or os.environ.get("LLM_MODEL", "gemma4-26b-a4b")
         self.token = token or api_key or os.environ.get("LLM_TOKEN") or os.environ.get("OPENAI_API_KEY")
         self.context_size = context_size or int(os.environ.get("LLM_CONTEXT_SIZE", "16384"))
-        self.timeout = timeout
+        self.default_timeout = timeout
 
         self.ensure_reachable()
 
@@ -59,7 +124,6 @@ class LLMClient:
         if self._ping_url(self.base_url):
             return True
 
-        # If the user explicitly passed a host/base_url, never overwrite it with default WSL fallback
         if self.user_specified_host:
             return False
 
@@ -118,14 +182,23 @@ class LLMClient:
         """Verify server connectivity."""
         return self.ensure_reachable()
 
-    def chat_completion(self, messages: List[Dict[str, str]], json_mode: bool = False, max_retries: int = 3) -> str:
+    def chat_completion(
+        self,
+        messages: List[Dict[str, str]],
+        json_mode: bool = False,
+        max_tokens: int = 1024,
+        timeout: Optional[int] = None,
+        max_retries: int = 2,
+    ) -> str:
         """Execute chat completion request with retry loop, token authentication, and context size limits."""
         url = f"{self.base_url}/chat/completions"
+        req_timeout = timeout or self.default_timeout
+
         payload: Dict[str, Any] = {
             "model": self.model,
             "messages": messages,
             "temperature": 0.2,
-            "max_tokens": 4096,
+            "max_tokens": max_tokens,
             "options": {
                 "num_ctx": self.context_size,
             },
@@ -142,7 +215,7 @@ class LLMClient:
                 req.add_header("Authorization", f"Bearer {self.token}")
 
             try:
-                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                with urllib.request.urlopen(req, timeout=req_timeout) as resp:
                     res_data = json.loads(resp.read().decode("utf-8"))
                     return res_data["choices"][0]["message"]["content"]
             except urllib.error.HTTPError as he:
@@ -153,16 +226,15 @@ class LLMClient:
                     pass
                 detail = f": {err_body}" if err_body else ""
                 last_error = f"HTTP Error {he.code}: {he.reason}{detail}"
-                # If 404 (e.g. model not found), don't retry in vain
                 if he.code == 404:
                     break
                 if attempt < max_retries:
-                    time.sleep(1.5 * attempt)
+                    time.sleep(1.0 * attempt)
                     continue
             except Exception as e:
                 last_error = str(e)
                 if attempt < max_retries:
-                    time.sleep(1.5 * attempt)
+                    time.sleep(1.0 * attempt)
                     continue
 
         raise LLMError(f"LLM API call failed ({url}, model={self.model}): {last_error}")
@@ -181,9 +253,10 @@ class LLMClient:
         language: str = "English",
         allow_fallback: bool = False,
     ) -> Dict[str, str]:
-        """Generate explanations for a symbol in JSON format in the requested natural language."""
-        prompt = f"""Please analyze the following {lang} symbol `{name}` ({kind}) and output its design intent and technical specifications in JSON format.
-Output Language: {language} (Write all text values in {language}).
+        """Generate concise explanations for a symbol in JSON format in the requested natural language."""
+        prompt = f"""Please analyze the following {lang} symbol `{name}` ({kind}) and output concise design intent and specifications in JSON format.
+Output Language: {language} (Write all explanation text in {language}).
+Be concise and avoid repetition.
 
 ### Signature / Type:
 `{signature}`
@@ -204,12 +277,12 @@ Output Language: {language} (Write all text values in {language}).
 {code}
 ```
 
-Return ONLY a valid JSON object matching this schema (all string values in {language}):
+Return ONLY a valid JSON object matching these keys (values written in {language}):
 {{
-  "purpose": "Core design intent and fundamental purpose of this symbol (1-2 sentences)",
-  "inputs": "Meaning, constraints, or pre-conditions of input arguments",
-  "outputs": "Meaning of return value or side effects / state mutations",
-  "overview": "Clear human-readable summary of the implementation and lifecycle (2-4 sentences)"
+  "purpose": "Core design intent and purpose (1-2 concise sentences in {language})",
+  "inputs": "Input parameters description (or None)",
+  "outputs": "Return value or side effects description",
+  "overview": "Concise summary of implementation and role (1-3 sentences in {language})"
 }}
 """
         messages = [
@@ -218,19 +291,18 @@ Return ONLY a valid JSON object matching this schema (all string values in {lang
         ]
 
         try:
-            raw_res = self.chat_completion(messages, json_mode=True)
-            m = re.search(r"\{[\s\S]*\}", raw_res)
-            if m:
-                return json.loads(m.group(0))
-            return json.loads(raw_res)
+            raw_res = self.chat_completion(messages, json_mode=False, max_tokens=768, timeout=60)
+            parsed = extract_json_from_text(raw_res)
+            return normalize_llm_json_dict(parsed)
         except Exception as e:
             if not allow_fallback:
                 raise LLMError(f"Failed to analyze symbol `{name}`: {e}") from e
+            is_ja = language in ("Japanese", "日本語")
             return {
-                "purpose": f"Executes `{name}` operations.",
-                "inputs": "Accepts input parameters.",
-                "outputs": "Returns result value.",
-                "overview": f"Basic operation for `{name}`.",
+                "purpose": f"`{name}` の処理を実行する。" if is_ja else f"Executes `{name}` operations.",
+                "inputs": "パラメータを受け取る。" if is_ja else "Accepts input parameters.",
+                "outputs": "結果値を返す。" if is_ja else "Returns result value.",
+                "overview": f"`{name}` の基本処理。" if is_ja else f"Basic operation for `{name}`.",
             }
 
     def refine_variable_top_down(
@@ -253,6 +325,7 @@ Return ONLY a valid JSON object matching this schema (all string values in {lang
 
         prompt = f"""Please analyze the following {lang} variable/field `{var_name}` in the context of the caller/parent functions that reference it.
 Output Language: {language} (Write all text in {language}).
+Be concise and avoid repetition.
 
 ### Variable: `{var_name}` ({var_kind})
 - Signature/Type: `{var_signature}`
@@ -260,7 +333,7 @@ Output Language: {language} (Write all text in {language}).
 ### Referencing Caller Functions:
 {chr(10).join(context_lines)}
 
-Return ONLY a valid JSON object matching this schema (all string values in {language}):
+Return ONLY a valid JSON object (all string values in {language}):
 {{
   "significance": "Essential significance and design rationale in context of callers (1-2 sentences)",
   "usage_scenario": "Data passing lifecycle and state management role across parent functions (1-2 sentences)",
@@ -273,16 +346,15 @@ Return ONLY a valid JSON object matching this schema (all string values in {lang
         ]
 
         try:
-            raw_res = self.chat_completion(messages, json_mode=True)
-            m = re.search(r"\{[\s\S]*\}", raw_res)
-            if m:
-                return json.loads(m.group(0))
-            return json.loads(raw_res)
+            raw_res = self.chat_completion(messages, json_mode=False, max_tokens=768, timeout=60)
+            parsed = extract_json_from_text(raw_res)
+            return normalize_llm_json_dict(parsed)
         except Exception as e:
             if not allow_fallback:
                 raise LLMError(f"Failed top-down variable refinement for `{var_name}`: {e}") from e
+            is_ja = language in ("Japanese", "日本語")
             return {
-                "significance": f"`{var_name}` is a state/data utilized by callers.",
-                "usage_scenario": "Passed and transformed across calling functions.",
-                "top_down_summary": f"Design summary for `{var_name}`.",
+                "significance": f"`{var_name}` は呼び出し元で使用される状態/データ。" if is_ja else f"`{var_name}` is a state/data utilized by callers.",
+                "usage_scenario": "呼び出し元関数間で受け渡される。" if is_ja else "Passed and transformed across calling functions.",
+                "top_down_summary": f"`{var_name}` の設計概要。" if is_ja else f"Design summary for `{var_name}`.",
             }

{pystdoc-0.2.0 → pystdoc-0.4.0/src/pystdoc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pystdoc
-Version: 0.2.0
+Version: 0.4.0
 Summary: Python Structural & Topological Documentation Engine (pystdoc): High-precision, LLM-powered hierarchical codebase and architecture documentation generator.
 Author: tab4moji
 License: MIT
@@ -120,7 +120,7 @@ designgen --dir ./my_project/
 | `--model` | `-m`, `LLM_MODEL` | `gemma4-26b-a4b` | LLM model identifier |
 | `--token` | `--api-key`, `OPENAI_API_KEY` | `None` | API Bearer token |
 | `--context-size` | `--ctx-size` | `16384` | Context window size |
-| `--concurrency` | `-j` | `4` | Number of parallel LLM workers |
+| `--concurrency` | `-j` | `1` | Number of parallel LLM workers |
 | `--force` | `-f` | `false` | Force regenerate all documents ignoring cache |
 | `--compile-commands` | | `None` | Path to `compile_commands.json` |