deepdoc 2.3.5__tar.gz → 2.3.6__tar.gz

{deepdoc-2.3.5 → deepdoc-2.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdoc
-Version: 2.3.5
+Version: 2.3.6
 Summary: Auto-generate beautiful docs from any codebase
 Author: Pranav Kumar
 License: MIT

{deepdoc-2.3.5 → deepdoc-2.3.6}/deepdoc/generator/__init__.py

@@ -1,3 +1,4 @@
+from .consistency import CrossBucketConsistencyPass
 from .evidence import AssembledEvidence, FileEvidenceCard, EvidenceAssembler
 from .generation import (
     PageGenerator,

deepdoc-2.3.6/deepdoc/generator/consistency.py

@@ -0,0 +1,126 @@
+"""Post-generation cross-bucket consistency pass.
+
+After all pages are generated independently, makes a single LLM call to
+identify cross-linking gaps — pages that discuss concepts documented
+elsewhere but don't link to them — and injects a "See also" callout.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+
+from ..llm import LLMClient
+from .generation import GenerationResult
+
+console = Console()
+
+CONSISTENCY_SYSTEM = (
+    "You are a documentation reviewer. "
+    "Your job is to identify cross-linking gaps between independently generated "
+    "documentation pages. A gap exists when page A discusses concepts that are clearly "
+    "documented on page B but contains no link to page B. "
+    "Return only valid JSON — no prose, no markdown fences."
+)
+
+_H2_RE = re.compile(r"^## (.+)", re.MULTILINE)
+
+
+class CrossBucketConsistencyPass:
+    """Single post-generation LLM pass to detect and patch cross-link gaps."""
+
+    def __init__(self, llm: LLMClient, output_dir: Path, cfg: dict[str, Any]) -> None:
+        self.llm = llm
+        self.output_dir = output_dir
+        self.cfg = cfg
+
+    def run(self, results: list[GenerationResult]) -> int:
+        """Detect cross-link gaps and inject 'See also' callouts.
+
+        Returns the number of pages patched (0 if nothing to do or LLM fails).
+        """
+        if not self.cfg.get("consistency_pass", True):
+            return 0
+
+        successful = [r for r in results if r.content and not r.error]
+        if len(successful) < 2:
+            return 0
+
+        slug_to_title = {r.bucket.slug: r.bucket.title for r in successful}
+
+        page_summaries = self._build_summaries(successful)
+        user_prompt = self._build_prompt(page_summaries)
+
+        try:
+            response = self.llm.complete(CONSISTENCY_SYSTEM, user_prompt)
+        except Exception as exc:
+            console.print(f"[dim yellow]  consistency pass: LLM call failed ({exc})[/dim yellow]")
+            return 0
+
+        cross_links = self._parse_response(response)
+        if cross_links is None:
+            return 0
+
+        patched = 0
+        for item in cross_links:
+            from_slug = item.get("from_slug", "")
+            to_slug = item.get("to_slug", "")
+            reason = item.get("reason", "")
+            if not from_slug or not to_slug or from_slug == to_slug:
+                continue
+            if to_slug not in slug_to_title:
+                continue
+            page_path = self.output_dir / f"{from_slug}.md"
+            if not page_path.exists():
+                continue
+            content = page_path.read_text(encoding="utf-8")
+            if f"/{to_slug}" in content:
+                continue
+            to_title = slug_to_title[to_slug]
+            callout = f"\n:::note[See also]\n- [{to_title}](/{to_slug}) — {reason}\n:::\n"
+            page_path.write_text(content.rstrip() + callout, encoding="utf-8")
+            patched += 1
+
+        return patched
+
+    # ── helpers ──────────────────────────────────────────────────────────
+
+    def _build_summaries(self, results: list[GenerationResult]) -> str:
+        lines: list[str] = []
+        for r in results:
+            headings = _H2_RE.findall(r.content or "")[:6]
+            heading_str = ", ".join(headings) if headings else "(no sections)"
+            lines.append(
+                f"- slug={r.bucket.slug} | title={r.bucket.title} "
+                f"| type={r.bucket.bucket_type} | sections=[{heading_str}]"
+            )
+        return "\n".join(lines)
+
+    def _build_prompt(self, page_summaries: str) -> str:
+        return (
+            f"You have the following documentation pages ({page_summaries.count(chr(10)) + 1} total).\n"
+            "Identify pairs where page A discusses concepts clearly documented on page B "
+            "but has no link to it.\n\n"
+            f"Pages:\n{page_summaries}\n\n"
+            'Return JSON: {"cross_links": [{"from_slug": "...", "to_slug": "...", "reason": "..."}]}\n\n'
+            "Rules:\n"
+            "- Only suggest links genuinely useful to a developer reading page A\n"
+            "- Do not suggest obvious/redundant links (e.g. intro → everything)\n"
+            "- Maximum 20 suggestions total"
+        )
+
+    def _parse_response(self, response: str) -> list[dict[str, str]] | None:
+        text = response.strip()
+        if text.startswith("```"):
+            lines = [ln for ln in text.splitlines() if not ln.strip().startswith("```")]
+            text = "\n".join(lines).strip()
+        try:
+            data = json.loads(text)
+            return data.get("cross_links", [])
+        except Exception:
+            console.print("[dim yellow]  consistency pass: could not parse LLM response[/dim yellow]")
+            return None

{deepdoc-2.3.5 → deepdoc-2.3.6}/deepdoc/generator/evidence.py

@@ -97,6 +97,13 @@ class FileEvidenceCard:
     targeted_snippet: str = ""
 
 
+def _unowned_ratio(symbols: list[Symbol], owned: set[str]) -> float:
+    """Fraction of file symbols not owned by the current bucket."""
+    if not symbols:
+        return 0.0
+    return sum(1 for s in symbols if s.name not in owned) / len(symbols)
+
+
 class EvidenceAssembler:
     """Gathers and formats evidence for a single bucket from the full scan output.
 
@@ -291,7 +298,17 @@ class EvidenceAssembler:
 
             # Choose tier
             if line_count <= self.large_file_lines:
-                code = content
+                if (
+                    owned_symbols_set
+                    and parsed
+                    and parsed.symbols
+                    and _unowned_ratio(parsed.symbols, owned_symbols_set) > 0.5
+                ):
+                    code = self._extract_owned_symbol_bodies(
+                        parsed, content, owned_symbols_set
+                    )
+                else:
+                    code = content
             elif line_count <= self.giant_file_lines:
                 code = self._extract_signatures(parsed, content)
             else:
@@ -1039,6 +1056,60 @@ class EvidenceAssembler:
 
         return header + "\n".join(sig_lines)
 
+    def _extract_owned_symbol_bodies(
+        self,
+        parsed: ParsedFile,
+        content: str,
+        owned_symbols: set[str],
+    ) -> str:
+        """Tier 0.5: file header + full bodies of owned symbols only.
+
+        Activated for Tier 1 files when owned_symbols is set and more than half
+        the file's symbols are unowned — avoids sending irrelevant functions to
+        the LLM.
+        """
+        lines = content.splitlines()
+
+        # File header: everything up to the first def/class/async def (≤60 lines)
+        header_end = 0
+        for i, line in enumerate(lines[:60]):
+            stripped = line.strip()
+            if stripped.startswith(("def ", "class ", "async def ")):
+                header_end = i
+                break
+        header = "\n".join(lines[:header_end]) if header_end else ""
+
+        # Precompute symbol end lines — prefer Symbol.end_line, fall back to
+        # next symbol's start minus 1 (same pattern as _extract_key_sections).
+        def _sym_end(idx: int) -> int:
+            sym = parsed.symbols[idx]
+            if sym.has_known_range():
+                return sym.end_line
+            if idx + 1 < len(parsed.symbols):
+                return parsed.symbols[idx + 1].start_line - 1
+            return len(lines)
+
+        body_parts: list[str] = []
+        for idx, sym in enumerate(parsed.symbols):
+            if sym.name not in owned_symbols:
+                continue
+            start = max(0, sym.start_line - 1)
+            end = _sym_end(idx)
+            body_parts.append("\n".join(lines[start:end]))
+
+        if not body_parts:
+            # owned_symbols listed names that don't match any parsed symbol in
+            # this file — include header + a safe leading chunk so the page
+            # still has some source context.
+            fallback = "\n".join(lines[:min(60, len(lines))])
+            return f"{header}\n\n{fallback}" if header else fallback
+
+        total = len(parsed.symbols)
+        owned_count = len(body_parts)
+        marker = f"# [Owned symbols only — {owned_count} of {total} in file]"
+        joined = "\n\n".join(body_parts)
+        return f"{header}\n\n{marker}\n\n{joined}" if header else f"{marker}\n\n{joined}"
+
     # ── Endpoint detail ──────────────────────────────────────────────────
 
     def _build_endpoints_detail(self, bucket: DocBucket) -> str:

{deepdoc-2.3.5 → deepdoc-2.3.6}/deepdoc/generator/post_processors.py

@@ -875,32 +875,42 @@ def fix_bare_mermaid_fences(content: str) -> str:
 
 
 def fix_bare_language_markers(content: str) -> str:
-    """Repair lines where the LLM appended ':language' instead of opening a fence.
+    """Repair lines where the LLM wrote a bare language name instead of opening a fence.
 
-    The LLM sometimes writes:
-        Some description text:typescript
-        interface Foo { ... }
-        ```
+    Two variants the LLM produces:
 
-    instead of:
-        Some description text
-        ```typescript
-        interface Foo { ... }
-        ```
+    1. Suffix variant — language appended after a colon:
+           Some description text:typescript
+           interface Foo { ... }
+           ```
+
+    2. Standalone variant — language on its own line:
+           #### Example Usage
+           typescript
+           <Component ... />
+           ```
 
-    The bare ':language' suffix leaves the code content in free MDX body,
-    causing acorn parse errors on any {expression} inside.
+    Both leave the code content in free MDX body causing acorn parse errors.
     """
     _LANGS = (
         r"typescript|javascript|python|bash|json|yaml|tsx|jsx"
         r"|go|rust|java|css|html|sql|sh|text|plaintext|ruby|php|c|cpp|swift"
     )
-    return re.sub(
+    # Variant 1: text ending in :language
+    content = re.sub(
         rf"^(.*\S):({_LANGS})\s*$",
         lambda m: f"{m.group(1)}\n```{m.group(2)}",
         content,
         flags=re.MULTILINE,
     )
+    # Variant 2: language word alone on its own line (must be preceded by non-code line)
+    content = re.sub(
+        rf"^({_LANGS})\n",
+        lambda m: f"```{m.group(1)}\n",
+        content,
+        flags=re.MULTILINE,
+    )
+    return content
 
 
 def fix_leaf_card_directives(content: str) -> str:

{deepdoc-2.3.5 → deepdoc-2.3.6}/deepdoc/pipeline_v2.py

@@ -378,6 +378,12 @@ class PipelineV2:
         gen_results = engine.generate_all(force=force)
         phase_timings["generate"] = time.perf_counter() - phase_start
         engine.update_manifest(gen_results)
+
+        from .generator.consistency import CrossBucketConsistencyPass
+        injected = CrossBucketConsistencyPass(self.llm, self.output_dir, self.cfg).run(gen_results)
+        if injected:
+            console.print(f"[dim]  ↳ consistency pass: {injected} cross-link(s) injected[/dim]")
+
         generation_summary = summarize_generation_results(gen_results)
         stats["pages_generated"] = generation_summary.succeeded
         stats["pages_failed"] = generation_summary.failed

{deepdoc-2.3.5 → deepdoc-2.3.6}/deepdoc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdoc
-Version: 2.3.5
+Version: 2.3.6
 Summary: Auto-generate beautiful docs from any codebase
 Author: Pranav Kumar
 License: MIT

{deepdoc-2.3.5 → deepdoc-2.3.6}/deepdoc.egg-info/SOURCES.txt

@@ -44,6 +44,7 @@ deepdoc/chatbot/source_archive.py
 deepdoc/chatbot/symbol_index.py
 deepdoc/chatbot/types.py
 deepdoc/generator/__init__.py
+deepdoc/generator/consistency.py
 deepdoc/generator/evidence.py
 deepdoc/generator/generation.py
 deepdoc/generator/post_processors.py
@@ -128,6 +129,7 @@ tests/test_classify.py
 tests/test_cli_generate.py
 tests/test_cli_serve.py
 tests/test_cli_update.py
+tests/test_consistency_pass.py
 tests/test_flow_candidates.py
 tests/test_framework_fixtures.py
 tests/test_framework_support.py

{deepdoc-2.3.5 → deepdoc-2.3.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deepdoc"
-version = "2.3.5"
+version = "2.3.6"
 description = "Auto-generate beautiful docs from any codebase"
 readme = "README.md"
 authors = [

deepdoc-2.3.6/tests/test_consistency_pass.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from unittest.mock import MagicMock
+
+from deepdoc.generator.consistency import CrossBucketConsistencyPass
+from deepdoc.generator.generation import GenerationResult
+from tests.conftest import make_bucket
+
+
+def _make_result(slug: str, title: str, content: str) -> GenerationResult:
+    bucket = make_bucket(title, slug, [])
+    return GenerationResult(bucket=bucket, content=content)
+
+
+def _make_llm(response: str) -> MagicMock:
+    llm = MagicMock()
+    llm.complete.return_value = response
+    return llm
+
+
+def test_consistency_pass_injects_missing_link(tmp_path):
+    """LLM returns a cross-link gap — callout is appended to the source page."""
+    output_dir = tmp_path / "docs"
+    output_dir.mkdir()
+
+    orders_content = "# Order Fulfillment\n\n## Overview\n\nplace_order calls charge_card.\n"
+    payments_content = "# Payments & Billing\n\n## Overview\n\nStripe integration.\n"
+
+    (output_dir / "order-fulfillment.md").write_text(orders_content)
+    (output_dir / "payments-billing.md").write_text(payments_content)
+
+    results = [
+        _make_result("order-fulfillment", "Order Fulfillment", orders_content),
+        _make_result("payments-billing", "Payments & Billing", payments_content),
+    ]
+
+    llm_response = json.dumps({
+        "cross_links": [
+            {
+                "from_slug": "order-fulfillment",
+                "to_slug": "payments-billing",
+                "reason": "mentions charge_card which is documented here",
+            }
+        ]
+    })
+    llm = _make_llm(llm_response)
+    cfg = {}
+
+    injected = CrossBucketConsistencyPass(llm, output_dir, cfg).run(results)
+
+    assert injected == 1
+    patched = (output_dir / "order-fulfillment.md").read_text()
+    assert ":::note[See also]" in patched
+    assert "/payments-billing" in patched
+    assert "charge_card" in patched
+    # Payments page untouched
+    assert (output_dir / "payments-billing.md").read_text() == payments_content
+
+
+def test_consistency_pass_skips_existing_link(tmp_path):
+    """LLM suggests a link that already exists in the page — no change, returns 0."""
+    output_dir = tmp_path / "docs"
+    output_dir.mkdir()
+
+    orders_content = (
+        "# Order Fulfillment\n\n"
+        "See [Payments & Billing](/payments-billing) for charge details.\n"
+    )
+    (output_dir / "order-fulfillment.md").write_text(orders_content)
+    (output_dir / "payments-billing.md").write_text("# Payments\n")
+
+    results = [
+        _make_result("order-fulfillment", "Order Fulfillment", orders_content),
+        _make_result("payments-billing", "Payments & Billing", "# Payments\n"),
+    ]
+
+    llm_response = json.dumps({
+        "cross_links": [
+            {"from_slug": "order-fulfillment", "to_slug": "payments-billing", "reason": "related"}
+        ]
+    })
+    llm = _make_llm(llm_response)
+
+    injected = CrossBucketConsistencyPass(llm, output_dir, {}).run(results)
+
+    assert injected == 0
+    # Content unchanged
+    assert (output_dir / "order-fulfillment.md").read_text() == orders_content
+
+
+def test_consistency_pass_handles_llm_failure_gracefully(tmp_path):
+    """LLM returns unparseable garbage — pass returns 0 without raising."""
+    output_dir = tmp_path / "docs"
+    output_dir.mkdir()
+
+    (output_dir / "page-a.md").write_text("# Page A\n")
+    (output_dir / "page-b.md").write_text("# Page B\n")
+
+    results = [
+        _make_result("page-a", "Page A", "# Page A\n"),
+        _make_result("page-b", "Page B", "# Page B\n"),
+    ]
+
+    llm = _make_llm("not valid json at all !!!")
+
+    injected = CrossBucketConsistencyPass(llm, output_dir, {}).run(results)
+
+    assert injected == 0
+    # Files untouched
+    assert (output_dir / "page-a.md").read_text() == "# Page A\n"

{deepdoc-2.3.5 → deepdoc-2.3.6}/tests/test_generation_evidence.py

@@ -2256,3 +2256,175 @@ def test_call_graph_context_prefers_exact_method_symbol_and_counts_extra_context
     assert evidence.total_evidence_chars >= (
         len(evidence.call_graph_context) + len(evidence.config_env_context)
     )
+
+
+# ── Tier 0.5: symbol-level evidence pack tests ───────────────────────────────
+
+
+def test_tier1_owned_symbol_bodies_extracts_only_owned(tmp_path):
+    """Tier 1 file with 5 fns, bucket owns 2 → only owned bodies in source_context."""
+    repo_root = tmp_path / "repo"
+    repo_root.mkdir()
+
+    src = "\n".join([
+        "import os",
+        "",
+        "def alpha():",
+        "    return 'alpha'",
+        "",
+        "def beta():",
+        "    return 'beta'",
+        "",
+        "def gamma():",
+        "    return 'gamma'",
+        "",
+        "def delta():",
+        "    return 'delta'",
+        "",
+        "def epsilon():",
+        "    return 'epsilon'",
+    ])
+    (repo_root / "mod.py").write_text(src)
+
+    symbols = [
+        Symbol(name="alpha",   kind="function", signature="def alpha():",   start_line=3,  end_line=4),
+        Symbol(name="beta",    kind="function", signature="def beta():",    start_line=6,  end_line=7),
+        Symbol(name="gamma",   kind="function", signature="def gamma():",   start_line=9,  end_line=10),
+        Symbol(name="delta",   kind="function", signature="def delta():",   start_line=12, end_line=13),
+        Symbol(name="epsilon", kind="function", signature="def epsilon():", start_line=15, end_line=16),
+    ]
+    parsed = ParsedFile(path=Path("mod.py"), language="python", symbols=symbols, imports=["os"])
+
+    scan = RepoScan(
+        file_tree={"": ["mod.py"]},
+        file_summaries={"mod.py": "utility module"},
+        file_contents={"mod.py": src},
+        parsed_files={"mod.py": parsed},
+        file_line_counts={"mod.py": len(src.splitlines())},
+        api_endpoints=[],
+        languages={"python": 1},
+        has_openapi=False,
+        openapi_paths=[],
+        total_files=1,
+        frameworks_detected=[],
+        entry_points=[],
+        config_files=[],
+    )
+
+    bucket = make_bucket("Mod", "mod", ["mod.py"])
+    bucket.owned_symbols = ["beta", "delta"]
+    plan = make_plan([bucket])
+
+    evidence = EvidenceAssembler(repo_root, scan, plan, dict(DEFAULT_CONFIG)).assemble(bucket)
+
+    # Owned function bodies present
+    assert "return 'beta'" in evidence.source_context
+    assert "return 'delta'" in evidence.source_context
+    # Unowned function bodies absent
+    assert "return 'alpha'" not in evidence.source_context
+    assert "return 'gamma'" not in evidence.source_context
+    assert "return 'epsilon'" not in evidence.source_context
+    assert "Owned symbols only" in evidence.source_context
+
+
+def test_tier1_owned_symbol_bodies_falls_through_for_low_unowned_ratio(tmp_path):
+    """Tier 1 file with 4 fns, bucket owns 3 (ratio=0.25) → full source included."""
+    repo_root = tmp_path / "repo"
+    repo_root.mkdir()
+
+    src = "\n".join([
+        "def alpha():",
+        "    return 'alpha'",
+        "",
+        "def beta():",
+        "    return 'beta'",
+        "",
+        "def gamma():",
+        "    return 'gamma'",
+        "",
+        "def delta():",
+        "    return 'delta'",
+    ])
+    (repo_root / "mod.py").write_text(src)
+
+    symbols = [
+        Symbol(name="alpha", kind="function", signature="def alpha():", start_line=1, end_line=2),
+        Symbol(name="beta",  kind="function", signature="def beta():",  start_line=4, end_line=5),
+        Symbol(name="gamma", kind="function", signature="def gamma():", start_line=7, end_line=8),
+        Symbol(name="delta", kind="function", signature="def delta():", start_line=10, end_line=11),
+    ]
+    parsed = ParsedFile(path=Path("mod.py"), language="python", symbols=symbols, imports=[])
+
+    scan = RepoScan(
+        file_tree={"": ["mod.py"]},
+        file_summaries={"mod.py": ""},
+        file_contents={"mod.py": src},
+        parsed_files={"mod.py": parsed},
+        file_line_counts={"mod.py": len(src.splitlines())},
+        api_endpoints=[],
+        languages={"python": 1},
+        has_openapi=False,
+        openapi_paths=[],
+        total_files=1,
+        frameworks_detected=[],
+        entry_points=[],
+        config_files=[],
+    )
+
+    bucket = make_bucket("Mod", "mod", ["mod.py"])
+    bucket.owned_symbols = ["alpha", "beta", "gamma"]  # 3 of 4 owned → ratio=0.25
+    plan = make_plan([bucket])
+
+    evidence = EvidenceAssembler(repo_root, scan, plan, dict(DEFAULT_CONFIG)).assemble(bucket)
+
+    # All four function names should appear — full source was included
+    for name in ("alpha", "beta", "gamma", "delta"):
+        assert name in evidence.source_context
+    assert "Owned symbols only" not in evidence.source_context
+
+
+def test_tier1_owned_symbol_bodies_falls_through_when_owned_symbols_empty(tmp_path):
+    """Tier 1 file with owned_symbols=[] → full source included unchanged."""
+    repo_root = tmp_path / "repo"
+    repo_root.mkdir()
+
+    src = "\n".join([
+        "def alpha():",
+        "    return 1",
+        "",
+        "def beta():",
+        "    return 2",
+    ])
+    (repo_root / "mod.py").write_text(src)
+
+    symbols = [
+        Symbol(name="alpha", kind="function", signature="def alpha():", start_line=1, end_line=2),
+        Symbol(name="beta",  kind="function", signature="def beta():",  start_line=4, end_line=5),
+    ]
+    parsed = ParsedFile(path=Path("mod.py"), language="python", symbols=symbols, imports=[])
+
+    scan = RepoScan(
+        file_tree={"": ["mod.py"]},
+        file_summaries={"mod.py": ""},
+        file_contents={"mod.py": src},
+        parsed_files={"mod.py": parsed},
+        file_line_counts={"mod.py": len(src.splitlines())},
+        api_endpoints=[],
+        languages={"python": 1},
+        has_openapi=False,
+        openapi_paths=[],
+        total_files=1,
+        frameworks_detected=[],
+        entry_points=[],
+        config_files=[],
+    )
+
+    bucket = make_bucket("Mod", "mod", ["mod.py"])
+    # owned_symbols defaults to [] — no narrowing should happen
+    plan = make_plan([bucket])
+
+    evidence = EvidenceAssembler(repo_root, scan, plan, dict(DEFAULT_CONFIG)).assemble(bucket)
+
+    assert "alpha" in evidence.source_context
+    assert "beta" in evidence.source_context
+    assert "Owned symbols only" not in evidence.source_context