source-kb 0.2.2__py3-none-any.whl

core/prompt/renderer.py

@@ -0,0 +1,236 @@
+"""Prompt renderer — template loading, variable computation, rendering.
+
+Delegates content assembly (source, skeleton, file list) to an injected
+PromptAssembler strategy. Core renderer is mode-agnostic.
+
+Usage:
+    from core.prompt.renderer import render_prompt
+
+    prompt = render_prompt(
+        template_path="skill/kb-init/templates/subagent-business.md",
+        config=config, kb_name="my-kb", module_name="my-service",
+        doc_type="business-logic", assembler=my_assembler,
+    )
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import PromptAssembler
+
+logger = logging.getLogger(__name__)
+
+
+def render_prompt(
+    template_path: str | Path,
+    config: dict[str, Any],
+    kb_name: str,
+    module_name: str,
+    doc_type: str,
+    assembler: PromptAssembler,
+    extras: dict[str, str] | None = None,
+    execution_snippet: str = "",
+    preset: dict[str, Any] | None = None,
+) -> str:
+    """Render a sub-agent prompt from template + computed variables.
+
+    Args:
+        template_path: Path to the .md template file
+        config: Full kb-project.yaml config dict
+        kb_name: Knowledge base name
+        module_name: Module name
+        doc_type: Document type key
+        assembler: PromptAssembler strategy (inline or reference)
+        extras: Additional variables to inject
+        execution_snippet: Mode-specific execution guidance text
+        preset: Preset config dict (for doc_type filename resolution)
+
+    Returns:
+        Fully rendered prompt string
+    """
+    template_path = Path(template_path)
+    if not template_path.exists():
+        raise FileNotFoundError(f"Template not found: {template_path}")
+
+    template = template_path.read_text(encoding="utf-8")
+    extras = extras or {}
+
+    # Compute all template variables
+    kb_config = config["knowledge_bases"][kb_name]
+    base_dir = Path(config.get("_config_dir", ".")).resolve()
+
+    knowledge_dir = Path(kb_config["knowledge_dir"])
+    if not knowledge_dir.is_absolute():
+        knowledge_dir = (base_dir / knowledge_dir).resolve()
+
+    source = kb_config.get("source", {})
+    cache_dir = Path(source.get("cache_dir", "./.source-cache"))
+    if not cache_dir.is_absolute():
+        cache_dir = (base_dir / cache_dir).resolve()
+
+    # Determine module_dir and source_cache
+    repo_info = _find_repo(config, kb_name, module_name)
+    if repo_info.get("path") == ".":
+        module_dir = knowledge_dir
+    else:
+        module_dir = knowledge_dir / module_name
+
+    if source.get("structure") == "monorepo":
+        repo_name = source.get("repo_name", "repo")
+        module_path = repo_info.get("path", module_name)
+        source_cache = cache_dir / repo_name / module_path
+    else:
+        source_cache = cache_dir / module_name
+
+    # Delegate content assembly to the injected strategy
+    file_list_override = (extras or {}).get("file_list_override")
+    file_list = assembler.resolve_file_list(module_dir, doc_type, file_list_override=file_list_override)
+    if file_list_override and Path(file_list_override).exists():
+        override_content = Path(file_list_override).read_text(encoding="utf-8").strip()
+        source_content = assembler.resolve_source_content_from_paths(
+            module_dir, doc_type, source_cache, override_content.splitlines()
+        )
+    else:
+        source_content = assembler.resolve_source_content(module_dir, doc_type, source_cache)
+    skeleton_content = assembler.resolve_skeleton_content(module_dir)
+
+    # Compute metadata
+    from core.skeleton.metadata import load_pregenerated
+    from core.prompt.content import compute_high_methods, scan_generated_docs, compute_sibling_modules
+    from core.paths import resolve_skeleton, resolve_skeleton_summary
+
+    # Compute skeleton path/size for template variables
+    # R2 rule: skeleton reading strategy based on size
+    #   < 50KB  → read skeleton.json directly
+    #   50-200KB → read summary.json only
+    #   > 200KB  → read summary only, batch offset/limit
+    skel_path_str = ""
+    skel_size_kb = 0
+    skel_read_instruction = "Read directly"
+    skel_summary = resolve_skeleton_summary(module_dir)
+    skel_full = resolve_skeleton(module_dir)
+
+    # Determine actual skeleton size (full skeleton is the reference for R2 threshold)
+    full_size_kb = 0
+    if skel_full and skel_full.exists():
+        full_size_kb = round(skel_full.stat().st_size / 1024, 1)
+
+    if full_size_kb > 200 and skel_summary and skel_summary.exists():
+        # > 200KB: must use summary, batch if needed
+        skel_path_str = str(skel_summary).replace("\\", "/")
+        skel_size_kb = round(skel_summary.stat().st_size / 1024, 1)
+        skel_read_instruction = "Skeleton too large, reading summary file only"
+    elif full_size_kb > 50 and skel_summary and skel_summary.exists():
+        # 50-200KB: use summary
+        skel_path_str = str(skel_summary).replace("\\", "/")
+        skel_size_kb = round(skel_summary.stat().st_size / 1024, 1)
+        skel_read_instruction = "Please read this summary file"
+    elif skel_full and skel_full.exists():
+        # < 50KB: read full skeleton directly
+        skel_path_str = str(skel_full).replace("\\", "/")
+        skel_size_kb = full_size_kb
+        skel_read_instruction = "Can read the full skeleton directly"
+    elif skel_summary and skel_summary.exists():
+        # No full skeleton but summary exists
+        skel_path_str = str(skel_summary).replace("\\", "/")
+        skel_size_kb = round(skel_summary.stat().st_size / 1024, 1)
+        skel_read_instruction = "Please read this summary file"
+
+    variables: dict[str, str] = {
+        "module_name": module_name,
+        "module_description": repo_info.get("description", f"{module_name} module"),
+        "module_dir": str(module_dir.relative_to(base_dir)).replace("\\", "/") if module_dir.is_relative_to(base_dir) else str(module_dir).replace("\\", "/"),
+        "doc_type": doc_type,
+        "source_cache_path": str(source_cache.relative_to(base_dir)).replace("\\", "/") if source_cache.is_relative_to(base_dir) else str(source_cache).replace("\\", "/"),
+        "output_path": str(module_dir.relative_to(base_dir)).replace("\\", "/") if module_dir.is_relative_to(base_dir) else str(module_dir).replace("\\", "/"),
+        "file_list": file_list,
+        "source_content": source_content,
+        "skeleton_content": skeleton_content,
+        "skeleton_path": skel_path_str if not Path(skel_path_str).is_absolute() else (str(Path(skel_path_str).relative_to(base_dir)).replace("\\", "/") if Path(skel_path_str).is_relative_to(base_dir) else skel_path_str),
+        "skeleton_size": str(skel_size_kb),
+        "skeleton_read_instruction": skel_read_instruction,
+        "high_methods": compute_high_methods(module_dir),
+        "generated_docs": scan_generated_docs(module_dir),
+        "generated_docs_files": scan_generated_docs(module_dir),
+        "sibling_modules": compute_sibling_modules(config, kb_name, module_name),
+        "global_metadata": load_pregenerated(module_dir),
+        "prior_docs_context": "",
+        "branch": repo_info.get("branch", "main"),
+        "module_type": repo_info.get("type", "service"),
+    }
+
+    # Inject prior docs context for dependent doc types
+    from core.prompt.content import build_prior_docs_context
+    from core.preset import load_preset
+    preset_name = kb_config.get("preset", "generic")
+    preset = load_preset(preset_name)
+    prior = build_prior_docs_context(module_dir, doc_type, preset=preset)
+    if prior:
+        variables["prior_docs_context"] = prior
+
+    # Merge extras (user-provided overrides)
+    variables.update({k: v for k, v in extras.items() if not k.startswith("__")})
+
+    # Inject execution guidance
+    if execution_snippet:
+        if "{execution_guidance}" in template:
+            template = template.replace("{execution_guidance}", execution_snippet)
+        else:
+            marker = "## Rules you must follow"
+            if marker in template:
+                pos = template.index(marker)
+                template = template[:pos] + execution_snippet + "\n\n" + template[pos:]
+
+    # Variable substitution
+    used: set[str] = set()
+
+    def replacer(m: re.Match) -> str:
+        key = m.group(1)
+        if key in variables:
+            used.add(key)
+            return variables[key]
+        return m.group(0)
+
+    rendered = re.sub(r"\{([a-z0-9_]+)\}", replacer, template)
+
+    # Append source content if assembler says so and not already used
+    if assembler.should_append_source():
+        appendix = []
+        src = variables.get("source_content", "")
+        if src and "source_content" not in used:
+            appendix.append(f"\n\n## Source file content\n\n{src}")
+        skel = variables.get("skeleton_content", "")
+        if skel and "skeleton_content" not in used and "[truncated" in src:
+            appendix.append(f"\n\n## Source skeleton (method signatures of truncated files only)\n\n{skel}")
+        if appendix:
+            rendered += "".join(appendix)
+
+    # Append global context if not already used in template
+    context_parts = []
+    for key in ("global_metadata", "prior_docs_context"):
+        val = variables.get(key, "")
+        if val and key not in used:
+            context_parts.append(f"\n\n{val}")
+    if context_parts:
+        rendered += "".join(context_parts)
+
+    return rendered
+
+
+def _find_repo(config: dict, kb_name: str, module_name: str) -> dict:
+    """Find repo/module config entry by name."""
+    kb = config["knowledge_bases"][kb_name]
+    source = kb["source"]
+    if source.get("structure") == "monorepo":
+        for mod in source.get("modules", []):
+            if mod["name"] == module_name:
+                return mod
+    else:
+        for repo in source.get("repos", []):
+            if repo["name"] == module_name:
+                return repo
+    return {"name": module_name}

core/prompt/response_parser.py

@@ -0,0 +1,274 @@
+"""LLM response parsing and validation.
+
+Handles malformed JSON, markdown code blocks, and content validation
+for both audit and sync responses.
+
+Usage:
+    from core.prompt.response_parser import parse_audit_response, validate_sync_response
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Literal
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Finding:
+    """Single audit finding from LLM response."""
+
+    dimension: str
+    status: Literal["pass", "fail"]
+    detail: str = ""
+    fix: str = ""
+
+
+@dataclass
+class ParseResult:
+    """Result of response parsing attempt."""
+
+    success: bool
+    findings: list[Finding] = field(default_factory=list)
+    raw_length: int = 0
+    parse_method: str = ""  # "direct", "code_block", "json_fix", "fallback"
+    error: str = ""
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def parse_audit_response(raw: str) -> ParseResult:
+    """Parse LLM audit response into structured findings.
+
+    Fallback chain:
+    1. json.loads(raw) directly
+    2. Extract from ```json ... ``` code block
+    3. Fix common JSON errors (trailing commas, unquoted keys, single quotes)
+    4. Return empty findings with error logged
+
+    Args:
+        raw: Raw LLM response string
+
+    Returns:
+        ParseResult with findings list and parse metadata
+    """
+    if not raw or not raw.strip():
+        return ParseResult(success=False, raw_length=0, error="Empty response")
+
+    raw_length = len(raw)
+
+    # Attempt 1: Direct JSON parse
+    data = _try_parse_json(raw.strip())
+    if data is not None:
+        findings = _extract_findings(data)
+        return ParseResult(
+            success=True, findings=findings,
+            raw_length=raw_length, parse_method="direct",
+        )
+
+    # Attempt 2: Extract from code block
+    extracted = _extract_json_from_code_block(raw)
+    if extracted:
+        data = _try_parse_json(extracted)
+        if data is not None:
+            findings = _extract_findings(data)
+            return ParseResult(
+                success=True, findings=findings,
+                raw_length=raw_length, parse_method="code_block",
+            )
+
+    # Attempt 3: Fix common errors
+    fixed = _fix_common_json_errors(raw.strip())
+    data = _try_parse_json(fixed)
+    if data is not None:
+        findings = _extract_findings(data)
+        return ParseResult(
+            success=True, findings=findings,
+            raw_length=raw_length, parse_method="json_fix",
+        )
+
+    # Also try fixing the code block content
+    if extracted:
+        fixed_extracted = _fix_common_json_errors(extracted)
+        data = _try_parse_json(fixed_extracted)
+        if data is not None:
+            findings = _extract_findings(data)
+            return ParseResult(
+                success=True, findings=findings,
+                raw_length=raw_length, parse_method="json_fix",
+            )
+
+    # All attempts failed
+    logger.warning("[response_parser] Failed to parse audit response (%d chars)", raw_length)
+    return ParseResult(
+        success=False, raw_length=raw_length,
+        parse_method="fallback", error="All parse attempts failed",
+    )
+
+
+def validate_sync_response(
+    raw: str,
+    original_content: str,
+    *,
+    max_expansion_ratio: float = 2.0,
+) -> tuple[bool, str]:
+    """Validate LLM sync response (section content).
+
+    Checks:
+    1. Non-empty response
+    2. Contains at least one heading or paragraph
+    3. Does not echo back prompt instructions
+    4. Length within max_expansion_ratio of original
+
+    Args:
+        raw: LLM response (proposed section content)
+        original_content: Original section content for comparison
+        max_expansion_ratio: Maximum allowed length ratio vs original
+
+    Returns:
+        (is_valid, validated_content_or_error_message)
+    """
+    if not raw or not raw.strip():
+        return False, "Empty response"
+
+    content = raw.strip()
+
+    # Strip markdown code fences if the entire response is wrapped
+    content = _strip_outer_fence(content)
+
+    # Check for prompt echo (common LLM failure mode)
+    prompt_markers = [
+        "You are a knowledge base maintainer",
+        "Rewrite this section",
+        "Document type:",
+        "Current section content:",
+        "Changed source files:",
+    ]
+    for marker in prompt_markers:
+        if marker in content[:500]:
+            return False, f"Response echoes prompt instructions: '{marker}'"
+
+    # Check minimum content
+    lines = [l for l in content.splitlines() if l.strip()]
+    if len(lines) < 1:
+        return False, "Response has no meaningful content"
+
+    # Check expansion ratio (only meaningful for substantial original content)
+    if original_content.strip():
+        original_len = len(original_content.strip())
+        new_len = len(content)
+        # Only apply ratio check when original is substantial (>100 chars)
+        # For short originals, allow more expansion since a few lines can legitimately grow
+        if original_len > 100 and new_len > original_len * max_expansion_ratio:
+            return False, (
+                f"Response too long: {new_len} chars vs original {original_len} "
+                f"(ratio {new_len/original_len:.1f}x > {max_expansion_ratio}x)"
+            )
+
+    return True, content
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _try_parse_json(text: str) -> dict | list | None:
+    """Attempt JSON parse, return None on failure."""
+    try:
+        return json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+
+def _extract_json_from_code_block(text: str) -> str | None:
+    """Extract JSON content from markdown code block."""
+    # Match ```json ... ``` or ``` ... ```
+    pattern = r"```(?:json)?\s*\n(.*?)\n\s*```"
+    match = re.search(pattern, text, re.DOTALL)
+    if match:
+        return match.group(1).strip()
+
+    # Try without newline requirement (single-line blocks)
+    pattern2 = r"```(?:json)?\s*(.*?)\s*```"
+    match2 = re.search(pattern2, text, re.DOTALL)
+    if match2:
+        return match2.group(1).strip()
+
+    return None
+
+
+def _fix_common_json_errors(text: str) -> str:
+    """Attempt to fix common LLM JSON formatting errors."""
+    # Remove trailing commas before } or ]
+    text = re.sub(r",\s*([}\]])", r"\1", text)
+
+    # Replace single quotes with double quotes (careful with apostrophes)
+    # Only do this if the text looks like it uses single quotes for strings
+    if text.count("'") > text.count('"') and "{" in text:
+        text = re.sub(r"'([^']*)'", r'"\1"', text)
+
+    # Fix unquoted keys: { key: "value" } → { "key": "value" }
+    text = re.sub(r'{\s*(\w+)\s*:', r'{"\1":', text)
+    text = re.sub(r',\s*(\w+)\s*:', r',"\1":', text)
+
+    return text
+
+
+def _extract_findings(data: dict | list) -> list[Finding]:
+    """Extract Finding objects from parsed JSON data."""
+    items: list[dict] = []
+
+    if isinstance(data, list):
+        items = data
+    elif isinstance(data, dict):
+        # Try common wrapper keys
+        for key in ("findings", "results", "audit", "items", "data"):
+            if key in data and isinstance(data[key], list):
+                items = data[key]
+                break
+        if not items:
+            # Single finding as dict
+            if "dimension" in data or "status" in data:
+                items = [data]
+
+    findings: list[Finding] = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        dimension = item.get("dimension", item.get("name", item.get("check", "")))
+        status = item.get("status", "pass")
+        if status not in ("pass", "fail"):
+            status = "fail" if status in ("failed", "error", "no", "false") else "pass"
+        detail = item.get("detail", item.get("message", item.get("description", "")))
+        fix = item.get("fix", item.get("suggestion", item.get("fix_content", "")))
+
+        findings.append(Finding(
+            dimension=str(dimension),
+            status=status,
+            detail=str(detail),
+            fix=str(fix) if fix else "",
+        ))
+
+    return findings
+
+
+def _strip_outer_fence(content: str) -> str:
+    """Strip markdown code fence if the entire content is wrapped in one."""
+    lines = content.splitlines()
+    if len(lines) >= 2:
+        if lines[0].startswith("```") and lines[-1].strip() == "```":
+            return "\n".join(lines[1:-1])
+    return content