code-finder 0.1.0__py3-none-any.whl

claude_context/synthesis/prompt_builder.py

@@ -0,0 +1,439 @@
+from typing import List, Dict, Any, Optional
+
+
+def _format_docstring_evidence_block(docstring_evidence: List[Dict]) -> str:
+    """
+    Format extracted docstrings as high-priority evidence (TIER 2.5).
+
+    Docstrings represent the author's documented intent and often contain:
+    - Parameter descriptions with types and defaults
+    - Algorithm explanations
+    - Academic paper references (arXiv, etc.)
+    - Trade-off discussions
+    """
+    if not docstring_evidence:
+        return ""
+
+    lines = ["=" * 70]
+    lines.append("CODE DOCSTRINGS & DESCRIPTIONS (High Priority - Author's Intent):")
+    lines.append("=" * 70)
+    lines.append("")
+    lines.append("These docstrings represent the author's documented explanations.")
+    lines.append("Use them for parameter descriptions, algorithms, and trade-offs.")
+    lines.append("")
+
+    for doc in docstring_evidence[:10]:  # Limit to top 10
+        source = doc.get("source", "unknown")
+        chunk_type = doc.get("chunk_type", "code")
+        name = doc.get("name", "unnamed")
+        docstring = doc.get("docstring", "")
+
+        lines.append(f"[{source}] ({chunk_type}: {name}):")
+        # Truncate long docstrings to 600 chars
+        if len(docstring) > 600:
+            lines.append(docstring[:600] + "...")
+        else:
+            lines.append(docstring)
+        lines.append("")
+
+    lines.append("⚠️  INSTRUCTIONS FOR DOCSTRINGS:")
+    lines.append("   • Use docstring descriptions for parameters and return values")
+    lines.append("   • Include any paper references mentioned (arXiv, DOI, etc.)")
+    lines.append("   • Cite as: [CITE:filepath] or [CITE:docstring]")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def _format_qa_insights_block(qa_insights: List[Dict]) -> str:
+    """
+    Format Q&A insights for prompt (TIER 2 - High Priority).
+    These represent user-identified topics and questions.
+    """
+    if not qa_insights or len(qa_insights) == 0:
+        return ""
+
+    lines = []
+    lines.append("=" * 70)
+    lines.append("USER Q&A INSIGHTS (High Priority - User-Identified Topics):")
+    lines.append("=" * 70)
+    lines.append("")
+    lines.append("The user explored these topics during interactive session.")
+    lines.append("These indicate what users find important/confusing.")
+    lines.append("")
+
+    for i, qa in enumerate(qa_insights[:5], 1):  # Limit to top 5
+        conf = qa.get("confidence", 0)
+        conf_label = "HIGH" if conf > 0.7 else "MEDIUM"
+
+        lines.append(f"{i}. Q: {qa['question']}")
+        lines.append(f"   A: {qa['answer'][:300]}..." if len(qa['answer']) > 300 else f"   A: {qa['answer']}")
+        lines.append(f"   Confidence: {conf_label} ({conf:.0%})")
+        if qa.get("context_used"):
+            lines.append(f"   [Based on external context + code analysis]")
+        lines.append("")
+
+    lines.append("⚠️  INSTRUCTIONS FOR Q&A:")
+    lines.append("   • Incorporate these insights where relevant to the section")
+    lines.append("   • Cite Q&A as: [CITE:Q&A Session]")
+    lines.append("   • Focus on high-confidence insights")
+    lines.append("   • Address questions that users actually asked")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def _format_essentials_block(essentials: Dict[str, Any]) -> str:
+    """
+    Format essential content (installation, quickstart) as MANDATORY block.
+    This takes priority in the prompt.
+    """
+    if not essentials:
+        return ""
+    
+    lines = ["=" * 70]
+    lines.append("MANDATORY CONTENT (include these FIRST in your output):")
+    lines.append("=" * 70)
+    lines.append("")
+    
+    # Installation
+    if essentials.get("installation"):
+        inst = essentials["installation"]
+        lines.append("1. INSTALLATION (include EXACTLY as shown):")
+        lines.append(f"   Command: {inst['command']}")
+        if inst.get("requirements"):
+            lines.append(f"   Requirements: {', '.join(inst['requirements'])}")
+        lines.append(f"   → Include in a code block in your output")
+        lines.append("")
+    
+    # Quickstart
+    if essentials.get("quickstart"):
+        qs = essentials["quickstart"]
+        lines.append("2. QUICKSTART EXAMPLE (include EXACTLY as shown):")
+        lines.append(f"   Language: {qs['language']}")
+        lines.append(f"   Code ({len(qs['code'].split(chr(10)))} lines):")
+        lines.append(f"```{qs['language']}")
+        lines.append(qs['code'][:500] + "..." if len(qs['code']) > 500 else qs['code'])
+        lines.append("```")
+        lines.append(f"   → Include this COMPLETE code block in your output")
+        lines.append("")
+    
+    # Authentication
+    if essentials.get("authentication"):
+        auth = essentials["authentication"]
+        lines.append("3. AUTHENTICATION (include if relevant):")
+        lines.append(f"```{auth['language']}")
+        lines.append(auth['code'][:300])
+        lines.append("```")
+        lines.append("")
+
+    # Overview/Description
+    if essentials.get("overview"):
+        lines.append("4. PROJECT OVERVIEW:")
+        lines.append(f"   {essentials['overview'][:500]}")
+        lines.append("   → Use this to introduce the project")
+        lines.append("")
+
+    # Feature list
+    if essentials.get("features"):
+        features = essentials["features"]
+        lines.append(f"5. KEY FEATURES (from '{features.get('section', 'Features')}' section):")
+        for item in features.get("items", [])[:10]:
+            lines.append(f"   • {item}")
+        lines.append("   → Include these when describing capabilities")
+        lines.append("")
+
+    lines.append("")
+    lines.append("⚠️  CRITICAL REQUIREMENTS:")
+    lines.append("   • You MUST include ALL mandatory content above")
+    lines.append("   • Include commands and code EXACTLY as shown (no paraphrasing)")
+    lines.append("   • Mandatory content comes FIRST, before architecture/internals")
+    lines.append("   • If you skip or paraphrase mandatory content, validation WILL FAIL")
+    lines.append("=" * 70)
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def _format_evidence_block(title: str, items: List[Dict]) -> str:
+    if not items:
+        return f"{title}:\n  (none)\n"
+    lines = [f"{title}:"]
+    for i, it in enumerate(items, 1):
+        src = it.get("source", f"item-{i}")
+        snip = (it.get("snippet") or "").strip()
+        lines.append(f"- [{src}]")
+        if snip:
+            lines.append(f"  {snip[:800]}")
+    return "\n".join(lines) + "\n"
+
+
+def _format_rationale_block(rationale: Dict[str, Any]) -> str:
+    if not rationale:
+        return ""
+
+    logic = rationale.get("logic", [])
+    decisions = rationale.get("decisions", [])
+    qa_items = rationale.get("qa", [])
+
+    if not (logic or decisions or qa_items):
+        return ""
+
+    lines: List[str] = ["=" * 70]
+    lines.append("RATIONALE & DECISIONS (Explain the WHY):")
+    lines.append("=" * 70)
+    lines.append("")
+
+    if logic:
+        lines.append("LogicExplainer summaries (use these to explain intent):")
+        for entry in logic[:3]:
+            lines.append(f"• {entry.get('unit_id')}: {entry.get('rationale', '')[:220]}")
+            trade_offs = entry.get("trade_offs") or []
+            if trade_offs:
+                lines.append(f"   Trade-offs: {', '.join(trade_offs[:2])}")
+            lines.append("   Cite as [CITE:rationale]")
+        lines.append("")
+
+    if decisions:
+        lines.append("Documented decisions / ADRs:")
+        for decision in decisions[:3]:
+            lines.append(f"• {decision.get('type','decision').upper()}: {decision.get('summary','')[:200]}")
+            if decision.get("source"):
+                lines.append(f"   Source: {decision['source']}")
+        lines.append("")
+
+    if qa_items:
+        lines.append("Relevant Q&A insights about rationale:")
+        for qa in qa_items[:3]:
+            lines.append(f"• Q: {qa.get('question','')[:150]}")
+            lines.append(f"  A: {qa.get('answer','')[:200]}")
+        lines.append("")
+
+    lines.append("⚠️  Explain decisions, trade-offs, and impacts.")
+    lines.append("   • Tie rationale to specific code or context.")
+    lines.append("   • Use the provided citations when referencing rationale.")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def build_section_prompt(section_name: str,
+                         instructions: str,
+                         code_evidence: List[Dict],
+                         context_evidence: List[Dict],
+                         rules: Dict,
+                         max_words: int = 250,
+                         structured_evidence: Optional[Dict[str, Any]] = None) -> str:
+    """
+    Build an evidence-based prompt for one documentation section.
+    
+    NOW WITH EVIDENCE-FIRST approach:
+    - If structured_evidence provided, formats essentials as MANDATORY
+    - Ensures installation/quickstart appear before code details
+    
+    Args:
+        section_name: Name of the section
+        instructions: Template instructions
+        code_evidence: Code snippets (legacy format)
+        context_evidence: Context snippets (legacy format)
+        rules: Validation rules
+        max_words: Word limit
+        structured_evidence: NEW - Tiered evidence (essentials, usage, implementation)
+    """
+    
+    # NEW: Use structured evidence if available
+    if structured_evidence:
+        prompt_parts = []
+        
+        # Add essentials block (MANDATORY - TIER 1)
+        essentials_block = _format_essentials_block(structured_evidence.get("essentials", {}))
+        if essentials_block:
+            prompt_parts.append(essentials_block)
+
+        # Add Q&A insights (HIGH PRIORITY - TIER 2)
+        usage = structured_evidence.get("usage", {})
+        if usage.get("qa_insights"):
+            qa_block = _format_qa_insights_block(usage["qa_insights"])
+            if qa_block:
+                prompt_parts.append(qa_block)
+
+        # Add docstring evidence (HIGH PRIORITY - TIER 2.5)
+        if usage.get("docstrings"):
+            docstring_block = _format_docstring_evidence_block(usage["docstrings"])
+            if docstring_block:
+                prompt_parts.append(docstring_block)
+
+        # Add usage examples (HIGH PRIORITY - TIER 2)
+        if usage.get("readme_examples"):
+            prompt_parts.append("USAGE EXAMPLES FROM README (include if relevant):")
+            for i, ex in enumerate(usage["readme_examples"][:3], 1):
+                prompt_parts.append(f"{i}. {ex['context']}:")
+                prompt_parts.append(f"```{ex['language']}\n{ex['code'][:300]}\n```")
+            prompt_parts.append("")
+
+        # Add tables (compatibility matrices, status tables) - TIER 2
+        if usage.get("tables"):
+            prompt_parts.append("=" * 70)
+            prompt_parts.append("TABLES FROM README (include EXACTLY as shown if relevant):")
+            prompt_parts.append("=" * 70)
+            prompt_parts.append("")
+            for table in usage["tables"][:3]:
+                has_status = table.get("has_status_indicators", False)
+                status_note = " [CONTAINS STATUS INDICATORS - preserve exactly]" if has_status else ""
+                prompt_parts.append(f"Table from '{table['section']}'{status_note}:")
+                prompt_parts.append(table["raw_content"])
+                prompt_parts.append("")
+            prompt_parts.append("⚠️  IMPORTANT: Include these tables EXACTLY as shown.")
+            prompt_parts.append("   • Do NOT summarize or paraphrase tables")
+            prompt_parts.append("   • Preserve all status indicators (✅/❌/⏳)")
+            prompt_parts.append("   • Include full table structure with headers")
+            prompt_parts.append("")
+
+        # Add callouts (coming soon, experimental, warnings) - TIER 2
+        if usage.get("callouts"):
+            prompt_parts.append("=" * 70)
+            prompt_parts.append("STATUS NOTICES AND CALLOUTS (must be reflected in docs):")
+            prompt_parts.append("=" * 70)
+            prompt_parts.append("")
+            for callout in usage["callouts"][:10]:
+                ctype = callout["callout_type"].upper().replace("_", " ")
+                prompt_parts.append(f"[{ctype}]: {callout['text']}")
+            prompt_parts.append("")
+            prompt_parts.append("⚠️  CRITICAL: These notices affect documentation accuracy:")
+            prompt_parts.append("   • 'COMING SOON' features must NOT be documented as available")
+            prompt_parts.append("   • 'EXPERIMENTAL' features need appropriate warnings")
+            prompt_parts.append("   • 'DEPRECATED' features should be clearly marked")
+            prompt_parts.append("   • Reflect these statuses accurately in your output")
+            prompt_parts.append("")
+
+        # Add module READMEs (for How To Guides section) - TIER 2
+        if usage.get("module_readmes"):
+            prompt_parts.append("=" * 70)
+            prompt_parts.append("MODULE-SPECIFIC DOCUMENTATION (from subdirectory READMEs):")
+            prompt_parts.append("=" * 70)
+            prompt_parts.append("")
+            prompt_parts.append("Use this evidence for detailed how-to guides and module-specific examples.")
+            prompt_parts.append("These provide usage patterns for specific subsystems/components.")
+            prompt_parts.append("")
+
+            for module in usage["module_readmes"][:10]:  # Limit to 10 modules
+                module_path = module.get('path', 'unknown')
+                prompt_parts.append(f"### Module: {module_path}")
+
+                if module.get('overview'):
+                    overview_text = module['overview'][:800]  # Limit overview length
+                    fallback_note = " [raw extraction]" if module.get('raw_fallback') else ""
+                    prompt_parts.append(f"Overview{fallback_note}: {overview_text}")
+                    prompt_parts.append("")
+
+                if module.get('features'):
+                    features = module['features']
+                    prompt_parts.append(f"Features ({features.get('section', 'Features')}):")
+                    for item in features.get('items', [])[:5]:  # Limit to 5 features
+                        prompt_parts.append(f"  • {item}")
+                    prompt_parts.append("")
+
+                if module.get('examples'):
+                    prompt_parts.append("Code Examples:")
+                    for ex in module['examples'][:2]:  # Limit to 2 examples per module
+                        prompt_parts.append(f"```{ex['language']}\n{ex['code'][:300]}\n```")
+                    prompt_parts.append("")
+
+                if module.get('tables'):
+                    for table in module['tables'][:1]:  # Limit to 1 table per module
+                        prompt_parts.append(f"Table ({table.get('section', 'table')}):")
+                        prompt_parts.append(table.get('raw_content', '')[:500])
+                        prompt_parts.append("")
+
+                prompt_parts.append("---")
+                prompt_parts.append("")
+
+            prompt_parts.append("⚠️  INSTRUCTION: Incorporate module-specific examples into 'How To Guides'.")
+            prompt_parts.append("   • Use module paths as context (e.g., 'examples/quantization/')")
+            prompt_parts.append("   • Reference module READMEs when documenting specific features")
+            prompt_parts.append("   • Don't duplicate root README content already in Getting Started")
+            prompt_parts.append("")
+
+        rationale_block = _format_rationale_block(structured_evidence.get("rationale", {}))
+        if rationale_block:
+            prompt_parts.append(rationale_block)
+
+        # Add implementation details (SUPPLEMENTARY - TIER 3)
+        impl = structured_evidence.get("implementation", {})
+        if impl.get("code_patterns"):
+            prompt_parts.append("CODE IMPLEMENTATION DETAILS (optional, use for depth):")
+            for pattern in impl["code_patterns"][:3]:
+                prompt_parts.append(f"- [{pattern['source']}]")
+                prompt_parts.append(f"  {pattern['snippet'][:400]}")
+            prompt_parts.append("")
+        
+        evidence_text = "\n".join(prompt_parts)
+    
+    else:
+        # Legacy format (backward compatibility)
+        evidence_blocks = []
+        evidence_blocks.append(_format_evidence_block("CODE EVIDENCE", code_evidence))
+        evidence_blocks.append(_format_evidence_block("CONTEXT EVIDENCE", context_evidence))
+        evidence_text = "\n".join(evidence_blocks)
+
+    # Build rules section
+    citation_style = rules.get("citation_style") or "[CITE:source]"
+    min_citations = rules.get("min_citations", 1)
+
+    rule_lines = [
+        f"Section: {section_name}",
+        "Rules:",
+        f"1) CITATION REQUIREMENT (MANDATORY - output will be rejected without citations):",
+        f"   • Every factual claim MUST cite its source using: {citation_style}",
+        f"   • Minimum {min_citations} citation(s) required in this section",
+        f"   • Format: [CITE:filepath:line-range] or [CITE:README] or [CITE:source_name]",
+        f"   • Example: 'The default value is 5 [CITE:src/config.py:42-45]'",
+    ]
+    if rules.get("mark_inference", True):
+        rule_lines.append("2) Mark any speculation with [INFERENCE]")
+    else:
+        rule_lines.append("2) Do not mark speculation")
+    rule_lines.append("3) Follow required elements exactly:")
+    for req in rules.get("required_elements", []):
+        rule_lines.append(f"   - {req}")
+    rule_lines.append(f"4) Maximum {max_words} words.")
+    
+    # Add mandatory content rule if structured evidence has essentials
+    if structured_evidence and structured_evidence.get("essentials"):
+        rule_lines.append("")
+        rule_lines.append("5) EVIDENCE-FIRST REQUIREMENT (CRITICAL - FAILURE TO COMPLY = VALIDATION FAILURE):")
+        rule_lines.append("   • If MANDATORY CONTENT appears above, it MUST be in your output")
+        rule_lines.append("   • Installation commands: Copy EXACTLY, use code blocks")
+        rule_lines.append("   • Quickstart code: Include COMPLETE example, not paraphrased")
+        rule_lines.append("   • Order: Mandatory content FIRST, then optional details")
+        rule_lines.append("   • Validation will REJECT output missing mandatory content")
+
+    lower_name = section_name.lower()
+
+    if "usage" in lower_name or "api" in lower_name:
+        rule_lines.append("")
+        rule_lines.append("6) RATIONALE REQUIREMENT:")
+        rule_lines.append("   • Explain at least one design choice, trade-off, or side effect drawn from the rationale block.")
+        rule_lines.append("   • Cite that explanation using [CITE:rationale] (this citation is mandatory).")
+
+    # Build final prompt with citation reminder
+    citation_reminder = ""
+    if rules.get("require_citations", True):
+        citation_reminder = f"""
+
+REMINDER: Your output MUST include at least {min_citations} citation(s) in the format {citation_style}.
+Citations should reference the evidence provided above. Without citations, your output will be rejected."""
+
+    return "\n".join([
+        "Based on the following evidence, write the requested section.",
+        "",
+        evidence_text,
+        "",
+        "Instructions:",
+        instructions.strip(),
+        "",
+        "\n".join([r for r in rule_lines if r]),
+        citation_reminder,
+    ])
+
+

claude_context/synthesis/providers.py

@@ -0,0 +1,115 @@
+import os
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class TextGenerator:
+    def generate(self, system_prompt: str, user_prompt: str, model: Optional[str] = None, temperature: float = 0.2, max_tokens: int = 1200) -> str:
+        raise NotImplementedError
+
+
+class OpenAITextGenerator(TextGenerator):
+    def __init__(self, api_key: Optional[str] = None, default_model: str = "gpt-4o-mini"):
+        try:
+            from openai import OpenAI  # type: ignore
+        except Exception as e:
+            raise RuntimeError("OpenAI SDK not installed. pip install openai") from e
+        self._OpenAI = OpenAI
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise RuntimeError("OPENAI_API_KEY is required for OpenAI provider")
+        self.client = self._OpenAI(api_key=self.api_key)
+        self.default_model = default_model
+
+    def generate(self, system_prompt: str, user_prompt: str, model: Optional[str] = None, temperature: float = 0.2, max_tokens: int = 1200) -> str:
+        mdl = model or self.default_model
+        resp = self.client.chat.completions.create(
+            model=mdl,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+        )
+        content = resp.choices[0].message.content or ""
+        if not content.strip():
+            raise RuntimeError("OpenAI returned empty content")
+        return content
+
+
+class AnthropicTextGenerator(TextGenerator):
+    def __init__(self, api_key: Optional[str] = None, default_model: str = "claude-sonnet-4-6"):
+        try:
+            import anthropic  # type: ignore
+        except Exception as e:
+            raise RuntimeError("Anthropic SDK not installed. pip install anthropic") from e
+        self._anthropic = anthropic
+        self.api_key = api_key or os.getenv("ANTHROPIC_API_KEY")
+        if not self.api_key:
+            raise RuntimeError("ANTHROPIC_API_KEY is required for Anthropic provider")
+        self.client = anthropic.Anthropic(api_key=self.api_key)
+        self.default_model = default_model
+
+    def generate(self, system_prompt: str, user_prompt: str, model: Optional[str] = None, temperature: float = 0.2, max_tokens: int = 1200) -> str:
+        mdl = model or self.default_model
+        msg = self.client.messages.create(
+            model=mdl,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            system=system_prompt,
+            messages=[{"role": "user", "content": user_prompt}],
+        )
+        parts = getattr(msg, "content", [])
+        text = ""
+        for p in parts:
+            if getattr(p, "type", "") == "text":
+                text += getattr(p, "text", "")
+        if not text.strip():
+            raise RuntimeError("Anthropic returned empty content")
+        return text
+
+
+def create_generator(provider: Optional[str] = None, model: Optional[str] = None) -> TextGenerator:
+    """
+    Provider selection with Anthropic-first detection when unspecified.
+    Order:
+      1) Explicit provider arg if provided
+      2) V2D_LLM_PROVIDER env
+      3) If ANTHROPIC_API_KEY set -> Anthropic
+      4) Else if OPENAI_API_KEY set -> OpenAI
+      5) Else fail fast
+
+    Model selection order:
+      1) Explicit model arg if provided
+      2) VIBE2DOC_MODEL env variable
+      3) Provider's default model
+    """
+    prov = (provider or os.getenv("V2D_LLM_PROVIDER") or "").lower().strip()
+
+    if not prov:
+        if os.getenv("ANTHROPIC_API_KEY"):
+            prov = "anthropic"
+        elif os.getenv("OPENAI_API_KEY"):
+            prov = "openai"
+
+    # Read model from environment if not explicitly provided
+    model_to_use = model or os.getenv("VIBE2DOC_MODEL")
+
+    if prov == "anthropic":
+        gen = AnthropicTextGenerator()
+        if model_to_use:
+            gen.default_model = model_to_use
+        return gen
+    elif prov == "openai":
+        gen = OpenAITextGenerator()
+        if model_to_use:
+            gen.default_model = model_to_use
+        return gen
+
+    raise RuntimeError("No LLM provider configured. Set V2D_LLM_PROVIDER or ANTHROPIC_API_KEY/OPENAI_API_KEY")
+
+
+