code-finder 0.1.0__py3-none-any.whl

claude_context/synthesis/validators.py

@@ -0,0 +1,458 @@
+import re
+import logging
+from typing import List, Dict, Tuple, Any, Optional, Set
+
+logger = logging.getLogger(__name__)
+
+
+def check_essentials_present(text: str, essentials: Dict[str, Any]) -> Tuple[bool, str]:
+    """
+    Validate that essential content (installation, quickstart) appears in output.
+
+    This is CRITICAL - fail if essentials in evidence but missing from output.
+    Prevents LLM from writing about obscure functions while skipping install commands.
+
+    MANDATORY EVIDENCE-FIRST APPROACH:
+    - Installation commands from README MUST appear in output
+    - Quickstart code examples from README MUST appear in output
+    - No paraphrasing - show EXACT commands/code
+    """
+    if not essentials:
+        return True, ""
+
+    violations = []
+
+    # CRITICAL: Check installation command (EXACT match required)
+    if essentials.get("installation"):
+        cmd = essentials["installation"].get("command", "")
+        if cmd:
+            # Must contain the EXACT command
+            if cmd not in text:
+                violations.append(f"CRITICAL: Missing installation command from README: {cmd}")
+            # Detect paraphrasing anti-pattern
+            if "can be installed" in text.lower() and cmd not in text:
+                violations.append(f"CRITICAL: Installation paraphrased instead of showing exact command: {cmd}")
+
+    # CRITICAL: Check quickstart code block presence
+    if essentials.get("quickstart"):
+        qs_code = essentials["quickstart"].get("code", "")
+        # Should have at least one code block
+        if "```" not in text:
+            violations.append("CRITICAL: Missing quickstart code block (found in README but not in output)")
+        # Prefer checking if substantial portion of quickstart is present
+        elif qs_code:
+            # Check if at least 50% of quickstart code is present
+            code_lines = [line.strip() for line in qs_code.split('\n') if line.strip() and not line.strip().startswith('#')]
+            if code_lines:
+                present_count = sum(1 for line in code_lines if line in text)
+                if present_count < len(code_lines) * 0.5:
+                    violations.append(f"CRITICAL: Quickstart example incomplete (only {present_count}/{len(code_lines)} lines present)")
+
+    # Check authentication if present (WARNING only - not blocking)
+    warnings = []
+    if essentials.get("authentication"):
+        auth = essentials["authentication"]
+        # At minimum, should mention auth if README does
+        if "auth" not in text.lower() and "authentication" not in text.lower():
+            warnings.append("WARNING: Authentication mentioned in README but not in output")
+
+    # Only CRITICAL violations block generation
+    if violations:
+        # Include warnings for context, but still fail on critical
+        all_issues = violations + warnings
+        return False, "; ".join(all_issues)
+
+    # If only warnings, still pass but return warnings for logging
+    if warnings:
+        return True, "; ".join(warnings)
+
+    return True, ""
+
+
+def check_max_words(text: str, max_words: int) -> Tuple[bool, str]:
+    words = len((text or "").strip().split())
+    if words <= max_words:
+        return True, ""
+    return False, f"Too long: {words} words > {max_words}"
+
+
+def check_citations(text: str, min_citations: int, citation_style: str) -> Tuple[bool, str]:
+    """
+    Check for citations in various formats.
+    
+    Supports:
+    - [CITE:source]
+    - [SOURCE:name]
+    - [API:endpoint]
+    - [file.py:10-20]
+    - And any [WORD:something] pattern
+    """
+    if not text:
+        return False, f"Missing citations: found 0, require >= {min_citations}"
+    
+    # Try to extract the prefix from citation_style (e.g., "CITE" from "[CITE:source]")
+    style_prefix = None
+    if citation_style and ":" in citation_style:
+        # Extract word between [ and :
+        match = re.search(r'\[([A-Z]+):', citation_style)
+        if match:
+            style_prefix = match.group(1)
+    
+    # Build pattern based on style
+    if style_prefix:
+        # Look for specific style like [SOURCE:...], [CITE:...], [API:...]
+        pattern = rf"\[{style_prefix}:[^\]]+\]"
+    else:
+        # Generic: look for any [WORD:...] pattern
+        pattern = r"\[([A-Z]+|[a-z_]+\.[a-z]+):[^\]]+\]"
+    
+    found = re.findall(pattern, text, re.IGNORECASE)
+    
+    if len(found) >= int(min_citations):
+        return True, ""
+    return False, f"Missing citations: found {len(found)}, require >= {min_citations}"
+
+
+def check_required_elements(text: str, required_elements: List[str]) -> Tuple[bool, str]:
+    missing = []
+    for req in required_elements or []:
+        if re.search(re.escape(req), text or "", re.IGNORECASE) is None:
+            missing.append(req)
+    if not missing:
+        return True, ""
+    return False, f"Missing required elements: {', '.join(missing)}"
+
+
+def validate_section_output(
+    text: str,
+    rules: Dict,
+    max_words: int,
+    essentials: Optional[Dict[str, Any]] = None,
+    section_name: Optional[str] = None,
+    structured_evidence: Optional[Dict[str, Any]] = None
+) -> List[str]:
+    """
+    Validate section output against rules.
+
+    EVIDENCE-FIRST VALIDATION (MANDATORY):
+    - CRITICAL violations (missing essentials) FAIL immediately
+    - Other violations (citations, word count) are warnings but still fail
+    - NEW: Anti-hallucination validation for API docs (WARNING level)
+
+    Priority:
+    1. Essential content (install, quickstart) - CRITICAL
+    2. Citations and required elements - HIGH
+    3. API reference validation (anti-hallucination) - MEDIUM
+    4. Word count - MEDIUM (can be slightly exceeded for completeness)
+    """
+    violations: List[str] = []
+    critical_violations: List[str] = []
+
+    # PRIORITY 1: Check essentials FIRST (CRITICAL - must pass)
+    # Only check essentials on sections where they belong (getting started, installation, quickstart)
+    # Skip for reference sections like "Sources and References", "API Reference", "Troubleshooting"
+    essentials_sections = {
+        'getting started', 'getting_started', 'installation', 'quickstart',
+        'quick start', 'quick_start', 'overview', 'introduction'
+    }
+    section_name_lower = (section_name or "").lower().replace("-", " ").replace("_", " ")
+
+    should_check_essentials = essentials and any(
+        es in section_name_lower for es in essentials_sections
+    )
+
+    if should_check_essentials:
+        ok, msg = check_essentials_present(text, essentials)
+        if not ok:
+            # Split into critical vs warning
+            parts = msg.split("; ")
+            for part in parts:
+                if "CRITICAL:" in part:
+                    critical_violations.append(part)
+                else:
+                    violations.append(part)
+
+    # PRIORITY 2: Citations and required elements (HIGH - should pass)
+    if rules.get("require_citations", True):
+        ok, msg = check_citations(text, rules.get("min_citations", 1), rules.get("citation_style", "[CITE:source]"))
+        if not ok:
+            violations.append(msg)
+
+    if rules.get("required_elements"):
+        ok, msg = check_required_elements(text, rules["required_elements"])
+        if not ok:
+            violations.append(msg)
+
+    # PRIORITY 2.5: Anti-hallucination validation for API docs (NEW!)
+    if section_name and "api" in section_name.lower() and structured_evidence:
+        # Extract entities from structured evidence
+        extracted_entities = {}
+
+        # Check implementation tier for entities
+        impl = structured_evidence.get("implementation", {})
+        if impl:
+            # Entities might be in code_patterns or directly in implementation
+            code_patterns = impl.get("code_patterns", [])
+
+            # Also check if entities are stored elsewhere in evidence
+            usage = structured_evidence.get("usage", {})
+            if usage and "entities" in usage:
+                extracted_entities = usage.get("entities", {})
+
+        # Only validate if we have entities to check against
+        if extracted_entities:
+            passed, confidence, warnings = validate_api_references(
+                text, extracted_entities, section_name
+            )
+
+            if warnings:
+                # Add as warnings, not critical violations
+                for warning in warnings:
+                    violations.append(f"API-VALIDATION: {warning}")
+
+            # Log confidence for monitoring
+            if confidence < 1.0:
+                logger.info(f"{section_name} API validation: {confidence:.0%} confidence, {len(warnings)} warnings")
+
+    rationale_data = structured_evidence.get("rationale") if structured_evidence else None
+    if rationale_data and any(rationale_data.get(key) for key in ("logic", "decisions", "qa")):
+        if "[CITE:rationale" not in (text or ""):
+            violations.append("Missing rationale citation: include at least one [CITE:rationale] entry")
+
+    # PRIORITY 3: Word count (MEDIUM - can be flexible)
+    ok, msg = check_max_words(text, max_words)
+    if not ok:
+        # Allow 10% overage if content is good
+        words = len((text or "").strip().split())
+        if words <= max_words * 1.1:
+            violations.append(f"Note: {words} words (slightly over {max_words} limit, acceptable)")
+        else:
+            violations.append(msg)
+
+    # Return critical violations first (fail-fast on these)
+    return critical_violations + violations
+
+
+def feedback_instructions(violations: List[str]) -> str:
+    if not violations:
+        return ""
+    bullet = "\n".join(f"- {v}" for v in violations)
+    return (
+        "Fix the following issues without adding new claims:\n"
+        f"{bullet}\n"
+        "Only make minimal edits to satisfy the rules."
+    )
+
+
+# Anti-hallucination validation - prevents docs from referencing non-existent APIs
+
+# Whitelist of safe terms that shouldn't be validated as entities
+SAFE_API_TERMS = {
+    # Generic types
+    'string', 'str', 'int', 'integer', 'float', 'double', 'bool', 'boolean',
+    'list', 'dict', 'dictionary', 'tuple', 'set', 'array', 'vector',
+    'object', 'function', 'class', 'method', 'property', 'attribute',
+    'type', 'none', 'null', 'undefined', 'any', 'optional',
+
+    # Common patterns
+    'api', 'endpoint', 'route', 'handler', 'service', 'client', 'server',
+    'request', 'response', 'error', 'exception', 'warning',
+    'config', 'configuration', 'settings', 'options', 'parameters', 'params',
+    'args', 'kwargs', 'context', 'state', 'data', 'result', 'output',
+
+    # External frameworks - PyTorch
+    'tensor', 'model', 'module', 'optimizer', 'scheduler', 'loss',
+    'dataset', 'dataloader', 'batch', 'epoch', 'checkpoint',
+    'cuda', 'device', 'gpu', 'cpu', 'dtype', 'shape',
+
+    # External frameworks - NumPy/Pandas
+    'numpy', 'ndarray', 'pandas', 'dataframe', 'series', 'index',
+    'sklearn', 'scipy', 'matplotlib', 'seaborn',
+
+    # External frameworks - Web
+    'flask', 'django', 'fastapi', 'requests', 'http', 'https',
+    'json', 'xml', 'yaml', 'csv', 'html', 'url', 'uri',
+
+    # External frameworks - TensorFlow
+    'tensorflow', 'keras', 'layer', 'sequential', 'functional',
+
+    # Documentation/general terms
+    'example', 'usage', 'overview', 'architecture', 'installation',
+    'quickstart', 'tutorial', 'guide', 'reference', 'documentation',
+    'readme', 'license', 'changelog', 'version', 'release',
+
+    # Common verbs/actions (not entities)
+    'create', 'read', 'update', 'delete', 'get', 'set', 'add', 'remove',
+    'initialize', 'configure', 'start', 'stop', 'run', 'execute',
+    'load', 'save', 'export', 'import', 'parse', 'format', 'validate'
+}
+
+
+def extract_api_references(content: str) -> Set[str]:
+    """
+    Extract potential API references from documentation content.
+
+    Looks for:
+    - Function calls: word()
+    - Class references: CapitalizedWord
+    - Code references: `backtick_word`
+    - Method calls: object.method
+
+    Returns:
+        Set of potential entity names
+    """
+    potential_refs = set()
+
+    # Pattern 1: Function calls (word followed by parentheses)
+    function_pattern = r'\b([a-zA-Z_][a-zA-Z0-9_]*)\s*\('
+    potential_refs.update(re.findall(function_pattern, content))
+
+    # Pattern 2: Class references (capitalized words, likely classes)
+    # Only in code contexts (between backticks or in code blocks)
+    code_contexts = re.findall(r'`([^`]+)`', content)
+    code_blocks = re.findall(r'```[a-z]*\n(.*?)```', content, re.DOTALL)
+
+    for code_text in code_contexts + code_blocks:
+        class_pattern = r'\b([A-Z][a-zA-Z0-9_]*)\b'
+        potential_refs.update(re.findall(class_pattern, code_text))
+
+    # Pattern 3: Method references (word.method)
+    method_pattern = r'\.([a-zA-Z_][a-zA-Z0-9_]*)\b'
+    potential_refs.update(re.findall(method_pattern, content))
+
+    return potential_refs
+
+
+def matches_entity(ref: str, extracted_entities: Dict, fuzzy: bool = True) -> bool:
+    """
+    Check if a reference matches a real extracted entity.
+
+    Args:
+        ref: The reference to check (e.g., "SmoothQuant")
+        extracted_entities: Extracted entities from code analysis
+        fuzzy: Enable fuzzy matching for partial matches
+
+    Returns:
+        True if the reference matches a real entity
+    """
+    if not extracted_entities:
+        return False
+
+    # Exact match
+    for file_data in extracted_entities.values():
+        entities = file_data.get('entities', [])
+        for entity in entities:
+            entity_name = entity.get('name', '')
+
+            # Exact match
+            if ref == entity_name:
+                return True
+
+    # Fuzzy match (if enabled)
+    if fuzzy:
+        ref_lower = ref.lower()
+
+        for file_data in extracted_entities.values():
+            entities = file_data.get('entities', [])
+            for entity in entities:
+                entity_name = entity.get('name', '')
+                entity_lower = entity_name.lower()
+
+                # Partial match: "SmoothQuant" matches "SmoothQuantModifier"
+                if ref_lower in entity_lower or entity_lower in ref_lower:
+                    return True
+
+                # Case-insensitive exact match
+                if ref_lower == entity_lower:
+                    return True
+
+    return False
+
+
+def validate_api_references(
+    content: str,
+    extracted_entities: Dict[str, Any],
+    doc_type: str
+) -> Tuple[bool, float, List[str]]:
+    """
+    Validate API references in documentation to prevent hallucinations.
+
+    This is a CONSERVATIVE check with:
+    - Whitelisted safe terms (generic types, external libraries)
+    - Fuzzy matching for partial names
+    - Confidence scoring instead of binary pass/fail
+    - Warnings instead of hard rejections
+
+    Args:
+        content: Documentation content to validate
+        extracted_entities: Dict of extracted code entities
+        doc_type: Type of documentation (e.g., "API", "Overview")
+
+    Returns:
+        (passed: bool, confidence: float, warnings: List[str])
+    """
+    if not content or not extracted_entities:
+        # No entities extracted = can't validate, but that's ok
+        return True, 1.0, []
+
+    # Extract potential API references
+    potential_refs = extract_api_references(content)
+
+    if not potential_refs:
+        # No references found = no risk of hallucination
+        return True, 1.0, []
+
+    # Classify references
+    verified_refs = []
+    unverified_refs = []
+    warnings = []
+
+    for ref in potential_refs:
+        ref_lower = ref.lower()
+
+        # Skip whitelisted safe terms
+        if ref_lower in SAFE_API_TERMS:
+            continue
+
+        # Skip very short refs (likely not entities)
+        if len(ref) <= 2:
+            continue
+
+        # Skip pure numbers
+        if ref.isdigit():
+            continue
+
+        # Check if it matches real entities (with fuzzy matching)
+        if matches_entity(ref, extracted_entities, fuzzy=True):
+            verified_refs.append(ref)
+        else:
+            unverified_refs.append(ref)
+            warnings.append(f"Unverified API reference: '{ref}'")
+
+    # Calculate confidence score
+    total_refs = len(verified_refs) + len(unverified_refs)
+
+    if total_refs == 0:
+        # All refs were whitelisted = perfect
+        confidence = 1.0
+    else:
+        confidence = len(verified_refs) / total_refs if total_refs > 0 else 1.0
+
+    # Graduated response based on confidence
+    if confidence >= 0.8:
+        # High confidence - pass with no warnings
+        return True, confidence, []
+
+    elif confidence >= 0.6:
+        # Medium confidence - pass with warnings
+        logger.warning(f"{doc_type} validation: {confidence:.0%} confidence ({len(unverified_refs)} unverified refs)")
+        return True, confidence, warnings[:5]  # Limit warnings
+
+    else:
+        # Low confidence - pass with strong warnings (but don't block)
+        # Too many false positives to make this blocking
+        logger.warning(f"{doc_type} validation: LOW confidence {confidence:.0%} ({len(unverified_refs)}/{total_refs} unverified)")
+        return True, confidence, warnings[:5]  # Still pass, but log warnings
+
+
+