kernhell 0.1.0__py3-none-any.whl

kernhell/pruner/dom_pruner.py

@@ -0,0 +1,240 @@
+"""DOM Pruner — strips noise from raw HTML for LLM consumption.
+
+The live DOM captured by Playwright is typically 200–500 KB of raw HTML.
+Sending all of that to an LLM is wasteful, slow, and often exceeds context
+limits.  The pruner retains only **semantically meaningful** elements and
+attributes, cutting token usage by 80–90%.
+
+**What we keep:**
+  - Interactive elements: ``<button>``, ``<a>``, ``<input>``, ``<select>``,
+    ``<textarea>``, ``<form>``, ``<label>``
+  - Semantic landmarks: ``<header>``, ``<nav>``, ``<main>``, ``<footer>``,
+    ``<section>``, ``<article>``, ``<aside>``
+  - Content elements: ``<h1>``–``<h6>``, ``<p>``, ``<li>``, ``<td>``, ``<th>``
+  - Important attributes: ``id``, ``class``, ``role``, ``aria-*``,
+    ``data-testid``, ``data-test``, ``name``, ``type``, ``href``, ``value``,
+    ``placeholder``, ``title``, ``alt``
+  - Visible text content (trimmed)
+
+**What we strip:**
+  - ``<script>``, ``<style>``, ``<svg>``, ``<noscript>``, ``<link>``, ``<meta>``
+  - Inline ``style`` attributes
+  - ``data:`` URIs and base64 content
+  - Empty/whitespace-only text nodes
+  - Deeply nested non-semantic ``<div>``/``<span>`` wrappers (flattened)
+"""
+
+from __future__ import annotations
+
+import re
+
+import structlog
+from selectolax.lexbor import LexborHTMLParser, LexborNode
+
+from kernhell.core.models import PrunedDOM
+
+logger = structlog.get_logger()
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+# Tags to remove entirely (including children)
+_STRIP_TAGS: frozenset[str] = frozenset({
+    "script", "style", "svg", "noscript", "link", "meta",
+    "iframe", "object", "embed", "template", "picture",
+})
+
+# Tags whose children we keep but the tag itself may be meaningful
+_SEMANTIC_TAGS: frozenset[str] = frozenset({
+    # Interactive
+    "button", "a", "input", "select", "textarea", "form", "label", "option",
+    # Landmarks
+    "header", "nav", "main", "footer", "section", "article", "aside", "dialog",
+    # Content
+    "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "ul", "ol",
+    "table", "thead", "tbody", "tr", "td", "th",
+    "img", "figure", "figcaption", "details", "summary",
+    # Structure (kept when they have semantic attributes)
+    "div", "span",
+})
+
+# Attributes worth preserving
+_KEEP_ATTRS: frozenset[str] = frozenset({
+    "id", "class", "role", "name", "type", "href", "value",
+    "placeholder", "title", "alt", "for", "action", "method",
+    "data-testid", "data-test", "data-cy",
+})
+
+# Prefix-based attribute preservation (aria-*)
+_KEEP_ATTR_PREFIXES: tuple[str, ...] = ("aria-", "data-testid", "data-test", "data-cy")
+
+# Regex for base64 / data URIs
+_DATA_URI_PATTERN = re.compile(r"data:[^;]+;base64,[A-Za-z0-9+/=]+")
+
+# Maximum text length to keep per element
+_MAX_TEXT_LENGTH = 120
+
+
+# ---------------------------------------------------------------------------
+# Pruner Implementation
+# ---------------------------------------------------------------------------
+
+
+def _should_keep_attr(name: str) -> bool:
+    """Determine if an HTML attribute should be preserved."""
+    if name in _KEEP_ATTRS:
+        return True
+    return any(name.startswith(prefix) for prefix in _KEEP_ATTR_PREFIXES)
+
+
+def _clean_attr_value(value: str) -> str:
+    """Clean an attribute value — strip data URIs, excessive whitespace."""
+    if not value:
+        return value
+    # Remove base64 data URIs
+    value = _DATA_URI_PATTERN.sub("[data-uri]", value)
+    # Collapse whitespace
+    value = " ".join(value.split())
+    # Truncate extremely long values (e.g. inline SVG paths)
+    if len(value) > 200:
+        value = value[:200] + "…"
+    return value
+
+
+def _has_semantic_value(node: LexborNode) -> bool:
+    """Check if a div/span has attributes that make it semantically meaningful.
+
+    Naked ``<div>`` and ``<span>`` wrappers with no id, class, role, or
+    aria attributes are noise — we flatten them (keep children, drop the tag).
+    """
+    if node.tag in ("div", "span"):
+        attrs = node.attributes
+        if not attrs:
+            return False
+        return any(_should_keep_attr(name) for name in attrs)
+    return True
+
+
+def _prune_node(node: LexborNode) -> str:
+    """Recursively prune a DOM node, returning cleaned HTML."""
+    tag = node.tag
+
+    # Skip stripped tags entirely
+    if tag and tag.lower() in _STRIP_TAGS:
+        return ""
+
+    # Handle text nodes
+    if tag is None or tag == "-text":
+        text = (node.text(deep=False) or "").strip()
+        if not text:
+            return ""
+        # Truncate long text
+        if len(text) > _MAX_TEXT_LENGTH:
+            text = text[:_MAX_TEXT_LENGTH] + "…"
+        return text
+
+    # For non-semantic div/span: flatten (emit children only)
+    if tag.lower() in ("div", "span") and not _has_semantic_value(node):
+        children_html = ""
+        child = node.child
+        while child is not None:
+            children_html += _prune_node(child)
+            child = child.next
+        return children_html
+
+    # Build the pruned tag with only meaningful attributes
+    attrs_str = ""
+    raw_attrs = node.attributes
+    if raw_attrs:
+        kept_attrs: list[str] = []
+        for name, value in raw_attrs.items():
+            if _should_keep_attr(name):
+                clean_value = _clean_attr_value(value or "")
+                if name == "style":
+                    continue  # Always strip inline styles
+                kept_attrs.append(f'{name}="{clean_value}"')
+        if kept_attrs:
+            attrs_str = " " + " ".join(kept_attrs)
+
+    # Recurse into children
+    children_html = ""
+    child = node.child
+    while child is not None:
+        children_html += _prune_node(child)
+        child = child.next
+
+    # Self-closing tags
+    if tag.lower() in ("input", "img", "br", "hr"):
+        return f"<{tag}{attrs_str}/>"
+
+    # Get direct text content
+    text = (node.text(deep=False) or "").strip()
+    if text and len(text) > _MAX_TEXT_LENGTH:
+        text = text[:_MAX_TEXT_LENGTH] + "…"
+
+    inner = text + children_html
+    if not inner.strip() and tag.lower() not in _SEMANTIC_TAGS:
+        return ""
+
+    return f"<{tag}{attrs_str}>{inner}</{tag}>"
+
+
+def prune_dom(raw_html: str) -> PrunedDOM:
+    """Prune raw HTML to a semantic skeleton for LLM consumption.
+
+    Parameters
+    ----------
+    raw_html:
+        The full ``page.content()`` HTML captured by Playwright.
+
+    Returns
+    -------
+    PrunedDOM
+        The pruned HTML along with size reduction metrics.
+
+    Examples
+    --------
+    >>> result = prune_dom("<html><body><script>...</script><button id='buy'>Buy</button></body></html>")
+    >>> "script" not in result.html
+    True
+    >>> "buy" in result.html.lower()
+    True
+    """
+    original_size = len(raw_html.encode("utf-8"))
+    logger.debug("pruner.start", original_size=original_size)
+
+    if not raw_html.strip():
+        return PrunedDOM(html="", original_size=0, pruned_size=0)
+
+    parser = LexborHTMLParser(raw_html)
+    body = parser.body
+
+    if body is None:
+        # Fallback: try the root
+        body = parser.root
+        if body is None:
+            logger.warning("pruner.no_body")
+            return PrunedDOM(html="", original_size=original_size, pruned_size=0)
+
+    pruned_html = _prune_node(body)
+
+    # Clean up excessive whitespace and empty lines in the result
+    pruned_html = re.sub(r"\n{3,}", "\n\n", pruned_html)
+    pruned_html = pruned_html.strip()
+
+    pruned_size = len(pruned_html.encode("utf-8"))
+
+    result = PrunedDOM(
+        html=pruned_html,
+        original_size=original_size,
+        pruned_size=pruned_size,
+    )
+
+    logger.info(
+        "pruner.done",
+        original_size=original_size,
+        pruned_size=pruned_size,
+        reduction_pct=result.reduction_pct,
+    )
+    return result

kernhell/reporter/__init__.py

@@ -0,0 +1,5 @@
+"""Reporter package — terminal and export formats."""
+
+from kernhell.reporter.reporter import HealingReporter
+
+__all__ = ["HealingReporter"]

kernhell/reporter/reporter.py

@@ -0,0 +1,124 @@
+"""Reporter module — produces beautiful terminal summaries and machine-readable JSON exports."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+from rich import box
+from rich.console import Console
+from rich.panel import Panel
+from rich.syntax import Syntax
+from rich.table import Table
+
+from kernhell.core.models import HealingReport, HealingResult
+
+console = Console(stderr=True)
+
+
+class HealingReporter:
+    """Renders healing run results to terminal and files."""
+
+    @staticmethod
+    def render_diff(old_code: str, new_code: str, filename: str = "") -> None:
+        """Display a syntax-highlighted diff of changes."""
+        import difflib
+
+        diff_lines = list(
+            difflib.unified_diff(
+                old_code.splitlines(),
+                new_code.splitlines(),
+                fromfile=f"a/{filename}" if filename else "original",
+                tofile=f"b/{filename}" if filename else "healed",
+                lineterm="",
+            )
+        )
+        if not diff_lines:
+            return
+
+        diff_text = "\n".join(diff_lines)
+        syntax = Syntax(diff_text, "diff", theme="monokai", line_numbers=False)
+        panel = Panel(
+            syntax,
+            title="[bold green]Surgical AST Diff (Comments & Spacing Preserved)[/bold green]",
+            border_style="green",
+            box=box.ROUNDED,
+        )
+        console.print(panel)
+
+    @staticmethod
+    def print_summary(report: HealingReport) -> None:
+        """Print full healing summary table to terminal."""
+        table = Table(
+            title=f"KernHell Healing Report: {report.test_file.name}",
+            box=box.ROUNDED,
+            show_lines=True,
+        )
+        table.add_column("#", style="dim", width=4)
+        table.add_column("Original Selector", style="red")
+        table.add_column("Healed Selector", style="bold green")
+        table.add_column("Strategy", style="cyan")
+        table.add_column("Confidence", justify="right")
+        table.add_column("Source", style="yellow")
+        table.add_column("Verified", justify="center")
+
+        for i, res in enumerate(report.results, start=1):
+            if res.fix:
+                source = "Semantic Cache" if res.from_cache else "AI Healer"
+                verified = "✅ YES" if res.fix.verified else "❌ NO"
+                conf_str = f"{int(res.fix.confidence * 100)}%"
+                table.add_row(
+                    str(i),
+                    res.failure.broken_selector,
+                    res.fix.new_selector,
+                    res.fix.strategy.value,
+                    conf_str,
+                    source,
+                    verified,
+                )
+            else:
+                table.add_row(
+                    str(i),
+                    res.failure.broken_selector,
+                    "[yellow](SKIPPED)[/yellow]",
+                    "-",
+                    "-",
+                    "-",
+                    f"[dim]{res.skip_reason[:30]}...[/dim]" if res.skip_reason else "-",
+                )
+
+        console.print(table)
+        console.print()
+
+    @staticmethod
+    def export_json(report: HealingReport, output_path: Optional[Path] = None) -> str:
+        """Export healing report to structured JSON."""
+        data = {
+            "test_file": str(report.test_file),
+            "total_failures": report.total_failures,
+            "healed_count": report.healed_count,
+            "skipped_count": report.skipped_count,
+            "cache_hits": report.cache_hits,
+            "llm_calls": report.llm_calls,
+            "verification_passed": report.verification_passed,
+            "duration_ms": report.duration_ms,
+            "results": [
+                {
+                    "broken_selector": r.failure.broken_selector,
+                    "line_number": r.failure.line_number,
+                    "healed_selector": r.fix.new_selector if r.fix else None,
+                    "strategy": r.fix.strategy.value if r.fix else None,
+                    "confidence": r.fix.confidence if r.fix else None,
+                    "from_cache": r.from_cache,
+                    "skipped": r.skipped,
+                    "skip_reason": r.skip_reason,
+                    "verified": r.fix.verified if r.fix else False,
+                }
+                for r in report.results
+            ],
+        }
+        json_str = json.dumps(data, indent=2)
+        if output_path:
+            output_path.write_text(json_str, encoding="utf-8")
+        return json_str

kernhell/scanner/__init__.py

@@ -0,0 +1 @@
+"""Scanner package — test execution and failure capture."""

kernhell/scanner/parser.py

@@ -0,0 +1,276 @@
+"""Playwright error output parser.
+
+Parses structured error output from Playwright/pytest to extract:
+  - The broken selector string
+  - The line number in the test file where the failure occurred
+  - The type of selector (CSS, XPath, text, role, etc.)
+  - The error type (timeout, not found, strict mode, etc.)
+
+This parser handles the standard Playwright error format from both
+``pytest-playwright`` and raw Playwright Python output.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import structlog
+
+from kernhell.core.models import ErrorType, FailureContext, SelectorType
+from kernhell.utils.file_utils import extract_surrounding_lines, read_file
+
+logger = structlog.get_logger()
+
+# ---------------------------------------------------------------------------
+# Regex patterns for Playwright error parsing
+# ---------------------------------------------------------------------------
+
+# Matches: "locator.click: Timeout 5000ms exceeded."
+_TIMEOUT_PATTERN = re.compile(
+    r"(locator\.\w+|page\.\w+):\s*Timeout\s+\d+ms\s+exceeded",
+    re.IGNORECASE,
+)
+
+# Matches: "waiting for locator("button#buy-now")" or 'locator("#buy-now")'
+_LOCATOR_PATTERN = re.compile(
+    r"""locator\(\s*["'](.+?)["']\s*\)""",
+    re.IGNORECASE,
+)
+
+# Matches: page.click("#buy-now") or page.fill("input.email", ...)
+_PAGE_ACTION_PATTERN = re.compile(
+    r"""page\.(?:click|fill|type|check|uncheck|select_option|hover|dblclick|focus)\(\s*["'](.+?)["']""",
+    re.IGNORECASE,
+)
+
+# Matches: page.get_by_role("button", name="Buy Now")
+_GET_BY_ROLE_PATTERN = re.compile(
+    r"""page\.get_by_role\(\s*["'](\w+)["'](?:\s*,\s*name\s*=\s*["'](.+?)["'])?\s*\)""",
+    re.IGNORECASE,
+)
+
+# Matches: page.get_by_text("Buy Now")
+_GET_BY_TEXT_PATTERN = re.compile(
+    r"""page\.get_by_text\(\s*["'](.+?)["']\s*\)""",
+    re.IGNORECASE,
+)
+
+# Matches: page.get_by_test_id("checkout-btn")
+_GET_BY_TESTID_PATTERN = re.compile(
+    r"""page\.get_by_test_id\(\s*["'](.+?)["']\s*\)""",
+    re.IGNORECASE,
+)
+
+# Matches: page.get_by_label("Email")
+_GET_BY_LABEL_PATTERN = re.compile(
+    r"""page\.get_by_label\(\s*["'](.+?)["']\s*\)""",
+    re.IGNORECASE,
+)
+
+# Matches: page.get_by_placeholder("Enter email")
+_GET_BY_PLACEHOLDER_PATTERN = re.compile(
+    r"""page\.get_by_placeholder\(\s*["'](.+?)["']\s*\)""",
+    re.IGNORECASE,
+)
+
+# Matches pytest traceback lines like: "tests/test_checkout.py:15: TimeoutError"
+_PYTEST_LOCATION_PATTERN = re.compile(
+    r"^(.+\.py):(\d+):\s*(TimeoutError|Error|AssertionError)",
+    re.MULTILINE,
+)
+
+# Matches: "strict mode violation" errors
+_STRICT_MODE_PATTERN = re.compile(
+    r"strict\s+mode\s+violation",
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class ParsedFailure:
+    """Intermediate result from parsing a single Playwright error block."""
+
+    selector: str = ""
+    selector_type: SelectorType = SelectorType.UNKNOWN
+    error_type: ErrorType = ErrorType.OTHER
+    line_number: int = 0
+    error_message: str = ""
+
+
+def classify_selector(selector: str) -> SelectorType:
+    """Determine the type of a Playwright selector string.
+
+    Examples
+    --------
+    >>> classify_selector("#buy-now")
+    SelectorType.CSS
+    >>> classify_selector("//div[@class='cart']")
+    SelectorType.XPATH
+    >>> classify_selector("text=Buy Now")
+    SelectorType.TEXT
+    """
+    s = selector.strip()
+    if s.startswith("//") or s.startswith("(//"):
+        return SelectorType.XPATH
+    if s.startswith("text=") or s.startswith('"') or s.startswith("'"):
+        return SelectorType.TEXT
+    if s.startswith("role="):
+        return SelectorType.ROLE
+    if s.startswith("data-testid=") or s.startswith("[data-testid"):
+        return SelectorType.TEST_ID
+    # Default to CSS for #id, .class, tag, [attr] patterns
+    return SelectorType.CSS
+
+
+def _detect_error_type(text: str) -> ErrorType:
+    """Classify the Playwright error type from the error output."""
+    if _STRICT_MODE_PATTERN.search(text):
+        return ErrorType.STRICT_MODE
+    if _TIMEOUT_PATTERN.search(text):
+        return ErrorType.TIMEOUT
+    if "waiting for" in text.lower() or "not found" in text.lower():
+        return ErrorType.NOT_FOUND
+    if "navigation" in text.lower():
+        return ErrorType.NAVIGATION
+    return ErrorType.OTHER
+
+
+def _extract_selector(text: str) -> tuple[str, SelectorType]:
+    """Extract the broken selector and its type from error output.
+
+    Tries multiple patterns in priority order and returns the first match.
+    """
+    # Try get_by_role first (most semantic)
+    m = _GET_BY_ROLE_PATTERN.search(text)
+    if m:
+        role = m.group(1)
+        name = m.group(2) or ""
+        selector = f'get_by_role("{role}", name="{name}")' if name else f'get_by_role("{role}")'
+        return selector, SelectorType.ROLE
+
+    # get_by_text
+    m = _GET_BY_TEXT_PATTERN.search(text)
+    if m:
+        return f'get_by_text("{m.group(1)}")', SelectorType.TEXT
+
+    # get_by_test_id
+    m = _GET_BY_TESTID_PATTERN.search(text)
+    if m:
+        return f'get_by_test_id("{m.group(1)}")', SelectorType.TEST_ID
+
+    # get_by_label
+    m = _GET_BY_LABEL_PATTERN.search(text)
+    if m:
+        return f'get_by_label("{m.group(1)}")', SelectorType.LABEL
+
+    # get_by_placeholder
+    m = _GET_BY_PLACEHOLDER_PATTERN.search(text)
+    if m:
+        return f'get_by_placeholder("{m.group(1)}")', SelectorType.PLACEHOLDER
+
+    # locator("selector")
+    m = _LOCATOR_PATTERN.search(text)
+    if m:
+        sel = m.group(1)
+        return sel, classify_selector(sel)
+
+    # page.click("selector") etc.
+    m = _PAGE_ACTION_PATTERN.search(text)
+    if m:
+        sel = m.group(1)
+        return sel, classify_selector(sel)
+
+    return "", SelectorType.UNKNOWN
+
+
+def _extract_line_number(error_text: str, test_file: Path) -> int:
+    """Extract the failing line number from pytest traceback output."""
+    filename = test_file.name
+    for m in _PYTEST_LOCATION_PATTERN.finditer(error_text):
+        if filename in m.group(1):
+            return int(m.group(2))
+
+    # Fallback: look for any ":<number>:" pattern near the test filename
+    fallback = re.search(
+        rf"{re.escape(filename)}:(\d+)",
+        error_text,
+    )
+    if fallback:
+        return int(fallback.group(1))
+
+    return 0
+
+
+def parse_playwright_errors(
+    error_output: str,
+    test_file: Path,
+) -> list[FailureContext]:
+    """Parse Playwright/pytest error output into structured FailureContexts.
+
+    Parameters
+    ----------
+    error_output:
+        The combined stdout+stderr from a failed ``pytest`` run.
+    test_file:
+        Path to the test file that was executed.
+
+    Returns
+    -------
+    list[FailureContext]
+        One context per detected selector failure.  May be empty if no
+        Playwright selector errors are found in the output.
+    """
+    logger.debug("parser.start", test_file=str(test_file), output_len=len(error_output))
+
+    failures: list[FailureContext] = []
+
+    # Split on "FAILED" or "ERROR" markers to handle multiple failures.
+    # Playwright test output typically has sections separated by "_ _ _" or "==="
+    sections = re.split(r"_{3,}|={3,}", error_output)
+
+    # Also try parsing the whole output as a single failure
+    if len(sections) <= 1:
+        sections = [error_output]
+
+    seen_selectors: set[str] = set()
+
+    for section in sections:
+        selector, sel_type = _extract_selector(section)
+        if not selector or selector in seen_selectors:
+            continue
+
+        seen_selectors.add(selector)
+        error_type = _detect_error_type(section)
+        line_number = _extract_line_number(section, test_file)
+
+        # Read surrounding code context if we have a valid line number
+        surrounding_code = ""
+        if line_number > 0 and test_file.is_file():
+            try:
+                content = read_file(test_file)
+                surrounding_code = extract_surrounding_lines(content, line_number)
+            except Exception:
+                logger.warning("parser.surrounding_code.failed", line=line_number)
+
+        ctx = FailureContext(
+            test_file=test_file,
+            line_number=line_number,
+            broken_selector=selector,
+            selector_type=sel_type,
+            error_message=section.strip()[:500],  # Cap at 500 chars
+            error_type=error_type,
+            surrounding_code=surrounding_code,
+        )
+        failures.append(ctx)
+        logger.info(
+            "parser.failure_found",
+            selector=selector,
+            selector_type=sel_type.value,
+            line=line_number,
+            error_type=error_type.value,
+        )
+
+    logger.debug("parser.done", failures_found=len(failures))
+    return failures