ref-agents 1.0.0__py3-none-any.whl

ref_agents/tools/brownfield_populator.py

@@ -0,0 +1,529 @@
+"""Brownfield Context Graph Populator.
+
+Orchestrates discovery-time population of the context graph from existing codebase.
+Layers:
+1. Structural: File nodes + AST extraction via tree-sitter (Universal)
+2. Archaeological: Git history -> Decision nodes (L1/L2 only)
+3. Synthesized: LLM -> Enriched Decision nodes (L1/L2 only)
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Literal
+
+import structlog
+
+from ref_agents.tools.context_graph import (
+    ContextNode,
+    get_context_graph,
+)
+
+logger = structlog.get_logger(__name__)
+
+# Mapping file extensions → tree-sitter language module name
+_LANG_LOADERS: dict[str, str] = {
+    ".py": "tree_sitter_python",
+    ".ts": "tree_sitter_typescript",
+    ".tsx": "tree_sitter_typescript",
+}
+
+# Node types to extract per language (tree-sitter type name → our node category)
+_CLASS_TYPES: dict[str, set[str]] = {
+    "python": {"class_definition"},
+    "typescript": {"class_declaration"},
+}
+_FUNC_TYPES: dict[str, set[str]] = {
+    "python": {"function_definition"},
+    "typescript": {"function_declaration", "method_definition"},
+}
+_IMPORT_TYPES: dict[str, set[str]] = {
+    "python": {"import_statement", "import_from_statement"},
+    "typescript": {"import_statement"},
+}
+_EXT_TO_LANG: dict[str, str] = {
+    ".py": "python",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+}
+
+
+def _get_language(ext: str) -> Any | None:
+    """Load tree-sitter Language for a file extension.
+
+    Args:
+        ext: File extension including dot (e.g. '.py').
+
+    Returns:
+        tree_sitter.Language or None if unsupported.
+    """
+    loader = _LANG_LOADERS.get(ext)
+    if not loader:
+        return None
+    try:
+        import importlib
+
+        from tree_sitter import Language
+
+        mod = importlib.import_module(loader)
+        return Language(mod.language())
+    except (ImportError, AttributeError) as e:
+        logger.debug("tree_sitter_lang_load_failed", ext=ext, error=str(e))
+        return None
+
+
+def _walk(node: Any, target_types: set[str]) -> list[Any]:
+    """Walk tree-sitter node tree, collecting nodes matching target_types.
+
+    Args:
+        node: tree_sitter.Node root to walk.
+        target_types: Set of node type strings to collect.
+
+    Returns:
+        List of matching nodes.
+    """
+    results = []
+    cursor = node.walk()
+    visited_children = False
+    while True:
+        if not visited_children:
+            if cursor.node.type in target_types:
+                results.append(cursor.node)
+            if not cursor.goto_first_child():
+                visited_children = True
+        elif cursor.goto_next_sibling():
+            visited_children = False
+        elif not cursor.goto_parent():
+            break
+    return results
+
+
+def _node_name(node: Any) -> str | None:
+    """Extract identifier name from a class/function definition node.
+
+    Args:
+        node: tree_sitter.Node (class_definition, function_definition, etc.)
+
+    Returns:
+        Name string or None if not found.
+    """
+    name_node = node.child_by_field_name("name")
+    if name_node is not None and name_node.text:
+        return name_node.text.decode("utf-8", errors="replace")
+    return None
+
+
+def _import_modules(node: Any, lang: str) -> list[str]:
+    """Extract module names from an import node.
+
+    Args:
+        node: tree_sitter.Node for an import statement.
+        lang: Language name ('python' or 'typescript').
+
+    Returns:
+        List of module name strings.
+    """
+    names: list[str] = []
+    if lang == "python":
+        if node.type == "import_statement":
+            for child in node.children:
+                if child.type in ("dotted_name", "aliased_import"):
+                    n = child.child_by_field_name("name") or child
+                    if n.text:
+                        names.append(n.text.decode("utf-8", errors="replace"))
+        elif node.type == "import_from_statement":
+            mod = node.child_by_field_name("module_name")
+            if mod and mod.text:
+                names.append(mod.text.decode("utf-8", errors="replace"))
+    elif lang == "typescript":
+        src = node.child_by_field_name("source")
+        if src and src.text:
+            raw = src.text.decode("utf-8", errors="replace").strip("'\"")
+            names.append(raw)
+    return names
+
+
+def _get_decorators(node: Any) -> list[str]:
+    """Collect decorator strings from siblings preceding a class/function node.
+
+    tree-sitter represents decorators as preceding siblings of the decorated
+    node (not children), so we walk backwards through prev_sibling chain.
+    Works for both Python and TypeScript — both use node type "decorator".
+
+    Args:
+        node: tree_sitter.Node for a class or function definition.
+
+    Returns:
+        List of decorator strings in declaration order (outermost first).
+    """
+    # TODO: TypeScript method decorators (@Injectable, @Get) may use a different
+    # node type path depending on TS grammar version — verify with tree-sitter-typescript.
+    decorator_type = "decorator"
+    decorators: list[str] = []
+    sibling = node.prev_sibling
+    while sibling is not None and sibling.type == decorator_type:
+        text = sibling.text
+        if text:
+            decorators.append(text.decode("utf-8", errors="replace").strip())
+        sibling = sibling.prev_sibling
+    decorators.reverse()
+    return decorators
+
+
+CriticalityLevel = Literal["L1", "L2", "L3"]
+
+
+class BrownfieldPopulator:
+    """Populates context graph during discovery."""
+
+    def __init__(self, directory: Path | None = None):
+        self.root = directory or Path.cwd()
+        self.graph = get_context_graph(self.root)
+
+    def populate(
+        self,
+        levels: list[CriticalityLevel] | None = None,
+        include_git: bool = True,
+        include_llm: bool = True,
+    ) -> dict:
+        """Run full brownfield population.
+
+        Args:
+            levels: List of criticality levels to process for deep analysis (git/llm).
+                   Defaults to ["L1", "L2"].
+            include_git: Whether to mine git history.
+            include_llm: Whether to use LLM synthesis.
+
+        Returns:
+            Dict with counts of nodes/edges added.
+        """
+        if levels is None:
+            levels = ["L1", "L2"]
+
+        logger.info("brownfield_population_start", root=str(self.root), levels=levels)
+
+        # Clear stale AST nodes so re-populate is idempotent
+        self._clear_ast_nodes()
+
+        # 1. Structural Scan (All files)
+        file_nodes = self._scan_structural()
+
+        # 2. Classify Criticality
+        classification = self._classify_criticality(file_nodes)
+
+        # Filter files for deep analysis based on levels
+        deep_analysis_files = []
+        for level in levels:
+            deep_analysis_files.extend(classification.get(level, []))
+
+        # 3. Archaeological Mining (Git)
+        if include_git:
+            self._mine_git_decisions(deep_analysis_files)
+
+        # 4. Synthesized Decisions (LLM)
+        if include_llm:
+            self._synthesize_decisions(deep_analysis_files)
+
+        # 5. Export for Dashboard (handled by save)
+        self.graph.save()
+
+        return {
+            "files_scanned": len(file_nodes),
+            "deep_analysis_files": len(deep_analysis_files),
+            "total_nodes": self.graph.node_count,
+            "total_edges": self.graph.edge_count,
+        }
+
+    def _clear_ast_nodes(self) -> None:
+        """Remove all AST-origin nodes from graph before repopulation.
+
+        Clears by node_type (not origin) so old-format nodes without origin field
+        are also removed. file/module/class/function are always AST-derived.
+        Context nodes (decision/pattern/issue/story/agent_session) preserved.
+        """
+        from ref_agents.tools.context_graph import AST_NODE_TYPES
+
+        ast_node_ids = [
+            n
+            for n in list(self.graph._graph.nodes)
+            if self.graph._graph.nodes[n].get("node_type") in AST_NODE_TYPES
+        ]
+        self.graph._graph.remove_nodes_from(ast_node_ids)
+        # Also nuke stale ast_graph.json cache to avoid re-merging on next load
+        ast_cache = self.root / ".ref_cache" / "ast_graph.json"
+        if ast_cache.exists():
+            ast_cache.unlink()
+        logger.info("ast_nodes_cleared", count=len(ast_node_ids))
+
+    def _scan_structural(self) -> list[str]:
+        """Layer 1: Create file + AST nodes via tree-sitter extraction."""
+        from tree_sitter import Parser
+
+        added_files = []
+        _skip_dirs = {
+            ".git",
+            "__pycache__",
+            ".venv",
+            "venv",
+            ".ref_cache",
+            "node_modules",
+            ".pytest_cache",
+        }
+
+        for path in self.root.rglob("*"):
+            if not path.is_file():
+                continue
+
+            rel_path = str(path.relative_to(self.root))
+            # Check skip dirs against RELATIVE path parts only
+            if any(p in _skip_dirs or p.startswith(".") for p in Path(rel_path).parts):
+                continue
+            file_node_id = f"file-{rel_path.replace('/', '-').replace('.', '_')}"
+
+            file_node = ContextNode(
+                id=file_node_id,
+                node_type="file",
+                content=rel_path,
+                origin="ast",
+                metadata={"path": rel_path, "ext": path.suffix},
+            )
+            self.graph.add_node(file_node)
+            added_files.append(rel_path)
+
+            lang_name = _EXT_TO_LANG.get(path.suffix)
+            lang = _get_language(path.suffix)
+            if lang is None or lang_name is None:
+                continue
+
+            try:
+                source = path.read_bytes()
+            except OSError:
+                continue
+
+            try:
+                parser = Parser(lang)
+                tree = parser.parse(source)
+            except (ValueError, RuntimeError) as e:
+                logger.debug("tree_sitter_parse_failed", file=rel_path, error=str(e))
+                continue
+
+            self._emit_ast_nodes(tree.root_node, file_node_id, rel_path, lang_name)
+
+        return added_files
+
+    def _emit_ast_nodes(
+        self,
+        root_node: Any,
+        file_node_id: str,
+        rel_path: str,
+        lang_name: str,
+    ) -> None:
+        """Create module/class/function nodes and edges via tree walker.
+
+        Functions nested inside a class are parented to that class node
+        (contains edge), not to the module. Decorators are captured in
+        node metadata.
+
+        Args:
+            root_node: tree_sitter.Node root of parsed tree.
+            file_node_id: Parent file node ID.
+            rel_path: Relative path for node ID generation.
+            lang_name: Language name ('python' or 'typescript').
+        """
+        base = rel_path.replace("/", "-").replace(".", "_")
+
+        # Module node — one per file
+        mod_id = f"module-{base}"
+        if mod_id not in self.graph._graph:
+            mod_node = ContextNode(
+                id=mod_id,
+                node_type="module",
+                content=rel_path,
+                origin="ast",
+                metadata={"path": rel_path},
+            )
+            self.graph.add_node(mod_node)
+            try:
+                self.graph.add_edge(mod_id, file_node_id, "contains", confidence=1.0)
+            except ValueError:
+                pass
+
+        class_types = _CLASS_TYPES.get(lang_name, set())
+        func_types = _FUNC_TYPES.get(lang_name, set())
+        import_types = _IMPORT_TYPES.get(lang_name, set())
+
+        all_target_types = class_types | func_types | import_types
+        matched_nodes = _walk(root_node, all_target_types)
+
+        # Two-pass: first pass builds class_id lookup keyed by ts_node id,
+        # so second pass can resolve enclosing class without re-walking.
+        class_counter: dict[str, int] = {}
+        func_counter: dict[str, int] = {}
+
+        # Map tree-sitter node id → graph class_id (for method parent resolution)
+        ts_node_to_class_id: dict[int, str] = {}
+
+        # Pass 1: emit class nodes
+        for ts_node in matched_nodes:
+            if ts_node.type not in class_types:
+                continue
+            name = _node_name(ts_node)
+            if not name:
+                continue
+            i = class_counter.get(name, 0)
+            class_counter[name] = i + 1
+            class_id = f"class-{base}-{name}-{i}"
+            ts_node_to_class_id[ts_node.id] = class_id
+            if class_id not in self.graph._graph:
+                decorators = _get_decorators(ts_node)
+                cls_node = ContextNode(
+                    id=class_id,
+                    node_type="class",
+                    content=f"{rel_path}::{name}",
+                    origin="ast",
+                    metadata={"name": name, "path": rel_path, "decorators": decorators},
+                )
+                self.graph.add_node(cls_node)
+                try:
+                    self.graph.add_edge(mod_id, class_id, "contains", confidence=1.0)
+                except ValueError:
+                    pass
+
+        # Pass 2: emit function nodes, resolve parent (class or module)
+        for ts_node in matched_nodes:
+            ntype = ts_node.type
+            if ntype in func_types:
+                name = _node_name(ts_node)
+                if not name:
+                    continue
+                i = func_counter.get(name, 0)
+                func_counter[name] = i + 1
+                func_id = f"func-{base}-{name}-{i}"
+                if func_id not in self.graph._graph:
+                    decorators = _get_decorators(ts_node)
+                    func_node = ContextNode(
+                        id=func_id,
+                        node_type="function",
+                        content=f"{rel_path}::{name}",
+                        origin="ast",
+                        metadata={
+                            "name": name,
+                            "path": rel_path,
+                            "decorators": decorators,
+                        },
+                    )
+                    self.graph.add_node(func_node)
+                    # Find enclosing class by walking ancestor chain
+                    parent_id = mod_id
+                    ancestor = ts_node.parent
+                    while ancestor is not None:
+                        if ancestor.id in ts_node_to_class_id:
+                            parent_id = ts_node_to_class_id[ancestor.id]
+                            break
+                        ancestor = ancestor.parent
+                    try:
+                        self.graph.add_edge(
+                            parent_id, func_id, "contains", confidence=1.0
+                        )
+                    except ValueError:
+                        pass
+
+            elif ntype in import_types:
+                for imp in _import_modules(ts_node, lang_name):
+                    candidate_id = (
+                        "file-" + imp.replace(".", "-").replace("/", "-") + "_py"
+                    )
+                    if candidate_id in self.graph._graph:
+                        try:
+                            self.graph.add_edge(
+                                mod_id, candidate_id, "imports", confidence=0.8
+                            )
+                        except ValueError:
+                            pass
+
+    def _scan_structural_single(self, path: Path, rel_path: str) -> None:
+        """Incremental re-extraction for a single changed file (used by file watcher).
+
+        Args:
+            path: Absolute path to the changed file.
+            rel_path: Path relative to project root.
+        """
+        from tree_sitter import Parser
+
+        file_node_id = f"file-{rel_path.replace('/', '-').replace('.', '_')}"
+
+        # Remove stale AST nodes for this file before re-adding
+        stale = [
+            n
+            for n in list(self.graph._graph.nodes)
+            if self.graph._graph.nodes[n].get("metadata", {}).get("path") == rel_path
+            and self.graph._graph.nodes[n].get("origin") == "ast"
+        ]
+        for s in stale:
+            self.graph._graph.remove_node(s)
+
+        file_node = ContextNode(
+            id=file_node_id,
+            node_type="file",
+            content=rel_path,
+            origin="ast",
+            metadata={"path": rel_path, "ext": path.suffix},
+        )
+        self.graph.add_node(file_node)
+
+        lang_name = _EXT_TO_LANG.get(path.suffix)
+        lang = _get_language(path.suffix)
+        if lang is None or lang_name is None:
+            return
+
+        try:
+            source = path.read_bytes()
+        except OSError:
+            return
+
+        try:
+            from tree_sitter import Parser
+
+            parser = Parser(lang)
+            tree = parser.parse(source)
+        except (ValueError, RuntimeError) as e:
+            logger.debug("tree_sitter_parse_failed", file=rel_path, error=str(e))
+            return
+
+        self._emit_ast_nodes(tree.root_node, file_node_id, rel_path, lang_name)
+
+    def _classify_criticality(
+        self, files: list[str]
+    ) -> dict[CriticalityLevel, list[str]]:
+        """Classify files into L1/L2/L3."""
+        classified: dict[CriticalityLevel, list[str]] = {"L1": [], "L2": [], "L3": []}
+
+        for rel_path in files:
+            # Heuristics
+            level: CriticalityLevel = "L2"  # Default
+
+            parts = Path(rel_path).parts
+            name = Path(rel_path).name
+
+            if name in ["main.py", "app.py", "server.py", "manage.py"]:
+                level = "L1"
+            elif "tests" in parts or name.startswith("test_"):
+                level = "L3"
+            elif name.endswith((".md", ".json", ".yaml", ".toml", ".txt")):
+                level = "L3"
+            elif "core" in parts or "api" in parts:
+                level = "L1"
+
+            classified[level].append(rel_path)
+
+        return classified
+
+    def _mine_git_decisions(self, files: list[str]) -> None:
+        """Layer 2: Create decision nodes from git history."""
+        # Placeholder for git miner integration
+        pass
+
+    def _synthesize_decisions(self, files: list[str]) -> None:
+        """Layer 3: Create decision nodes from LLM analysis."""
+        # Placeholder for LLM synthesizer integration
+        pass

ref_agents/tools/browser/__init__.py

@@ -0,0 +1,50 @@
+"""Browser Tools for REF Agents.
+
+This package provides browser automation capabilities for frontend E2E testing
+using Playwright MCP. It includes:
+
+- playwright_mcp_client: Client wrapper for Playwright MCP tools
+- execution_logger: Audit trail logging for test execution
+- evidence_verifier: Verify screenshots, logs, and timestamps
+- screenshot_utils: Screenshot capture and management
+- test_executor: Execute conceptual tests via browser tools
+- report_generator: Generate test reports with evidence
+"""
+
+from ref_agents.tools.browser.evidence_verifier import (
+    EvidenceVerifier,
+    VerificationResult,
+)
+from ref_agents.tools.browser.execution_logger import (
+    ExecutionLogEntry,
+    ExecutionLogger,
+)
+from ref_agents.tools.browser.screenshot_utils import (
+    get_screenshot_dir,
+    get_screenshot_path,
+    save_screenshot,
+    verify_screenshot_content,
+)
+from ref_agents.tools.browser.test_executor import (
+    BrowserTestExecutor,
+    ExecutionResult,
+    StepResult,
+)
+
+__all__ = [
+    # Evidence verification
+    "EvidenceVerifier",
+    "VerificationResult",
+    # Execution logging
+    "ExecutionLogEntry",
+    "ExecutionLogger",
+    # Screenshot utilities
+    "get_screenshot_dir",
+    "get_screenshot_path",
+    "save_screenshot",
+    "verify_screenshot_content",
+    # Test execution
+    "BrowserTestExecutor",
+    "ExecutionResult",
+    "StepResult",
+]