attocode-code-intel 0.1.0__py3-none-any.whl

attocode_intel/rules/executor.py

@@ -0,0 +1,531 @@
+"""Rule executor — runs rules against source files to produce findings."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from attocode_intel.rules.combinators import MatchContext
+from attocode_intel.rules.metavar import (
+    apply_metavar_fix,
+    check_metavar_constraints,
+    interpolate_message,
+)
+from attocode_intel.rules.model import (
+    EnrichedFinding,
+    RuleTier,
+    UnifiedRule,
+)
+
+logger = logging.getLogger(__name__)
+
+# Per-rule subprocess timeout for ast-grep (seconds). Structural patterns
+# can be expensive on huge files; cap to keep ``analyze`` responsive.
+_AST_GREP_TIMEOUT = 30.0
+
+# Languages where ast-grep ships a tree-sitter grammar. When a structural
+# rule targets a language outside this set we either skip the rule (no
+# fallback regex) or fall back to ``rule.pattern`` if the YAML defined one.
+_AST_GREP_LANGS = frozenset({
+    "bash", "c", "cpp", "csharp", "css", "elixir", "go", "html", "java",
+    "javascript", "json", "kotlin", "lua", "php", "python", "ruby", "rust",
+    "scala", "shell", "swift", "tsx", "typescript", "yaml",
+})
+
+# Extension to language mapping
+_EXT_LANG: dict[str, str] = {
+    ".py": "python", ".pyi": "python",
+    ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript",
+    ".ts": "typescript", ".tsx": "typescript", ".mts": "typescript", ".cts": "typescript",
+    ".go": "go",
+    ".rs": "rust",
+    ".java": "java",
+    ".kt": "kotlin", ".kts": "kotlin",
+    ".rb": "ruby",
+    ".php": "php",
+    ".swift": "swift",
+    ".cs": "csharp",
+    ".c": "c", ".h": "c",
+    ".cpp": "cpp", ".hpp": "cpp", ".cc": "cpp", ".cxx": "cpp",
+    ".lua": "lua",
+    ".sh": "shell", ".bash": "shell", ".zsh": "shell",
+    ".scala": "scala",
+    ".ex": "elixir", ".exs": "elixir",
+}
+
+# Comment prefix detection per language
+_COMMENT_PREFIX: dict[str, list[str]] = {
+    "python": ["#"],
+    "javascript": ["//"],
+    "typescript": ["//"],
+    "go": ["//"],
+    "rust": ["//"],
+    "java": ["//"],
+    "kotlin": ["//"],
+    "ruby": ["#"],
+    "php": ["//", "#"],
+    "swift": ["//"],
+    "csharp": ["//"],
+    "c": ["//"],
+    "cpp": ["//"],
+    "lua": ["--"],
+    "shell": ["#"],
+    "scala": ["//"],
+    "elixir": ["#"],
+}
+
+
+def detect_language(file_path: str) -> str:
+    """Detect language from file extension."""
+    ext = os.path.splitext(file_path)[1].lower()
+    return _EXT_LANG.get(ext, "")
+
+
+def _is_comment_line(line: str, language: str) -> bool:
+    stripped = line.lstrip()
+    for prefix in _COMMENT_PREFIX.get(language, []):
+        if stripped.startswith(prefix):
+            return True
+    return False
+
+
+def _ast_grep_available() -> bool:
+    """Return True when an ``sg`` or ``ast-grep`` binary is on PATH."""
+    return shutil.which("sg") is not None or shutil.which("ast-grep") is not None
+
+
+def _ast_grep_binary() -> str:
+    """Resolve the ast-grep binary, preferring the short ``sg`` alias."""
+    return shutil.which("sg") or shutil.which("ast-grep") or "sg"
+
+
+# ast-grep's ``language`` field uses TitleCase ("Go", "Python") whereas
+# our internal language tags are lowercase. Mapping covers the cases that
+# differ from a naive ``.title()``.
+_AST_GREP_LANG_NAMES: dict[str, str] = {
+    "javascript": "JavaScript",
+    "typescript": "TypeScript",
+    "tsx": "Tsx",
+    "csharp": "CSharp",
+    "cpp": "Cpp",
+    "html": "Html",
+    "css": "Css",
+    "json": "Json",
+    "yaml": "Yaml",
+}
+
+
+def _ast_grep_lang_name(lang: str) -> str:
+    return _AST_GREP_LANG_NAMES.get(lang, lang.title())
+
+
+def _yaml_quote(value: str) -> str:
+    """Render ``value`` as a YAML double-quoted scalar with escapes.
+
+    Double-quoted scalars handle every printable + control character via
+    backslash escapes. Single-quoted scalars cannot represent literal
+    newlines or tabs, so synthesised / evolved patterns containing those
+    (which are legal ast-grep input) would corrupt the inline-rules YAML.
+    """
+    escaped = (
+        value.replace("\\", "\\\\")
+        .replace('"', '\\"')
+        .replace("\n", "\\n")
+        .replace("\r", "\\r")
+        .replace("\t", "\\t")
+    )
+    return '"' + escaped + '"'
+
+
+def _build_inline_rule_yaml(rule: UnifiedRule, lang: str) -> str:
+    """Build the minimal ast-grep YAML rule for ``sg scan --inline-rules``.
+
+    Used when the rule provides ``structural_context`` and/or
+    ``structural_selector`` to disambiguate the parse.
+
+    ast-grep accepts two pattern shapes under ``rule.pattern``:
+
+    * a bare string — the simple pattern, no disambiguation
+    * an object with ``context`` (the full snippet that parses correctly)
+      and ``selector`` (the AST kind to extract from inside it). The plain
+      ``pattern`` field on the object is *not* used when context drives
+      matching; including it makes the YAML invalid.
+    """
+    parts: list[str] = [
+        f"id: {rule.id or 'inline'}",
+        f"language: {_ast_grep_lang_name(lang)}",
+        "rule:",
+    ]
+    if rule.structural_context:
+        # Context without selector produces invalid ast-grep YAML — the
+        # parser needs a selector to know which AST kind to extract from
+        # the wrapping context. Loader-level validation rejects pack rules
+        # in this shape; this guard catches programmatically-constructed
+        # rules (synthesis / evolution / direct UnifiedRule(...) callers)
+        # so they fail loudly instead of silently no-op'ing.
+        if not rule.structural_selector:
+            raise ValueError(
+                f"rule {rule.qualified_id!r} has structural_context but no "
+                "structural_selector; ast-grep needs both to extract a "
+                "node from the wrapping context"
+            )
+        parts.append("  pattern:")
+        parts.append(f"    context: {_yaml_quote(rule.structural_context)}")
+        parts.append(f"    selector: {rule.structural_selector}")
+    elif rule.structural_selector:
+        # Selector without context: pair it with the pattern as object.
+        parts.append("  pattern:")
+        parts.append(f"    pattern: {_yaml_quote(rule.structural_pattern)}")
+        parts.append(f"    selector: {rule.structural_selector}")
+    else:
+        parts.append(f"  pattern: {_yaml_quote(rule.structural_pattern)}")
+    return "\n".join(parts) + "\n"
+
+
+def _execute_structural_rule(
+    rule: UnifiedRule,
+    files_by_lang: dict[str, list[str]],
+    *,
+    project_dir: str = "",
+) -> list[EnrichedFinding]:
+    """Run a single structural rule across applicable files via ast-grep.
+
+    Per language: shells out to ``sg run --pattern <pat> --lang <lang>
+    --json=stream <files...>`` and parses one match per JSON line. Files
+    in languages without an ast-grep grammar are silently skipped — the
+    caller is responsible for supplying ``rule.pattern`` if they want a
+    Tier-1 fallback for those languages.
+
+    Returns the list of (partially populated) ``EnrichedFinding`` objects.
+    """
+    if not rule.structural_pattern:
+        return []
+
+    # Determine which languages this rule applies to. An empty
+    # ``rule.languages`` means "all languages we have files for".
+    target_langs: list[str]
+    if rule.languages:
+        target_langs = [lang for lang in rule.languages if lang in files_by_lang]
+    else:
+        target_langs = list(files_by_lang.keys())
+
+    binary = _ast_grep_binary()
+    findings: list[EnrichedFinding] = []
+
+    for lang in target_langs:
+        if lang not in _AST_GREP_LANGS:
+            continue
+        files = files_by_lang.get(lang, [])
+        if not files:
+            continue
+
+        # When the rule needs context/selector to disambiguate parsing
+        # (e.g. ``fmt.Println($X)`` in Go), shell out to ``sg scan
+        # --inline-rules`` with a YAML rule body. Otherwise the simpler
+        # ``sg run --pattern`` path is faster.
+        if rule.structural_context or rule.structural_selector:
+            inline_yaml = _build_inline_rule_yaml(rule, lang)
+            cmd = [
+                binary, "scan",
+                "--inline-rules", inline_yaml,
+                "--json=stream",
+                *files,
+            ]
+        else:
+            cmd = [
+                binary, "run",
+                "--pattern", rule.structural_pattern,
+                "--lang", lang,
+                "--json=stream",
+                *files,
+            ]
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=_AST_GREP_TIMEOUT,
+                check=False,
+            )
+        except (subprocess.TimeoutExpired, FileNotFoundError) as exc:
+            logger.warning(
+                "ast-grep failed for rule %s lang=%s: %s",
+                rule.qualified_id, lang, exc,
+            )
+            continue
+
+        # ast-grep returns non-zero when *no* matches are found. Only treat
+        # stderr-with-actual-content as an error signal.
+        if result.returncode not in (0, 1) and result.stderr.strip():
+            logger.warning(
+                "ast-grep error for rule %s lang=%s: %s",
+                rule.qualified_id, lang, result.stderr.strip()[:200],
+            )
+            continue
+
+        for line in result.stdout.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                match = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            file_path = match.get("file", "")
+            start = match.get("range", {}).get("start", {})
+            line_no = int(start.get("line", 0)) + 1  # ast-grep is 0-based
+            snippet = (match.get("lines") or "").rstrip()[:200]
+
+            captures: dict[str, str] = {}
+            metavars = match.get("metaVariables") or match.get("metavariables") or {}
+            singles = metavars.get("single") or {}
+            for var_name, info in singles.items():
+                text = info.get("text") if isinstance(info, dict) else None
+                if isinstance(text, str):
+                    captures[var_name] = text
+            # ast-grep emits ``$$$X`` (variadic) bindings under ``multi``.
+            # Concatenate the segment texts so description templates that
+            # reference ``$X`` still resolve to something readable.
+            multis = metavars.get("multi") or {}
+            for var_name, segments in multis.items():
+                if var_name in captures or not isinstance(segments, list):
+                    continue
+                parts = [
+                    s.get("text", "") for s in segments
+                    if isinstance(s, dict) and isinstance(s.get("text"), str)
+                ]
+                if parts:
+                    captures[var_name] = "".join(parts)
+
+            description = rule.description
+            if captures:
+                description = interpolate_message(description, captures)
+
+            suggested_fix = ""
+            if rule.fix:
+                if rule.fix.uses_metavars and captures:
+                    sf_search, sf_replace = apply_metavar_fix(
+                        rule.fix.search, rule.fix.replace, captures,
+                    )
+                    suggested_fix = f"{sf_search} → {sf_replace}"
+                else:
+                    suggested_fix = f"{rule.fix.search} → {rule.fix.replace}"
+
+            rel_path = file_path
+            if project_dir:
+                try:
+                    rel_path = os.path.relpath(file_path, project_dir)
+                except ValueError:
+                    pass
+
+            findings.append(EnrichedFinding(
+                rule_id=rule.qualified_id,
+                rule_name=rule.name,
+                severity=rule.severity,
+                category=rule.category,
+                confidence=rule.confidence,
+                file=rel_path,
+                line=line_no,
+                code_snippet=snippet,
+                description=description,
+                explanation=rule.explanation,
+                recommendation=rule.recommendation,
+                examples=list(rule.examples),
+                suggested_fix=suggested_fix,
+                captures=captures,
+                cwe=rule.cwe,
+                pack=rule.pack,
+                tags=list(rule.tags),
+            ))
+    return findings
+
+
+def execute_rules(
+    files: list[str],
+    rules: list[UnifiedRule],
+    *,
+    project_dir: str = "",
+) -> list[EnrichedFinding]:
+    """Execute rules against a list of files.
+
+    Args:
+        files: Absolute file paths to scan.
+        rules: Rules to execute (should be pre-filtered to enabled only).
+        project_dir: Project root for relative path computation.
+
+    Returns:
+        List of EnrichedFinding (partially populated — use enricher for full context).
+    """
+    findings: list[EnrichedFinding] = []
+
+    # ----- Tier 2: structural rules via ast-grep --------------------------
+    structural_rules = [
+        r for r in rules
+        if r.tier == RuleTier.STRUCTURAL and r.structural_pattern
+    ]
+    if structural_rules:
+        if _ast_grep_available():
+            files_by_lang: dict[str, list[str]] = {}
+            for fp in files:
+                lang = detect_language(fp)
+                if lang:
+                    files_by_lang.setdefault(lang, []).append(fp)
+            for rule in structural_rules:
+                findings.extend(_execute_structural_rule(
+                    rule, files_by_lang, project_dir=project_dir,
+                ))
+        else:
+            logger.info(
+                "ast-grep not on PATH; skipping %d structural rule(s). "
+                "Install with: cargo install ast-grep || brew install ast-grep",
+                len(structural_rules),
+            )
+
+    # Pre-group rules by language for fast lookup. Tier-1 rules always run
+    # here; Tier-2 (structural) rules also run here as a regex fallback
+    # for languages where ast-grep can't help — either because the language
+    # has no ast-grep grammar, or because the binary is missing entirely.
+    # Roadmap Phase 1: "Fallback to regex Tier 1 for languages without
+    # ast-grep grammars".
+    ast_grep_ok = _ast_grep_available()
+    lang_rules: dict[str, list[UnifiedRule]] = {}
+    universal_rules: list[UnifiedRule] = []
+
+    for rule in rules:
+        if rule.pattern is None and rule.composite_pattern is None:
+            continue  # no executable pattern
+
+        regex_langs: list[str]
+        if rule.tier == RuleTier.REGEX:
+            regex_langs = list(rule.languages)
+            regex_universal = not rule.languages
+        elif rule.tier == RuleTier.STRUCTURAL:
+            # Only run the regex fallback when ast-grep can't cover the rule.
+            if ast_grep_ok:
+                if not rule.languages:
+                    continue  # universal structural — ast-grep handled it
+                regex_langs = [
+                    lang for lang in rule.languages
+                    if lang not in _AST_GREP_LANGS
+                ]
+                regex_universal = False
+                if not regex_langs:
+                    continue
+            else:
+                # ast-grep missing — fall back to regex everywhere the rule
+                # was meant to apply.
+                regex_langs = list(rule.languages)
+                regex_universal = not rule.languages
+        else:
+            continue  # plugin tier
+
+        if regex_universal:
+            universal_rules.append(rule)
+        else:
+            for lang in regex_langs:
+                lang_rules.setdefault(lang, []).append(rule)
+
+    for file_path in files:
+        lang = detect_language(file_path)
+        applicable = list(universal_rules)
+        if lang:
+            applicable.extend(lang_rules.get(lang, []))
+        if not applicable:
+            continue
+
+        try:
+            content = Path(file_path).read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+
+        lines = content.splitlines()
+        rel_path = file_path
+        if project_dir:
+            try:
+                rel_path = os.path.relpath(file_path, project_dir)
+            except ValueError:
+                pass
+
+        for i, line in enumerate(lines):
+            line_no = i + 1
+            is_comment = _is_comment_line(line, lang)
+
+            for rule in applicable:
+                if is_comment and not rule.scan_comments:
+                    continue
+
+                captures: dict[str, str] = {}
+
+                # --- Composite pattern path ---
+                if rule.composite_pattern is not None:
+                    ctx = MatchContext(
+                        line=line, line_no=line_no, all_lines=lines,
+                    )
+                    if not rule.composite_pattern.evaluate(ctx):
+                        continue
+                    captures = ctx.captures
+
+                # --- Simple pattern path ---
+                elif rule.pattern is not None:
+                    match = rule.pattern.search(line)
+                    if not match:
+                        continue
+                    if rule.metavars:
+                        captures = {
+                            k: v for k, v in match.groupdict().items()
+                            if v is not None
+                        }
+                        if not check_metavar_constraints(
+                            captures, rule.metavar_regex, rule.metavar_comparison,
+                        ):
+                            continue
+                else:
+                    continue
+
+                # Build description with metavar interpolation
+                description = rule.description
+                if captures:
+                    description = interpolate_message(description, captures)
+
+                # Build suggested fix
+                suggested_fix = ""
+                if rule.fix:
+                    if rule.fix.uses_metavars and captures:
+                        sf_search, sf_replace = apply_metavar_fix(
+                            rule.fix.search, rule.fix.replace, captures,
+                        )
+                        suggested_fix = f"{sf_search} \u2192 {sf_replace}"
+                    else:
+                        suggested_fix = f"{rule.fix.search} \u2192 {rule.fix.replace}"
+
+                findings.append(EnrichedFinding(
+                    rule_id=rule.qualified_id,
+                    rule_name=rule.name,
+                    severity=rule.severity,
+                    category=rule.category,
+                    confidence=rule.confidence,
+                    file=rel_path,
+                    line=line_no,
+                    # Provider redaction must see the complete literal before
+                    # imposing a payload budget (a truncated key may not match).
+                    code_snippet=line.rstrip(),
+                    description=description,
+                    explanation=rule.explanation,
+                    recommendation=rule.recommendation,
+                    examples=list(rule.examples),
+                    suggested_fix=suggested_fix,
+                    captures=dict(captures),
+                    cwe=rule.cwe,
+                    pack=rule.pack,
+                    tags=list(rule.tags),
+                ))
+
+    # Sort: severity first, then file, then line
+    _sev_order = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4}
+    findings.sort(key=lambda f: (_sev_order.get(f.severity, 9), f.file, f.line))
+    return findings

attocode_intel/rules/filters/__init__.py

@@ -0,0 +1,5 @@
+"""Deterministic pre-filter pipeline for findings.
+
+Reduces noise before the connected agent sees findings.
+No LLM needed — these are all mechanical, deterministic filters.
+"""

attocode_intel/rules/filters/pipeline.py

@@ -0,0 +1,151 @@
+"""Pre-filter pipeline — orchestrates deterministic finding filters.
+
+Reduces noise before the connected coding agent sees findings:
+1. Dedup — remove exact duplicates and merge overlapping findings
+2. Test file adjuster — lower severity for test/spec files
+3. Confidence threshold — drop findings below minimum confidence
+
+Deterministic by default. Setting ATTOCODE_FLAG_CONFIDENCE adds one model call
+per finding between stages 2 and 3, which second-guesses the constants.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+
+from attocode_intel.rules.model import EnrichedFinding, RuleSeverity
+
+logger = logging.getLogger(__name__)
+
+# Test file patterns
+_TEST_FILE_PATTERNS = [
+    re.compile(r"test[_/]"),
+    re.compile(r"_test\.\w+$"),
+    re.compile(r"\.test\.\w+$"),
+    re.compile(r"\.spec\.\w+$"),
+    re.compile(r"__tests__/"),
+    re.compile(r"tests?/"),
+    re.compile(r"conftest\.py$"),
+    re.compile(r"fixtures?/"),
+]
+
+
+def _is_test_file(file_path: str) -> bool:
+    normalized = file_path.replace(os.sep, "/")
+    return any(p.search(normalized) for p in _TEST_FILE_PATTERNS)
+
+
+# Severity demotion map for test files
+_SEVERITY_DEMOTION = {
+    RuleSeverity.CRITICAL: RuleSeverity.HIGH,
+    RuleSeverity.HIGH: RuleSeverity.MEDIUM,
+    RuleSeverity.MEDIUM: RuleSeverity.LOW,
+    RuleSeverity.LOW: RuleSeverity.INFO,
+}
+
+
+def dedup_findings(findings: list[EnrichedFinding]) -> list[EnrichedFinding]:
+    """Remove exact duplicates and merge overlapping findings.
+
+    Two findings overlap if they're on the same file:line with the same
+    category. Keep the one with higher confidence.
+    """
+    seen: dict[tuple[str, int, str], EnrichedFinding] = {}
+    for f in findings:
+        key = (f.file, f.line, str(f.category))
+        existing = seen.get(key)
+        if existing is None or f.confidence >= existing.confidence:
+            if existing is not None and existing.rule_id != f.rule_id:
+                f.related_findings = list(set(f.related_findings + [existing.rule_id]))
+            seen[key] = f
+        elif existing.rule_id != f.rule_id:
+            # f lost the confidence race — record its existence on the winner
+            existing.related_findings = list(set(existing.related_findings + [f.rule_id]))
+
+    return list(seen.values())
+
+
+def adjust_test_file_severity(
+    findings: list[EnrichedFinding],
+) -> list[EnrichedFinding]:
+    """Lower severity for findings in test files."""
+    for f in findings:
+        if _is_test_file(f.file):
+            new_sev = _SEVERITY_DEMOTION.get(f.severity)
+            if new_sev:
+                f.severity = new_sev
+                f.confidence *= 0.8  # reduce confidence for test files
+    return findings
+
+
+def filter_by_confidence(
+    findings: list[EnrichedFinding],
+    min_confidence: float = 0.5,
+) -> list[EnrichedFinding]:
+    """Drop findings below confidence threshold."""
+    return [f for f in findings if f.confidence >= min_confidence]
+
+
+def run_pipeline(
+    findings: list[EnrichedFinding],
+    *,
+    min_confidence: float = 0.5,
+    adjust_test_severity: bool = True,
+    project_dir: str = "",
+) -> list[EnrichedFinding]:
+    """Run the full deterministic pre-filter pipeline.
+
+    Pipeline stages:
+    1. Dedup overlapping findings
+    2. Adjust severity for test files
+    2b. Estimate p(true positive), if ATTOCODE_FLAG_CONFIDENCE is set
+    3. Apply confidence threshold
+
+    Args:
+        findings: Raw findings from executor.
+        min_confidence: Minimum confidence to keep (default 0.5).
+        adjust_test_severity: Whether to demote test file severity.
+
+    Returns:
+        Filtered findings ready for the agent.
+    """
+    original_count = len(findings)
+
+    # 1. Dedup
+    findings = dedup_findings(findings)
+    dedup_removed = original_count - len(findings)
+
+    # 2. Test file severity adjustment
+    if adjust_test_severity:
+        findings = adjust_test_file_severity(findings)
+
+    # 2b. Second-guess the constants (no-op unless ATTOCODE_FLAG_CONFIDENCE is set).
+    #     Before the threshold on purpose: in live mode the threshold gates on
+    #     the estimate rather than on the constant.
+    from attocode_intel import confidence
+    from attocode_intel.confidence.settings import workspace
+
+    with workspace(project_dir):
+        if confidence.settings.selected() != "off" and project_dir:
+            from attocode_intel.rules.enricher import enrich_findings
+
+            enrich_findings(findings, project_dir=project_dir)
+        confidence.score(findings, min_confidence=min_confidence)
+
+    # 3. Confidence threshold
+    findings = filter_by_confidence(findings, min_confidence)
+    conf_removed = original_count - dedup_removed - len(findings)
+
+    if dedup_removed > 0 or conf_removed > 0:
+        logger.info(
+            "Pre-filter: %d → %d findings (-%d dedup, -%d low-confidence)",
+            original_count, len(findings), dedup_removed, max(0, conf_removed),
+        )
+
+    # Re-sort after filtering
+    _sev_order = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4}
+    findings.sort(key=lambda f: (_sev_order.get(f.severity, 9), f.file, f.line))
+
+    return findings