attocode-code-intel 0.1.0__py3-none-any.whl

attocode_intel/rules/importers/semgrep.py

@@ -0,0 +1,382 @@
+"""Semgrep rule importer — converts Semgrep YAML rules to attocode format.
+
+Handles Semgrep's rule format including:
+- ``pattern``, ``patterns``, ``pattern-either``, ``pattern-not``
+- ``metavariable-regex`` and ``metavariable-comparison``
+- ``fix`` (autofix)
+- ``metadata`` (CWE, confidence, OWASP)
+
+Limitations:
+- Semgrep patterns that use ``$X`` metavariables in AST-aware mode
+  cannot be directly converted to regex. These are flagged as needing
+  the structural tier (ast-grep).
+- ``pattern-inside`` with full AST scope is approximated as line-based.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Semgrep severity -> attocode severity
+_SEVERITY_MAP = {
+    "ERROR": "high",
+    "WARNING": "medium",
+    "INFO": "low",
+    "INVENTORY": "info",
+}
+
+# Semgrep category hints -> attocode category
+_CATEGORY_MAP = {
+    "security": "security",
+    "correctness": "correctness",
+    "best-practice": "style",
+    "performance": "performance",
+    "maintainability": "complexity",
+}
+
+
+def _has_ast_metavars(pattern: str) -> bool:
+    """Check if a Semgrep pattern uses AST-aware metavariables.
+
+    Semgrep metavars like ``$X`` in AST patterns work differently from
+    regex patterns — they match entire AST nodes. We can only convert
+    simple patterns where ``$X`` maps to ``\\w+`` or similar.
+    """
+    # Patterns with ellipsis operators (... in Semgrep) are AST-aware
+    if "..." in pattern and "$" in pattern:
+        return True
+    # Patterns with type constraints or deep patterns
+    if "<..." in pattern or "...>" in pattern:
+        return True
+    return False
+
+
+def _extract_cwe(metadata: dict) -> str:
+    """Extract CWE ID from Semgrep metadata."""
+    # Semgrep stores CWE in various places
+    cwe = metadata.get("cwe", "")
+    if isinstance(cwe, list):
+        cwe = cwe[0] if cwe else ""
+    cwe = str(cwe)
+
+    # Normalize to "CWE-NNN" format
+    m = re.search(r"CWE-?(\d+)", cwe, re.IGNORECASE)
+    if m:
+        return f"CWE-{m.group(1)}"
+    return ""
+
+
+def _extract_confidence(metadata: dict) -> float:
+    """Extract confidence from Semgrep metadata."""
+    conf = metadata.get("confidence", "")
+    if isinstance(conf, (int, float)):
+        return float(conf) if 0 <= conf <= 1 else conf / 100
+    conf_str = str(conf).lower()
+    if conf_str == "high":
+        return 0.9
+    elif conf_str == "medium":
+        return 0.7
+    elif conf_str == "low":
+        return 0.5
+    return 0.8
+
+
+def convert_semgrep_rule(rule: dict[str, Any]) -> dict[str, Any] | None:
+    """Convert a single Semgrep rule to attocode YAML format.
+
+    Args:
+        rule: A single rule dict from Semgrep YAML.
+
+    Returns:
+        Attocode-format rule dict, or None if unconvertible.
+    """
+    rule_id = rule.get("id", "")
+    if not rule_id:
+        return None
+
+    metadata = rule.get("metadata", {})
+    severity = _SEVERITY_MAP.get(rule.get("severity", "WARNING"), "medium")
+
+    # Determine category
+    category = "security"  # default for Semgrep rules
+    for tag in metadata.get("category", []) if isinstance(metadata.get("category"), list) else [str(metadata.get("category", ""))]:
+        tag_lower = tag.lower()
+        if tag_lower in _CATEGORY_MAP:
+            category = _CATEGORY_MAP[tag_lower]
+            break
+
+    # Extract pattern(s)
+    attocode_rule: dict[str, Any] = {
+        "id": rule_id,
+        "name": rule_id.replace("-", " ").replace("_", " ").title()[:60],
+        "message": rule.get("message", ""),
+        "severity": severity,
+        "category": category,
+        "confidence": _extract_confidence(metadata),
+    }
+
+    # CWE
+    cwe = _extract_cwe(metadata)
+    if cwe:
+        attocode_rule["cwe"] = cwe
+
+    # Languages
+    languages = rule.get("languages", [])
+    if languages:
+        attocode_rule["languages"] = languages
+
+    # Tags
+    tags = metadata.get("tags", [])
+    if isinstance(tags, list):
+        attocode_rule["tags"] = tags
+
+    # Handle pattern types
+    needs_structural = False
+
+    if "pattern" in rule:
+        pattern = str(rule["pattern"])
+        if _has_ast_metavars(pattern):
+            needs_structural = True
+            attocode_rule["structural_pattern"] = pattern
+            # Still provide regex approximation
+            regex_approx = _semgrep_to_regex(pattern)
+            if regex_approx:
+                attocode_rule["pattern"] = regex_approx
+        else:
+            regex = _semgrep_to_regex(pattern)
+            if regex:
+                attocode_rule["pattern"] = regex
+            else:
+                return None  # can't convert
+
+    elif "patterns" in rule:
+        # Semgrep composite patterns -> attocode patterns list
+        patterns_list = _convert_patterns(rule["patterns"])
+        if patterns_list:
+            attocode_rule["patterns"] = patterns_list
+        else:
+            return None
+
+    elif "pattern-either" in rule:
+        either = rule["pattern-either"]
+        converted = []
+        for p in either:
+            if isinstance(p, str):
+                regex = _semgrep_to_regex(p)
+                if regex:
+                    converted.append(regex)
+            elif isinstance(p, dict) and "pattern" in p:
+                regex = _semgrep_to_regex(str(p["pattern"]))
+                if regex:
+                    converted.append(regex)
+        if converted:
+            attocode_rule["pattern-either"] = converted
+        else:
+            return None
+
+    else:
+        return None  # no pattern to convert
+
+    # Metavariable constraints — convert Semgrep format to attocode format
+    # Semgrep: {"metavariable": "$VAR", "regex": "..."} or {"metavariable": "$VAR", "comparison": "..."}
+    # Attocode: {"VAR": "..."} (key is var name without $)
+    for item in rule.get("patterns", []):
+        if isinstance(item, dict):
+            if "metavariable-regex" in item:
+                mv = item["metavariable-regex"]
+                if isinstance(mv, dict) and "metavariable" in mv and "regex" in mv:
+                    var = str(mv["metavariable"]).lstrip("$")
+                    attocode_rule.setdefault("metavariable-regex", {})[var] = str(mv["regex"])
+                elif isinstance(mv, dict):
+                    # Already in attocode format (key: regex pairs)
+                    attocode_rule["metavariable-regex"] = mv
+            if "metavariable-comparison" in item:
+                mv = item["metavariable-comparison"]
+                if isinstance(mv, dict) and "metavariable" in mv and "comparison" in mv:
+                    var = str(mv["metavariable"]).lstrip("$")
+                    attocode_rule.setdefault("metavariable-comparison", {})[var] = str(mv["comparison"])
+                elif isinstance(mv, dict):
+                    attocode_rule["metavariable-comparison"] = mv
+
+    # Fix
+    fix = rule.get("fix", "")
+    if fix:
+        attocode_rule["fix"] = {
+            "search": rule.get("pattern", ""),
+            "replace": fix,
+        }
+
+    # Generate explanation stub from message
+    message = rule.get("message", "")
+    if message and "explanation" not in attocode_rule:
+        attocode_rule["explanation"] = message
+    if message and "recommendation" not in attocode_rule:
+        attocode_rule["recommendation"] = f"Address: {message[:100]}"
+
+    # Mark as needing structural tier
+    if needs_structural:
+        attocode_rule.setdefault("tags", []).append("needs-structural-tier")
+
+    return attocode_rule
+
+
+def _semgrep_to_regex(pattern: str) -> str | None:
+    """Convert a Semgrep pattern to a regex pattern.
+
+    Simple Semgrep patterns (without deep AST matching) can often
+    be converted to regex with metavariable support.
+    """
+    if not pattern.strip():
+        return None
+
+    # Remove leading/trailing whitespace and ellipsis
+    p = pattern.strip()
+
+    # Replace Semgrep ellipsis with regex wildcard
+    p = p.replace("...", ".*?")
+
+    # Escape regex special chars EXCEPT those in metavar patterns
+    # We do a targeted escape: escape chars that are regex-special
+    # but preserve $VAR metavariables
+    result = []
+    i = 0
+    while i < len(p):
+        if p[i] == "$" and i + 1 < len(p) and (p[i + 1].isalpha() or p[i + 1] == "_"):
+            # Keep $VAR as-is (metavar)
+            j = i + 1
+            while j < len(p) and (p[j].isalnum() or p[j] == "_"):
+                j += 1
+            result.append(p[i:j])
+            i = j
+        elif p[i] in r"\.+*?[]{}()^|":
+            # Escape regex special chars
+            if p[i] == "." and i + 2 < len(p) and p[i:i+3] == ".*?":
+                result.append(".*?")
+                i += 3
+            else:
+                result.append("\\" + p[i])
+                i += 1
+        else:
+            result.append(p[i])
+            i += 1
+
+    regex = "".join(result)
+
+    # Verify it compiles
+    try:
+        re.compile(regex)
+        return regex
+    except re.error:
+        return None
+
+
+def _convert_patterns(patterns: list) -> list[dict[str, str]] | None:
+    """Convert Semgrep patterns list to attocode patterns list."""
+    result: list[dict[str, str]] = []
+
+    for item in patterns:
+        if not isinstance(item, dict):
+            continue
+
+        if "pattern" in item:
+            regex = _semgrep_to_regex(str(item["pattern"]))
+            if regex:
+                result.append({"pattern": regex})
+
+        elif "pattern-not" in item:
+            regex = _semgrep_to_regex(str(item["pattern-not"]))
+            if regex:
+                result.append({"pattern-not": regex})
+
+        elif "pattern-inside" in item:
+            regex = _semgrep_to_regex(str(item["pattern-inside"]))
+            if regex:
+                result.append({"pattern-inside": regex})
+
+        elif "pattern-not-inside" in item:
+            regex = _semgrep_to_regex(str(item["pattern-not-inside"]))
+            if regex:
+                result.append({"pattern-not-inside": regex})
+
+        elif "pattern-either" in item:
+            either_list = item["pattern-either"]
+            converted = []
+            for p in either_list:
+                if isinstance(p, str):
+                    regex = _semgrep_to_regex(p)
+                    if regex:
+                        converted.append(regex)
+                elif isinstance(p, dict) and "pattern" in p:
+                    regex = _semgrep_to_regex(str(p["pattern"]))
+                    if regex:
+                        converted.append(regex)
+            if converted:
+                result.append({"pattern-either": converted})
+
+    return result if result else None
+
+
+# ---------------------------------------------------------------------------
+# Batch conversion
+# ---------------------------------------------------------------------------
+
+
+def convert_semgrep_file(yaml_content: str) -> tuple[list[dict], list[str]]:
+    """Convert a Semgrep YAML file (possibly with multiple rules) to attocode format.
+
+    Args:
+        yaml_content: Raw YAML content from a Semgrep rule file.
+
+    Returns:
+        (converted_rules, errors) — list of attocode rule dicts and error messages.
+    """
+    try:
+        import yaml  # type: ignore[import-untyped]
+        data = yaml.safe_load(yaml_content)
+    except Exception as exc:
+        return [], [f"Failed to parse YAML: {exc}"]
+
+    if not isinstance(data, dict) or "rules" not in data:
+        return [], ["Expected Semgrep format with top-level 'rules' key"]
+
+    converted: list[dict] = []
+    errors: list[str] = []
+
+    for i, rule in enumerate(data["rules"]):
+        if not isinstance(rule, dict):
+            errors.append(f"Rule at index {i}: expected dict")
+            continue
+        result = convert_semgrep_rule(rule)
+        if result:
+            converted.append(result)
+        else:
+            errors.append(f"Rule '{rule.get('id', f'index-{i}')}': could not convert")
+
+    return converted, errors
+
+
+def convert_semgrep_to_yaml(yaml_content: str) -> str:
+    """Convert Semgrep YAML to attocode YAML string.
+
+    Convenience function for the MCP tool.
+    """
+    converted, errors = convert_semgrep_file(yaml_content)
+
+    try:
+        import yaml  # type: ignore[import-untyped]
+    except ImportError:
+        return "Error: PyYAML not installed"
+
+    parts: list[str] = []
+    if converted:
+        parts.append(yaml.dump(converted, default_flow_style=False, sort_keys=False))
+    if errors:
+        parts.append("\n# Conversion errors:")
+        for e in errors:
+            parts.append(f"#   {e}")
+
+    return "\n".join(parts) if parts else "No rules converted."

attocode_intel/rules/loader.py

@@ -0,0 +1,292 @@
+"""Rule loader — loads rules from builtins, YAML files, and packs."""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+
+from attocode_intel.rules.metavar import compile_metavar_pattern, has_metavars
+from attocode_intel.rules.model import (
+    AutoFix,
+    FewShotExample,
+    RuleCategory,
+    RuleSeverity,
+    RuleSource,
+    RuleTier,
+    UnifiedRule,
+    from_bug_pattern,
+    from_security_pattern,
+)
+
+logger = logging.getLogger(__name__)
+
+# Category mapping for YAML rules
+_YAML_CATEGORY_MAP: dict[str, RuleCategory] = {
+    "correctness": RuleCategory.CORRECTNESS,
+    "suspicious": RuleCategory.SUSPICIOUS,
+    "complexity": RuleCategory.COMPLEXITY,
+    "performance": RuleCategory.PERFORMANCE,
+    "style": RuleCategory.STYLE,
+    "security": RuleCategory.SECURITY,
+    "deprecated": RuleCategory.DEPRECATED,
+    # Legacy compat (must match from_bug_pattern and from_security_pattern mappings)
+    "anti_pattern": RuleCategory.SUSPICIOUS,
+    "secret": RuleCategory.SECURITY,
+    "logic_error": RuleCategory.CORRECTNESS,
+    "error_handling": RuleCategory.CORRECTNESS,
+    "edge_case": RuleCategory.SUSPICIOUS,
+    "concurrency": RuleCategory.SUSPICIOUS,
+    "resource_leak": RuleCategory.CORRECTNESS,
+    "type_safety": RuleCategory.CORRECTNESS,
+}
+
+_VALID_SEVERITIES = frozenset(s.value for s in RuleSeverity)
+
+
+def load_builtin_rules() -> list[UnifiedRule]:
+    """Load builtin rules from bug_finder and security patterns."""
+    rules: list[UnifiedRule] = []
+
+    # 1. Bug finder patterns
+    try:
+        from attocode_intel.bug_finder import _PATTERNS
+
+        for regex, cat, sev, desc, conf in _PATTERNS:
+            try:
+                rule = from_bug_pattern(regex, cat.value, sev.value, desc, conf)
+                rules.append(rule)
+            except Exception as exc:
+                logger.warning("Failed to adapt builtin bug pattern '%s': %s", desc[:40], exc)
+    except ImportError:
+        logger.debug("bug_finder not available, skipping builtin bug rules")
+
+    # 2. Security patterns (secrets + anti-patterns)
+    try:
+        from attocode_intel._internal.integrations.security.patterns import (
+            ANTI_PATTERNS,
+            SECRET_PATTERNS,
+        )
+
+        for pat in SECRET_PATTERNS:
+            try:
+                rules.append(from_security_pattern(pat))
+            except Exception as exc:
+                logger.warning("Failed to adapt security pattern '%s': %s", getattr(pat, "name", "?"), exc)
+        for pat in ANTI_PATTERNS:
+            try:
+                rules.append(from_security_pattern(pat))
+            except Exception as exc:
+                logger.warning("Failed to adapt security pattern '%s': %s", getattr(pat, "name", "?"), exc)
+    except ImportError:
+        logger.debug("security patterns not available, skipping builtin security rules")
+
+    logger.info("Loaded %d builtin rules", len(rules))
+    return rules
+
+
+def load_yaml_rules(
+    rules_dir: str | Path,
+    *,
+    source: RuleSource = RuleSource.USER,
+    pack: str = "",
+) -> list[UnifiedRule]:
+    """Load rules from YAML files in a directory.
+
+    Supports the existing .attocode/rules/*.yaml format plus the
+    expanded format with explanation and examples fields.
+
+    Args:
+        rules_dir: Directory containing \\*.yaml rule files.
+        source: RuleSource tag for loaded rules.
+        pack: Pack name to associate with loaded rules.
+
+    Returns:
+        List of parsed UnifiedRule instances. Invalid rules are skipped.
+    """
+    rules_path = Path(rules_dir)
+    if not rules_path.is_dir():
+        return []
+
+    try:
+        import yaml  # type: ignore[import-untyped]
+    except ImportError:
+        logger.debug("PyYAML not installed — skipping YAML rules from %s", rules_dir)
+        return []
+
+    rules: list[UnifiedRule] = []
+    for yaml_file in sorted(rules_path.glob("*.yaml")):
+        try:
+            content = yaml_file.read_text(encoding="utf-8")
+            data = yaml.safe_load(content)
+            if data is None:
+                continue
+            items = [data] if isinstance(data, dict) else data if isinstance(data, list) else []
+            for i, item in enumerate(items):
+                try:
+                    rule = _parse_yaml_rule(
+                        item, source=source, pack=pack, origin=f"{yaml_file.name}[{i}]",
+                    )
+                    if rule is not None:
+                        rules.append(rule)
+                except Exception as exc:
+                    logger.warning("Invalid rule in %s at index %d: %s", yaml_file.name, i, exc)
+        except Exception as exc:
+            logger.warning("Failed to load rules from %s: %s", yaml_file, exc)
+
+    if rules:
+        logger.info("Loaded %d YAML rules from %s", len(rules), rules_dir)
+    return rules
+
+
+def _parse_yaml_rule(
+    data: dict,
+    *,
+    source: RuleSource = RuleSource.USER,
+    pack: str = "",
+    origin: str = "",
+) -> UnifiedRule | None:
+    """Parse a single YAML rule dict into a UnifiedRule."""
+    # ``pattern`` is required unless one of: ``patterns``/``pattern-either``
+    # provides a composite, OR ``structural_pattern`` provides a Tier-2
+    # ast-grep matcher (message-only structural rules are valid).
+    has_composite = "patterns" in data or "pattern-either" in data
+    has_structural = bool(data.get("structural_pattern"))
+    # Allow ``description`` as a synonym for ``message`` so structural-only
+    # rules don't have to repeat themselves.
+    if "message" not in data and "description" in data:
+        data = {**data, "message": data["description"]}
+    required = ["id", "message", "severity"]
+    if not (has_composite or has_structural):
+        required.append("pattern")
+    missing = [f for f in required if f not in data]
+    if missing:
+        logger.warning("Rule %s missing required fields: %s", origin, ", ".join(missing))
+        return None
+
+    sev_str = str(data["severity"]).lower()
+    if sev_str not in _VALID_SEVERITIES:
+        logger.warning("Rule %s has invalid severity '%s'", origin, data["severity"])
+        return None
+
+    # Category (default to security for backward compat)
+    cat_str = str(data.get("category", "security")).lower()
+    category = _YAML_CATEGORY_MAP.get(cat_str, RuleCategory.SECURITY)
+
+    # Build composite pattern if "patterns" or "pattern-either" is present
+    composite_pattern = None
+    if has_composite:
+        from attocode_intel.rules.combinators import build_composite_from_yaml
+        if "patterns" in data:
+            composite_src = data["patterns"]  # list of dicts
+        else:
+            # Top-level pattern-either: wrap as dict for build_composite_from_yaml
+            composite_src = {"pattern-either": data["pattern-either"]}
+        composite_pattern = build_composite_from_yaml(composite_src)
+        if composite_pattern is None:
+            logger.warning("Rule %s has invalid composite patterns", origin)
+            return None
+
+    # Compile primary pattern (may be absent for composite-only rules)
+    compiled: re.Pattern[str] | None = None
+    metavar_names: list[str] = []
+    if "pattern" in data:
+        pattern_str = str(data["pattern"])
+        try:
+            if has_metavars(pattern_str):
+                compiled, metavar_names = compile_metavar_pattern(pattern_str)
+            else:
+                compiled = re.compile(pattern_str)
+        except re.error as exc:
+            logger.warning("Rule %s has invalid regex: %s", origin, exc)
+            return None
+
+    tier = RuleTier.STRUCTURAL if data.get("structural_pattern") else RuleTier.REGEX
+
+    # ast-grep's inline-rule YAML accepts a ``pattern`` object with either
+    # a bare string OR ``{context, selector}``. Specifying ``context``
+    # without ``selector`` produces an invalid rule that ast-grep rejects
+    # at parse time, which silently no-ops the rule. Catch it at load.
+    if data.get("structural_context") and not data.get("structural_selector"):
+        logger.warning(
+            "Rule %s sets structural_context without structural_selector; "
+            "ast-grep needs both — skipping this rule",
+            origin,
+        )
+        return None
+
+    # Parse metavariable constraints
+    metavar_regex: dict[str, str] = {}
+    raw_mv_regex = data.get("metavariable-regex") or data.get("metavariable_regex")
+    if isinstance(raw_mv_regex, dict):
+        metavar_regex = {str(k): str(v) for k, v in raw_mv_regex.items()}
+
+    metavar_comparison: dict[str, str] = {}
+    raw_mv_cmp = data.get("metavariable-comparison") or data.get("metavariable_comparison")
+    if isinstance(raw_mv_cmp, dict):
+        metavar_comparison = {str(k): str(v) for k, v in raw_mv_cmp.items()}
+
+    fix = None
+    fix_data = data.get("fix")
+    if isinstance(fix_data, dict) and "search" in fix_data and "replace" in fix_data:
+        fix_search = str(fix_data["search"])
+        fix_replace = str(fix_data["replace"])
+        uses_metavars = has_metavars(fix_search) or has_metavars(fix_replace)
+        fix = AutoFix(search=fix_search, replace=fix_replace, uses_metavars=uses_metavars)
+
+    examples: list[FewShotExample] = []
+    for ex in data.get("examples", []):
+        if isinstance(ex, dict) and "bad" in ex and "good" in ex:
+            examples.append(FewShotExample(
+                bad_code=str(ex["bad"]),
+                good_code=str(ex["good"]),
+                explanation=str(ex.get("explanation", "")),
+            ))
+
+    # Normalize languages and tags to lists (YAML allows string shorthand)
+    raw_langs = data.get("languages", [])
+    languages = [raw_langs] if isinstance(raw_langs, str) else list(raw_langs)
+    raw_tags = data.get("tags", [])
+    tags = [raw_tags] if isinstance(raw_tags, str) else list(raw_tags)
+
+    # External references (CWE links, blog posts, etc.) — preserved on import
+    raw_refs = data.get("references", [])
+    references = [raw_refs] if isinstance(raw_refs, str) else list(raw_refs)
+
+    return UnifiedRule(
+        id=str(data["id"]),
+        name=str(data.get("name", data["id"])),
+        description=str(data["message"]),
+        severity=RuleSeverity(sev_str),
+        category=category,
+        languages=languages,
+        pattern=compiled,
+        structural_pattern=str(data.get("structural_pattern", "")),
+        structural_context=str(data.get("structural_context", "")),
+        structural_selector=str(data.get("structural_selector", "")),
+        cwe=str(data.get("cwe", "")),
+        tags=tags,
+        source=source,
+        tier=tier,
+        confidence=float(data.get("confidence", 0.8)),
+        fix=fix,
+        enabled=bool(data.get("enabled", True)),
+        pack=pack,
+        explanation=str(data.get("explanation", "")),
+        examples=examples,
+        recommendation=str(data.get("recommendation", "")),
+        scan_comments=bool(data.get("scan_comments", False)),
+        metavars=metavar_names,
+        metavar_regex=metavar_regex,
+        metavar_comparison=metavar_comparison,
+        composite_pattern=composite_pattern,
+        references=references,
+    )
+
+
+def load_user_rules(project_dir: str) -> list[UnifiedRule]:
+    """Load user-defined rules from .attocode/rules/*.yaml."""
+    return load_yaml_rules(
+        Path(project_dir) / ".attocode" / "rules",
+        source=RuleSource.USER,
+    )