empathy-framework 4.7.0__py3-none-any.whl → 4.8.0__py3-none-any.whl

empathy_os/workflows/security_audit_phase3.py

@@ -0,0 +1,328 @@
+"""Phase 3 Scanner Improvements - AST-based Command Injection Detection
+
+This module provides AST-based analysis for detecting actual eval/exec usage
+vs mentions in comments, docstrings, and documentation.
+
+Created: 2026-01-26
+Related: docs/SECURITY_PHASE2_COMPLETE.md
+"""
+
+import ast
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class EvalExecDetector(ast.NodeVisitor):
+    """AST visitor that detects actual eval() and exec() calls.
+
+    This visitor walks the AST to find real function calls to eval() and exec(),
+    distinguishing them from:
+    - String literals mentioning eval/exec
+    - Comments mentioning eval/exec
+    - Docstrings documenting security policies
+    - Detection code checking for eval/exec patterns
+    """
+
+    def __init__(self, file_path: str):
+        """Initialize detector.
+
+        Args:
+            file_path: Path to file being analyzed (for context)
+        """
+        self.file_path = file_path
+        self.findings: list[dict[str, Any]] = []
+        self._in_docstring = False
+        self._current_function = None
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+        """Visit function definition to track context."""
+        self._current_function = node.name
+        self.generic_visit(node)
+        self._current_function = None
+
+    def visit_Call(self, node: ast.Call) -> None:
+        """Visit function call nodes to detect eval/exec."""
+        # Check if this is a call to eval() or exec()
+        func_name = None
+
+        if isinstance(node.func, ast.Name):
+            func_name = node.func.id
+        elif isinstance(node.func, ast.Attribute):
+            # Handle attribute access like obj.exec()
+            func_name = node.func.attr
+
+        if func_name in ("eval", "exec"):
+            # Found a real eval/exec call!
+            self.findings.append({
+                "type": "command_injection",
+                "function": func_name,
+                "line": node.lineno,
+                "col": node.col_offset,
+                "context": self._current_function,
+            })
+
+        self.generic_visit(node)
+
+
+def analyze_file_for_eval_exec(file_path: str | Path) -> list[dict[str, Any]]:
+    """Analyze a Python file for actual eval/exec usage using AST.
+
+    Args:
+        file_path: Path to Python file to analyze
+
+    Returns:
+        List of findings (actual eval/exec calls)
+
+    Example:
+        >>> findings = analyze_file_for_eval_exec("myfile.py")
+        >>> for finding in findings:
+        ...     print(f"{finding['function']} at line {finding['line']}")
+    """
+    file_path = Path(file_path)
+
+    if not file_path.exists():
+        return []
+
+    try:
+        content = file_path.read_text(encoding="utf-8", errors="ignore")
+        tree = ast.parse(content, filename=str(file_path))
+
+        detector = EvalExecDetector(str(file_path))
+        detector.visit(tree)
+
+        return detector.findings
+
+    except SyntaxError as e:
+        logger.debug(f"Syntax error parsing {file_path}: {e}")
+        return []
+    except Exception as e:
+        logger.debug(f"Error analyzing {file_path}: {e}")
+        return []
+
+
+def is_scanner_implementation_file(file_path: str) -> bool:
+    """Check if file is part of security scanner implementation.
+
+    Scanner files legitimately contain eval/exec patterns for detection
+    purposes and should not be flagged.
+
+    Args:
+        file_path: Path to check
+
+    Returns:
+        True if this is a scanner implementation file
+    """
+    scanner_indicators = [
+        # Scanner implementation files
+        "bug_predict",
+        "security_audit",
+        "security_scan",
+        "vulnerability_scan",
+        "owasp",
+        "secrets_detector",
+        "pii_scrubber",
+
+        # Pattern/rule definition files
+        "patterns.py",
+        "rules.py",
+        "checks.py",
+
+        # Test files for security scanners
+        "test_bug_predict",
+        "test_security",
+        "test_scanner",
+    ]
+
+    path_lower = file_path.lower()
+    return any(indicator in path_lower for indicator in scanner_indicators)
+
+
+def is_in_docstring_or_comment(line_content: str, file_content: str, line_num: int) -> bool:
+    """Enhanced check if line is in docstring or comment.
+
+    Phase 3 Enhancement: More robust detection of documentation context.
+
+    Args:
+        line_content: The line to check
+        file_content: Full file content
+        line_num: Line number (1-indexed)
+
+    Returns:
+        True if line is in docstring or comment
+    """
+    line = line_content.strip()
+
+    # Check for comment lines
+    if line.startswith("#"):
+        return True
+
+    # Check for inline comments
+    if "#" in line_content and line_content.index("#") < line_content.find("eval") if "eval" in line_content else True:
+        return True
+
+    # Parse file as AST to find docstrings
+    try:
+        tree = ast.parse(file_content)
+
+        # Get all docstrings
+        docstrings = []
+        for node in ast.walk(tree):
+            docstring = ast.get_docstring(node)
+            if docstring:
+                docstrings.append(docstring)
+
+        # Check if any docstring contains this line content
+        for docstring in docstrings:
+            if line_content.strip() in docstring:
+                return True
+
+    except SyntaxError:
+        pass
+
+    # Check for security policy patterns
+    security_patterns = [
+        "no eval",
+        "no exec",
+        "never use eval",
+        "never use exec",
+        "avoid eval",
+        "avoid exec",
+        "security:",
+        "- no eval",
+        "- no exec",
+    ]
+
+    line_lower = line.lower()
+    if any(pattern in line_lower for pattern in security_patterns):
+        return True
+
+    return False
+
+
+def enhanced_command_injection_detection(
+    file_path: str,
+    original_findings: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """Enhanced command injection detection with AST-based filtering.
+
+    Phase 3: Uses AST to distinguish actual eval/exec calls from mentions
+    in documentation, comments, and scanner implementation.
+
+    Args:
+        file_path: Path to file being analyzed
+        original_findings: Findings from regex-based detection
+
+    Returns:
+        Filtered list of actual vulnerabilities (not false positives)
+    """
+    # Step 1: Check if this is a scanner implementation file
+    if is_scanner_implementation_file(file_path):
+        return []  # Scanner files are allowed to mention eval/exec
+
+    # Step 2: For Python files, use AST-based detection
+    if file_path.endswith(".py"):
+        try:
+            ast_findings = analyze_file_for_eval_exec(file_path)
+
+            # Convert AST findings to format compatible with original
+            filtered = []
+            for finding in ast_findings:
+                filtered.append({
+                    "type": "command_injection",
+                    "file": file_path,
+                    "line": finding["line"],
+                    "match": f"{finding['function']}(",
+                    "severity": "critical",
+                    "owasp": "A03:2021 Injection",
+                    "context": finding.get("context", ""),
+                })
+
+            return filtered
+
+        except Exception as e:
+            logger.debug(f"AST analysis failed for {file_path}, falling back to regex: {e}")
+            # Fall back to original findings if AST fails
+            pass
+
+    # Step 3: For non-Python files or if AST fails, filter original findings
+    try:
+        file_content = Path(file_path).read_text(encoding="utf-8", errors="ignore")
+
+        filtered = []
+        for finding in original_findings:
+            line_num = finding.get("line", 0)
+            lines = file_content.split("\n")
+
+            if 0 < line_num <= len(lines):
+                line_content = lines[line_num - 1]
+
+                # Skip if in docstring or comment
+                if is_in_docstring_or_comment(line_content, file_content, line_num):
+                    continue
+
+                filtered.append(finding)
+
+        return filtered
+
+    except Exception as e:
+        logger.debug(f"Enhanced filtering failed for {file_path}: {e}")
+        return original_findings
+
+
+# =============================================================================
+# Integration with SecurityAuditWorkflow
+# =============================================================================
+
+
+def apply_phase3_filtering(findings: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Apply Phase 3 AST-based filtering to command injection findings.
+
+    This is the main entry point for Phase 3 improvements.
+
+    Args:
+        findings: List of command injection findings from regex-based detection
+                 (should only contain command_injection type)
+
+    Returns:
+        Filtered list with false positives removed
+    """
+    if not findings:
+        return []
+
+    # Group findings by file
+    by_file: dict[str, list[dict[str, Any]]] = {}
+    for finding in findings:
+        file_path = finding.get("file", "")
+        if file_path not in by_file:
+            by_file[file_path] = []
+        by_file[file_path].append(finding)
+
+    # Apply enhanced detection per file
+    filtered_findings = []
+    for file_path, file_findings in by_file.items():
+        enhanced = enhanced_command_injection_detection(file_path, file_findings)
+        filtered_findings.extend(enhanced)
+
+    return filtered_findings
+
+
+if __name__ == "__main__":
+    # Test on known files
+    test_files = [
+        "src/empathy_os/workflows/bug_predict.py",
+        "src/empathy_os/orchestration/execution_strategies.py",
+        "tests/test_bug_predict_workflow.py",
+    ]
+
+    for file in test_files:
+        if Path(file).exists():
+            findings = analyze_file_for_eval_exec(file)
+            print(f"\n{file}:")
+            print(f"  Actual eval/exec calls: {len(findings)}")
+            for f in findings:
+                print(f"    Line {f['line']}: {f['function']}() in {f.get('context', 'module')}")
+        else:
+            print(f"\n{file}: Not found")

empathy_os/workflows/telemetry_mixin.py

@@ -0,0 +1,269 @@
+"""Telemetry Mixin for Workflow LLM Call Tracking
+
+Extracted from BaseWorkflow to improve maintainability and reusability.
+Provides telemetry tracking for LLM calls and workflow executions.
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from empathy_os.models import (
+        TelemetryBackend,
+    )
+
+logger = logging.getLogger(__name__)
+
+# Try to import UsageTracker
+try:
+    from empathy_os.telemetry import UsageTracker
+
+    TELEMETRY_AVAILABLE = True
+except ImportError:
+    TELEMETRY_AVAILABLE = False
+    UsageTracker = None  # type: ignore
+
+
+class TelemetryMixin:
+    """Mixin that provides telemetry tracking for workflow LLM calls.
+
+    This mixin extracts telemetry logic from BaseWorkflow to improve
+    maintainability and enable reuse in other contexts.
+
+    Attributes:
+        _telemetry_backend: Backend for storing telemetry records
+        _telemetry_tracker: UsageTracker singleton for tracking
+        _enable_telemetry: Whether telemetry is enabled
+        _run_id: Current workflow run ID for correlation
+
+    Usage:
+        class MyWorkflow(TelemetryMixin, BaseWorkflow):
+            pass
+
+        # TelemetryMixin methods are now available
+        workflow._track_telemetry(...)
+        workflow._emit_call_telemetry(...)
+        workflow._emit_workflow_telemetry(...)
+    """
+
+    # Instance variables (set by __init__ or subclass)
+    _telemetry_backend: TelemetryBackend | None = None
+    _telemetry_tracker: UsageTracker | None = None
+    _enable_telemetry: bool = True
+    _run_id: str | None = None
+
+    # These must be provided by the class using this mixin
+    name: str = "unknown"
+    _provider_str: str = "unknown"
+
+    def _init_telemetry(self, telemetry_backend: TelemetryBackend | None = None) -> None:
+        """Initialize telemetry tracking.
+
+        Call this from __init__ to set up telemetry.
+
+        Args:
+            telemetry_backend: Optional backend for storing telemetry records.
+                             Defaults to TelemetryStore (JSONL file backend).
+        """
+        from empathy_os.models import get_telemetry_store
+
+        self._telemetry_backend = telemetry_backend or get_telemetry_store()
+        self._telemetry_tracker = None
+        self._enable_telemetry = True
+
+        if TELEMETRY_AVAILABLE and UsageTracker is not None:
+            try:
+                self._telemetry_tracker = UsageTracker.get_instance()
+            except (OSError, PermissionError) as e:
+                # File system errors - log but disable telemetry
+                logger.debug(f"Failed to initialize telemetry tracker (file system error): {e}")
+                self._enable_telemetry = False
+            except (AttributeError, TypeError, ValueError) as e:
+                # Configuration or initialization errors
+                logger.debug(f"Failed to initialize telemetry tracker (config error): {e}")
+                self._enable_telemetry = False
+
+    def _track_telemetry(
+        self,
+        stage: str,
+        tier: Any,  # ModelTier
+        model: str,
+        cost: float,
+        tokens: dict[str, int],
+        cache_hit: bool,
+        cache_type: str | None,
+        duration_ms: int,
+    ) -> None:
+        """Track telemetry for an LLM call.
+
+        Args:
+            stage: Stage name
+            tier: Model tier used (ModelTier enum)
+            model: Model ID used
+            cost: Cost in USD
+            tokens: Dictionary with "input" and "output" token counts
+            cache_hit: Whether this was a cache hit
+            cache_type: Cache type if cache hit
+            duration_ms: Duration in milliseconds
+        """
+        if not self._enable_telemetry or self._telemetry_tracker is None:
+            return
+
+        try:
+            provider_str = getattr(self, "_provider_str", "unknown")
+            self._telemetry_tracker.track_llm_call(
+                workflow=self.name,
+                stage=stage,
+                tier=tier.value.upper() if hasattr(tier, "value") else str(tier).upper(),
+                model=model,
+                provider=provider_str,
+                cost=cost,
+                tokens=tokens,
+                cache_hit=cache_hit,
+                cache_type=cache_type,
+                duration_ms=duration_ms,
+            )
+        except (AttributeError, TypeError, ValueError) as e:
+            # INTENTIONAL: Telemetry tracking failures should never crash workflows
+            logger.debug(f"Failed to track telemetry (config/data error): {e}")
+        except (OSError, PermissionError) as e:
+            # File system errors - log but never crash workflow
+            logger.debug(f"Failed to track telemetry (file system error): {e}")
+
+    def _emit_call_telemetry(
+        self,
+        step_name: str,
+        task_type: str,
+        tier: str,
+        model_id: str,
+        input_tokens: int,
+        output_tokens: int,
+        cost: float,
+        latency_ms: int,
+        success: bool = True,
+        error_message: str | None = None,
+        fallback_used: bool = False,
+    ) -> None:
+        """Emit an LLMCallRecord to the telemetry backend.
+
+        Args:
+            step_name: Name of the workflow step
+            task_type: Task type used for routing
+            tier: Model tier used
+            model_id: Model ID used
+            input_tokens: Input token count
+            output_tokens: Output token count
+            cost: Estimated cost
+            latency_ms: Latency in milliseconds
+            success: Whether the call succeeded
+            error_message: Error message if failed
+            fallback_used: Whether fallback was used
+        """
+        from empathy_os.models import LLMCallRecord
+
+        record = LLMCallRecord(
+            call_id=str(uuid.uuid4()),
+            timestamp=datetime.now().isoformat(),
+            workflow_name=self.name,
+            step_name=step_name,
+            task_type=task_type,
+            provider=getattr(self, "_provider_str", "unknown"),
+            tier=tier,
+            model_id=model_id,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            estimated_cost=cost,
+            latency_ms=latency_ms,
+            success=success,
+            error_message=error_message,
+            fallback_used=fallback_used,
+            metadata={"run_id": self._run_id},
+        )
+        try:
+            if self._telemetry_backend is not None:
+                self._telemetry_backend.log_call(record)
+        except (AttributeError, ValueError, TypeError):
+            # Telemetry backend errors - log but don't crash workflow
+            logger.debug("Failed to log call telemetry (backend error)")
+        except OSError:
+            # File system errors - log but don't crash workflow
+            logger.debug("Failed to log call telemetry (file system error)")
+        except Exception:  # noqa: BLE001
+            # INTENTIONAL: Telemetry is optional diagnostics - never crash workflow
+            logger.debug("Unexpected error logging call telemetry")
+
+    def _emit_workflow_telemetry(self, result: Any) -> None:
+        """Emit a WorkflowRunRecord to the telemetry backend.
+
+        Args:
+            result: The WorkflowResult to record
+        """
+        from empathy_os.models import WorkflowRunRecord, WorkflowStageRecord
+
+        # Build stage records
+        stages = [
+            WorkflowStageRecord(
+                stage_name=s.name,
+                tier=s.tier.value if hasattr(s.tier, "value") else str(s.tier),
+                model_id=(
+                    self.get_model_for_tier(s.tier)
+                    if hasattr(self, "get_model_for_tier")
+                    else "unknown"
+                ),
+                input_tokens=s.input_tokens,
+                output_tokens=s.output_tokens,
+                cost=s.cost,
+                latency_ms=s.duration_ms,
+                success=not s.skipped and result.error is None,
+                skipped=s.skipped,
+                skip_reason=s.skip_reason,
+            )
+            for s in result.stages
+        ]
+
+        record = WorkflowRunRecord(
+            run_id=self._run_id or str(uuid.uuid4()),
+            workflow_name=self.name,
+            started_at=result.started_at.isoformat(),
+            completed_at=result.completed_at.isoformat(),
+            stages=stages,
+            total_input_tokens=sum(s.input_tokens for s in result.stages if not s.skipped),
+            total_output_tokens=sum(s.output_tokens for s in result.stages if not s.skipped),
+            total_cost=result.cost_report.total_cost,
+            baseline_cost=result.cost_report.baseline_cost,
+            savings=result.cost_report.savings,
+            savings_percent=result.cost_report.savings_percent,
+            total_duration_ms=result.total_duration_ms,
+            success=result.success,
+            error=result.error,
+            providers_used=[getattr(self, "_provider_str", "unknown")],
+            tiers_used=list(result.cost_report.by_tier.keys()),
+        )
+        try:
+            if self._telemetry_backend is not None:
+                self._telemetry_backend.log_workflow(record)
+        except (AttributeError, ValueError, TypeError):
+            # Telemetry backend errors - log but don't crash workflow
+            logger.debug("Failed to log workflow telemetry (backend error)")
+        except OSError:
+            # File system errors - log but don't crash workflow
+            logger.debug("Failed to log workflow telemetry (file system error)")
+        except Exception:  # noqa: BLE001
+            # INTENTIONAL: Telemetry is optional diagnostics - never crash workflow
+            logger.debug("Unexpected error logging workflow telemetry")
+
+    def _generate_run_id(self) -> str:
+        """Generate a new run ID for telemetry correlation.
+
+        Returns:
+            A new UUID string for the run
+        """
+        self._run_id = str(uuid.uuid4())
+        return self._run_id