ctrlcode 0.1.0__py3-none-any.whl

ctrlcode/agents/observability.py

@@ -0,0 +1,421 @@
+"""Observability tools for Executor agent."""
+
+import re
+from typing import Any
+from dataclasses import dataclass
+
+
+@dataclass
+class TestResult:
+    """Parsed test execution result."""
+
+    total: int
+    passed: int
+    failed: int
+    skipped: int
+    duration: float
+    status: str  # "pass" | "fail"
+    failures: list[dict[str, str]]
+
+
+@dataclass
+class PerformanceMetrics:
+    """Parsed performance metrics."""
+
+    p50: float | None = None
+    p75: float | None = None
+    p90: float | None = None
+    p95: float | None = None
+    p99: float | None = None
+    mean: float | None = None
+    throughput: float | None = None
+    error_rate: float | None = None
+
+
+@dataclass
+class LogAnalysis:
+    """Parsed log analysis."""
+
+    total_lines: int
+    errors: list[str]
+    warnings: list[str]
+    error_count: int
+    warning_count: int
+    health: str  # "healthy" | "warnings" | "errors"
+
+
+class TestOutputParser:
+    """Parser for test framework outputs."""
+
+    @staticmethod
+    def parse_pytest(output: str) -> TestResult:
+        """
+        Parse pytest output.
+
+        Args:
+            output: pytest stdout/stderr
+
+        Returns:
+            Parsed TestResult
+        """
+        # Extract test counts
+        # Format: "======================== 10 passed, 2 failed in 0.5s ========================"
+        match = re.search(
+            r'(\d+)\s+passed(?:,\s+(\d+)\s+failed)?(?:,\s+(\d+)\s+skipped)?.*?in\s+([\d.]+)s',
+            output
+        )
+
+        if match:
+            passed = int(match.group(1))
+            failed = int(match.group(2) or 0)
+            skipped = int(match.group(3) or 0)
+            duration = float(match.group(4))
+            total = passed + failed + skipped
+        else:
+            # Fallback parsing
+            total = 0
+            passed = 0
+            failed = 0
+            skipped = 0
+            duration = 0.0
+
+        # Extract failure details
+        failures = []
+        failure_pattern = r'FAILED\s+([\w/:.]+)\s+-\s+(.+)'
+        for match in re.finditer(failure_pattern, output):
+            failures.append({
+                "test": match.group(1),
+                "reason": match.group(2).strip()
+            })
+
+        status = "pass" if failed == 0 else "fail"
+
+        return TestResult(
+            total=total,
+            passed=passed,
+            failed=failed,
+            skipped=skipped,
+            duration=duration,
+            status=status,
+            failures=failures
+        )
+
+    @staticmethod
+    def parse_generic(output: str) -> TestResult:
+        """
+        Parse generic test output.
+
+        Args:
+            output: Test framework output
+
+        Returns:
+            Parsed TestResult
+        """
+        # Look for common patterns
+        passed = len(re.findall(r'\bPASS(ED)?\b', output, re.IGNORECASE))
+        failed = len(re.findall(r'\bFAIL(ED)?\b', output, re.IGNORECASE))
+        skipped = len(re.findall(r'\bSKIP(PED)?\b', output, re.IGNORECASE))
+
+        total = passed + failed + skipped
+        status = "pass" if failed == 0 else "fail"
+
+        return TestResult(
+            total=total,
+            passed=passed,
+            failed=failed,
+            skipped=skipped,
+            duration=0.0,
+            status=status,
+            failures=[]
+        )
+
+
+class PerformanceParser:
+    """Parser for performance benchmark outputs."""
+
+    @staticmethod
+    def parse_wrk(output: str) -> PerformanceMetrics:
+        """
+        Parse wrk benchmark output.
+
+        Args:
+            output: wrk stdout
+
+        Returns:
+            Parsed PerformanceMetrics
+        """
+        metrics = PerformanceMetrics()
+
+        # Parse latency distribution
+        # Format: "50%  145ms"
+        percentiles = {
+            '50%': 'p50',
+            '75%': 'p75',
+            '90%': 'p90',
+            '95%': 'p95',
+            '99%': 'p99',
+        }
+
+        for percentile, attr in percentiles.items():
+            pattern = rf'{re.escape(percentile)}\s+([\d.]+)(ms|s)'
+            match = re.search(pattern, output)
+            if match:
+                value = float(match.group(1))
+                unit = match.group(2)
+                if unit == 's':
+                    value *= 1000  # Convert to ms
+                setattr(metrics, attr, value)
+
+        # Parse throughput
+        # Format: "Requests/sec: 6.90"
+        match = re.search(r'Requests/sec:\s+([\d.]+)', output)
+        if match:
+            metrics.throughput = float(match.group(1))
+
+        return metrics
+
+    @staticmethod
+    def parse_ab(output: str) -> PerformanceMetrics:
+        """
+        Parse Apache Bench output.
+
+        Args:
+            output: ab stdout
+
+        Returns:
+            Parsed PerformanceMetrics
+        """
+        metrics = PerformanceMetrics()
+
+        # Parse mean latency
+        # Format: "Time per request:       145.0 [ms] (mean)"
+        match = re.search(r'Time per request:\s+([\d.]+)\s+\[ms\]\s+\(mean\)', output)
+        if match:
+            metrics.mean = float(match.group(1))
+
+        # Parse throughput
+        # Format: "Requests per second:    6.90 [#/sec] (mean)"
+        match = re.search(r'Requests per second:\s+([\d.]+)', output)
+        if match:
+            metrics.throughput = float(match.group(1))
+
+        # Parse failure rate
+        # Format: "Failed requests:        1"
+        total_match = re.search(r'Complete requests:\s+(\d+)', output)
+        failed_match = re.search(r'Failed requests:\s+(\d+)', output)
+
+        if total_match and failed_match:
+            total = int(total_match.group(1))
+            failed = int(failed_match.group(1))
+            metrics.error_rate = (failed / total) * 100 if total > 0 else 0.0
+
+        return metrics
+
+
+class LogParser:
+    """Parser for application logs."""
+
+    @staticmethod
+    def parse_structured_logs(output: str) -> LogAnalysis:
+        """
+        Parse structured application logs.
+
+        Args:
+            output: Log output
+
+        Returns:
+            Parsed LogAnalysis
+        """
+        lines = output.splitlines()
+        total_lines = len(lines)
+
+        errors = []
+        warnings = []
+
+        # Common log level patterns
+        error_pattern = r'\b(ERROR|FATAL|CRITICAL)\b'
+        warning_pattern = r'\bWARN(ING)?\b'
+
+        for line in lines:
+            if re.search(error_pattern, line, re.IGNORECASE):
+                errors.append(line.strip())
+            elif re.search(warning_pattern, line, re.IGNORECASE):
+                warnings.append(line.strip())
+
+        error_count = len(errors)
+        warning_count = len(warnings)
+
+        # Determine health
+        if error_count > 0:
+            health = "errors"
+        elif warning_count > 0:
+            health = "warnings"
+        else:
+            health = "healthy"
+
+        return LogAnalysis(
+            total_lines=total_lines,
+            errors=errors,
+            warnings=warnings,
+            error_count=error_count,
+            warning_count=warning_count,
+            health=health
+        )
+
+
+class ObservabilityTools:
+    """High-level observability tools for Executor agent."""
+
+    def __init__(self):
+        """Initialize observability tools."""
+        self.test_parser = TestOutputParser()
+        self.perf_parser = PerformanceParser()
+        self.log_parser = LogParser()
+
+    def analyze_test_output(
+        self,
+        output: str,
+        framework: str = "pytest"
+    ) -> dict[str, Any]:
+        """
+        Analyze test output and extract structured data.
+
+        Args:
+            output: Test framework output
+            framework: Test framework name (pytest, jest, etc.)
+
+        Returns:
+            Structured test analysis
+        """
+        if framework == "pytest":
+            result = self.test_parser.parse_pytest(output)
+        else:
+            result = self.test_parser.parse_generic(output)
+
+        return {
+            "status": result.status,
+            "summary": {
+                "total": result.total,
+                "passed": result.passed,
+                "failed": result.failed,
+                "skipped": result.skipped,
+                "duration": result.duration,
+            },
+            "failures": result.failures,
+            "recommendation": self._test_recommendation(result)
+        }
+
+    def analyze_performance(
+        self,
+        output: str,
+        tool: str = "wrk",
+        threshold_ms: float | None = None
+    ) -> dict[str, Any]:
+        """
+        Analyze performance benchmark output.
+
+        Args:
+            output: Benchmark tool output
+            tool: Tool name (wrk, ab, etc.)
+            threshold_ms: Optional performance threshold in ms
+
+        Returns:
+            Structured performance analysis
+        """
+        if tool == "wrk":
+            metrics = self.perf_parser.parse_wrk(output)
+        elif tool == "ab":
+            metrics = self.perf_parser.parse_ab(output)
+        else:
+            metrics = PerformanceMetrics()
+
+        # Determine status
+        status = "pass"
+        if threshold_ms and metrics.p99:
+            status = "pass" if metrics.p99 <= threshold_ms else "warning"
+
+        return {
+            "status": status,
+            "metrics": {
+                "p50": metrics.p50,
+                "p75": metrics.p75,
+                "p90": metrics.p90,
+                "p95": metrics.p95,
+                "p99": metrics.p99,
+                "mean": metrics.mean,
+                "throughput": metrics.throughput,
+                "error_rate": metrics.error_rate,
+            },
+            "threshold": threshold_ms,
+            "recommendation": self._perf_recommendation(metrics, threshold_ms)
+        }
+
+    def analyze_logs(
+        self,
+        output: str,
+        max_errors: int = 0
+    ) -> dict[str, Any]:
+        """
+        Analyze application logs.
+
+        Args:
+            output: Log output
+            max_errors: Maximum acceptable error count
+
+        Returns:
+            Structured log analysis
+        """
+        analysis = self.log_parser.parse_structured_logs(output)
+
+        status = "pass"
+        if analysis.error_count > max_errors:
+            status = "fail"
+        elif analysis.warning_count > 0:
+            status = "warning"
+
+        return {
+            "status": status,
+            "health": analysis.health,
+            "summary": {
+                "total_lines": analysis.total_lines,
+                "error_count": analysis.error_count,
+                "warning_count": analysis.warning_count,
+            },
+            "errors": analysis.errors[:10],  # Top 10 errors
+            "warnings": analysis.warnings[:10],  # Top 10 warnings
+            "recommendation": self._log_recommendation(analysis)
+        }
+
+    def _test_recommendation(self, result: TestResult) -> str:
+        """Generate recommendation from test result."""
+        if result.status == "pass":
+            return "approve"
+        elif result.failed <= 2:
+            return "investigate (few failures, likely fixable)"
+        else:
+            return "fix (multiple failures detected)"
+
+    def _perf_recommendation(
+        self,
+        metrics: PerformanceMetrics,
+        threshold: float | None
+    ) -> str:
+        """Generate recommendation from performance metrics."""
+        if not threshold or not metrics.p99:
+            return "baseline established"
+
+        if metrics.p99 <= threshold:
+            return "approve"
+        elif metrics.p99 <= threshold * 1.1:
+            return "approve (marginal exceedance, acceptable)"
+        else:
+            return "investigate (performance degradation detected)"
+
+    def _log_recommendation(self, analysis: LogAnalysis) -> str:
+        """Generate recommendation from log analysis."""
+        if analysis.health == "healthy":
+            return "approve"
+        elif analysis.health == "warnings":
+            return "approve (warnings present, monitor in production)"
+        else:
+            return "investigate (errors detected)"