specweave 0.1.0

package/src/agents/sre/scripts/log-analyzer.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+
+"""
+log-analyzer.py
+Parse application/system logs for error patterns and anomalies
+
+Usage: python3 log-analyzer.py /var/log/application.log
+       python3 log-analyzer.py /var/log/application.log --errors-only
+       python3 log-analyzer.py /var/log/application.log --since "2025-10-26 14:00"
+"""
+
+import re
+import sys
+import argparse
+from datetime import datetime, timedelta
+from collections import Counter, defaultdict
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Analyze log files for errors and patterns')
+    parser.add_argument('logfile', help='Path to log file')
+    parser.add_argument('--errors-only', action='store_true', help='Show only errors (ERROR, FATAL)')
+    parser.add_argument('--warnings', action='store_true', help='Include warnings')
+    parser.add_argument('--since', help='Show logs since timestamp (YYYY-MM-DD HH:MM)')
+    parser.add_argument('--until', help='Show logs until timestamp (YYYY-MM-DD HH:MM)')
+    parser.add_argument('--pattern', help='Search for specific pattern (regex)')
+    parser.add_argument('--top', type=int, default=10, help='Show top N errors (default: 10)')
+    return parser.parse_args()
+
+def parse_log_line(line):
+    """Parse common log formats"""
+    # Try different log formats
+    patterns = [
+        # JSON: {"timestamp":"2025-10-26T14:00:00Z","level":"ERROR","message":"..."}
+        r'\{"timestamp":"(?P<timestamp>[^"]+)".*"level":"(?P<level>[^"]+)".*"message":"(?P<message>[^"]+)"',
+
+        # Standard: [2025-10-26 14:00:00] ERROR: message
+        r'\[(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]\s+(?P<level>\w+):\s+(?P<message>.*)',
+
+        # Syslog: Oct 26 14:00:00 hostname application[1234]: ERROR message
+        r'(?P<timestamp>\w+ \d+ \d{2}:\d{2}:\d{2})\s+\S+\s+\S+:\s+(?P<level>\w+)\s+(?P<message>.*)',
+
+        # Simple: 2025-10-26 14:00:00 ERROR message
+        r'(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+(?P<level>\w+)\s+(?P<message>.*)',
+    ]
+
+    for pattern in patterns:
+        match = re.match(pattern, line)
+        if match:
+            return match.groupdict()
+
+    # If no pattern matched, return raw line
+    return {'timestamp': None, 'level': 'INFO', 'message': line.strip()}
+
+def parse_timestamp(ts_str):
+    """Parse various timestamp formats"""
+    if not ts_str:
+        return None
+
+    formats = [
+        '%Y-%m-%dT%H:%M:%SZ',
+        '%Y-%m-%d %H:%M:%S',
+        '%b %d %H:%M:%S',
+    ]
+
+    for fmt in formats:
+        try:
+            return datetime.strptime(ts_str, fmt)
+        except ValueError:
+            continue
+
+    return None
+
+def main():
+    args = parse_args()
+
+    # Parse filters
+    since = datetime.strptime(args.since, '%Y-%m-%d %H:%M') if args.since else None
+    until = datetime.strptime(args.until, '%Y-%m-%d %H:%M') if args.until else None
+
+    # Stats
+    total_lines = 0
+    error_count = 0
+    warning_count = 0
+    error_messages = Counter()
+    errors_by_hour = defaultdict(int)
+    error_timeline = []
+
+    print(f"Analyzing log file: {args.logfile}")
+    print("=" * 80)
+    print()
+
+    try:
+        with open(args.logfile, 'r', encoding='utf-8', errors='ignore') as f:
+            for line in f:
+                total_lines += 1
+
+                # Parse log line
+                parsed = parse_log_line(line)
+                level = parsed.get('level', '').upper()
+                message = parsed.get('message', '')
+                timestamp = parse_timestamp(parsed.get('timestamp'))
+
+                # Filter by time range
+                if since and timestamp and timestamp < since:
+                    continue
+                if until and timestamp and timestamp > until:
+                    continue
+
+                # Filter by pattern
+                if args.pattern and not re.search(args.pattern, message, re.IGNORECASE):
+                    continue
+
+                # Filter by level
+                if args.errors_only and level not in ['ERROR', 'FATAL', 'CRITICAL']:
+                    continue
+
+                # Count errors and warnings
+                if level in ['ERROR', 'FATAL', 'CRITICAL']:
+                    error_count += 1
+
+                    # Extract error message (first 100 chars)
+                    error_key = message[:100] if len(message) > 100 else message
+                    error_messages[error_key] += 1
+
+                    # Group by hour
+                    if timestamp:
+                        hour_key = timestamp.strftime('%Y-%m-%d %H:00')
+                        errors_by_hour[hour_key] += 1
+                        error_timeline.append((timestamp, message))
+
+                elif level in ['WARN', 'WARNING'] and args.warnings:
+                    warning_count += 1
+
+        # Print summary
+        print(f"📊 SUMMARY")
+        print(f"---------")
+        print(f"Total lines: {total_lines:,}")
+        print(f"Errors: {error_count:,}")
+        if args.warnings:
+            print(f"Warnings: {warning_count:,}")
+        print()
+
+        # Top errors
+        if error_messages:
+            print(f"🔥 TOP {args.top} ERRORS")
+            print(f"{'Count':<10} {'Message':<70}")
+            print("-" * 80)
+            for msg, count in error_messages.most_common(args.top):
+                msg_short = (msg[:67] + '...') if len(msg) > 70 else msg
+                print(f"{count:<10} {msg_short}")
+            print()
+
+        # Errors by hour
+        if errors_by_hour:
+            print(f"📈 ERRORS BY HOUR")
+            print(f"{'Hour':<20} {'Count':<10} {'Graph':<50}")
+            print("-" * 80)
+
+            max_errors = max(errors_by_hour.values())
+            for hour in sorted(errors_by_hour.keys()):
+                count = errors_by_hour[hour]
+                bar_length = int((count / max_errors) * 40)
+                bar = '█' * bar_length
+                print(f"{hour:<20} {count:<10} {bar}")
+            print()
+
+        # Error timeline (last 20)
+        if error_timeline:
+            print(f"⏱️  ERROR TIMELINE (Last 20)")
+            print(f"{'Timestamp':<20} {'Message':<60}")
+            print("-" * 80)
+
+            for timestamp, message in sorted(error_timeline, reverse=True)[:20]:
+                ts_str = timestamp.strftime('%Y-%m-%d %H:%M:%S')
+                msg_short = (message[:57] + '...') if len(message) > 60 else message
+                print(f"{ts_str:<20} {msg_short}")
+            print()
+
+        # Recommendations
+        print(f"💡 RECOMMENDATIONS")
+        print(f"-----------------")
+
+        if error_count == 0:
+            print("✅ No errors found. System looks healthy!")
+        elif error_count < 10:
+            print(f"⚠️  {error_count} errors found. Review above for details.")
+        elif error_count < 100:
+            print(f"⚠️  {error_count} errors found. Investigate top errors.")
+        else:
+            print(f"🚨 {error_count} errors found! Immediate investigation required.")
+            print("   - Check for cascading failures")
+            print("   - Review error timeline for spike")
+            print("   - Check related services")
+
+        if errors_by_hour:
+            # Find hour with most errors
+            peak_hour = max(errors_by_hour.items(), key=lambda x: x[1])
+            print(f"\n📍 Peak error hour: {peak_hour[0]} ({peak_hour[1]} errors)")
+            print(f"   - Review what happened at this time")
+            print(f"   - Check deployment, traffic spike, external dependency")
+
+        print()
+
+    except FileNotFoundError:
+        print(f"❌ Error: Log file not found: {args.logfile}")
+        sys.exit(1)
+    except PermissionError:
+        print(f"❌ Error: Permission denied: {args.logfile}")
+        print(f"   Try: sudo python3 {sys.argv[0]} {args.logfile}")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

package/src/agents/sre/scripts/metrics-collector.sh

@@ -0,0 +1,294 @@
+#!/bin/bash
+
+# metrics-collector.sh
+# Gather system metrics for incident diagnosis
+# Usage: ./metrics-collector.sh [output_file]
+
+set -e
+
+OUTPUT_FILE=${1:-"metrics-$(date +%Y%m%d-%H%M%S).txt"}
+
+echo "Collecting system metrics..."
+echo "Output: $OUTPUT_FILE"
+echo ""
+
+{
+    echo "========================================="
+    echo "SYSTEM METRICS COLLECTION"
+    echo "========================================="
+    echo "Date: $(date)"
+    echo "Hostname: $(hostname)"
+    echo "Uptime: $(uptime -p 2>/dev/null || uptime)"
+    echo ""
+
+    # 1. CPU Metrics
+    echo "========================================="
+    echo "1. CPU METRICS"
+    echo "========================================="
+    echo ""
+
+    echo "CPU Info:"
+    lscpu | grep -E "^Model name|^CPU\(s\)|^Thread|^Core|^Socket"
+    echo ""
+
+    echo "CPU Usage (snapshot):"
+    top -bn1 | head -20
+    echo ""
+
+    echo "Load Average:"
+    uptime
+    echo ""
+
+    if command -v mpstat &> /dev/null; then
+        echo "CPU by Core:"
+        mpstat -P ALL 1 1
+        echo ""
+    fi
+
+    # 2. Memory Metrics
+    echo "========================================="
+    echo "2. MEMORY METRICS"
+    echo "========================================="
+    echo ""
+
+    echo "Memory Overview:"
+    free -h
+    echo ""
+
+    echo "Memory Details:"
+    cat /proc/meminfo | head -20
+    echo ""
+
+    echo "Top Memory Processes:"
+    ps aux | sort -nrk 4,4 | head -10
+    echo ""
+
+    # 3. Disk Metrics
+    echo "========================================="
+    echo "3. DISK METRICS"
+    echo "========================================="
+    echo ""
+
+    echo "Disk Usage:"
+    df -h
+    echo ""
+
+    echo "Inode Usage:"
+    df -i
+    echo ""
+
+    if command -v iostat &> /dev/null; then
+        echo "Disk I/O Stats:"
+        iostat -x 1 5
+        echo ""
+    fi
+
+    echo "Disk Space by Directory (/):"
+    du -sh /* 2>/dev/null | sort -hr | head -20
+    echo ""
+
+    # 4. Network Metrics
+    echo "========================================="
+    echo "4. NETWORK METRICS"
+    echo "========================================="
+    echo ""
+
+    echo "Network Interfaces:"
+    ip addr show
+    echo ""
+
+    echo "Network Statistics:"
+    netstat -s | head -50
+    echo ""
+
+    echo "Active Connections:"
+    netstat -an | grep ESTABLISHED | wc -l
+    echo ""
+
+    echo "Top 10 IPs by Connection Count:"
+    netstat -ntu | awk '{print $5}' | cut -d: -f1 | sort | uniq -c | sort -nr | head -10
+    echo ""
+
+    if command -v ss &> /dev/null; then
+        echo "Socket Stats:"
+        ss -s
+        echo ""
+    fi
+
+    # 5. Process Metrics
+    echo "========================================="
+    echo "5. PROCESS METRICS"
+    echo "========================================="
+    echo ""
+
+    echo "Process Count:"
+    ps aux | wc -l
+    echo ""
+
+    echo "Top CPU Processes:"
+    ps aux | sort -nrk 3,3 | head -10
+    echo ""
+
+    echo "Top Memory Processes:"
+    ps aux | sort -nrk 4,4 | head -10
+    echo ""
+
+    echo "Zombie Processes:"
+    ps aux | grep -E "<defunct>|Z" | grep -v grep
+    echo ""
+
+    # 6. Database Metrics (PostgreSQL)
+    echo "========================================="
+    echo "6. DATABASE METRICS (PostgreSQL)"
+    echo "========================================="
+    echo ""
+
+    if command -v psql &> /dev/null; then
+        if sudo -u postgres psql -c "SELECT 1" &> /dev/null; then
+            echo "PostgreSQL Connection Count:"
+            sudo -u postgres psql -t -c "SELECT count(*) FROM pg_stat_activity;"
+            echo ""
+
+            echo "PostgreSQL Max Connections:"
+            sudo -u postgres psql -t -c "SHOW max_connections;"
+            echo ""
+
+            echo "PostgreSQL Active Queries:"
+            sudo -u postgres psql -x -c "SELECT pid, usename, application_name, state, query FROM pg_stat_activity WHERE state != 'idle' LIMIT 10;"
+            echo ""
+
+            echo "PostgreSQL Database Sizes:"
+            sudo -u postgres psql -c "SELECT datname, pg_size_pretty(pg_database_size(datname)) FROM pg_database WHERE datistemplate = false;"
+            echo ""
+
+            echo "PostgreSQL Table Sizes (top 10):"
+            sudo -u postgres psql -c "SELECT schemaname, tablename, pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size FROM pg_tables ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC LIMIT 10;"
+            echo ""
+
+            if command -v pg_stat_statements &> /dev/null; then
+                echo "PostgreSQL Slow Queries (top 5):"
+                sudo -u postgres psql -c "SELECT query, calls, total_exec_time, mean_exec_time FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 5;"
+                echo ""
+            fi
+        else
+            echo "PostgreSQL not accessible"
+            echo ""
+        fi
+    else
+        echo "PostgreSQL not installed"
+        echo ""
+    fi
+
+    # 7. Web Server Metrics (nginx)
+    echo "========================================="
+    echo "7. WEB SERVER METRICS (nginx)"
+    echo "========================================="
+    echo ""
+
+    if systemctl is-active --quiet nginx 2>/dev/null; then
+        echo "Nginx Status: Running"
+
+        if [ -f /var/log/nginx/access.log ]; then
+            echo ""
+            echo "Nginx Request Count (last 1000 lines):"
+            tail -1000 /var/log/nginx/access.log | wc -l
+
+            echo ""
+            echo "Nginx Status Codes (last 1000 lines):"
+            tail -1000 /var/log/nginx/access.log | awk '{print $9}' | sort | uniq -c | sort -nr
+
+            echo ""
+            echo "Nginx Top 10 URLs:"
+            tail -1000 /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -nr | head -10
+
+            echo ""
+            echo "Nginx Top 10 IPs:"
+            tail -1000 /var/log/nginx/access.log | awk '{print $1}' | sort | uniq -c | sort -nr | head -10
+        fi
+    else
+        echo "Nginx not running"
+    fi
+    echo ""
+
+    # 8. Application Metrics (customize as needed)
+    echo "========================================="
+    echo "8. APPLICATION METRICS"
+    echo "========================================="
+    echo ""
+
+    echo "Application Processes:"
+    ps aux | grep -E "node|java|python|ruby" | grep -v grep
+    echo ""
+
+    echo "Application Ports:"
+    netstat -tlnp 2>/dev/null | grep -E "node|java|python|ruby"
+    echo ""
+
+    # 9. System Logs (recent errors)
+    echo "========================================="
+    echo "9. RECENT SYSTEM ERRORS"
+    echo "========================================="
+    echo ""
+
+    echo "Recent Syslog Errors (last 50):"
+    if [ -f /var/log/syslog ]; then
+        grep -i "error\|fail\|critical" /var/log/syslog | tail -50
+    else
+        echo "Syslog not found"
+    fi
+    echo ""
+
+    echo "Recent Journal Errors (last 10 minutes):"
+    if command -v journalctl &> /dev/null; then
+        journalctl --since "10 minutes ago" --priority=err --no-pager | tail -50
+    else
+        echo "journalctl not available"
+    fi
+    echo ""
+
+    # 10. System Info
+    echo "========================================="
+    echo "10. SYSTEM INFORMATION"
+    echo "========================================="
+    echo ""
+
+    echo "OS Version:"
+    cat /etc/os-release 2>/dev/null || uname -a
+    echo ""
+
+    echo "Kernel Version:"
+    uname -r
+    echo ""
+
+    echo "System Time:"
+    date
+    echo ""
+
+    echo "Timezone:"
+    timedatectl 2>/dev/null || cat /etc/timezone
+    echo ""
+
+    # Summary
+    echo "========================================="
+    echo "COLLECTION COMPLETE"
+    echo "========================================="
+    echo "Collected at: $(date)"
+    echo "Metrics saved to: $OUTPUT_FILE"
+    echo ""
+
+} > "$OUTPUT_FILE" 2>&1
+
+# Print summary to console
+echo ""
+echo "✅ Metrics collection complete!"
+echo ""
+echo "Summary:"
+grep -E "CPU Usage|Memory Overview|Disk Usage|Active Connections|PostgreSQL Connection Count" "$OUTPUT_FILE" | head -20
+echo ""
+echo "Full report: $OUTPUT_FILE"
+echo ""
+echo "Next steps:"
+echo "  - Review metrics for anomalies"
+echo "  - Compare with baseline metrics"
+echo "  - Share with team for analysis"
+echo ""