debugger-help 3.0.0__py3-none-any.whl

debugger_help/__init__.py

@@ -0,0 +1,2 @@
+"""debugger.help VPS Agent — Deep system monitoring."""
+__version__ = "3.0.0"

debugger_help/agent.py

@@ -0,0 +1,1125 @@
+#!/usr/bin/env python3
+"""
+debugger.help VPS Agent v3 — Ultimate Deep Debugger
+
+Captures EVERYTHING from your VPS:
+- pm2 logs, status, process list, restart counts
+- GPU: VRAM, temp, power, utilization, ECC errors, throttling, driver version, CUDA version
+- CPU: per-core usage, load average, frequency, context switches
+- Memory: RAM, swap, shared, buffers, cached
+- Disk: usage per mount, I/O read/write rates, inode usage
+- Network: per-interface stats, connection counts by state, open ports, DNS resolution
+- Processes: top CPU/memory consumers, zombie/defunct processes, open file descriptors
+- Docker containers (if running): status, CPU, memory, restart counts
+- Systemd services: failed units
+- SSL certificates: expiry checks
+- File watchers: key log files (syslog, dmesg, pm2 logs, custom)
+- Image generation: Flux/SDXL/ComfyUI pipeline errors, model load times, inference timing
+- Environment: all relevant env vars (sanitized), Python packages, Node packages
+
+Usage:
+    pip install psutil requests
+    # Optional: pip install pynvml docker
+
+    export DEBUGGER_API_KEY="sk_your_api_key_here"
+    export DEBUGGER_INGEST_URL="https://your-project.supabase.co/functions/v1/ingest"
+    python debugger_agent.py
+"""
+
+import os
+import sys
+import time
+import json
+import socket
+import logging
+import traceback
+import threading
+import subprocess
+import io
+import re
+import glob
+import ssl
+from datetime import datetime, timezone
+from collections import deque
+from pathlib import Path
+
+try:
+    import psutil
+except ImportError:
+    print("ERROR: psutil required. Install with: pip install psutil")
+    sys.exit(1)
+
+try:
+    import requests
+    from requests.adapters import HTTPAdapter
+    from urllib3.util.retry import Retry
+except ImportError:
+    print("ERROR: requests required. Install with: pip install requests")
+    sys.exit(1)
+
+# Optional GPU monitoring
+try:
+    import pynvml
+    pynvml.nvmlInit()
+    GPU_AVAILABLE = True
+    GPU_DRIVER_VERSION = pynvml.nvmlSystemGetDriverVersion()
+    if isinstance(GPU_DRIVER_VERSION, bytes):
+        GPU_DRIVER_VERSION = GPU_DRIVER_VERSION.decode()
+    try:
+        GPU_CUDA_VERSION = pynvml.nvmlSystemGetCudaDriverVersion_v2()
+        GPU_CUDA_VERSION = f"{GPU_CUDA_VERSION // 1000}.{(GPU_CUDA_VERSION % 1000) // 10}"
+    except Exception:
+        GPU_CUDA_VERSION = "unknown"
+except (ImportError, Exception):
+    GPU_AVAILABLE = False
+    GPU_DRIVER_VERSION = None
+    GPU_CUDA_VERSION = None
+
+# Optional Docker monitoring
+try:
+    import docker
+    DOCKER_CLIENT = docker.from_env()
+    DOCKER_AVAILABLE = True
+except Exception:
+    DOCKER_CLIENT = None
+    DOCKER_AVAILABLE = False
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s"
+)
+logger = logging.getLogger("debugger-agent")
+
+# Configuration
+API_KEY = os.environ.get("DEBUGGER_API_KEY", "")
+INGEST_URL = os.environ.get("DEBUGGER_INGEST_URL", "")
+SOURCE_NAME = os.environ.get("DEBUGGER_SOURCE", f"vps-{socket.gethostname()}")
+PLATFORM = os.environ.get("DEBUGGER_PLATFORM", "Python (VPS)")
+INTERVAL = int(os.environ.get("DEBUGGER_INTERVAL", "10"))
+VERSION = "3.0.0"
+
+# Additional log files to watch
+WATCH_LOG_FILES = [
+    p for p in os.environ.get("DEBUGGER_WATCH_LOGS", "").split(",") if p.strip()
+] or []
+
+# SSL domains to check
+SSL_CHECK_DOMAINS = [
+    d for d in os.environ.get("DEBUGGER_SSL_DOMAINS", "").split(",") if d.strip()
+] or []
+
+if not API_KEY:
+    logger.error("DEBUGGER_API_KEY not set.")
+    sys.exit(1)
+if not INGEST_URL:
+    logger.error("DEBUGGER_INGEST_URL not set.")
+    sys.exit(1)
+
+# --- HTTP Session with Retry ---
+
+session = requests.Session()
+retry = Retry(total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
+session.mount("https://", HTTPAdapter(max_retries=retry))
+session.mount("http://", HTTPAdapter(max_retries=retry))
+
+HEADERS = {
+    "Authorization": f"Bearer {API_KEY}",
+    "Content-Type": "application/json",
+}
+
+
+def send(payload: dict) -> bool:
+    """Send payload with auto-retry."""
+    try:
+        resp = session.post(INGEST_URL, json=payload, headers=HEADERS, timeout=15)
+        return resp.status_code == 200
+    except Exception as e:
+        logger.debug(f"Send failed (will retry): {e}")
+        return False
+
+
+def run_cmd(cmd: str, timeout: int = 10) -> str:
+    """Run shell command and return output."""
+    try:
+        result = subprocess.run(
+            cmd, shell=True, capture_output=True, text=True, timeout=timeout
+        )
+        return (result.stdout + result.stderr).strip()
+    except subprocess.TimeoutExpired:
+        return f"[timeout after {timeout}s]"
+    except Exception as e:
+        return f"[error: {e}]"
+
+
+# --- Stdout/Stderr Capture ---
+
+class StreamCapture(io.TextIOBase):
+    """Captures writes to stdout/stderr and sends them as logs."""
+
+    def __init__(self, original_stream, level="info", max_buffer=500):
+        self.original = original_stream
+        self.level = level
+        self.buffer = deque(maxlen=max_buffer)
+        self.lock = threading.Lock()
+
+    def write(self, text):
+        self.original.write(text)
+        if text.strip():
+            with self.lock:
+                self.buffer.append(text.strip())
+                lower = text.lower()
+                detected_level = self.level
+                if any(kw in lower for kw in [
+                    "error", "exception", "traceback", "failed", "critical",
+                    "cuda", "oom", "killed", "segfault", "abort", "panic"
+                ]):
+                    detected_level = "error"
+                elif any(kw in lower for kw in ["warning", "warn", "deprecat"]):
+                    detected_level = "warn"
+
+                send({
+                    "type": "log",
+                    "source": SOURCE_NAME,
+                    "platform": PLATFORM,
+                    "version": VERSION,
+                    "level": detected_level,
+                    "message": f"[std{self.level}] {text.strip()[:2000]}",
+                    "context": {"capturedFrom": f"std{self.level}"},
+                })
+        return len(text)
+
+    def flush(self):
+        self.original.flush()
+
+    def get_recent(self, n=100):
+        with self.lock:
+            return list(self.buffer)[-n:]
+
+
+# --- GPU Metrics (Deep) ---
+
+def get_gpu_metrics() -> dict:
+    if not GPU_AVAILABLE:
+        return {}
+    try:
+        device_count = pynvml.nvmlDeviceGetCount()
+        gpus = []
+        for i in range(device_count):
+            handle = pynvml.nvmlDeviceGetHandleByIndex(i)
+            temp = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU)
+            mem = pynvml.nvmlDeviceGetMemoryInfo(handle)
+            power = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000
+            util = pynvml.nvmlDeviceGetUtilizationRates(handle)
+            name = pynvml.nvmlDeviceGetName(handle)
+            if isinstance(name, bytes):
+                name = name.decode("utf-8")
+
+            gpu_info = {
+                "index": i,
+                "name": name,
+                "temp_c": temp,
+                "vram_used_gb": round(mem.used / 1e9, 2),
+                "vram_total_gb": round(mem.total / 1e9, 2),
+                "vram_free_gb": round(mem.free / 1e9, 2),
+                "vram_pct": round((mem.used / mem.total) * 100, 1) if mem.total > 0 else 0,
+                "gpu_util_pct": util.gpu,
+                "mem_util_pct": util.memory,
+                "power_w": round(power, 1),
+            }
+
+            # Power limit
+            try:
+                power_limit = pynvml.nvmlDeviceGetPowerManagementLimit(handle) / 1000
+                gpu_info["power_limit_w"] = round(power_limit, 1)
+            except Exception:
+                pass
+
+            # Throttle reasons
+            try:
+                throttle = pynvml.nvmlDeviceGetCurrentClocksThrottleReasons(handle)
+                reasons = []
+                if throttle & 0x0000000000000002:
+                    reasons.append("idle")
+                if throttle & 0x0000000000000004:
+                    reasons.append("app_clocks")
+                if throttle & 0x0000000000000008:
+                    reasons.append("sw_power_cap")
+                if throttle & 0x0000000000000020:
+                    reasons.append("hw_slowdown")
+                if throttle & 0x0000000000000040:
+                    reasons.append("sync_boost")
+                if throttle & 0x0000000000000080:
+                    reasons.append("sw_thermal")
+                if throttle & 0x0000000000000100:
+                    reasons.append("hw_thermal")
+                if throttle & 0x0000000000000200:
+                    reasons.append("hw_power_brake")
+                gpu_info["throttle_reasons"] = reasons if reasons else ["none"]
+            except Exception:
+                pass
+
+            # Clock speeds
+            try:
+                gpu_info["clock_graphics_mhz"] = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_GRAPHICS)
+                gpu_info["clock_mem_mhz"] = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_MEM)
+            except Exception:
+                pass
+
+            # Running processes on GPU
+            try:
+                procs = pynvml.nvmlDeviceGetComputeRunningProcesses(handle)
+                gpu_procs = []
+                for p in procs[:10]:
+                    try:
+                        proc = psutil.Process(p.pid)
+                        gpu_procs.append({
+                            "pid": p.pid,
+                            "name": proc.name(),
+                            "vram_mb": round(p.usedGpuMemory / 1e6, 1) if p.usedGpuMemory else 0,
+                            "cmdline": " ".join(proc.cmdline()[:5])
+                        })
+                    except Exception:
+                        gpu_procs.append({"pid": p.pid, "vram_mb": round(p.usedGpuMemory / 1e6, 1) if p.usedGpuMemory else 0})
+                gpu_info["processes"] = gpu_procs
+            except Exception:
+                pass
+
+            # ECC errors
+            try:
+                ecc_single = pynvml.nvmlDeviceGetTotalEccErrors(handle, pynvml.NVML_SINGLE_BIT_ECC, pynvml.NVML_VOLATILE_ECC)
+                ecc_double = pynvml.nvmlDeviceGetTotalEccErrors(handle, pynvml.NVML_DOUBLE_BIT_ECC, pynvml.NVML_VOLATILE_ECC)
+                gpu_info["ecc_single_bit"] = ecc_single
+                gpu_info["ecc_double_bit"] = ecc_double
+            except Exception:
+                pass
+
+            gpus.append(gpu_info)
+
+        return {
+            "driver_version": GPU_DRIVER_VERSION,
+            "cuda_version": GPU_CUDA_VERSION,
+            "device_count": device_count,
+            "gpus": gpus,
+            # Flatten primary GPU for metrics table
+            "gpu_temp": gpus[0]["temp_c"] if gpus else None,
+            "gpu_vram": gpus[0]["vram_used_gb"] if gpus else None,
+        }
+    except Exception as e:
+        return {"error": str(e)}
+
+
+# --- PM2 Integration ---
+
+def get_pm2_status() -> dict:
+    """Get full pm2 process list with details."""
+    raw = run_cmd("pm2 jlist 2>/dev/null")
+    if not raw or raw.startswith("[error") or raw.startswith("[timeout"):
+        return {"available": False, "raw": raw}
+
+    try:
+        processes = json.loads(raw)
+        result = {"available": True, "processes": []}
+        for p in processes:
+            env = p.get("pm2_env", {})
+            monit = p.get("monit", {})
+            proc_info = {
+                "name": p.get("name"),
+                "pid": p.get("pid"),
+                "pm_id": p.get("pm_id"),
+                "status": env.get("status"),
+                "restart_count": env.get("restart_time", 0),
+                "unstable_restarts": env.get("unstable_restarts", 0),
+                "uptime_ms": int(time.time() * 1000) - env.get("pm_uptime", 0) if env.get("pm_uptime") else None,
+                "cpu_pct": monit.get("cpu"),
+                "memory_mb": round(monit.get("memory", 0) / 1e6, 1),
+                "exec_mode": env.get("exec_mode"),
+                "node_version": env.get("node_version"),
+                "script": env.get("pm_exec_path"),
+                "cwd": env.get("pm_cwd"),
+                "created_at": env.get("created_at"),
+                "instances": env.get("instances"),
+                "exit_code": env.get("exit_code"),
+            }
+            # Get recent logs for this process
+            recent_logs = run_cmd(f"pm2 logs {p.get('name')} --nostream --lines 30 2>/dev/null", timeout=5)
+            if recent_logs and not recent_logs.startswith("["):
+                proc_info["recent_logs"] = recent_logs[-3000:]  # Last 3KB
+            
+            # Get recent error logs
+            err_log_path = env.get("pm_err_log_path")
+            if err_log_path and os.path.exists(err_log_path):
+                try:
+                    with open(err_log_path, "r") as f:
+                        lines = f.readlines()
+                        proc_info["recent_errors"] = "".join(lines[-50:])[-3000:]
+                except Exception:
+                    pass
+
+            result["processes"].append(proc_info)
+        return result
+    except json.JSONDecodeError:
+        return {"available": True, "raw_output": raw[:2000]}
+
+
+def get_pm2_logs(lines: int = 100) -> str:
+    """Get combined pm2 logs."""
+    output = run_cmd(f"pm2 logs --nostream --lines {lines} 2>/dev/null", timeout=10)
+    return output[-5000:] if output else ""
+
+
+# --- System Metrics (Deep) ---
+
+def get_disk_io() -> dict:
+    """Get disk I/O stats."""
+    try:
+        io_counters = psutil.disk_io_counters(perdisk=False)
+        if io_counters:
+            return {
+                "read_mb": round(io_counters.read_bytes / 1e6, 1),
+                "write_mb": round(io_counters.write_bytes / 1e6, 1),
+                "read_count": io_counters.read_count,
+                "write_count": io_counters.write_count,
+                "read_time_ms": io_counters.read_time,
+                "write_time_ms": io_counters.write_time,
+            }
+    except Exception:
+        pass
+    return {}
+
+
+def get_network_details() -> dict:
+    """Get detailed network info."""
+    result = {}
+    
+    # Per-interface stats
+    try:
+        net_io = psutil.net_io_counters(pernic=True)
+        interfaces = {}
+        for name, counters in net_io.items():
+            if name == "lo":
+                continue
+            interfaces[name] = {
+                "sent_mb": round(counters.bytes_sent / 1e6, 1),
+                "recv_mb": round(counters.bytes_recv / 1e6, 1),
+                "packets_sent": counters.packets_sent,
+                "packets_recv": counters.packets_recv,
+                "errors_in": counters.errin,
+                "errors_out": counters.errout,
+                "drops_in": counters.dropin,
+                "drops_out": counters.dropout,
+            }
+        result["interfaces"] = interfaces
+    except Exception:
+        pass
+
+    # Connection states
+    try:
+        connections = psutil.net_connections(kind="tcp")
+        states = {}
+        for conn in connections:
+            s = conn.status
+            states[s] = states.get(s, 0) + 1
+        result["tcp_states"] = states
+        result["total_connections"] = len(connections)
+    except Exception:
+        pass
+
+    # Listening ports
+    try:
+        listening = []
+        for conn in psutil.net_connections(kind="tcp"):
+            if conn.status == "LISTEN":
+                listening.append({
+                    "port": conn.laddr.port,
+                    "addr": conn.laddr.ip,
+                    "pid": conn.pid,
+                })
+        result["listening_ports"] = listening[:30]
+    except Exception:
+        pass
+
+    # DNS check
+    try:
+        start = time.time()
+        socket.getaddrinfo("google.com", 80)
+        result["dns_resolve_ms"] = round((time.time() - start) * 1000, 1)
+    except Exception as e:
+        result["dns_error"] = str(e)
+
+    return result
+
+
+def get_top_processes(n: int = 15) -> list:
+    """Get top N processes by CPU and memory."""
+    procs = []
+    for p in psutil.process_iter(["pid", "name", "cpu_percent", "memory_percent", "status", "num_fds", "num_threads", "create_time", "cmdline"]):
+        try:
+            info = p.info
+            if info["cpu_percent"] is None:
+                continue
+            procs.append({
+                "pid": info["pid"],
+                "name": info["name"],
+                "cpu_pct": info["cpu_percent"],
+                "mem_pct": round(info["memory_percent"] or 0, 1),
+                "status": info["status"],
+                "fds": info.get("num_fds"),
+                "threads": info.get("num_threads"),
+                "cmd": " ".join((info.get("cmdline") or [])[:5])[:200],
+            })
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            pass
+    procs.sort(key=lambda x: x["cpu_pct"], reverse=True)
+    return procs[:n]
+
+
+def get_zombie_processes() -> list:
+    """Find zombie/defunct processes."""
+    zombies = []
+    for p in psutil.process_iter(["pid", "name", "status", "ppid"]):
+        try:
+            if p.info["status"] == psutil.STATUS_ZOMBIE:
+                zombies.append({
+                    "pid": p.info["pid"],
+                    "name": p.info["name"],
+                    "ppid": p.info["ppid"],
+                })
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            pass
+    return zombies
+
+
+def get_disk_details() -> list:
+    """Get all mount point usage and inode info."""
+    mounts = []
+    for part in psutil.disk_partitions(all=False):
+        try:
+            usage = psutil.disk_usage(part.mountpoint)
+            info = {
+                "mount": part.mountpoint,
+                "device": part.device,
+                "fstype": part.fstype,
+                "total_gb": round(usage.total / 1e9, 1),
+                "used_gb": round(usage.used / 1e9, 1),
+                "free_gb": round(usage.free / 1e9, 1),
+                "pct": usage.percent,
+            }
+            # Inode usage (Linux)
+            try:
+                inode_output = run_cmd(f"df -i {part.mountpoint} | tail -1", timeout=3)
+                parts = inode_output.split()
+                if len(parts) >= 5:
+                    info["inodes_used"] = parts[2]
+                    info["inodes_free"] = parts[3]
+                    info["inodes_pct"] = parts[4]
+            except Exception:
+                pass
+            mounts.append(info)
+        except Exception:
+            pass
+    return mounts
+
+
+def get_systemd_failed() -> list:
+    """Get failed systemd units."""
+    output = run_cmd("systemctl --failed --no-pager --plain 2>/dev/null", timeout=5)
+    if not output or "0 loaded" in output:
+        return []
+    
+    failed = []
+    for line in output.split("\n"):
+        line = line.strip()
+        if line and not line.startswith("UNIT") and not line.startswith("LOAD") and "loaded" not in line.lower():
+            parts = line.split()
+            if len(parts) >= 1:
+                failed.append(parts[0])
+    return failed[:20]
+
+
+def get_docker_status() -> dict:
+    """Get Docker container info."""
+    if not DOCKER_AVAILABLE:
+        return {"available": False}
+    try:
+        containers = []
+        for c in DOCKER_CLIENT.containers.list(all=True):
+            stats = {}
+            if c.status == "running":
+                try:
+                    s = c.stats(stream=False)
+                    cpu_delta = s["cpu_stats"]["cpu_usage"]["total_usage"] - s["precpu_stats"]["cpu_usage"]["total_usage"]
+                    system_delta = s["cpu_stats"]["system_cpu_usage"] - s["precpu_stats"]["system_cpu_usage"]
+                    cpu_pct = round((cpu_delta / system_delta) * 100, 1) if system_delta > 0 else 0
+                    mem_usage = s["memory_stats"].get("usage", 0)
+                    mem_limit = s["memory_stats"].get("limit", 1)
+                    stats = {
+                        "cpu_pct": cpu_pct,
+                        "memory_mb": round(mem_usage / 1e6, 1),
+                        "memory_limit_mb": round(mem_limit / 1e6, 1),
+                    }
+                except Exception:
+                    pass
+
+            containers.append({
+                "name": c.name,
+                "image": c.image.tags[0] if c.image.tags else str(c.image.id)[:12],
+                "status": c.status,
+                "restart_count": c.attrs.get("RestartCount", 0),
+                **stats,
+            })
+        return {"available": True, "containers": containers}
+    except Exception as e:
+        return {"available": False, "error": str(e)}
+
+
+def check_ssl_certs() -> list:
+    """Check SSL certificate expiry for configured domains."""
+    results = []
+    for domain in SSL_CHECK_DOMAINS:
+        try:
+            ctx = ssl.create_default_context()
+            with ctx.wrap_socket(socket.socket(), server_hostname=domain) as s:
+                s.settimeout(5)
+                s.connect((domain, 443))
+                cert = s.getpeercert()
+                expires = datetime.strptime(cert["notAfter"], "%b %d %H:%M:%S %Y %Z")
+                days_left = (expires - datetime.now()).days
+                results.append({
+                    "domain": domain,
+                    "expires": cert["notAfter"],
+                    "days_left": days_left,
+                    "issuer": dict(x[0] for x in cert.get("issuer", [])).get("organizationName", "unknown"),
+                    "warning": days_left < 30,
+                })
+        except Exception as e:
+            results.append({"domain": domain, "error": str(e)})
+    return results
+
+
+def get_system_logs() -> dict:
+    """Get recent system logs (dmesg, syslog, auth)."""
+    logs = {}
+    
+    # dmesg (kernel messages — GPU errors, OOM kills show up here)
+    dmesg = run_cmd("dmesg --time-format iso -T 2>/dev/null | tail -50", timeout=5)
+    if dmesg and not dmesg.startswith("[error"):
+        logs["dmesg"] = dmesg[-3000:]
+
+    # Check for OOM kills specifically
+    oom = run_cmd("dmesg | grep -i 'oom\\|killed process\\|out of memory' | tail -10 2>/dev/null", timeout=5)
+    if oom and not oom.startswith("[error"):
+        logs["oom_kills"] = oom
+
+    # journalctl recent errors
+    journal = run_cmd("journalctl -p err --since '1 hour ago' --no-pager -q 2>/dev/null | tail -30", timeout=5)
+    if journal and not journal.startswith("[error"):
+        logs["journal_errors"] = journal[-2000:]
+
+    # Auth log (failed logins, suspicious activity)
+    auth = run_cmd("tail -20 /var/log/auth.log 2>/dev/null || tail -20 /var/log/secure 2>/dev/null", timeout=5)
+    if auth and not auth.startswith("[error"):
+        logs["auth_log"] = auth[-1000:]
+
+    return logs
+
+
+def get_firewall_rules() -> str:
+    """Get firewall rules summary."""
+    # Try ufw first, then iptables
+    ufw = run_cmd("ufw status verbose 2>/dev/null", timeout=5)
+    if ufw and "Status:" in ufw:
+        return ufw[:2000]
+    ipt = run_cmd("iptables -L -n --line-numbers 2>/dev/null | head -40", timeout=5)
+    return ipt[:2000] if ipt else ""
+
+
+def get_env_sanitized() -> dict:
+    """Get relevant environment variables with secrets redacted."""
+    relevant_prefixes = [
+        "NODE_", "PYTHON", "PATH", "HOME", "USER", "SHELL", "LANG",
+        "CUDA", "NVIDIA", "GPU", "LD_LIBRARY", "VIRTUAL_ENV", "CONDA",
+        "PM2_", "PORT", "HOST", "DISPLAY", "XDG_", "DBUS",
+    ]
+    env = {}
+    for key, value in os.environ.items():
+        if any(key.startswith(p) for p in relevant_prefixes):
+            # Redact anything that looks like a secret
+            if any(s in key.upper() for s in ["KEY", "SECRET", "TOKEN", "PASS", "AUTH"]):
+                env[key] = "[REDACTED]"
+            else:
+                env[key] = value[:500]
+    return env
+
+
+def get_installed_packages() -> dict:
+    """Get installed Python and Node packages."""
+    pkgs = {}
+    
+    # Python packages
+    pip_list = run_cmd("pip list --format=json 2>/dev/null", timeout=10)
+    if pip_list and pip_list.startswith("["):
+        try:
+            pkgs["python"] = {p["name"]: p["version"] for p in json.loads(pip_list)}
+        except Exception:
+            pass
+
+    # Node packages (global)
+    npm_list = run_cmd("npm list -g --depth=0 --json 2>/dev/null", timeout=10)
+    if npm_list and npm_list.startswith("{"):
+        try:
+            deps = json.loads(npm_list).get("dependencies", {})
+            pkgs["node_global"] = {k: v.get("version", "?") for k, v in deps.items()}
+        except Exception:
+            pass
+
+    # pm2 ecosystem config
+    pm2_conf = run_cmd("cat ecosystem.config.js 2>/dev/null || cat ecosystem.config.cjs 2>/dev/null", timeout=3)
+    if pm2_conf and not pm2_conf.startswith("[error"):
+        pkgs["pm2_ecosystem"] = pm2_conf[:3000]
+
+    return pkgs
+
+
+# --- File Watcher Thread ---
+
+class LogFileWatcher(threading.Thread):
+    """Watch log files for new lines and send them."""
+
+    def __init__(self, files: list):
+        super().__init__(daemon=True)
+        self.files = files
+        self.positions = {}
+
+    def run(self):
+        # Initialize positions to end of files
+        for f in self.files:
+            try:
+                self.positions[f] = os.path.getsize(f)
+            except Exception:
+                self.positions[f] = 0
+
+        while True:
+            for filepath in self.files:
+                try:
+                    size = os.path.getsize(filepath)
+                    if size > self.positions.get(filepath, 0):
+                        with open(filepath, "r") as fh:
+                            fh.seek(self.positions[filepath])
+                            new_lines = fh.read(10000)  # Max 10KB per read
+                            self.positions[filepath] = fh.tell()
+
+                        if new_lines.strip():
+                            level = "info"
+                            lower = new_lines.lower()
+                            if any(kw in lower for kw in ["error", "exception", "traceback", "failed", "critical"]):
+                                level = "error"
+                            elif "warn" in lower:
+                                level = "warn"
+
+                            send({
+                                "type": "log",
+                                "source": SOURCE_NAME,
+                                "platform": PLATFORM,
+                                "version": VERSION,
+                                "level": level,
+                                "message": f"[file:{os.path.basename(filepath)}] {new_lines.strip()[:2000]}",
+                                "context": {"capturedFrom": "file_watcher", "file": filepath},
+                            })
+                    elif size < self.positions.get(filepath, 0):
+                        # File was truncated/rotated
+                        self.positions[filepath] = 0
+                except Exception:
+                    pass
+            time.sleep(2)
+
+
+# --- Collect Everything ---
+
+def collect_full_metrics() -> dict:
+    """Collect absolutely everything."""
+    cpu = psutil.cpu_percent(interval=1)
+    cpu_per_core = psutil.cpu_percent(interval=0, percpu=True)
+    mem = psutil.virtual_memory()
+    swap = psutil.swap_memory()
+    
+    data = {
+        "cpu": cpu,
+        "memory": mem.percent,
+        "network_latency": 0,
+        "custom": {
+            # CPU deep
+            "cpu_per_core": cpu_per_core,
+            "cpu_count_logical": psutil.cpu_count(),
+            "cpu_count_physical": psutil.cpu_count(logical=False),
+            "load_avg": list(os.getloadavg()) if hasattr(os, "getloadavg") else [],
+            
+            # Memory deep
+            "memory_used_gb": round(mem.used / 1e9, 2),
+            "memory_total_gb": round(mem.total / 1e9, 2),
+            "memory_available_gb": round(mem.available / 1e9, 2),
+            "memory_cached_gb": round(getattr(mem, "cached", 0) / 1e9, 2),
+            "memory_buffers_gb": round(getattr(mem, "buffers", 0) / 1e9, 2),
+            "swap_used_gb": round(swap.used / 1e9, 2),
+            "swap_total_gb": round(swap.total / 1e9, 2),
+            "swap_pct": swap.percent,
+
+            # Disk I/O
+            "disk_io": get_disk_io(),
+
+            # OS info
+            "os": f"{os.uname().sysname} {os.uname().release}" if hasattr(os, "uname") else "unknown",
+            "hostname": socket.gethostname(),
+            "python_version": sys.version.split()[0],
+            "node_version": run_cmd("node --version 2>/dev/null"),
+            "uptime_hours": round((time.time() - psutil.boot_time()) / 3600, 1),
+        },
+    }
+
+    # CPU frequency
+    try:
+        freq = psutil.cpu_freq()
+        if freq:
+            data["custom"]["cpu_freq_mhz"] = round(freq.current, 0)
+            data["custom"]["cpu_freq_max_mhz"] = round(freq.max, 0)
+    except Exception:
+        pass
+
+    # Context switches
+    try:
+        ctx = psutil.cpu_stats()
+        data["custom"]["ctx_switches"] = ctx.ctx_switches
+        data["custom"]["interrupts"] = ctx.interrupts
+    except Exception:
+        pass
+
+    # GPU
+    gpu = get_gpu_metrics()
+    if gpu:
+        data["gpu_temp"] = gpu.get("gpu_temp")
+        data["gpu_vram"] = gpu.get("gpu_vram")
+        data["custom"]["gpu"] = gpu
+
+    return data
+
+
+def collect_deep_snapshot() -> dict:
+    """Full system snapshot for variable inspector."""
+    snapshot = {
+        "system": {
+            "hostname": socket.gethostname(),
+            "uptime_hours": round((time.time() - psutil.boot_time()) / 3600, 1),
+            "cpu_count": psutil.cpu_count(),
+            "memory_total_gb": round(psutil.virtual_memory().total / 1e9, 2),
+            "python_version": sys.version.split()[0],
+            "os": f"{os.uname().sysname} {os.uname().release}" if hasattr(os, "uname") else "unknown",
+            "kernel": run_cmd("uname -r 2>/dev/null"),
+        },
+        "gpu": get_gpu_metrics() if GPU_AVAILABLE else {"available": False},
+        "pm2": get_pm2_status(),
+        "disks": get_disk_details(),
+        "network": get_network_details(),
+        "top_processes": get_top_processes(15),
+        "zombie_processes": get_zombie_processes(),
+        "docker": get_docker_status(),
+        "systemd_failed": get_systemd_failed(),
+        "firewall": get_firewall_rules(),
+        "environment": get_env_sanitized(),
+    }
+
+    # Job state
+    state = job_tracker.get_state()
+    if state:
+        snapshot["current_job"] = state
+
+    return snapshot
+
+
+# --- Job State Tracking ---
+
+class JobTracker:
+    """Track job states from BullMQ or similar queue systems."""
+
+    def __init__(self):
+        self.current_job = None
+        self.job_history = deque(maxlen=50)
+        self.lock = threading.Lock()
+
+    def update(self, job_id: str, status: str, progress: float = 0,
+               last_action: str = "", error: str | None = None,
+               metadata: dict | None = None):
+        with self.lock:
+            job = {
+                "job_id": job_id,
+                "status": status,
+                "progress": progress,
+                "last_action": last_action,
+                "error": error,
+                "metadata": metadata or {},
+                "updated_at": datetime.now(timezone.utc).isoformat(),
+            }
+            self.current_job = job
+            self.job_history.append(job)
+
+    def get_state(self) -> dict | None:
+        with self.lock:
+            return self.current_job.copy() if self.current_job else None
+
+    def get_history(self, n: int = 20) -> list:
+        with self.lock:
+            return list(self.job_history)[-n:]
+
+    def send_update(self):
+        state = self.get_state()
+        if not state:
+            return
+        send({
+            "type": "inspect",
+            "source": SOURCE_NAME,
+            "platform": PLATFORM,
+            "version": VERSION,
+            "variables": {"current_job": state, "job_history": self.get_history()},
+        })
+
+
+# --- Public API for External Use ---
+
+job_tracker = JobTracker()
+stdout_capture = None
+stderr_capture = None
+
+
+def send_log(level: str, message: str, context: dict = None):
+    send({
+        "type": "log",
+        "source": SOURCE_NAME,
+        "platform": PLATFORM,
+        "version": VERSION,
+        "level": level,
+        "message": message,
+        "context": context or {},
+    })
+
+
+def send_error(title: str, stack_trace: str = "", context: dict = None):
+    send({
+        "type": "error",
+        "source": SOURCE_NAME,
+        "platform": PLATFORM,
+        "version": VERSION,
+        "title": title,
+        "stackTrace": stack_trace,
+        "context": context or {},
+    })
+
+
+def update_job(job_id: str, status: str, progress: float = 0,
+               last_action: str = "", error: str = None, metadata: dict = None):
+    job_tracker.update(job_id, status, progress, last_action, error, metadata)
+    job_tracker.send_update()
+
+    if error:
+        send_error(f"Job {job_id} failed: {error}", context={
+            "job_id": job_id, "status": status, "last_action": last_action,
+            **(metadata or {}),
+        })
+
+
+def capture_image_gen(job_id: str, model: str, params: dict,
+                      result: dict = None, error: str = None,
+                      duration_s: float = None):
+    """Specialized capture for image generation pipelines."""
+    context = {
+        "job_id": job_id,
+        "model": model,
+        "params": params,
+        "duration_s": duration_s,
+    }
+    if result:
+        context["result"] = result
+        send_log("info", f"Image gen complete: {model} ({duration_s:.1f}s)", context)
+    if error:
+        context["error"] = error
+        send_error(f"Image gen failed: {model} — {error}", context=context)
+    
+    update_job(job_id, "completed" if result else "failed",
+               progress=100 if result else 0,
+               last_action=f"generate:{model}",
+               error=error,
+               metadata=context)
+
+
+# --- Main Loop ---
+
+def main():
+    global stdout_capture, stderr_capture
+
+    # Capture stdout/stderr
+    stdout_capture = StreamCapture(sys.stdout, "info")
+    stderr_capture = StreamCapture(sys.stderr, "error")
+    sys.stdout = stdout_capture
+    sys.stderr = stderr_capture
+
+    logger.info(f"debugger.help Agent v{VERSION} — Ultimate Deep Debugger")
+    logger.info(f"Source: {SOURCE_NAME} | GPU: {'yes' if GPU_AVAILABLE else 'no'} | Docker: {'yes' if DOCKER_AVAILABLE else 'no'}")
+    logger.info(f"Interval: {INTERVAL}s | Endpoint: {INGEST_URL}")
+
+    # Start file watchers
+    watch_files = list(WATCH_LOG_FILES)
+    # Auto-discover pm2 log files
+    pm2_log_dir = os.path.expanduser("~/.pm2/logs")
+    if os.path.isdir(pm2_log_dir):
+        pm2_logs = glob.glob(os.path.join(pm2_log_dir, "*.log"))
+        watch_files.extend(pm2_logs)
+        logger.info(f"Watching {len(pm2_logs)} pm2 log files")
+    
+    if watch_files:
+        watcher = LogFileWatcher(watch_files)
+        watcher.start()
+        logger.info(f"File watcher started for {len(watch_files)} files")
+
+    # Initial connection
+    send({"type": "heartbeat", "source": SOURCE_NAME, "platform": PLATFORM, "version": VERSION})
+    send_log("info", f"Agent v{VERSION} started on {socket.gethostname()}", {
+        "hostname": socket.gethostname(),
+        "python_version": sys.version,
+        "gpu_available": GPU_AVAILABLE,
+        "docker_available": DOCKER_AVAILABLE,
+        "pid": os.getpid(),
+        "gpu_driver": GPU_DRIVER_VERSION,
+        "cuda_version": GPU_CUDA_VERSION,
+    })
+
+    # Send initial deep snapshot with packages
+    try:
+        pkgs = get_installed_packages()
+        send({
+            "type": "inspect",
+            "source": SOURCE_NAME,
+            "platform": PLATFORM,
+            "version": VERSION,
+            "variables": {"installed_packages": pkgs},
+        })
+    except Exception:
+        pass
+
+    tick = 0
+    consecutive_failures = 0
+
+    while True:
+        try:
+            # Metrics every tick
+            metrics = collect_full_metrics()
+            ok = send({
+                "type": "metric",
+                "source": SOURCE_NAME,
+                "platform": PLATFORM,
+                "version": VERSION,
+                **metrics,
+            })
+
+            if ok:
+                consecutive_failures = 0
+            else:
+                consecutive_failures += 1
+
+            # Auto-reconnect backoff
+            if consecutive_failures > 5:
+                backoff = min(consecutive_failures * 5, 60)
+                logger.warning(f"Connection issues. Backing off {backoff}s...")
+                time.sleep(backoff)
+                send({"type": "heartbeat", "source": SOURCE_NAME, "platform": PLATFORM, "version": VERSION})
+                consecutive_failures = 0
+                continue
+
+            # Deep snapshot every ~1 min
+            if tick % 6 == 0:
+                snapshot = collect_deep_snapshot()
+                send({
+                    "type": "inspect",
+                    "source": SOURCE_NAME,
+                    "platform": PLATFORM,
+                    "version": VERSION,
+                    "variables": snapshot,
+                })
+
+            # PM2 logs check every ~30 seconds
+            if tick % 3 == 0:
+                pm2_logs = get_pm2_logs(50)
+                if pm2_logs:
+                    # Check for errors in pm2 logs
+                    for line in pm2_logs.split("\n"):
+                        lower = line.lower()
+                        if any(kw in lower for kw in ["error", "exception", "failed", "crash", "enoent", "eacces", "killed"]):
+                            send_log("error", f"[pm2] {line.strip()[:500]}", {"capturedFrom": "pm2_logs"})
+
+            # System logs check every ~2 min
+            if tick % 12 == 0:
+                sys_logs = get_system_logs()
+                if sys_logs:
+                    send({
+                        "type": "inspect",
+                        "source": SOURCE_NAME,
+                        "platform": PLATFORM,
+                        "version": VERSION,
+                        "variables": {"system_logs": sys_logs},
+                    })
+
+            # SSL cert check every ~10 min
+            if tick % 60 == 0 and SSL_CHECK_DOMAINS:
+                ssl_results = check_ssl_certs()
+                for r in ssl_results:
+                    if r.get("warning") or r.get("error"):
+                        send_log("warn", f"SSL cert issue: {r.get('domain')} — {r.get('error') or f'{r.get(\"days_left\")} days left'}", r)
+                send({
+                    "type": "inspect",
+                    "source": SOURCE_NAME,
+                    "platform": PLATFORM,
+                    "version": VERSION,
+                    "variables": {"ssl_certs": ssl_results},
+                })
+
+            # GPU warnings every ~30 seconds
+            if GPU_AVAILABLE and tick % 3 == 0:
+                gpu = get_gpu_metrics()
+                gpus = gpu.get("gpus", [])
+                for g in gpus:
+                    if g.get("temp_c", 0) > 85:
+                        send_log("warn", f"GPU {g['index']} temperature critical: {g['temp_c']}°C", g)
+                    if g.get("vram_pct", 0) > 90:
+                        send_log("warn", f"GPU {g['index']} VRAM critical: {g['vram_pct']}% ({g['vram_used_gb']}/{g['vram_total_gb']} GB)", g)
+                    throttle = g.get("throttle_reasons", [])
+                    if throttle and throttle != ["none"] and throttle != ["idle"]:
+                        send_log("warn", f"GPU {g['index']} throttling: {', '.join(throttle)}", g)
+
+            # Heartbeat every ~5 min
+            if tick % 30 == 0 and tick > 0:
+                send({"type": "heartbeat", "source": SOURCE_NAME, "platform": PLATFORM, "version": VERSION})
+
+            # Packages update every ~30 min
+            if tick % 180 == 0 and tick > 0:
+                pkgs = get_installed_packages()
+                send({
+                    "type": "inspect",
+                    "source": SOURCE_NAME,
+                    "platform": PLATFORM,
+                    "version": VERSION,
+                    "variables": {"installed_packages": pkgs},
+                })
+
+            tick += 1
+            time.sleep(INTERVAL)
+
+        except KeyboardInterrupt:
+            send_log("info", "Agent shutting down gracefully")
+            logger.info("Shutting down...")
+            sys.stdout = stdout_capture.original
+            sys.stderr = stderr_capture.original
+            break
+        except Exception as e:
+            logger.error(f"Main loop error: {e}")
+            send_error(str(e), traceback.format_exc())
+            time.sleep(INTERVAL)
+
+
+if __name__ == "__main__":
+    main()

debugger_help-3.0.0.dist-info/METADATA

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: debugger-help
+Version: 3.0.0
+Summary: debugger.help VPS Agent — Deep system monitoring for logs, GPU, PM2, Docker, and more
+Author: debugger.help
+License: MIT
+Project-URL: Homepage, https://debugger.help
+Project-URL: Repository, https://github.com/YOUR_ORG/debugger-help
+Keywords: debugger,monitoring,logging,gpu,vps,observability
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Monitoring
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: psutil>=5.9.0
+Requires-Dist: requests>=2.28.0
+Provides-Extra: gpu
+Requires-Dist: pynvml>=11.0.0; extra == "gpu"
+Provides-Extra: docker
+Requires-Dist: docker>=6.0.0; extra == "docker"
+Provides-Extra: all
+Requires-Dist: pynvml>=11.0.0; extra == "all"
+Requires-Dist: docker>=6.0.0; extra == "all"
+
+# debugger-help
+
+Deep VPS monitoring agent for [debugger.help](https://debugger.help). Captures PM2, GPU metrics, Docker containers, system health, and streams everything to your dashboard in real time.
+
+## Install
+
+```bash
+pip install debugger-help
+```
+
+With GPU monitoring:
+```bash
+pip install debugger-help[gpu]
+```
+
+With everything:
+```bash
+pip install debugger-help[all]
+```
+
+## Quick Start
+
+```bash
+export DEBUGGER_API_KEY="your-api-key"
+export DEBUGGER_INGEST_URL="your-ingest-url"
+debugger-agent
+```
+
+Or keep it running with PM2:
+```bash
+pm2 start "debugger-agent" --name debugger-agent
+pm2 save
+```
+
+## What it captures
+
+- PM2 process states, restart counts, logs, error logs
+- GPU: VRAM per-process, temperature, power, utilization, clock speeds, throttle reasons, ECC errors
+- CPU: per-core usage, load average, frequency, context switches
+- Memory: RAM, swap, shared, buffers, cached
+- Disk: usage per mount, I/O rates, inode usage
+- Network: per-interface stats, TCP connection states, open ports, DNS resolution
+- Processes: top CPU/memory consumers, zombies, open file descriptors
+- Docker containers: status, CPU, memory, restart counts
+- Systemd: failed units
+- SSL certificates: expiry checks
+- System logs: dmesg, OOM kills, journalctl errors
+- File watchers: syslog, PM2 logs, custom log files
+
+## Environment Variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `DEBUGGER_API_KEY` | Yes | Your debugger.help API key |
+| `DEBUGGER_INGEST_URL` | Yes | Your ingest endpoint URL |
+| `DEBUGGER_SOURCE` | No | Source name (default: `vps-{hostname}`) |
+| `DEBUGGER_INTERVAL` | No | Collection interval in seconds (default: `10`) |
+| `DEBUGGER_WATCH_LOGS` | No | Comma-separated extra log file paths |
+| `DEBUGGER_SSL_DOMAINS` | No | Comma-separated domains for SSL checks |
+
+## License
+
+MIT

debugger_help-3.0.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+debugger_help/__init__.py,sha256=LvFFmKY3_tMH7u9l7pif6OQX9YLTzXD-lAtpT16bB60,80
+debugger_help/agent.py,sha256=NWwOZlL-0PafD1ZfRNGnr6j1DGLJ7x_W-a3JcYuVgkU,40545
+debugger_help-3.0.0.dist-info/METADATA,sha256=QFtQNHPDsFCGntsmXJB7ZQBjf_bqaI65qsTZNPUKfZo,2803
+debugger_help-3.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+debugger_help-3.0.0.dist-info/entry_points.txt,sha256=TbCJip8NfrK6ArjhHyLBAJLMDVtKTxm-IYwRNULNXqo,60
+debugger_help-3.0.0.dist-info/top_level.txt,sha256=Mvew_obR30M1IogknxotpcHXyI4x4EqwWUMUyG94MaU,14
+debugger_help-3.0.0.dist-info/RECORD,,

debugger_help-3.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

debugger_help-3.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+debugger-agent = debugger_help.agent:main

debugger_help-3.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+debugger_help