castrel-proxy 0.1.3__tar.gz → 0.1.4__tar.gz

{castrel_proxy-0.1.3 → castrel_proxy-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: castrel-proxy
-Version: 0.1.3
+Version: 0.1.4
 Summary: A lightweight remote command execution bridge client with MCP integration
 Project-URL: Homepage, https://github.com/castrel-ai/castrel-proxy
 Project-URL: Documentation, https://github.com/castrel-ai/castrel-proxy#readme
@@ -44,6 +44,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Networking
 Requires-Python: >=3.10
 Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: file-read-backwards>=2.0.0
 Requires-Dist: langchain-mcp-adapters>=0.2.1
 Requires-Dist: mcp>=1.0.0
 Requires-Dist: pyyaml>=6.0.1

{castrel_proxy-0.1.3 → castrel_proxy-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "castrel-proxy"
-version = "0.1.3"
+version = "0.1.4"
 description = "A lightweight remote command execution bridge client with MCP integration"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -28,6 +28,7 @@ dependencies = [
     "aiohttp>=3.9.0",
     "mcp>=1.0.0",
     "langchain-mcp-adapters>=0.2.1",
+    "file-read-backwards>=2.0.0",
 ]
 
 [project.optional-dependencies]

{castrel_proxy-0.1.3 → castrel_proxy-0.1.4}/src/castrel_proxy/core/__init__.py

@@ -4,6 +4,7 @@ from .client_id import get_client_id, get_machine_metadata
 from .config import Config, ConfigError, get_config
 from .daemon import DaemonManager, get_daemon_manager
 from .executor import CommandExecutor, ExecutionResult
+from .openclaw import OpenClawChecker, HealthStatus
 
 __all__ = [
     "get_client_id",
@@ -15,4 +16,6 @@ __all__ = [
     "get_daemon_manager",
     "CommandExecutor",
     "ExecutionResult",
+    "OpenClawChecker",
+    "HealthStatus",
 ]

{castrel_proxy-0.1.3 → castrel_proxy-0.1.4}/src/castrel_proxy/core/config.py

@@ -4,7 +4,7 @@ Configuration File Management Module
 Handles reading, writing, and validating the ~/.castrel/config.yaml configuration file
 """
 
-from datetime import datetime
+from datetime import datetime, UTC
 from pathlib import Path
 from typing import Optional
 
@@ -56,14 +56,58 @@ class Config:
         """
         self._ensure_config_dir()
 
+        # Load existing config to preserve openclaw settings
+        existing_config = {}
+        if self.config_file.exists():
+            try:
+                with open(self.config_file, "r", encoding="utf-8") as f:
+                    existing_config = yaml.safe_load(f) or {}
+            except Exception:
+                # If loading fails, start with empty dict
+                existing_config = {}
+
         config_data = {
             "server_url": server_url,
             "verification_code": verification_code,
             "client_id": client_id,
             "workspace_id": workspace_id,
-            "paired_at": datetime.utcnow().isoformat() + "Z",
+            "paired_at": datetime.now(UTC).isoformat() + "Z",
         }
 
+        # Preserve or initialize openclaw configuration
+        # If exists in existing config, preserve it; otherwise use default values from getters
+        # Note: openclaw_runtime_log_path should not be saved if empty, to allow date-based rotation
+        
+        # openclaw_check_enabled
+        if "openclaw_check_enabled" in existing_config:
+            config_data["openclaw_check_enabled"] = existing_config["openclaw_check_enabled"]
+        else:
+            config_data["openclaw_check_enabled"] = False
+        
+        # openclaw_config_path
+        if "openclaw_config_path" in existing_config:
+            config_data["openclaw_config_path"] = existing_config["openclaw_config_path"]
+        else:
+            config_data["openclaw_config_path"] = str(Path.home() / ".openclaw" / "openclaw.json")
+        
+        # openclaw_runtime_log_path - only save if not empty
+        if "openclaw_runtime_log_path" in existing_config:
+            runtime_log_path = existing_config["openclaw_runtime_log_path"]
+            if runtime_log_path:  # Only save if not empty
+                config_data["openclaw_runtime_log_path"] = runtime_log_path
+        
+        # openclaw_gateway_log_path
+        if "openclaw_gateway_log_path" in existing_config:
+            config_data["openclaw_gateway_log_path"] = existing_config["openclaw_gateway_log_path"]
+        else:
+            config_data["openclaw_gateway_log_path"] = str(Path.home() / ".openclaw" / "logs" / "gateway.err.log")
+        
+        # openclaw_agents_dir
+        if "openclaw_agents_dir" in existing_config:
+            config_data["openclaw_agents_dir"] = existing_config["openclaw_agents_dir"]
+        else:
+            config_data["openclaw_agents_dir"] = str(Path.home() / ".openclaw" / "agents")
+
         try:
             with open(self.config_file, "w", encoding="utf-8") as f:
                 yaml.safe_dump(config_data, f, default_flow_style=False, allow_unicode=True)
@@ -148,6 +192,59 @@ class Config:
         """Get workspace ID"""
         return self.load()["workspace_id"]
 
+    def get_openclaw_check_enabled(self) -> bool:
+        """
+        Get OpenClaw check enabled status
+        
+        Returns:
+            bool: True if OpenClaw check is enabled, False otherwise (default)
+        """
+        try:
+            config = self.load()
+            return config.get("openclaw_check_enabled", False)
+        except ConfigError:
+            # If config doesn't exist, return False
+            return False
+
+    def get_openclaw_config_path(self) -> str:
+        """
+        Get OpenClaw config file path
+        
+        Returns:
+            str: Path to OpenClaw config file (~/.openclaw/openclaw.json)
+        """
+        try:
+            config = self.load()
+            return config.get("openclaw_config_path", str(Path.home() / ".openclaw" / "openclaw.json"))
+        except ConfigError:
+            return str(Path.home() / ".openclaw" / "openclaw.json")
+
+    def get_openclaw_gateway_log_path(self) -> str:
+        """
+        Get OpenClaw gateway error log path
+        
+        Returns:
+            str: Path to OpenClaw gateway error log file
+        """
+        try:
+            config = self.load()
+            return config.get("openclaw_gateway_log_path", str(Path.home() / ".openclaw" / "logs" / "gateway.err.log"))
+        except ConfigError:
+            return str(Path.home() / ".openclaw" / "logs" / "gateway.err.log")
+
+    def get_openclaw_agents_dir(self) -> str:
+        """
+        Get OpenClaw agents directory path
+        
+        Returns:
+            str: Path to OpenClaw agents directory
+        """
+        try:
+            config = self.load()
+            return config.get("openclaw_agents_dir", str(Path.home() / ".openclaw" / "agents"))
+        except ConfigError:
+            return str(Path.home() / ".openclaw" / "agents")
+
 
 # Global configuration instance
 _config = Config()

castrel_proxy-0.1.4/src/castrel_proxy/core/openclaw.py

@@ -0,0 +1,562 @@
+"""
+OpenClaw Check Module
+
+Responsible for performing OpenClaw health checks and returning structured status information
+"""
+
+import json
+import logging
+import re
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import file_read_backwards
+
+from ..core.config import get_config
+
+logger = logging.getLogger(__name__)
+
+
+class HealthStatus:
+    """Health status result"""
+
+    def __init__(
+        self,
+        status: str,
+        message: str,
+        details: Optional[Dict] = None,
+    ):
+        """
+        Initialize health status
+
+        Args:
+            status: Status level ("healthy", "warning", "error")
+            message: Human-readable status message
+            details: Additional check-specific parameters
+        """
+        self.status = status
+        self.message = message
+        self.details = details or {}
+
+    def to_dict(self) -> Dict:
+        """Convert to dictionary format"""
+        return {
+            "status": self.status,
+            "message": self.message,
+            "details": self.details,
+        }
+
+
+class OpenClawChecker:
+    """OpenClaw health checker"""
+
+    def __init__(self):
+        """
+        Initialize OpenClaw checker
+        """
+        self.config = get_config()
+        
+        # Get paths from config (runtime_log_path will be dynamic, see _get_runtime_log_path)
+        self.openclaw_config_path = Path(self.config.get_openclaw_config_path())
+        self.gateway_log_path = Path(self.config.get_openclaw_gateway_log_path())
+        self.agents_dir = Path(self.config.get_openclaw_agents_dir())
+        
+        # Store configured runtime log path (may be empty for date-based rotation)
+        self._configured_runtime_log_path = None
+        self._load_configured_runtime_log_path()
+        
+        # Load runtime log rules from file
+        self.runtime_log_rules = self._load_runtime_log_rules()
+        
+        # Get maximum window minutes from rules
+        self._max_window_minutes = max(
+            (rule.get("window_minutes", 1) for rule in self.runtime_log_rules),
+            default=10
+        )
+
+    def _load_runtime_log_rules(self) -> List[Dict]:
+        """
+        Load runtime log rules from data file
+
+        Returns:
+            List[Dict]: List of check rules
+        """
+        try:
+            # Get path to rules file
+            rules_file = Path(__file__).parent.parent / "data" / "openclaw_runtime_log_rules.json"
+            with open(rules_file, "r", encoding="utf-8") as f:
+                rules = json.load(f)
+                logger.info(f"Loaded {len(rules)} OpenClaw runtime log rules")
+                return rules
+        except Exception as e:
+            logger.error(f"Failed to load runtime log rules: {e}", exc_info=True)
+            # Return empty list if rules file cannot be loaded
+            return []
+
+    def _load_configured_runtime_log_path(self):
+        """
+        Load configured runtime log path from config or OpenClaw config file
+        
+        Note: If openclaw_runtime_log_path is explicitly set to empty string in castrel config,
+        it means use date-based rotation, so we won't try OpenClaw config file.
+        """
+        # First, try to get from castrel config
+        try:
+            config_data = self.config.load()
+            # Check if key exists (even if value is empty string)
+            if "openclaw_runtime_log_path" in config_data:
+                configured_path = config_data["openclaw_runtime_log_path"]
+                if configured_path:  # Only set if not empty
+                    self._configured_runtime_log_path = Path(configured_path)
+                    logger.debug(f"Loaded runtime log path from castrel config: {self._configured_runtime_log_path}")
+                # If empty string, it means use date-based rotation, so don't try OpenClaw config
+                return
+        except Exception:
+            pass
+        
+        # If not in castrel config, try OpenClaw config file
+        try:
+            if self.openclaw_config_path.exists():
+                with open(self.openclaw_config_path, "r", encoding="utf-8") as f:
+                    config = json.load(f)
+                    logging_config = config.get("logging", {})
+                    log_file = logging_config.get("file")
+                    if log_file:
+                        self._configured_runtime_log_path = Path(log_file)
+                        logger.debug(f"Loaded runtime log path from OpenClaw config: {self._configured_runtime_log_path}")
+        except Exception as e:
+            logger.warning(f"Failed to read OpenClaw config for log path: {e}")
+    
+    def _get_runtime_log_path(self) -> Path:
+        """
+        Get current runtime log path (dynamic, based on current date if not configured)
+        
+        Returns:
+            Path: Current runtime log path
+        """
+        # If configured path exists, use it
+        if self._configured_runtime_log_path:
+            return self._configured_runtime_log_path
+        
+        # Otherwise, use date-based path (YYYY-MM-DD format, based on local time)
+        local_date = datetime.now().strftime("%Y-%m-%d")
+        return Path(f"/tmp/openclaw/openclaw-{local_date}.log")
+
+    def _parse_timestamp(self, timestamp_str: str) -> Optional[datetime]:
+        """
+        Parse timestamp from string (ISO format or Unix timestamp)
+
+        Args:
+            timestamp_str: Timestamp string
+
+        Returns:
+            Optional[datetime]: Parsed datetime (timezone-aware, UTC) or None if failed
+        """
+        try:
+            # Try ISO format first
+            dt = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00"))
+            # Convert to UTC if timezone-aware, otherwise assume UTC
+            if dt.tzinfo is not None:
+                dt = dt.astimezone(timezone.utc)
+            else:
+                dt = dt.replace(tzinfo=timezone.utc)
+            return dt
+        except (ValueError, AttributeError):
+            try:
+                # Try Unix timestamp (milliseconds or seconds)
+                ts = float(timestamp_str)
+                if ts > 1e12:  # Milliseconds
+                    ts = ts / 1000
+                # fromtimestamp returns local time, convert to UTC
+                dt = datetime.fromtimestamp(ts, tz=timezone.utc)
+                return dt
+            except (ValueError, OSError):
+                return None
+
+    def _parse_jsonl_line(self, line: str) -> Optional[Dict]:
+        """
+        Parse a JSONL line
+
+        Args:
+            line: JSONL line string
+
+        Returns:
+            Optional[Dict]: Parsed JSON object or None if failed
+        """
+        try:
+            return json.loads(line.strip())
+        except (json.JSONDecodeError, AttributeError):
+            return None
+
+    def _read_lines_for_window(self, file_path: Path, now: datetime) -> List[str]:
+        """
+        Read lines from file covering the maximum window time using reverse reading
+        
+        Strategy:
+        - Read backwards from end of file
+        - Collect lines until we reach a line older than (now - max_window_minutes)
+        - Reverse the collected lines to get chronological order
+        
+        Args:
+            file_path: Path to file
+            now: Current datetime
+            
+        Returns:
+            List[str]: List of lines covering the time window (in chronological order)
+        """
+        lines = []
+        
+        if not file_path.exists():
+            return lines
+
+        try:
+            # Calculate cutoff time: lines older than this won't be included
+            cutoff_time = now - timedelta(minutes=self._max_window_minutes)
+            
+            # Read backwards from end of file
+            # Limit to reasonable number of lines to avoid reading entire file if timestamps are missing
+            max_lines = 100000  # Safety limit
+            line_count = 0
+            
+            with file_read_backwards.FileReadBackwards(file_path, encoding="utf-8") as f:
+                for line in f:
+                    line_count += 1
+                    if line_count > max_lines:
+                        logger.warning(f"Reached max_lines limit ({max_lines}) when reading {file_path}")
+                        break
+                    
+                    # Parse line to check timestamp
+                    log_entry = self._parse_jsonl_line(line)
+                    if log_entry:
+                        time_str = log_entry.get("time")
+                        if time_str:
+                            log_time = self._parse_timestamp(time_str)
+                            if log_time:
+                                # Stop if we've gone past the cutoff time
+                                if log_time < cutoff_time:
+                                    break
+                                # Add line to collection (newest first, no reversal needed)
+                                lines.append(line)
+                            else:
+                                # Timestamp parsing failed, include the line anyway
+                                lines.append(line)
+                        else:
+                            # No timestamp, include the line anyway (might be recent)
+                            lines.append(line)
+                    else:
+                        # Failed to parse JSON, include the line anyway
+                        lines.append(line)
+            
+            # Keep lines in reverse order (newest first) - no reversal needed
+                
+        except Exception as e:
+            logger.error(f"Failed to read lines from {file_path}: {e}", exc_info=True)
+
+        return lines
+
+    def _check_runtime_log(self, now: datetime) -> List[Dict]:
+        """
+        Check runtime log for issues
+
+        Args:
+            now: Current datetime
+
+        Returns:
+            List[Dict]: List of detected issues
+        """
+        issues = []
+        
+        runtime_log_path = self._get_runtime_log_path()
+
+        if not runtime_log_path.exists():
+            return issues
+
+        try:
+            # Read lines covering the maximum window time
+            lines = self._read_lines_for_window(runtime_log_path, now)
+            
+            if not lines:
+                return issues
+
+            # Check each rule
+            # Lines are in reverse order (newest first), so we iterate from newest to oldest
+            for rule in self.runtime_log_rules:
+                window_start = now - timedelta(minutes=rule["window_minutes"])
+                matches = 0
+
+                # Iterate from newest to oldest (lines are already in reverse order)
+                for line in lines:
+                    # Parse JSONL line
+                    log_entry = self._parse_jsonl_line(line)
+                    if not log_entry:
+                        continue
+
+                    # Get timestamp
+                    time_str = log_entry.get("time")
+                    if not time_str:
+                        continue
+
+                    log_time = self._parse_timestamp(time_str)
+                    if not log_time:
+                        continue
+                    
+                    # If we've gone past the window start, we can break (lines are in reverse order)
+                    if log_time < window_start:
+                        break
+
+                    # Check if line matches pattern
+                    log_text = json.dumps(log_entry, ensure_ascii=False)
+                    if re.search(rule["pattern"], log_text, re.IGNORECASE):
+                        matches += 1
+
+                # Check threshold
+                if matches >= rule["threshold"]:
+                    issues.append(
+                        {
+                            "status": "error",
+                            "message": rule["name"],
+                            "details": {
+                                "rule": rule["name"],
+                                "matches": matches,
+                                "threshold": rule["threshold"],
+                                "window_minutes": rule["window_minutes"],
+                            },
+                        }
+                    )
+
+        except Exception as e:
+            logger.error(f"Failed to check runtime log: {e}", exc_info=True)
+
+        return issues
+
+    def _check_gateway_log(self, now: datetime) -> Optional[Dict]:
+        """
+        Check gateway error log
+
+        Args:
+            now: Current datetime
+
+        Returns:
+            Optional[Dict]: Issue if detected, None otherwise
+        """
+        if not self.gateway_log_path.exists():
+            return None
+
+        try:
+            # Check if file was modified in the last minute
+            mtime = datetime.fromtimestamp(self.gateway_log_path.stat().st_mtime, tz=timezone.utc)
+            if mtime >= now - timedelta(minutes=1):
+                return {
+                    "status": "error",
+                    "message": "Daemon process error",
+                    "details": {
+                        "log_path": str(self.gateway_log_path),
+                        "last_modified": mtime.isoformat(),
+                    },
+                }
+        except Exception as e:
+            logger.error(f"Failed to check gateway log: {e}", exc_info=True)
+
+        return None
+
+    def _check_agent_logs(self, now: datetime) -> Optional[Dict]:
+        """
+        Check agent session logs
+
+        Args:
+            now: Current datetime
+
+        Returns:
+            Optional[Dict]: Issue if detected, None otherwise
+        """
+        if not self.agents_dir.exists():
+            return None
+
+        try:
+            # Find all JSONL files in sessions directories
+            session_files = []
+            for agent_dir in self.agents_dir.iterdir():
+                if not agent_dir.is_dir():
+                    continue
+                sessions_dir = agent_dir / "sessions"
+                if not sessions_dir.exists():
+                    continue
+
+                for jsonl_file in sessions_dir.glob("*.jsonl"):
+                    # Check if file was modified in the last minute
+                    try:
+                        mtime = datetime.fromtimestamp(jsonl_file.stat().st_mtime, tz=timezone.utc)
+                        if mtime >= now - timedelta(minutes=1):
+                            session_files.append(jsonl_file)
+                    except OSError:
+                        continue
+
+            if not session_files:
+                return None
+
+            # Check logs from the past 5 minutes
+            window_start = now - timedelta(minutes=5)
+            total_lines = 0
+            error_lines = 0
+
+            for jsonl_file in session_files:
+                try:
+                    # Read backwards from end of file, check lines from past 5 minutes
+                    with file_read_backwards.FileReadBackwards(jsonl_file, encoding="utf-8") as f:
+                        for line in f:
+                            log_entry = self._parse_jsonl_line(line)
+                            if not log_entry:
+                                continue
+
+                            # Get timestamp
+                            timestamp = log_entry.get("timestamp")
+                            if not timestamp:
+                                continue
+
+                            log_time = self._parse_timestamp(str(timestamp))
+                            if not log_time:
+                                continue
+                            
+                            # If we've gone past the window start, we can break (reading backwards)
+                            if log_time < window_start:
+                                break
+
+                            total_lines += 1
+
+                            # Check if it's an error line
+                            error_message = log_entry.get("errorMessage")
+                            is_error = log_entry.get("isError", False)
+
+                            if (error_message and error_message.strip()) or is_error:
+                                error_lines += 1
+
+                except Exception as e:
+                    logger.warning(f"Failed to read agent log {jsonl_file}: {e}")
+
+            # Check error rate
+            if total_lines > 0:
+                error_rate = error_lines / total_lines
+                if error_rate >= 0.3:
+                    return {
+                        "status": "error",
+                        "message": "Agent failure rate too high",
+                        "details": {
+                            "error_rate": error_rate,
+                            "error_lines": error_lines,
+                            "total_lines": total_lines,
+                            "checked_files": len(session_files),
+                        },
+                    }
+
+        except Exception as e:
+            logger.error(f"Failed to check agent logs: {e}", exc_info=True)
+
+        return None
+
+    async def check_all(self) -> HealthStatus:
+        """
+        Perform all OpenClaw checks
+
+        Returns:
+            HealthStatus: Overall health status with aggregated details
+        """
+        try:
+            all_details = {}
+            overall_status = "healthy"
+            messages = []
+
+            # Perform OpenClaw check
+            openclaw_status = await self.check_openclaw_status()
+            if openclaw_status:
+                all_details["openclaw"] = openclaw_status
+                if openclaw_status.get("status") != "healthy":
+                    overall_status = openclaw_status.get("status", "warning")
+                    messages.append(openclaw_status.get("message", ""))
+
+            # Determine overall message
+            if overall_status == "healthy":
+                message = "All checks passed"
+            else:
+                message = "; ".join(messages) if messages else "OpenClaw check issues detected"
+
+            return HealthStatus(
+                status=overall_status,
+                message=message,
+                details=all_details,
+            )
+
+        except Exception as e:
+            logger.error(f"OpenClaw check failed: {e}", exc_info=True)
+            return HealthStatus(
+                status="error",
+                message=f"OpenClaw check exception: {str(e)}",
+                details={},
+            )
+
+    async def check_openclaw_status(self) -> Optional[Dict]:
+        """
+        Check OpenClaw status (by reading files)
+
+        Returns:
+            Optional[Dict]: OpenClaw status, or None if check cannot be performed
+        """
+        try:
+            now = datetime.now(timezone.utc)
+            issues = []
+
+            # Check runtime log
+            runtime_issues = self._check_runtime_log(now)
+            issues.extend(runtime_issues)
+
+            # Check gateway log
+            gateway_issue = self._check_gateway_log(now)
+            if gateway_issue:
+                issues.append(gateway_issue)
+
+            # Check agent logs
+            agent_issue = self._check_agent_logs(now)
+            if agent_issue:
+                issues.append(agent_issue)
+
+            # Determine overall status
+            if issues:
+                # Get the most severe status
+                statuses = [issue.get("status", "warning") for issue in issues]
+                if "error" in statuses:
+                    overall_status = "error"
+                else:
+                    overall_status = "warning"
+
+                # Combine messages
+                messages = [issue.get("message", "") for issue in issues]
+                message = "; ".join(messages)
+
+                return {
+                    "status": overall_status,
+                    "message": message,
+                    "details": {
+                        "issues": issues,
+                        "runtime_log_path": str(self._get_runtime_log_path()),
+                        "gateway_log_path": str(self.gateway_log_path),
+                        "agents_dir": str(self.agents_dir),
+                    },
+                }
+            else:
+                return {
+                    "status": "healthy",
+                    "message": "All OpenClaw checks passed",
+                    "details": {
+                        "runtime_log_path": str(self._get_runtime_log_path()),
+                        "gateway_log_path": str(self.gateway_log_path),
+                        "agents_dir": str(self.agents_dir),
+                    },
+                }
+
+        except Exception as e:
+            logger.error(f"OpenClaw status check failed: {e}", exc_info=True)
+            return {
+                "status": "error",
+                "message": f"OpenClaw status check failed: {str(e)}",
+                "details": {},
+            }

castrel_proxy-0.1.4/src/castrel_proxy/data/openclaw_runtime_log_rules.json

@@ -0,0 +1,50 @@
+[
+  {
+    "pattern": "panic:|fatal|runtime error|segmentation fault|exit status",
+    "window_minutes": 1,
+    "threshold": 1,
+    "name": "Gateway process crash restart"
+  },
+  {
+    "pattern": "rpc server stopped|failed to serve rpc|listen tcp :4399",
+    "window_minutes": 1,
+    "threshold": 1,
+    "name": "RPC port disconnection"
+  },
+  {
+    "pattern": "all llm providers failed|All models failed|no available llm provider",
+    "window_minutes": 1,
+    "threshold": 1,
+    "name": "LLM model permanent failure"
+  },
+  {
+    "pattern": "rate limited|429 Too Many Requests|503 Service Unavailable",
+    "window_minutes": 1,
+    "threshold": 8,
+    "name": "Single model continuous 503/429"
+  },
+  {
+    "pattern": "tool execution timeout|context deadline exceeded.*tool",
+    "window_minutes": 2,
+    "threshold": 5,
+    "name": "Tool call continuous timeout"
+  },
+  {
+    "pattern": "created new page.*chrome",
+    "window_minutes": 5,
+    "threshold": 75,
+    "name": "Browser instance leak"
+  },
+  {
+    "pattern": "session ended unexpectedly|agent terminated by signal",
+    "window_minutes": 5,
+    "threshold": 5,
+    "name": "Agent session abnormal interruption"
+  },
+  {
+    "pattern": "tool.*failed permanently|(max retries exceeded.*(browser|google_search|fetch)|(browser|google_search|fetch).*max retries exceeded)",
+    "window_minutes": 10,
+    "threshold": 10,
+    "name": "Critical tool permanent failure"
+  }
+]

{castrel_proxy-0.1.3 → castrel_proxy-0.1.4}/src/castrel_proxy/network/websocket_client.py

@@ -16,6 +16,8 @@ import aiohttp
 
 from ..operations import document
 from ..core.executor import CommandExecutor
+from ..core.openclaw import OpenClawChecker
+from ..core.config import get_config
 from ..mcp.manager import get_mcp_manager
 from ..security.whitelist import get_whitelist_file_path, is_command_allowed
 
@@ -55,6 +57,12 @@ class WebSocketClient:
         self.session: Optional[aiohttp.ClientSession] = None
         self.heartbeat_task: Optional[asyncio.Task] = None
         self.heartbeat_interval = 30.0  # 30seconds，Ensure enough heartbeats before server timeout
+        # OpenClaw check related
+        self.openclaw_checker = OpenClawChecker()
+        self.openclaw_check_enabled = get_config().get_openclaw_check_enabled()
+        self.openclaw_check_task: Optional[asyncio.Task] = None
+        self.openclaw_check_interval = 60.0  # OpenClaw check interval (seconds)
+        self.last_openclaw_status = None  # Track last status to avoid duplicate notifications
 
     def _get_ws_url(self) -> str:
         """Get WebSocket URL"""
@@ -333,6 +341,119 @@ class WebSocketClient:
 
         logger.info(f"[CLIENT-HEARTBEAT-STOP] Heartbeat task stopped: client_id={self.client_id}")
 
+    async def _perform_openclaw_check(self) -> dict:
+        """
+        Perform OpenClaw check
+        
+        Returns:
+            dict: OpenClaw status with the following structure:
+                {
+                    "status": "healthy|warning|error",
+                    "message": "Status description",
+                    "details": {
+                        # Check-specific parameters from OpenClawChecker
+                    }
+                }
+        """
+        openclaw_status = await self.openclaw_checker.check_all()
+        return openclaw_status.to_dict()
+
+    async def _openclaw_check_loop(self):
+        """OpenClaw check loop - independent coroutine that sends notifications when issues are detected"""
+        logger.info(
+            f"[CLIENT-OPENCLAW-CHECK-START] OpenClaw check task started: "
+            f"interval={self.openclaw_check_interval}s, client_id={self.client_id}"
+        )
+
+        while self.running and self.ws and not self.ws.closed:
+            try:
+                # Perform OpenClaw check
+                status = await self._perform_openclaw_check()
+                
+                # If issues detected and status changed, send notification
+                if status.get("status") != "healthy":
+                    # Check if status changed (avoid duplicate notifications)
+                    status_key = (status.get("status"), status.get("message"))
+                    if status_key != self.last_openclaw_status:
+                        await self._send_openclaw_notification(status)
+                        self.last_openclaw_status = status_key
+                else:
+                    # When recovered, if there was a previous issue, also send a notification
+                    if self.last_openclaw_status is not None:
+                        await self._send_openclaw_notification(status)
+                        self.last_openclaw_status = None
+
+                # Wait for next check
+                await asyncio.sleep(self.openclaw_check_interval)
+
+            except Exception as e:
+                logger.error(
+                    f"[CLIENT-OPENCLAW-CHECK-ERROR] OpenClaw check error: {e}, "
+                    f"client_id={self.client_id}",
+                    exc_info=True
+                )
+                # Continue running even if check fails
+                await asyncio.sleep(self.openclaw_check_interval)
+
+        logger.info(
+            f"[CLIENT-OPENCLAW-CHECK-STOP] OpenClaw check task stopped: "
+            f"client_id={self.client_id}"
+        )
+
+    async def _send_openclaw_notification(self, status: dict):
+        """
+        Send OpenClaw status notification message
+        
+        Args:
+            status: OpenClaw status dictionary with structure:
+                {
+                    "status": "healthy|warning|error",
+                    "message": "Status description",
+                    "details": {
+                        # Any check-specific parameters
+                    }
+                }
+        """
+        if not self.ws or self.ws.closed:
+            logger.warning(
+                f"[CLIENT-OPENCLAW-NOTIFY] WebSocket not connected, skipping notification: "
+                f"client_id={self.client_id}"
+            )
+            return
+
+        try:
+            notification_msg = {
+                "id": str(uuid.uuid4()),
+                "type": "health_status",  # Keep "health_status" for protocol compatibility
+                "timestamp": int(time.time() * 1000),
+                "data": {
+                    "status": status.get("status"),
+                    "message": status.get("message"),
+                    "details": status.get("details", {}),
+                }
+            }
+            
+            logger.info(
+                f"[CLIENT-OPENCLAW-NOTIFY] Sending OpenClaw notification: "
+                f"status={status.get('status')}, message_id={notification_msg['id']}, "
+                f"client_id={self.client_id}"
+            )
+            
+            # Send message directly
+            await self.ws.send_json(notification_msg)
+            
+            logger.debug(
+                f"[CLIENT-OPENCLAW-NOTIFY] OpenClaw notification sent: "
+                f"message_id={notification_msg['id']}, client_id={self.client_id}"
+            )
+            
+        except Exception as e:
+            logger.error(
+                f"[CLIENT-OPENCLAW-NOTIFY-ERROR] Failed to send OpenClaw notification: "
+                f"error={e}, client_id={self.client_id}",
+                exc_info=True
+            )
+
     async def _execute_local_command(
         self,
         message_id: str,
@@ -1029,6 +1150,19 @@ class WebSocketClient:
                 f"client_id={self.client_id}"
             )
 
+            # Start OpenClaw check task (if enabled)
+            if self.openclaw_check_enabled:
+                self.openclaw_check_task = asyncio.create_task(self._openclaw_check_loop())
+                logger.info(
+                    f"[CLIENT-OPENCLAW-CHECK-TASK] OpenClaw check task started: "
+                    f"interval={self.openclaw_check_interval}s, client_id={self.client_id}"
+                )
+            else:
+                logger.debug(
+                    f"[CLIENT-OPENCLAW-CHECK-TASK] OpenClaw check disabled, skipping task start: "
+                    f"client_id={self.client_id}"
+                )
+
             return True
 
         except Exception as e:
@@ -1044,6 +1178,22 @@ class WebSocketClient:
         """Disconnect WebSocket connection"""
         logger.info(f"[CLIENT-DISCONNECT-START] Starting disconnect process: client_id={self.client_id}")
 
+        # Stop OpenClaw check task
+        if self.openclaw_check_task and not self.openclaw_check_task.done():
+            logger.debug(
+                f"[CLIENT-DISCONNECT-OPENCLAW-CHECK] Cancelling OpenClaw check task: "
+                f"client_id={self.client_id}"
+            )
+            self.openclaw_check_task.cancel()
+            try:
+                await self.openclaw_check_task
+            except asyncio.CancelledError:
+                pass
+            logger.info(
+                f"[CLIENT-DISCONNECT-OPENCLAW-CHECK] OpenClaw check task stopped: "
+                f"client_id={self.client_id}"
+            )
+
         # Stop heartbeat task
         if self.heartbeat_task and not self.heartbeat_task.done():
             logger.debug(f"[CLIENT-DISCONNECT-HEARTBEAT] Cancelling heartbeat task: client_id={self.client_id}")