python-corekit 0.1.0__py3-none-any.whl

corekit/log_monitor/models.py

@@ -0,0 +1,150 @@
+"""
+Configuration models for the log monitor.
+
+A configuration is a list of containers, each with rules; a rule is a regex and
+the actions to take when a log line matches it. These parse straight from YAML.
+"""
+
+from typing import Literal, Union
+
+from pydantic import BaseModel, Field
+
+from corekit.schemas.enum import StringEnum
+
+
+class Severity(StringEnum):
+    """Severity levels for log pattern matches"""
+
+    INFO = "info"
+    WARNING = "warning"
+    CRITICAL = "critical"
+
+
+class ActionType(StringEnum):
+    """Types of actions that can be executed"""
+
+    LOG = "log"
+    NOTIFY = "notify"
+    RESTART_CONTAINER = "restart_container"
+    PAUSE_CONTAINER = "pause_container"
+    EXECUTE_COMMAND = "execute_command"
+
+
+class RateLimitPeriod(StringEnum):
+    """Time periods for rate limiting"""
+
+    MINUTE = "minute"
+    HOUR = "hour"
+    DAY = "day"
+
+
+# Action Models
+class BaseAction(BaseModel):
+    """Base class for all action types"""
+
+    type: ActionType
+
+
+class LogAction(BaseAction):
+    """Log a message when pattern matches"""
+
+    type: Literal[ActionType.LOG] = ActionType.LOG
+    message: str = Field(..., description="Message template with {container} and {text} placeholders")
+
+
+class NotifyAction(BaseAction):
+    """Send notification with throttling"""
+
+    type: Literal[ActionType.NOTIFY] = ActionType.NOTIFY
+    throttle: int = Field(default=0, description="Minimum seconds between notifications for same container")
+
+
+class RestartContainerAction(BaseAction):
+    """Restart a container with safeguards"""
+
+    type: Literal[ActionType.RESTART_CONTAINER] = ActionType.RESTART_CONTAINER
+    max_restarts: int = Field(default=3, description="Maximum restarts within restart_window")
+    restart_window: int = Field(default=3600, description="Time window in seconds for max_restarts")
+    delay: int = Field(default=0, description="Delay in seconds before restarting")
+
+
+class PauseContainerAction(BaseAction):
+    """Pause a container"""
+
+    type: Literal[ActionType.PAUSE_CONTAINER] = ActionType.PAUSE_CONTAINER
+
+
+class ExecuteCommandAction(BaseAction):
+    """Execute a shell command"""
+
+    type: Literal[ActionType.EXECUTE_COMMAND] = ActionType.EXECUTE_COMMAND
+    command: str = Field(..., description="Command template with {container} placeholder")
+
+
+# Union type for all actions
+Action = Union[LogAction, NotifyAction, RestartContainerAction, PauseContainerAction, ExecuteCommandAction]
+
+
+# Rule Model
+class Rule(BaseModel):
+    """Log monitoring rule with pattern matching and actions"""
+
+    name: str = Field(..., description="Human-readable rule name")
+    pattern: str = Field(..., description="Regex pattern to match against log lines")
+    severity: Severity = Field(default=Severity.INFO, description="Severity level of matches")
+    actions: list[Action] = Field(default_factory=list, description="Actions to execute on match")
+    send_notification: bool = Field(
+        default=False, description="Whether to send notification on match (uses notification service)"
+    )
+
+
+# Container Config Model
+class ContainerConfig(BaseModel):
+    """Configuration for monitoring a specific container or pattern"""
+
+    name: str = Field(..., description="Container name or regex pattern, '*' for all")
+    enabled: bool = Field(default=True, description="Whether this config is active")
+    rules: list[Rule] = Field(default_factory=list, description="Rules to apply to this container")
+
+
+# Advanced Config
+class RateLimit(BaseModel):
+    """Rate limit configuration"""
+
+    count: int = Field(..., description="Number of actions allowed")
+    period: RateLimitPeriod = Field(..., description="Time period for the limit")
+
+    @property
+    def period_seconds(self) -> int:
+        """Convert period to seconds"""
+        mapping = {
+            RateLimitPeriod.MINUTE: 60,
+            RateLimitPeriod.HOUR: 3600,
+            RateLimitPeriod.DAY: 86400,
+        }
+        return mapping[self.period]
+
+
+class AdvancedConfig(BaseModel):
+    """Advanced monitoring configuration"""
+
+    rate_limits: dict[ActionType, RateLimit] = Field(default_factory=dict, description="Rate limits per action type")
+    ignore_patterns: list[str] = Field(default_factory=list, description="Regex patterns to ignore in logs")
+
+
+# Global Config
+class GlobalConfig(BaseModel):
+    """Global monitoring settings"""
+
+    log_buffer_size: int = Field(default=1000, description="Max log lines to buffer per container")
+
+
+# Main Config Model
+class LogMonitorConfig(BaseModel):
+    """Complete log monitor configuration"""
+
+    global_config: GlobalConfig = Field(alias="global", default_factory=GlobalConfig)
+    containers: list[ContainerConfig] = Field(default_factory=list)
+    advanced: AdvancedConfig = Field(default_factory=AdvancedConfig)
+
+    model_config = {"populate_by_name": True}

corekit/log_monitor/service.py

@@ -0,0 +1,418 @@
+"""
+React to what containers write to their logs.
+
+A YAML file maps containers to rules; each rule is a regex and the actions to
+take when a log line matches -- log it, notify, restart or pause the container,
+or run a command. Actions are rate limited, and restarts are additionally capped
+per container so a crash loop cannot become a restart loop.
+
+    LogMonitor.run("config.yaml")
+
+"""
+
+import re
+import signal
+import subprocess
+import threading
+import time
+from collections import defaultdict, deque
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import Any
+
+import yaml
+from docker.errors import APIError, NotFound
+from docker.models.containers import Container
+
+from corekit.docker.watchdog import Watchdog
+from corekit.log_monitor import constants as log_monitor_constants
+from corekit.log_monitor.models import (
+    Action,
+    ActionType,
+    ExecuteCommandAction,
+    LogAction,
+    LogMonitorConfig,
+    NotifyAction,
+    PauseContainerAction,
+    RestartContainerAction,
+    Rule,
+    Severity,
+)
+from corekit.notifications import BaseNotificationService, Notification, NotificationType
+
+
+class LogMonitor(Watchdog):
+    """
+    Monitors Docker container logs and triggers actions based on pattern matching.
+    Inherits Docker operations from Watchdog, composes with notification services.
+    """
+
+    def __init__(
+        self,
+        config_path: str | None = None,
+        config: LogMonitorConfig | None = None,
+        notification_service: BaseNotificationService | None = None,
+        enforce_label: bool = False,
+        max_workers: int | None = None,
+        docker_host: str | None = None,
+        handle_signals: bool = False,
+    ) -> None:
+        """
+        :param config_path: YAML configuration to load.
+        :param config: an already-built configuration, instead of a path.
+        :param notification_service: where notifications go; logs by default.
+        :param enforce_label: only act on containers carrying the managed label.
+        :param handle_signals: install SIGTERM and SIGINT handlers. Off by
+            default because installing them is process-global and only the
+            program's entry point should decide that; ``run()`` turns it on.
+        """
+        super().__init__(docker_host=docker_host, enforce_label=enforce_label)
+
+        if config is not None:
+            self.config = config
+        else:
+            path = config_path or log_monitor_constants.DEFAULT_CONFIG_PATH
+            with open(path) as handle:
+                self.config = LogMonitorConfig(**yaml.safe_load(handle))
+
+        # Notification service composition (not reimplementation)
+        self._notification_service = notification_service or BaseNotificationService()
+
+        # Thread management for graceful shutdown
+        self._shutdown_event = threading.Event()
+        self._max_workers = max_workers or log_monitor_constants.DEFAULT_MAX_WORKERS
+        self._executor: ThreadPoolExecutor | None = None
+        self._futures: dict[str, Future] = {}
+
+        if handle_signals:
+            self.install_signal_handlers()
+
+        # Log monitoring state (LogMonitor-specific)
+        # TODO: determine if these should be a constant or part of the config
+        # TODO: some of this stuff might make a LOT more sense being stored in redis...
+        self.action_counts: dict[ActionType, deque[float]] = defaultdict(
+            lambda: deque(maxlen=log_monitor_constants.ACTION_COUNTS_MAX_LEN)
+        )
+        self.restart_counts: dict[str, deque[float]] = defaultdict(
+            lambda: deque(maxlen=log_monitor_constants.RESTART_COUNTS_MAX_LEN)
+        )
+        self.log_buffers: dict[str, deque[str]] = defaultdict(
+            lambda: deque(maxlen=self.config.global_config.log_buffer_size)
+        )
+        # Properly track notification throttling per container
+        self.last_notify_times: dict[str, float] = {}
+
+    def _signal_handler(self, signum: int, frame: Any) -> None:
+        """Handle SIGTERM/SIGINT for graceful shutdown"""
+        signal_name = signal.Signals(signum).name
+        self.warning(f"Received {signal_name}, initiating graceful shutdown...")
+        self.shutdown()
+
+    @staticmethod
+    def _severity_to_notification_type(severity: Severity) -> NotificationType:
+        """Map log monitor severity to notification type"""
+        mapping = {
+            Severity.INFO: NotificationType.INFO,
+            Severity.WARNING: NotificationType.WARNING,
+            Severity.CRITICAL: NotificationType.CRITICAL,
+        }
+        return mapping.get(severity, NotificationType.INFO)
+
+    def _send_notification(self, message: str, severity: Severity) -> None:
+        """Send notification using the notification service"""
+        notification = Notification(
+            message=message, type=self._severity_to_notification_type(severity), meta={"source": "LogMonitor"}
+        )
+        self._notification_service.notify(notification)
+
+    def check_rate_limit(self, action_type: ActionType) -> bool:
+        """Check if action is within rate limits"""
+        if action_type not in self.config.advanced.rate_limits:
+            return True
+
+        rate_limit = self.config.advanced.rate_limits[action_type]
+
+        # Clean old entries
+        cutoff = time.time() - rate_limit.period_seconds
+        self.action_counts[action_type] = deque(
+            [t for t in self.action_counts[action_type] if t > cutoff],
+            maxlen=log_monitor_constants.ACTION_COUNTS_MAX_LEN,
+        )
+
+        return len(self.action_counts[action_type]) < rate_limit.count
+
+    def _execute_log_action(self, action: LogAction, container_name: str, matched_text: str) -> None:
+        """Execute log action - write formatted message to log"""
+        message = action.message.format(container=container_name, text=matched_text)
+        self.info(f"[LogAction] {message}")
+
+    def _execute_notify_action(self, action: NotifyAction, container_name: str) -> None:
+        """Execute notify action - check throttle only (actual notification sent separately)"""
+        # Check throttle
+        last_notify = self.last_notify_times.get(container_name, 0.0)
+        if time.time() - last_notify < action.throttle:
+            self.debug(f"Notification throttled for {container_name}")
+            return
+        self.last_notify_times[container_name] = time.time()
+
+    def _execute_restart_action(self, action: RestartContainerAction, container_name: str) -> None:
+        """Execute restart action - delegate to Watchdog parent"""
+
+        # Check restart count
+        cutoff = time.time() - action.restart_window
+        self.restart_counts[container_name] = deque(
+            [t for t in self.restart_counts[container_name] if t > cutoff],
+            maxlen=log_monitor_constants.RESTART_COUNTS_MAX_LEN,
+        )
+
+        if len(self.restart_counts[container_name]) >= action.max_restarts:
+            self.warning(f"Max restarts ({action.max_restarts}) reached for {container_name}, skipping")
+            return
+
+        if action.delay > 0:
+            self.info(f"Waiting {action.delay}s before restarting {container_name}")
+            time.sleep(action.delay)
+
+        try:
+            # Delegate to parent Watchdog method
+            container = self._client.containers.get(container_name)
+            container.restart()
+            self.restart_counts[container_name].append(time.time())
+            self.info(f"Restarted container: {container_name}")
+        except NotFound:
+            self.error(f"Container '{container_name}' not found")
+        except APIError as e:
+            self.error(f"Failed to restart {container_name}: {e}")
+
+    def _execute_pause_action(self, action: PauseContainerAction, container_name: str) -> None:
+        """Execute pause action - delegate to Watchdog parent"""
+        self.pause_container_by_name(container_name)
+
+    def _execute_command_action(self, action: ExecuteCommandAction, container_name: str) -> None:
+        """Execute shell command action"""
+        command = action.command.format(container=container_name)
+        try:
+            result = subprocess.run(
+                command,
+                shell=True,
+                capture_output=True,
+                text=True,
+                timeout=30,  # 30 second timeout for safety
+            )
+            if result.returncode == 0:
+                self.info(f"Executed: {command}\nOutput: {result.stdout}")
+            else:
+                self.error(f"Command failed: {command}\nError: {result.stderr}")
+        except subprocess.TimeoutExpired:
+            self.error(f"Command timed out: {command}")
+        except Exception as e:
+            self.error(f"Failed to execute command: {e}")
+
+    def execute_action(self, action: Action, container_name: str, matched_text: str = "") -> None:
+        """Execute configured actions by delegating to specific action handlers"""
+        action_type = action.type
+
+        # Check rate limits
+        if not self.check_rate_limit(action_type):
+            self.warning(f"Rate limit exceeded for {action_type.value}, skipping")
+            return
+
+        self.action_counts[action_type].append(time.time())
+
+        if action_type is ActionType.LOG:
+            self._execute_log_action(action, container_name, matched_text)
+        elif action_type is ActionType.NOTIFY:
+            self._execute_notify_action(action, container_name)
+        elif action_type is ActionType.RESTART_CONTAINER:
+            self._execute_restart_action(action, container_name)
+        elif action_type is ActionType.PAUSE_CONTAINER:
+            self._execute_pause_action(action, container_name)
+        elif action_type is ActionType.EXECUTE_COMMAND:
+            self._execute_command_action(action, container_name)
+        else:
+            self.warning(f"No handler for action type {action_type}")
+
+    def process_log_line(self, container_name: str, line: str, rules: list[Rule]) -> None:
+        """Process a single log line against all rules"""
+        for rule in rules:
+            match = re.search(rule.pattern, line)
+
+            if match:
+                # Log the match
+                log_message = f"{container_name}: {rule.name} - {line.strip()}"
+                if rule.severity == Severity.CRITICAL:
+                    self.error(log_message)
+                elif rule.severity == Severity.WARNING:
+                    self.warning(log_message)
+                else:
+                    self.info(log_message)
+
+                # Execute actions
+                for action in rule.actions:
+                    self.execute_action(action, container_name, line)
+
+                # Send notification if configured
+                if rule.send_notification:
+                    message = f"*{rule.name}* in `{container_name}`\n```{line.strip()}```"
+                    self._send_notification(message, rule.severity)
+
+    # FIXME: does this actually make more sense being part of watchdog?
+    def monitor_container(self, container_name: str, rules: list[Rule]) -> None:
+        """Monitor a specific container's logs with graceful shutdown support"""
+
+        try:
+            container: Container = self._client.containers.get(container_name)
+            self.info(f"Started monitoring: {container_name}")
+
+            # Stream logs with tail to avoid replaying entire history
+            log_stream = container.logs(stream=True, follow=True, tail=100)
+
+            for line in log_stream:
+                # Check for shutdown signal
+                if self._shutdown_event.is_set():
+                    self.info(f"Shutdown requested, stopping monitor for: {container_name}")
+                    break
+
+                decoded_line = line.decode("utf-8", errors="ignore")
+
+                # Check against ignore patterns FIRST before buffering
+                if any(re.search(pattern, decoded_line) for pattern in self.config.advanced.ignore_patterns):
+                    continue
+
+                # Only buffer lines that pass ignore patterns
+                self.log_buffers[container_name].append(decoded_line)
+
+                self.process_log_line(container_name, decoded_line, rules)
+
+        except NotFound:
+            self.error(f"Container '{container_name}' not found")
+        except APIError as e:
+            self.error(f"Docker API error monitoring {container_name}: {e}")
+        except Exception as e:
+            self.error(f"Unexpected error monitoring {container_name}: {e}")
+        finally:
+            self.info(f"Stopped monitoring: {container_name}")
+
+    def get_container_rules(self, container_name: str) -> list[Rule]:
+        """Get rules for a specific container"""
+        rules: list[Rule] = []
+
+        for container_config in self.config.containers:
+            if not container_config.enabled:
+                continue
+
+            name_pattern = container_config.name
+
+            # Check if this config applies to this container
+            if name_pattern == "*" or re.match(name_pattern, container_name):
+                rules.extend(container_config.rules)
+
+        return rules
+
+    def is_healthy(self) -> bool:
+        """Health check for container orchestration"""
+        if self._executor is None:
+            return False
+
+        # Check if shutdown has been requested
+        if self._shutdown_event.is_set():
+            return False
+
+        # Check if we have active monitors running
+        active_monitors = sum(1 for f in self._futures.values() if not f.done())
+
+        return active_monitors > 0
+
+    def get_monitor_status(self) -> dict[str, Any]:
+        """Get detailed monitoring status for observability"""
+        return {
+            "active_monitors": sum(1 for f in self._futures.values() if not f.done()),
+            "failed_monitors": sum(1 for f in self._futures.values() if f.done() and f.exception()),
+            "total_monitors": len(self._futures),
+            "shutdown_requested": self._shutdown_event.is_set(),
+            "max_workers": self._max_workers,
+        }
+
+    def start(self) -> None:
+        """Start monitoring all containers using thread pool"""
+        self.info("Docker Log Monitor starting...")
+        self.info(f"Config loaded: {len(self.config.containers)} container configs")
+        self.info(f"Thread pool max workers: {self._max_workers}")
+
+        # Initialize thread pool executor
+        self._executor = ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="log_monitor")
+
+        # Get all running containers (delegate to parent list_containers)
+        containers: list[Container] = self.list_containers()
+
+        for container in containers:
+            container_name = container.name
+            rules = self.get_container_rules(container_name)
+
+            if rules:
+                # Submit to thread pool instead of creating unlimited threads
+                future = self._executor.submit(self.monitor_container, container_name, rules)
+                self._futures[container_name] = future
+                self.info(f"Submitted monitoring task for: {container_name}")
+
+        self.info(f"Monitoring {len(self._futures)} containers")
+
+        # Keep main thread alive and periodically check monitor health
+        try:
+            while not self._shutdown_event.is_set():
+                time.sleep(5)
+
+                # Check for failed monitors
+                for container_name, future in list(self._futures.items()):
+                    if future.done():
+                        exception = future.exception()
+                        if exception:
+                            self.error(f"Monitor for {container_name} failed: {exception}")
+                        else:
+                            self.info(f"Monitor for {container_name} completed normally")
+
+                        # Remove from active futures
+                        del self._futures[container_name]
+
+                # Optional: Log status periodically (disabled to reduce log volume)
+                # if len(self._futures) > 0:
+                #     self.debug(f"Active monitors: {len(self._futures)}")
+
+        except KeyboardInterrupt:
+            self.info("KeyboardInterrupt received")
+            self.shutdown()
+
+    def shutdown(self) -> None:
+        """Gracefully shutdown all monitoring threads"""
+        if self._shutdown_event.is_set():
+            self.debug("Shutdown already requested, waiting for completion...")
+        else:
+            self.info("Initiating graceful shutdown...")
+            self._shutdown_event.set()
+
+        if self._executor:
+            active_count = sum(1 for f in self._futures.values() if not f.done())
+            if active_count > 0:
+                self.info(f"Waiting for {active_count} monitors to stop...")
+
+            # Wait for threads to finish gracefully
+            self._executor.shutdown(wait=True, cancel_futures=False)
+
+            self.info("All monitoring threads stopped")
+
+        self.info("LogMonitor shutdown complete")
+
+    def install_signal_handlers(self) -> None:
+        """
+        Shut down cleanly on SIGTERM and SIGINT.
+
+        Process-global, so only an entry point should call it.
+        """
+        signal.signal(signal.SIGTERM, self._signal_handler)
+        signal.signal(signal.SIGINT, self._signal_handler)
+
+    @classmethod
+    def run(cls, config_path: str | None = None, **kwargs: Any) -> None:
+        """
+        Run as a program: install signal handlers and monitor until stopped.
+        """
+        cls(config_path=config_path, handle_signals=True, **kwargs).start()

corekit/notifications/__init__.py

@@ -0,0 +1,8 @@
+"""
+Notification delivery and payloads.
+"""
+
+from corekit.notifications.base import BaseNotificationService
+from corekit.notifications.models import Notification, NotificationType
+
+__all__ = ["BaseNotificationService", "Notification", "NotificationType"]

corekit/notifications/base.py

@@ -0,0 +1,51 @@
+"""
+Notification delivery.
+
+Subclass ``BaseNotificationService`` and implement ``_send``::
+
+    class EmailNotificationService(BaseNotificationService):
+        '''
+        Sends notifications by email.
+        '''
+
+        def _send(self, message: str) -> None:
+            smtp.send(message)
+
+    service.notify(Notification(message="disk full", type=NotificationType.ERROR))
+
+Override ``_send``, not ``send``: ``notify`` formats the message and calls
+``_send``, so an override with any other name silently does nothing.
+"""
+
+from corekit.notifications.models import Notification
+from corekit.observability.loggable import Loggable
+
+__all__ = ["BaseNotificationService"]
+
+
+class BaseNotificationService(Loggable):
+    """
+    Formats notifications and hands them to a transport.
+
+    The default transport logs, so an unconfigured service is harmless rather
+    than broken.
+    """
+
+    @staticmethod
+    def _format(notification: Notification) -> str:
+        """
+        Render a notification as the text a transport will send.
+        """
+        return f"[{notification.type} Notification]: {notification.message}"
+
+    def _send(self, message: str) -> None:
+        """
+        Deliver formatted text. Override this in a subclass.
+        """
+        self.warning(message)
+
+    def notify(self, notification: Notification) -> None:
+        """
+        Format a notification and deliver it.
+        """
+        self._send(self._format(notification))

corekit/notifications/models.py

@@ -0,0 +1,34 @@
+"""
+Notification payloads.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from corekit.schemas.enum import StringEnum
+
+__all__ = ["Notification", "NotificationType"]
+
+
+class NotificationType(StringEnum):
+    """
+    Severity of a notification.
+    """
+
+    NOTE = "NOTE"
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+class Notification(BaseModel):
+    """
+    A message to deliver, with its severity and any surrounding context.
+    """
+
+    message: str
+    type: NotificationType = NotificationType.INFO
+    meta: dict[str, Any] = Field(default_factory=dict)

corekit/observability/__init__.py

@@ -0,0 +1,21 @@
+"""
+Logging and performance measurement.
+
+``Loggable`` gives a class a logger named after itself; ``Benchmarkable`` adds
+split timing on top. They live together because they are the same concern --
+knowing what a running system is doing.
+
+    class Importer(Benchmarkable):
+        def run(self) -> None:
+            self.timing()
+            self.info("starting")
+            ...
+            self.timing("finished")
+"""
+
+from corekit.observability.benchmarkable import Benchmarkable
+from corekit.observability.loggable import Loggable
+from corekit.observability.timing.split import Split
+from corekit.observability.timing.timer import Timer
+
+__all__ = ["Benchmarkable", "Loggable", "Split", "Timer"]