realtimex-deeptutor 0.5.0.post1__py3-none-any.whl

src/logging/adapters/lightrag.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+"""
+LightRAG Log Forwarder
+======================
+
+Forwards LightRAG and RAG-Anything logs to DeepTutor's unified logging system.
+"""
+
+from contextlib import contextmanager
+import logging
+from pathlib import Path
+from typing import Optional
+
+
+class LightRAGLogForwarder(logging.Handler):
+    """
+    Handler that forwards LightRAG logger messages to DeepTutor logger.
+    """
+
+    def __init__(self, ai_tutor_logger, add_prefix: bool = True):
+        """
+        Args:
+            ai_tutor_logger: DeepTutor Logger instance
+            add_prefix: Whether to add [LightRAG] prefix to messages
+        """
+        super().__init__()
+        self.ai_tutor_logger = ai_tutor_logger
+        self.add_prefix = add_prefix
+        # Capture all log levels
+        self.setLevel(logging.DEBUG)
+
+    def emit(self, record: logging.LogRecord):
+        """
+        Forward log record to DeepTutor logger.
+        All logs are forwarded as info level to maintain consistent format.
+        """
+        try:
+            # Get the original message directly without adding [LightRAG] prefix
+            # LightRAG already formats messages appropriately (e.g., "DEBUG: xxx" for debug logs)
+            message = record.getMessage()
+
+            # Use info() for all levels to maintain consistent format
+            # This ensures all logs appear as [RAGTool] ... (or [RAGTool] DEBUG: ... for debug)
+            self.ai_tutor_logger.info(message)
+
+        except Exception:
+            # Avoid errors in forwarding from affecting main flow
+            self.handleError(record)
+
+
+def get_lightrag_forwarding_config() -> dict:
+    """
+    Load LightRAG forwarding configuration from main.yaml.
+
+    Returns:
+        dict: Configuration dictionary with defaults if not found
+    """
+    try:
+        from src.services.config import load_config_with_main
+
+        # Use resolve() to get absolute path, ensuring correct project root regardless of working directory
+        project_root = Path(__file__).resolve().parent.parent.parent.parent
+        config = load_config_with_main("solve_config.yaml", project_root)
+
+        forwarding_config = config.get("logging", {}).get("lightrag_forwarding", {})
+
+        return {
+            "enabled": forwarding_config.get("enabled", True),
+            "min_level": forwarding_config.get("min_level", "INFO"),
+            "add_prefix": forwarding_config.get("add_prefix", True),
+            "logger_names": forwarding_config.get(
+                "logger_names", {"knowledge_init": "KnowledgeInit", "rag_tool": "RAGTool"}
+            ),
+        }
+    except Exception:
+        # Return defaults if config loading fails
+        return {
+            "enabled": True,
+            "min_level": "INFO",
+            "add_prefix": True,
+            "logger_names": {"knowledge_init": "KnowledgeInit", "rag_tool": "RAGTool"},
+        }
+
+
+@contextmanager
+def LightRAGLogContext(logger_name: Optional[str] = None, scene: Optional[str] = None):
+    """
+    Context manager for LightRAG log forwarding.
+
+    Automatically sets up and tears down log forwarding.
+
+    Args:
+        logger_name: Explicit logger name (overrides scene-based lookup)
+        scene: Scene name ('knowledge_init' or 'rag_tool') for logger name lookup
+
+    Usage:
+        with LightRAGLogContext("RAGTool"):
+            # RAG operations
+            rag = RAGAnything(...)
+    """
+    from ..logger import get_logger
+
+    # Get configuration
+    config = get_lightrag_forwarding_config()
+
+    # Check if forwarding is enabled
+    if not config.get("enabled", True):
+        # If disabled, just pass through without forwarding
+        yield
+        return
+
+    # Debug: Log that forwarding is being set up (only if we have a logger)
+    # This helps verify the context manager is being called
+    try:
+        debug_logger = get_logger("RAGForward")
+        debug_logger.debug(
+            f"Setting up LightRAG log forwarding (scene={scene}, logger_name={logger_name})"
+        )
+    except:
+        pass  # Ignore if logger setup fails
+
+    # Determine logger name
+    if logger_name is None:
+        if scene:
+            logger_names = config.get("logger_names", {})
+            logger_name = logger_names.get(scene, "Main")
+        else:
+            logger_name = "Main"
+
+    # Get DeepTutor logger
+    ai_tutor_logger = get_logger(logger_name)
+
+    # Get forwarding settings
+    add_prefix = config.get("add_prefix", True)
+    min_level_str = config.get("min_level", "INFO")
+    min_level = getattr(logging, min_level_str.upper(), logging.INFO)
+
+    # Get LightRAG logger
+    lightrag_logger = logging.getLogger("lightrag")
+
+    # Store original handlers and level to restore later if needed
+    original_handlers = lightrag_logger.handlers[:]  # Copy list
+    original_level = lightrag_logger.level
+
+    # Temporarily remove existing console handlers to avoid duplicate output
+    # We'll forward all logs through our handler instead
+    console_handlers_to_remove = []
+    for handler in original_handlers:
+        if isinstance(handler, logging.StreamHandler):
+            console_handlers_to_remove.append(handler)
+
+    for handler in console_handlers_to_remove:
+        lightrag_logger.removeHandler(handler)
+
+    # Ensure LightRAG logger level is set low enough to capture all logs
+    # The logger level controls which logs are created, handler level controls which are processed
+    # Set to DEBUG to ensure we capture everything, then filter at handler level
+    if lightrag_logger.level > logging.DEBUG:
+        lightrag_logger.setLevel(logging.DEBUG)
+
+    # Create and add forwarder
+    forwarder = LightRAGLogForwarder(ai_tutor_logger, add_prefix=add_prefix)
+    forwarder.setLevel(min_level)
+    lightrag_logger.addHandler(forwarder)
+
+    # Test that forwarding works by sending a test log
+    try:
+        test_msg = "LightRAG log forwarding enabled"
+        lightrag_logger.info(test_msg)
+    except:
+        pass  # Ignore test log errors
+
+    try:
+        yield
+    finally:
+        # Clean up: remove our forwarder
+        if forwarder in lightrag_logger.handlers:
+            lightrag_logger.removeHandler(forwarder)
+            forwarder.close()
+
+        # Restore original console handlers if they were removed
+        for handler in console_handlers_to_remove:
+            if handler not in lightrag_logger.handlers:
+                lightrag_logger.addHandler(handler)

src/logging/adapters/llamaindex.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+"""
+LlamaIndex Log Forwarder
+========================
+
+Forwards LlamaIndex logs to DeepTutor's unified logging system.
+"""
+
+from contextlib import contextmanager
+import logging
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class LlamaIndexLogForwarder(logging.Handler):
+    """
+    Handler that forwards LlamaIndex logger messages to DeepTutor logger.
+    """
+
+    def __init__(self, ai_tutor_logger, add_prefix: bool = True):
+        """
+        Args:
+            ai_tutor_logger: DeepTutor Logger instance
+            add_prefix: Whether to add prefix to messages
+        """
+        super().__init__()
+        self.ai_tutor_logger = ai_tutor_logger
+        self.add_prefix = add_prefix
+        # Capture all log levels
+        self.setLevel(logging.DEBUG)
+
+    def emit(self, record: logging.LogRecord):
+        """
+        Forward log record to DeepTutor logger.
+        """
+        try:
+            message = record.getMessage()
+
+            # Map log levels
+            level = record.levelno
+            if level >= logging.ERROR:
+                self.ai_tutor_logger.error(message)
+            elif level >= logging.WARNING:
+                self.ai_tutor_logger.warning(message)
+            elif level >= logging.INFO:
+                self.ai_tutor_logger.info(message)
+            else:
+                self.ai_tutor_logger.debug(message)
+
+        except Exception:
+            self.handleError(record)
+
+
+@contextmanager
+def LlamaIndexLogContext(
+    logger_name: Optional[str] = None,
+    scene: str = "llamaindex",
+    min_level: str = "INFO",
+):
+    """
+    Context manager for LlamaIndex log forwarding.
+
+    Automatically sets up and tears down log forwarding.
+
+    Args:
+        logger_name: Explicit logger name (defaults to scene name)
+        scene: Scene name for logger identification
+        min_level: Minimum log level to forward
+
+    Usage:
+        with LlamaIndexLogContext("VectorSearch"):
+            # LlamaIndex operations
+            index.query(query)
+    """
+    from ..logger import get_logger
+
+    # Determine logger name
+    if logger_name is None:
+        logger_name = scene.title().replace("_", "")
+
+    # Get DeepTutor logger
+    ai_tutor_logger = get_logger(logger_name)
+
+    # Get LlamaIndex loggers
+    llama_loggers = [
+        logging.getLogger("llama_index"),
+        logging.getLogger("llama_index.core"),
+        logging.getLogger("llama_index.vector_stores"),
+        logging.getLogger("llama_index.embeddings"),
+    ]
+
+    # Parse min level
+    min_level_int = getattr(logging, min_level.upper(), logging.INFO)
+
+    # Store original state
+    original_states: List[Dict[str, Any]] = []
+    forwarders: List[Tuple[logging.Logger, LlamaIndexLogForwarder]] = []
+
+    for llama_logger in llama_loggers:
+        original_states.append(
+            {
+                "logger": llama_logger,
+                "handlers": llama_logger.handlers[:],
+                "level": llama_logger.level,
+            }
+        )
+
+        # Temporarily remove console handlers
+        console_handlers_to_remove = []
+        for handler in llama_logger.handlers:
+            if isinstance(handler, logging.StreamHandler):
+                console_handlers_to_remove.append(handler)
+
+        for handler in console_handlers_to_remove:
+            llama_logger.removeHandler(handler)
+
+        # Set level
+        if llama_logger.level > logging.DEBUG:
+            llama_logger.setLevel(logging.DEBUG)
+
+        # Add forwarder
+        forwarder = LlamaIndexLogForwarder(ai_tutor_logger)
+        forwarder.setLevel(min_level_int)
+        llama_logger.addHandler(forwarder)
+        forwarders.append((llama_logger, forwarder))
+
+    try:
+        yield
+    finally:
+        # Clean up forwarders
+        for llama_logger, forwarder in forwarders:
+            if forwarder in llama_logger.handlers:
+                llama_logger.removeHandler(forwarder)
+                forwarder.close()
+
+        # Restore original state
+        for state in original_states:
+            llama_logger = state["logger"]
+            # Restore handlers that were removed
+            for handler in state["handlers"]:
+                if handler not in llama_logger.handlers:
+                    llama_logger.addHandler(handler)

src/logging/config.py

@@ -0,0 +1,80 @@
+"""
+Logging Configuration
+=====================
+
+Configuration settings for the logging system.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class LoggingConfig:
+    """Configuration for the logging system."""
+
+    # Output settings
+    console_output: bool = True
+    file_output: bool = True
+
+    # Log levels
+    console_level: str = "INFO"
+    file_level: str = "DEBUG"
+
+    # Log directory (relative to project root or absolute)
+    log_dir: Optional[str] = None
+
+    # File rotation settings
+    max_bytes: int = 10 * 1024 * 1024  # 10MB
+    backup_count: int = 5
+
+    # WebSocket streaming
+    websocket_enabled: bool = True
+    websocket_queue_size: int = 1000
+
+    # LightRAG forwarding
+    lightrag_forwarding_enabled: bool = True
+    lightrag_min_level: str = "INFO"
+    lightrag_add_prefix: bool = True
+
+
+def get_default_log_dir() -> Path:
+    """Get the default log directory."""
+    project_root = Path(__file__).resolve().parent.parent.parent
+    return project_root / "data" / "user" / "logs"
+
+
+def load_logging_config() -> LoggingConfig:
+    """
+    Load logging configuration from config files.
+
+    Returns:
+        LoggingConfig instance with loaded or default values.
+    """
+    try:
+        from src.services.config import get_path_from_config, load_config_with_main
+
+        project_root = Path(__file__).resolve().parent.parent.parent
+        config = load_config_with_main("solve_config.yaml", project_root)
+
+        logging_config = config.get("logging", {})
+
+        return LoggingConfig(
+            console_output=logging_config.get("console_output", True),
+            file_output=logging_config.get("file_output", True),
+            console_level=logging_config.get("console_level", "INFO"),
+            file_level=logging_config.get("file_level", "DEBUG"),
+            log_dir=get_path_from_config(config, "user_log_dir"),
+            lightrag_forwarding_enabled=logging_config.get("lightrag_forwarding", {}).get(
+                "enabled", True
+            ),
+            lightrag_min_level=logging_config.get("lightrag_forwarding", {}).get(
+                "min_level", "INFO"
+            ),
+            lightrag_add_prefix=logging_config.get("lightrag_forwarding", {}).get(
+                "add_prefix", True
+            ),
+        )
+    except Exception:
+        return LoggingConfig()

src/logging/handlers/__init__.py

@@ -0,0 +1,20 @@
+"""
+Log Handlers
+============
+
+Custom logging handlers for various output destinations.
+"""
+
+from .console import ConsoleHandler
+from .file import FileHandler, JSONFileHandler, RotatingFileHandler, create_task_logger
+from .websocket import LogInterceptor, WebSocketLogHandler
+
+__all__ = [
+    "ConsoleHandler",
+    "FileHandler",
+    "JSONFileHandler",
+    "RotatingFileHandler",
+    "WebSocketLogHandler",
+    "LogInterceptor",
+    "create_task_logger",
+]

src/logging/handlers/console.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+Console Log Handler
+===================
+
+Color-coded console output with symbols.
+"""
+
+import logging
+import sys
+
+
+class ConsoleFormatter(logging.Formatter):
+    """
+    Clean console formatter with colors and symbols.
+    Format: [Module]    Symbol Message
+    """
+
+    # ANSI color codes
+    COLORS = {
+        "DEBUG": "\033[90m",  # Gray
+        "INFO": "\033[37m",  # White
+        "SUCCESS": "\033[32m",  # Green
+        "WARNING": "\033[33m",  # Yellow
+        "ERROR": "\033[31m",  # Red
+        "CRITICAL": "\033[35m",  # Magenta
+    }
+    RESET = "\033[0m"
+    BOLD = "\033[1m"
+    DIM = "\033[2m"
+
+    # Symbols for different log types
+    SYMBOLS = {
+        "DEBUG": "·",
+        "INFO": "●",
+        "SUCCESS": "✓",
+        "WARNING": "⚠",
+        "ERROR": "✗",
+        "CRITICAL": "✗",
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        # Get module name (padded to 12 chars for alignment)
+        module = getattr(record, "module_name", record.name)
+        module_padded = f"[{module}]".ljust(14)
+
+        # Get symbol (can be overridden via record.symbol)
+        symbol = getattr(record, "symbol", self.SYMBOLS.get(record.levelname, "●"))
+
+        # Get color
+        level = getattr(record, "display_level", record.levelname)
+        color = self.COLORS.get(level, self.COLORS["INFO"])
+
+        # Format message
+        message = record.getMessage()
+
+        # Build output: [Module]    ● Message
+        return f"{self.DIM}{module_padded}{self.RESET} {color}{symbol}{self.RESET} {message}"
+
+
+class ConsoleHandler(logging.StreamHandler):
+    """
+    Console handler with color-coded output.
+    """
+
+    def __init__(self, level: int = logging.INFO):
+        """
+        Initialize console handler.
+
+        Args:
+            level: Minimum log level to display
+        """
+        super().__init__(sys.stdout)
+        self.setLevel(level)
+        self.setFormatter(ConsoleFormatter())

src/logging/handlers/file.py

@@ -0,0 +1,201 @@
+"""
+File Log Handlers
+=================
+
+File-based logging with rotation support.
+"""
+
+import asyncio
+from datetime import datetime
+import json
+import logging
+from logging.handlers import RotatingFileHandler as BaseRotatingFileHandler
+from pathlib import Path
+from typing import Optional
+
+
+class FileFormatter(logging.Formatter):
+    """
+    Detailed file formatter for log files.
+    Format: TIMESTAMP [LEVEL] [Module] Message
+    """
+
+    def __init__(self):
+        super().__init__(
+            fmt="%(asctime)s [%(levelname)-8s] [%(module_name)-12s] %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+
+    def format(self, record: logging.LogRecord) -> str:
+        # Ensure module_name exists
+        if not hasattr(record, "module_name"):
+            record.module_name = record.name
+        return super().format(record)
+
+
+class FileHandler(logging.FileHandler):
+    """
+    File handler with detailed formatting.
+    """
+
+    def __init__(
+        self,
+        filename: str,
+        level: int = logging.DEBUG,
+        encoding: str = "utf-8",
+    ):
+        """
+        Initialize file handler.
+
+        Args:
+            filename: Path to log file
+            level: Minimum log level
+            encoding: File encoding
+        """
+        # Ensure directory exists
+        Path(filename).parent.mkdir(parents=True, exist_ok=True)
+
+        super().__init__(filename, encoding=encoding)
+        self.setLevel(level)
+        self.setFormatter(FileFormatter())
+
+
+class RotatingFileHandler(BaseRotatingFileHandler):
+    """
+    Rotating file handler with size-based rotation.
+    """
+
+    def __init__(
+        self,
+        filename: str,
+        level: int = logging.DEBUG,
+        max_bytes: int = 10 * 1024 * 1024,  # 10MB
+        backup_count: int = 5,
+        encoding: str = "utf-8",
+    ):
+        """
+        Initialize rotating file handler.
+
+        Args:
+            filename: Path to log file
+            level: Minimum log level
+            max_bytes: Maximum file size before rotation
+            backup_count: Number of backup files to keep
+            encoding: File encoding
+        """
+        # Ensure directory exists
+        Path(filename).parent.mkdir(parents=True, exist_ok=True)
+
+        super().__init__(
+            filename,
+            maxBytes=max_bytes,
+            backupCount=backup_count,
+            encoding=encoding,
+        )
+        self.setLevel(level)
+        self.setFormatter(FileFormatter())
+
+
+class JSONFileHandler(logging.Handler):
+    """
+    A logging handler that writes structured JSON logs to a file.
+    Each line is a valid JSON object (JSONL format).
+
+    Useful for:
+    - LLM call logging
+    - Structured analysis
+    - Log parsing and analysis
+    """
+
+    def __init__(
+        self,
+        filepath: str,
+        level: int = logging.DEBUG,
+        encoding: str = "utf-8",
+    ):
+        """
+        Initialize JSON file handler.
+
+        Args:
+            filepath: Path to log file
+            level: Minimum log level
+            encoding: File encoding
+        """
+        super().__init__()
+
+        # Ensure directory exists
+        Path(filepath).parent.mkdir(parents=True, exist_ok=True)
+
+        self.filepath = filepath
+        self.encoding = encoding
+        self.setLevel(level)
+        self.setFormatter(logging.Formatter("%(message)s"))
+
+    def emit(self, record: logging.LogRecord):
+        """Emit a log record as JSON."""
+        try:
+            # Build JSON entry
+            entry = {
+                "timestamp": datetime.fromtimestamp(record.created).isoformat(),
+                "level": record.levelname,
+                "module": getattr(record, "module_name", record.name),
+                "message": self.format(record),
+            }
+
+            # Add extra fields if present
+            for key in ["symbol", "display_level", "tool_name", "elapsed_ms", "tokens"]:
+                if hasattr(record, key):
+                    entry[key] = getattr(record, key)
+
+            # Write to file
+            with open(self.filepath, "a", encoding=self.encoding) as f:
+                f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+        except Exception:
+            self.handleError(record)
+
+
+def create_task_logger(
+    task_id: str,
+    module_name: str,
+    log_dir: str,
+    queue: Optional["asyncio.Queue"] = None,
+) -> logging.Logger:
+    """
+    Create a logger for a specific task with file and optional WebSocket output.
+
+    Args:
+        task_id: Unique task identifier
+        module_name: Module name (e.g., "Solver", "Research")
+        log_dir: Directory for log files
+        queue: Optional asyncio.Queue for WebSocket streaming
+
+    Returns:
+        Configured logger
+    """
+    from .websocket import WebSocketLogHandler
+
+    # Create log directory
+    log_path = Path(log_dir)
+    log_path.mkdir(parents=True, exist_ok=True)
+
+    # Create logger
+    logger = logging.getLogger(f"ai_tutor.{module_name}.{task_id}")
+    logger.setLevel(logging.DEBUG)
+    logger.handlers.clear()
+    logger.propagate = False
+
+    # File handler
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    log_file = log_path / f"{module_name}_{task_id}_{timestamp}.log"
+
+    file_handler = FileHandler(str(log_file))
+    logger.addHandler(file_handler)
+
+    # WebSocket handler if queue provided
+    if queue is not None:
+        ws_handler = WebSocketLogHandler(queue)
+        ws_handler.setLevel(logging.INFO)
+        logger.addHandler(ws_handler)
+
+    return logger