realtimex-deeptutor 0.5.0.post1__py3-none-any.whl → 0.5.0.post3__py3-none-any.whl

src/knowledge/manager.py

@@ -6,11 +6,61 @@ Knowledge Base Manager
 Manages multiple knowledge bases and provides utilities for accessing them.
 """
 
+from contextlib import contextmanager
 from datetime import datetime
 import hashlib
 import json
+import os
 from pathlib import Path
 import shutil
+import sys
+
+from src.services.rag.components.routing import FileTypeRouter
+
+
+# Cross-platform file locking
+@contextmanager
+def file_lock_shared(file_handle):
+    """Acquire a shared (read) lock on a file - cross-platform."""
+    if sys.platform == "win32":
+        import msvcrt
+
+        msvcrt.locking(file_handle.fileno(), msvcrt.LK_NBLCK, 1)
+        try:
+            yield
+        finally:
+            file_handle.seek(0)
+            msvcrt.locking(file_handle.fileno(), msvcrt.LK_UNLCK, 1)
+    else:
+        import fcntl
+
+        fcntl.flock(file_handle.fileno(), fcntl.LOCK_SH)
+        try:
+            yield
+        finally:
+            fcntl.flock(file_handle.fileno(), fcntl.LOCK_UN)
+
+
+@contextmanager
+def file_lock_exclusive(file_handle):
+    """Acquire an exclusive (write) lock on a file - cross-platform."""
+    if sys.platform == "win32":
+        import msvcrt
+
+        msvcrt.locking(file_handle.fileno(), msvcrt.LK_NBLCK, 1)
+        try:
+            yield
+        finally:
+            file_handle.seek(0)
+            msvcrt.locking(file_handle.fileno(), msvcrt.LK_UNLCK, 1)
+    else:
+        import fcntl
+
+        fcntl.flock(file_handle.fileno(), fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(file_handle.fileno(), fcntl.LOCK_UN)
 
 
 class KnowledgeBaseManager:
@@ -27,42 +77,113 @@ class KnowledgeBaseManager:
     def _load_config(self) -> dict:
         """Load knowledge base configuration (kb_config.json only stores KB list)"""
         if self.config_file.exists():
-            with open(self.config_file, encoding="utf-8") as f:
-                config = json.load(f)
+            try:
+                with open(self.config_file, encoding="utf-8") as f:
+                    with file_lock_shared(f):
+                        content = f.read()
+                        if not content.strip():
+                            # Empty file, return default
+                            return {"knowledge_bases": {}}
+                        config = json.loads(content)
+
+                # Ensure knowledge_bases key exists
+                if "knowledge_bases" not in config:
+                    config["knowledge_bases"] = {}
+
                 # Migration: remove old "default" field if present
                 if "default" in config:
                     del config["default"]
-                    # Save cleaned config
-                    try:
-                        with open(self.config_file, "w", encoding="utf-8") as wf:
-                            json.dump(config, wf, indent=2, ensure_ascii=False)
-                    except Exception:
-                        pass
+                    # Note: Don't save during load to avoid recursion issues
+                    # The next _save_config() call will persist this change
+
                 return config
+            except (json.JSONDecodeError, Exception) as e:
+                print(f"[KnowledgeBaseManager] Error loading config: {e}")
+                return {"knowledge_bases": {}}
         return {"knowledge_bases": {}}
 
     def _save_config(self):
-        """Save knowledge base configuration"""
+        """Save knowledge base configuration (thread-safe with file locking)"""
+        # Use exclusive lock for writing
         with open(self.config_file, "w", encoding="utf-8") as f:
-            json.dump(self.config, f, indent=2, ensure_ascii=False)
+            with file_lock_exclusive(f):
+                json.dump(self.config, f, indent=2, ensure_ascii=False)
+                f.flush()
+                os.fsync(f.fileno())  # Ensure data is written to disk
+
+    def update_kb_status(
+        self,
+        name: str,
+        status: str,
+        progress: dict | None = None,
+    ):
+        """
+        Update knowledge base status and progress in kb_config.json.
+
+        Args:
+            name: Knowledge base name
+            status: Status string ("initializing", "processing", "ready", "error")
+            progress: Optional progress dict with keys like:
+                - stage: Current stage name
+                - message: Human-readable message
+                - percent: Progress percentage (0-100)
+                - current: Current item number
+                - total: Total items
+                - file_name: Current file being processed
+                - error: Error message (if status is "error")
+        """
+        # Reload config to get latest state
+        self.config = self._load_config()
+
+        if "knowledge_bases" not in self.config:
+            self.config["knowledge_bases"] = {}
+
+        if name not in self.config["knowledge_bases"]:
+            # Auto-register if not exists
+            self.config["knowledge_bases"][name] = {
+                "path": name,
+                "description": f"Knowledge base: {name}",
+            }
+
+        kb_config = self.config["knowledge_bases"][name]
+        kb_config["status"] = status
+        kb_config["updated_at"] = datetime.now().isoformat()
+
+        if progress is not None:
+            kb_config["progress"] = progress
+        elif status == "ready":
+            # Clear progress when ready
+            kb_config["progress"] = {
+                "stage": "completed",
+                "message": "Ready",
+                "percent": 100,
+            }
+
+        self._save_config()
+
+    def get_kb_status(self, name: str) -> dict | None:
+        """Get status and progress for a knowledge base."""
+        self.config = self._load_config()
+        kb_config = self.config.get("knowledge_bases", {}).get(name)
+        if not kb_config:
+            return None
+        return {
+            "status": kb_config.get("status", "unknown"),
+            "progress": kb_config.get("progress"),
+            "updated_at": kb_config.get("updated_at"),
+        }
 
     def list_knowledge_bases(self) -> list[str]:
         """List all available knowledge bases from kb_config.json"""
-        kb_list = []
+        # Always reload config from file to ensure we have the latest data
+        # This is important when new KBs are created by other processes/requests
+        self.config = self._load_config()
 
         # Read knowledge base list from config file (this is the authoritative source)
+        # Return all KBs in config, regardless of directory status
+        # (status field indicates if KB is ready or still initializing)
         config_kbs = self.config.get("knowledge_bases", {})
-
-        for kb_name in config_kbs.keys():
-            # Verify knowledge base directory exists
-            kb_dir = self.base_dir / kb_name
-            if kb_dir.exists() and kb_dir.is_dir():
-                kb_list.append(kb_name)
-            else:
-                # If in config but directory doesn't exist, log warning but don't add
-                print(
-                    f"Warning: Knowledge base '{kb_name}' is in config but directory does not exist: {kb_dir}"
-                )
+        kb_list = list(config_kbs.keys())
 
         # If no config file or config is empty, fallback to scanning directory (backward compatibility)
         if not kb_list and self.base_dir.exists():
@@ -183,83 +304,115 @@ class KnowledgeBaseManager:
 
         This method:
         1. Gets the KB name (from parameter or default)
-        2. Reads metadata.json from the KB directory
-        3. Collects statistics about files and RAG status
+        2. Reads status and progress from kb_config.json
+        3. Reads metadata.json from the KB directory (if exists)
+        4. Collects statistics about files and RAG status
         """
+        # Reload config to get latest status
+        self.config = self._load_config()
+
         kb_name = name or self.get_default()
         if kb_name is None:
             raise ValueError("No knowledge base name provided and no default set")
 
         # Get knowledge base path
         kb_dir = self.base_dir / kb_name
-        if not kb_dir.exists():
-            raise ValueError(f"Knowledge base directory does not exist: {kb_dir}")
 
-        # Verify knowledge base is in config (if not, give warning but don't block)
-        if kb_name not in self.config.get("knowledge_bases", {}):
-            print(
-                f"Warning: Knowledge base '{kb_name}' is not in kb_config.json, but directory exists"
-            )
+        # Get status and progress from kb_config.json
+        kb_config = self.config.get("knowledge_bases", {}).get(kb_name, {})
+        status = kb_config.get("status")
+        progress = kb_config.get("progress")
+
+        # KB might not have a directory yet if still initializing
+        dir_exists = kb_dir.exists()
+
+        # For old KBs without status field, determine status from rag_storage
+        if not status and dir_exists:
+            rag_storage_dir = kb_dir / "rag_storage"
+            if rag_storage_dir.exists() and any(rag_storage_dir.iterdir()):
+                status = "ready"
+            else:
+                status = "unknown"
+        elif not status:
+            status = "unknown"
 
         info = {
             "name": kb_name,
             "path": str(kb_dir),
             "is_default": kb_name == self.get_default(),
             "metadata": {},
+            "status": status,
+            "progress": progress,
         }
 
         # Read metadata.json (if exists)
-        metadata_file = kb_dir / "metadata.json"
-        if metadata_file.exists():
-            try:
-                with open(metadata_file, encoding="utf-8") as f:
-                    info["metadata"] = json.load(f)
-            except Exception as e:
-                print(f"Warning: Failed to read metadata.json for KB '{kb_name}': {e}")
-                info["metadata"] = {}
-        else:
-            # metadata.json doesn't exist, use empty dict
-            info["metadata"] = {}
+        if dir_exists:
+            metadata_file = kb_dir / "metadata.json"
+            if metadata_file.exists():
+                try:
+                    with open(metadata_file, encoding="utf-8") as f:
+                        info["metadata"] = json.load(f)
+                except Exception as e:
+                    print(f"Warning: Failed to read metadata.json for KB '{kb_name}': {e}")
+                    info["metadata"] = {}
 
         # Count files - handle errors gracefully
-        raw_dir = kb_dir / "raw"
-        images_dir = kb_dir / "images"
-        content_list_dir = kb_dir / "content_list"
-        rag_storage_dir = kb_dir / "rag_storage"
+        raw_dir = kb_dir / "raw" if dir_exists else None
+        images_dir = kb_dir / "images" if dir_exists else None
+        content_list_dir = kb_dir / "content_list" if dir_exists else None
+        rag_storage_dir = kb_dir / "rag_storage" if dir_exists else None
 
-        try:
-            raw_count = (
-                len([f for f in raw_dir.iterdir() if f.is_file()]) if raw_dir.exists() else 0
-            )
-        except Exception:
-            raw_count = 0
+        raw_count = 0
+        images_count = 0
+        content_lists_count = 0
 
-        try:
-            images_count = (
-                len([f for f in images_dir.iterdir() if f.is_file()]) if images_dir.exists() else 0
-            )
-        except Exception:
-            images_count = 0
+        if dir_exists:
+            try:
+                raw_count = (
+                    len([f for f in raw_dir.iterdir() if f.is_file()]) if raw_dir.exists() else 0
+                )
+            except Exception:
+                pass
 
-        try:
-            content_lists_count = (
-                len(list(content_list_dir.glob("*.json"))) if content_list_dir.exists() else 0
-            )
-        except Exception:
-            content_lists_count = 0
+            try:
+                images_count = (
+                    len([f for f in images_dir.iterdir() if f.is_file()])
+                    if images_dir.exists()
+                    else 0
+                )
+            except Exception:
+                pass
+
+            try:
+                content_lists_count = (
+                    len(list(content_list_dir.glob("*.json"))) if content_list_dir.exists() else 0
+                )
+            except Exception:
+                pass
 
         metadata = info["metadata"]
         rag_provider = metadata.get("rag_provider") if isinstance(metadata, dict) else None
+        # Also check kb_config for rag_provider (fallback)
+        if not rag_provider:
+            rag_provider = kb_config.get("rag_provider")
+
+        rag_initialized = (
+            dir_exists and rag_storage_dir and rag_storage_dir.exists() and rag_storage_dir.is_dir()
+        )
+
         info["statistics"] = {
             "raw_documents": raw_count,
             "images": images_count,
             "content_lists": content_lists_count,
-            "rag_initialized": rag_storage_dir.exists() and rag_storage_dir.is_dir(),
-            "rag_provider": rag_provider,  # Add RAG provider info
+            "rag_initialized": rag_initialized,
+            "rag_provider": rag_provider,
+            # Include status and progress in statistics for backward compatibility
+            "status": status,
+            "progress": progress,
         }
 
         # Try to get RAG statistics
-        if rag_storage_dir.exists() and rag_storage_dir.is_dir():
+        if rag_initialized:
             try:
                 entities_file = rag_storage_dir / "kv_store_full_entities.json"
                 relations_file = rag_storage_dir / "kv_store_full_relations.json"
@@ -406,14 +559,25 @@ class KnowledgeBaseManager:
         if not folder.is_dir():
             raise ValueError(f"Path is not a directory: {folder}")
 
-        # Get supported files in folder
-        supported_extensions = {".pdf", ".docx", ".doc", ".txt", ".md", ".markdown"}
+        # Get RAG provider from KB metadata to determine supported extensions
+        kb_dir = self.base_dir / kb_name
+        metadata_file = kb_dir / "metadata.json"
+        provider = "raganything"  # default to most comprehensive
+        if metadata_file.exists():
+            try:
+                with open(metadata_file, encoding="utf-8") as f:
+                    kb_meta = json.load(f)
+                    provider = kb_meta.get("rag_provider") or "raganything"
+            except Exception:
+                pass
+
+        # Get supported files in folder based on provider
+        supported_extensions = FileTypeRouter.get_extensions_for_provider(provider)
         files: list[Path] = []
         for ext in supported_extensions:
             files.extend(folder.glob(f"**/*{ext}"))
 
         # Generate folder ID
-        import hashlib
 
         folder_id = hashlib.md5(  # noqa: S324
             str(folder).encode(), usedforsecurity=False
@@ -523,12 +687,13 @@ class KnowledgeBaseManager:
 
         return True
 
-    def scan_linked_folder(self, folder_path: str) -> list[str]:
+    def scan_linked_folder(self, folder_path: str, provider: str = "raganything") -> list[str]:
         """
         Scan a linked folder and return list of supported file paths.
 
         Args:
             folder_path: Path to folder
+            provider: RAG provider to determine supported extensions (default: raganything)
 
         Returns:
             List of file paths (as strings)
@@ -538,7 +703,7 @@ class KnowledgeBaseManager:
         if not folder.exists() or not folder.is_dir():
             return []
 
-        supported_extensions = {".pdf", ".docx", ".doc", ".txt", ".md", ".markdown"}
+        supported_extensions = FileTypeRouter.get_extensions_for_provider(provider)
         files = []
 
         for ext in supported_extensions:
@@ -583,8 +748,20 @@ class KnowledgeBaseManager:
             except Exception:
                 pass
 
-        # Scan current files
-        supported_extensions = {".pdf", ".docx", ".doc", ".txt", ".md", ".markdown"}
+        # Get RAG provider from KB metadata to determine supported extensions
+        kb_dir = self.base_dir / kb_name
+        metadata_file = kb_dir / "metadata.json"
+        provider = "raganything"  # default to most comprehensive
+        if metadata_file.exists():
+            try:
+                with open(metadata_file, encoding="utf-8") as f:
+                    metadata = json.load(f)
+                    provider = metadata.get("rag_provider") or "raganything"
+            except Exception:
+                pass
+
+        # Scan current files based on provider's supported extensions
+        supported_extensions = FileTypeRouter.get_extensions_for_provider(provider)
         new_files = []
         modified_files = []
 

src/knowledge/progress_tracker.py

@@ -99,13 +99,54 @@ class ProgressTracker:
                 print(f"[ProgressTracker] Callback error: {e}")
 
     def _save_progress(self, progress: dict):
-        """Save progress to file"""
+        """Save progress to kb_config.json and local .progress.json file"""
+        # Save to kb_config.json (centralized config)
+        try:
+            from src.knowledge.manager import KnowledgeBaseManager
+
+            manager = KnowledgeBaseManager(base_dir=str(self.base_dir))
+
+            # Determine status based on stage
+            stage = progress.get("stage", "")
+            if stage == "completed":
+                status = "ready"
+            elif stage == "error":
+                status = "error"
+            elif stage in [
+                "initializing",
+                "processing_documents",
+                "processing_file",
+                "extracting_items",
+            ]:
+                status = "processing"
+            else:
+                status = "initializing"
+
+            # Update kb_config.json with status and progress
+            manager.update_kb_status(
+                name=self.kb_name,
+                status=status,
+                progress={
+                    "stage": progress.get("stage"),
+                    "message": progress.get("message"),
+                    "percent": progress.get("progress_percent", 0),
+                    "current": progress.get("current", 0),
+                    "total": progress.get("total", 0),
+                    "file_name": progress.get("file_name"),
+                    "error": progress.get("error"),
+                    "timestamp": progress.get("timestamp"),
+                },
+            )
+        except Exception as e:
+            print(f"[ProgressTracker] Failed to save progress to kb_config.json: {e}")
+
+        # Also save to local .progress.json file (for backward compatibility)
         try:
             self.kb_dir.mkdir(parents=True, exist_ok=True)
             with open(self.progress_file, "w", encoding="utf-8") as f:
                 json.dump(progress, f, indent=2, ensure_ascii=False)
         except Exception as e:
-            print(f"[ProgressTracker] Failed to save progress: {e}")
+            print(f"[ProgressTracker] Failed to save progress to local file: {e}")
 
     def update(
         self,

src/knowledge/start_kb.py

@@ -15,6 +15,8 @@ try:
     from .config import KNOWLEDGE_BASES_DIR, get_env_config, setup_paths
 
     setup_paths()
+    from src.services.rag.components.routing import FileTypeRouter
+
     from .extract_numbered_items import process_content_list
     from .initializer import KnowledgeBaseInitializer
     from .manager import KnowledgeBaseManager
@@ -28,6 +30,7 @@ except ImportError:
     from src.knowledge.extract_numbered_items import process_content_list
     from src.knowledge.initializer import KnowledgeBaseInitializer
     from src.knowledge.manager import KnowledgeBaseManager
+    from src.services.rag.components.routing import FileTypeRouter
 
 
 def list_knowledge_bases():
@@ -123,6 +126,12 @@ async def init_knowledge_base(args):
         return
 
     # Collect document files
+    # Use provider from env var or default to raganything (most comprehensive)
+    import os
+
+    provider = os.getenv("RAG_PROVIDER", "raganything")
+    glob_patterns = FileTypeRouter.get_glob_patterns_for_provider(provider)
+
     doc_files = []
     if args.docs:
         doc_files.extend(args.docs)
@@ -130,8 +139,8 @@ async def init_knowledge_base(args):
     if args.docs_dir:
         docs_dir = Path(args.docs_dir)
         if docs_dir.exists() and docs_dir.is_dir():
-            for ext in ["*.pdf", "*.docx", "*.doc", "*.txt", "*.md"]:
-                doc_files.extend([str(f) for f in docs_dir.glob(ext)])
+            for pattern in glob_patterns:
+                doc_files.extend([str(f) for f in docs_dir.glob(pattern)])
         else:
             print(f"✗ Error: Document directory does not exist: {args.docs_dir}\n")
             return

src/logging/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 Unified Logging System for DeepTutor
 =====================================
@@ -39,6 +40,7 @@ from .adapters import (
 from .config import (
     LoggingConfig,
     get_default_log_dir,
+    get_global_log_level,
     load_logging_config,
 )
 
@@ -58,6 +60,7 @@ from .logger import (
     LogLevel,
     get_logger,
     reset_logger,
+    set_default_service_prefix,
 )
 
 # Statistics tracking
@@ -75,6 +78,7 @@ __all__ = [
     "LogLevel",
     "get_logger",
     "reset_logger",
+    "set_default_service_prefix",
     "ConsoleFormatter",
     "FileFormatter",
     # Handlers
@@ -100,4 +104,5 @@ __all__ = [
     "LoggingConfig",
     "load_logging_config",
     "get_default_log_dir",
+    "get_global_log_level",
 ]

src/logging/adapters/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 Log Adapters
 ============

src/logging/adapters/lightrag.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 """
 LightRAG Log Forwarder
 ======================
 
 Forwards LightRAG and RAG-Anything logs to DeepTutor's unified logging system.
+Uses the unified global log level from config/main.yaml -> logging.level
 """
 
 from contextlib import contextmanager
@@ -31,17 +33,21 @@ class LightRAGLogForwarder(logging.Handler):
 
     def emit(self, record: logging.LogRecord):
         """
-        Forward log record to DeepTutor logger.
-        All logs are forwarded as info level to maintain consistent format.
+        Forward log record to DeepTutor logger with proper level mapping.
         """
         try:
-            # Get the original message directly without adding [LightRAG] prefix
-            # LightRAG already formats messages appropriately (e.g., "DEBUG: xxx" for debug logs)
             message = record.getMessage()
 
-            # Use info() for all levels to maintain consistent format
-            # This ensures all logs appear as [RAGTool] ... (or [RAGTool] DEBUG: ... for debug)
-            self.ai_tutor_logger.info(message)
+            # Map LightRAG log levels to DeepTutor logger methods
+            level = record.levelno
+            if level >= logging.ERROR:
+                self.ai_tutor_logger.error(message)
+            elif level >= logging.WARNING:
+                self.ai_tutor_logger.warning(message)
+            elif level >= logging.INFO:
+                self.ai_tutor_logger.info(message)
+            else:
+                self.ai_tutor_logger.debug(message)
 
         except Exception:
             # Avoid errors in forwarding from affecting main flow
@@ -50,7 +56,7 @@ class LightRAGLogForwarder(logging.Handler):
 
 def get_lightrag_forwarding_config() -> dict:
     """
-    Load LightRAG forwarding configuration from main.yaml.
+    Load LightRAG forwarding configuration from config/main.yaml.
 
     Returns:
         dict: Configuration dictionary with defaults if not found
@@ -58,27 +64,28 @@ def get_lightrag_forwarding_config() -> dict:
     try:
         from src.services.config import load_config_with_main
 
-        # Use resolve() to get absolute path, ensuring correct project root regardless of working directory
+        from ..config import get_global_log_level
+
         project_root = Path(__file__).resolve().parent.parent.parent.parent
         config = load_config_with_main("solve_config.yaml", project_root)
+        logging_config = config.get("logging", {})
 
-        forwarding_config = config.get("logging", {}).get("lightrag_forwarding", {})
+        # Use the unified global log level
+        level = get_global_log_level()
 
         return {
-            "enabled": forwarding_config.get("enabled", True),
-            "min_level": forwarding_config.get("min_level", "INFO"),
-            "add_prefix": forwarding_config.get("add_prefix", True),
-            "logger_names": forwarding_config.get(
-                "logger_names", {"knowledge_init": "KnowledgeInit", "rag_tool": "RAGTool"}
+            "enabled": True,
+            "min_level": level,
+            "logger_names": logging_config.get(
+                "rag_logger_names", {"knowledge_init": "RAG-Init", "rag_tool": "RAG"}
             ),
         }
     except Exception:
         # Return defaults if config loading fails
         return {
             "enabled": True,
-            "min_level": "INFO",
-            "add_prefix": True,
-            "logger_names": {"knowledge_init": "KnowledgeInit", "rag_tool": "RAGTool"},
+            "min_level": "DEBUG",
+            "logger_names": {"knowledge_init": "RAG-Init", "rag_tool": "RAG"},
         }
 
 

src/logging/adapters/llamaindex.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 """
 LlamaIndex Log Forwarder
 ========================