codebuff-agent 2.1.74__py3-none-any.whl

codebuff_core/knowledge/loader.py

@@ -0,0 +1,339 @@
+"""
+Knowledge Base - Deterministic library detection for API guides
+
+Keyword matching + regex based library detection for loading appropriate API guides.
+(No LLM calls - saves tokens and improves reliability)
+"""
+
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+# Library descriptions (reference)
+LIBRARY_DESCRIPTIONS: Dict[str, str] = {
+    "matplotlib": "Visualization, graphs, charts, plot, histogram, scatter plot, EDA, data visualization, used with seaborn",
+    "dask": "Large-scale data processing, pandas replacement, distributed processing, lazy evaluation, dd.read_csv",
+    "polars": "High-performance DataFrame, pandas replacement, Rust-based, pl.read_csv",
+    "pyspark": "Spark-based distributed processing, big data, SparkSession",
+    "vaex": "Large-scale data exploration, out-of-core processing",
+    "modin": "pandas acceleration, parallel processing",
+    "ray": "Distributed computing, parallel processing framework",
+}
+
+
+class LibraryDetector:
+    """
+    Deterministic library detection (no LLM calls).
+    Uses keyword matching + regex to detect required libraries from user requests.
+    """
+
+    # Explicit library mention patterns (highest priority)
+    EXPLICIT_PATTERNS: Dict[str, str] = {
+        r"\bdask\b": "dask",
+        r"\bpolars\b": "polars",
+        r"\bpyspark\b": "pyspark",
+        r"\bvaex\b": "vaex",
+        r"\bmodin\b": "modin",
+        r"\bray\b": "ray",
+        r"\bmatplotlib\b": "matplotlib",
+        r"\bseaborn\b": "matplotlib",  # seaborn -> matplotlib guide
+        r"\bplt\.": "matplotlib",
+        r"\bdd\.read": "dask",
+        r"\bpl\.read": "polars",
+        r"\bpl\.DataFrame": "polars",
+    }
+
+    # Keyword scores per library (0.0 ~ 1.0)
+    KEYWORD_SCORES: Dict[str, Dict[str, float]] = {
+        "dask": {
+            "대용량": 0.7,
+            "big data": 0.7,
+            "bigdata": 0.7,
+            "빅데이터": 0.7,
+            "lazy": 0.8,
+            "lazy evaluation": 0.9,
+            "out-of-core": 0.9,
+            "out of core": 0.9,
+            "분산 처리": 0.6,
+            "distributed": 0.6,
+            "parallel dataframe": 0.8,
+            "병렬 데이터프레임": 0.8,
+        },
+        "polars": {
+            "rust 기반": 0.9,
+            "rust-based": 0.9,
+            "fast dataframe": 0.7,
+            "고성능 dataframe": 0.7,
+            "빠른 데이터프레임": 0.7,
+        },
+        "matplotlib": {
+            "시각화": 0.7,
+            "visualization": 0.7,
+            "visualize": 0.7,
+            "plot": 0.7,
+            "chart": 0.7,
+            "graph": 0.6,
+            "그래프": 0.6,
+            "차트": 0.7,
+            "histogram": 0.8,
+            "히스토그램": 0.8,
+            "scatter": 0.8,
+            "산점도": 0.8,
+            "line plot": 0.8,
+            "라인 플롯": 0.8,
+            "bar chart": 0.8,
+            "막대 그래프": 0.8,
+            "eda": 0.5,
+            "탐색적 데이터 분석": 0.6,
+            "figure": 0.5,
+            "subplot": 0.8,
+            "heatmap": 0.7,
+            "히트맵": 0.7,
+        },
+        "pyspark": {
+            "spark": 0.9,
+            "sparksession": 0.95,
+            "spark session": 0.95,
+            "rdd": 0.9,
+            "hadoop": 0.7,
+            "클러스터": 0.6,
+            "cluster": 0.6,
+        },
+        "vaex": {
+            "vaex": 1.0,
+            "memory mapping": 0.8,
+            "메모리 매핑": 0.8,
+        },
+        "modin": {
+            "modin": 1.0,
+            "pandas 가속": 0.8,
+            "pandas acceleration": 0.8,
+        },
+        "ray": {
+            "ray": 0.9,
+            "분산 컴퓨팅": 0.7,
+            "distributed computing": 0.7,
+        },
+    }
+
+    # Score threshold
+    SCORE_THRESHOLD = 0.7
+
+    def detect(
+        self,
+        request: str,
+        available_libraries: List[str],
+        imported_libraries: List[str] = None,
+    ) -> List[str]:
+        """
+        Detect required libraries from user request.
+
+        Args:
+            request: User's natural language request
+            available_libraries: List of libraries with available guides
+            imported_libraries: Already imported libraries (optional)
+
+        Returns:
+            List of detected libraries
+        """
+        request_lower = request.lower()
+        detected: Set[str] = set()
+
+        # Step 1: Explicit pattern matching (highest priority)
+        for pattern, lib in self.EXPLICIT_PATTERNS.items():
+            if lib in available_libraries and re.search(
+                pattern, request, re.IGNORECASE
+            ):
+                detected.add(lib)
+
+        # Step 2: Keyword scoring
+        for lib, keywords in self.KEYWORD_SCORES.items():
+            if lib in detected or lib not in available_libraries:
+                continue
+
+            max_score = 0.0
+            for keyword, score in keywords.items():
+                if keyword.lower() in request_lower:
+                    max_score = max(max_score, score)
+
+            if max_score >= self.SCORE_THRESHOLD:
+                detected.add(lib)
+
+        # Step 3: Consider already imported libraries
+        if imported_libraries:
+            for lib in imported_libraries:
+                lib_lower = lib.lower()
+                # seaborn -> matplotlib
+                if lib_lower == "seaborn" and "matplotlib" in available_libraries:
+                    detected.add("matplotlib")
+                elif lib_lower in available_libraries:
+                    detected.add(lib_lower)
+
+        return list(detected)
+
+
+# LibraryDetector singleton instance
+_library_detector_instance: Optional[LibraryDetector] = None
+
+
+def get_library_detector() -> LibraryDetector:
+    """Get singleton LibraryDetector"""
+    global _library_detector_instance
+    if _library_detector_instance is None:
+        _library_detector_instance = LibraryDetector()
+    return _library_detector_instance
+
+
+# LLM library detection prompt
+LIBRARY_DETECTION_PROMPT = """Analyze the user's request and determine which libraries to use for code generation.
+
+## Available Library API Guides:
+{library_list}
+
+## User Request:
+{request}
+
+## Notebook Context:
+- Already imported libraries: {imported_libraries}
+
+## Instructions:
+1. Analyze the user request **semantically**
+2. Select only libraries that will actually be used in code generation
+3. Example: "Apply dask" → select dask
+4. Example: "EDA with visualization" → select matplotlib
+5. Example: "Use dask instead of pandas" → select dask
+
+## Output Format (JSON only):
+{{"libraries": ["library1", "library2"]}}
+
+Empty array is also valid: {{"libraries": []}}
+"""
+
+
+class KnowledgeBase:
+    """Library knowledge loader"""
+
+    def __init__(self, knowledge_dir: Optional[str] = None):
+        if knowledge_dir:
+            self.knowledge_dir = Path(knowledge_dir)
+        else:
+            # Default path: knowledge/libraries
+            self.knowledge_dir = Path(__file__).parent / "libraries"
+
+        self._cache: Dict[str, str] = {}
+
+    def get_library_list_for_prompt(self) -> str:
+        """Generate library list for LLM detection"""
+        available = self.list_available_libraries()
+        lines = []
+        for lib in available:
+            desc = LIBRARY_DESCRIPTIONS.get(lib, "Other library")
+            lines.append(f"- **{lib}**: {desc}")
+        return "\n".join(lines)
+
+    def get_detection_prompt(
+        self, request: str, imported_libraries: List[str] = None
+    ) -> str:
+        """Generate LLM library detection prompt"""
+        library_list = self.get_library_list_for_prompt()
+        imported = ", ".join(imported_libraries) if imported_libraries else "None"
+
+        return LIBRARY_DETECTION_PROMPT.format(
+            library_list=library_list, request=request, imported_libraries=imported
+        )
+
+    def load_library_guide(self, library: str) -> Optional[str]:
+        """
+        Load library guide file.
+
+        Args:
+            library: Library name (e.g., 'dask', 'polars')
+
+        Returns:
+            Guide content or None
+        """
+        # Check cache
+        if library in self._cache:
+            return self._cache[library]
+
+        # Load file
+        file_path = self.knowledge_dir / f"{library}.md"
+        if file_path.exists():
+            content = file_path.read_text(encoding="utf-8")
+            self._cache[library] = content
+            return content
+
+        return None
+
+    def load_libraries_knowledge(self, libraries: List[str]) -> str:
+        """
+        Load guides for specified libraries.
+
+        Args:
+            libraries: List of library names
+
+        Returns:
+            Combined guide string
+        """
+        if not libraries:
+            return ""
+
+        guides = []
+        for lib in sorted(libraries):
+            guide = self.load_library_guide(lib)
+            if guide:
+                guides.append(f"## {lib.upper()} Library API Guide\n\n{guide}")
+
+        if not guides:
+            return ""
+
+        return "\n\n---\n\n".join(guides)
+
+    def format_knowledge_section(self, libraries: List[str]) -> str:
+        """
+        Format knowledge section for prompt injection.
+
+        Args:
+            libraries: List of libraries detected by LLM
+
+        Returns:
+            Formatted knowledge section (empty string if none)
+        """
+        knowledge = self.load_libraries_knowledge(libraries)
+
+        if not knowledge:
+            return ""
+
+        return f"""
+## 📚 Library API Reference (MUST follow!)
+
+Follow the API usage in the guides below. Avoid ❌ incorrect code and use ✅ correct code.
+
+{knowledge}
+
+---
+"""
+
+    def list_available_libraries(self) -> List[str]:
+        """List available library guides"""
+        if not self.knowledge_dir.exists():
+            return []
+
+        return [f.stem for f in self.knowledge_dir.glob("*.md")]
+
+
+# Singleton instance
+_knowledge_base_instance: Optional[KnowledgeBase] = None
+
+
+def get_knowledge_base() -> KnowledgeBase:
+    """Get singleton KnowledgeBase"""
+    global _knowledge_base_instance
+    if _knowledge_base_instance is None:
+        _knowledge_base_instance = KnowledgeBase()
+    return _knowledge_base_instance
+
+
+# Alias (backward compatibility)
+KnowledgeLoader = KnowledgeBase
+get_knowledge_loader = get_knowledge_base

codebuff_core/knowledge/watchdog_service.py

@@ -0,0 +1,354 @@
+"""
+Watchdog Service - File system monitoring for automatic RAG re-indexing.
+
+Features:
+- Monitors knowledge base directory for changes
+- Debounces rapid file changes
+- Triggers incremental re-indexing
+- Pattern-based file filtering
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import threading
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Callable, Optional, Set
+
+if TYPE_CHECKING:
+    from codebuff_core.models.rag import WatchdogConfig
+
+logger = logging.getLogger(__name__)
+
+
+class WatchdogService:
+    """
+    File system monitoring service for RAG knowledge base.
+
+    Features:
+    - Asynchronous file change detection
+    - Debouncing to prevent excessive re-indexing
+    - Pattern-based filtering (*.md, *.py, etc.)
+    - Ignore patterns for cache/build directories
+    - Callback-based notification
+
+    Usage:
+        service = WatchdogService(config, on_change_callback)
+        service.start("/path/to/knowledge")
+        # ... later
+        service.stop()
+    """
+
+    def __init__(
+        self,
+        config: "WatchdogConfig",
+        on_change_callback: Optional[Callable[[Set[Path]], None]] = None,
+    ):
+        self._config = config
+        self._on_change = on_change_callback
+        self._observer = None
+        self._running = False
+        self._pending_changes: Set[Path] = set()
+        self._last_change_time: Optional[datetime] = None
+        self._debounce_task: Optional[asyncio.Task] = None
+        self._lock = threading.Lock()
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+
+    def start(self, watch_path: Path) -> bool:
+        """
+        Start monitoring the specified directory.
+
+        Args:
+            watch_path: Directory to monitor
+
+        Returns:
+            True if monitoring started successfully
+        """
+        if not self._config.enabled:
+            logger.info("Watchdog disabled by configuration")
+            return False
+
+        if self._running:
+            logger.warning("Watchdog already running")
+            return True
+
+        try:
+            from watchdog.events import FileSystemEventHandler
+            from watchdog.observers import Observer
+        except ImportError:
+            logger.warning(
+                "watchdog not installed, file monitoring disabled. "
+                "Install with: pip install watchdog"
+            )
+            return False
+
+        if not watch_path.exists():
+            logger.warning(f"Watch path does not exist: {watch_path}")
+            return False
+
+        # Store event loop for async callbacks
+        try:
+            self._loop = asyncio.get_running_loop()
+        except RuntimeError:
+            self._loop = None
+
+        # Create event handler
+        handler = self._create_event_handler(FileSystemEventHandler)
+
+        # Start observer
+        self._observer = Observer()
+        self._observer.schedule(handler, str(watch_path), recursive=True)
+        self._observer.start()
+        self._running = True
+
+        logger.info(f"Watchdog started monitoring: {watch_path}")
+        return True
+
+    def _create_event_handler(self, base_class):
+        """Create a watchdog event handler class."""
+        service = self
+
+        class RAGEventHandler(base_class):
+            def on_any_event(self, event):
+                if event.is_directory:
+                    return
+
+                # Get file path
+                file_path = Path(event.src_path)
+
+                # Check if file matches patterns
+                if not service._should_process(file_path):
+                    return
+
+                # Queue the change
+                service._queue_change(file_path)
+
+        return RAGEventHandler()
+
+    def _should_process(self, file_path: Path) -> bool:
+        """Check if file should trigger re-indexing."""
+        file_name = file_path.name
+
+        # Check ignore patterns
+        for pattern in self._config.ignore_patterns:
+            if file_name.startswith(pattern.replace("*", "")):
+                return False
+            if pattern.startswith("*") and file_name.endswith(pattern[1:]):
+                return False
+
+        # Check include patterns
+        for pattern in self._config.patterns:
+            if pattern.startswith("*"):
+                suffix = pattern[1:]
+                if file_name.endswith(suffix):
+                    return True
+            elif file_name == pattern:
+                return True
+
+        return False
+
+    def _queue_change(self, file_path: Path) -> None:
+        """Queue a file change for processing."""
+        with self._lock:
+            self._pending_changes.add(file_path)
+            self._last_change_time = datetime.now()
+
+        logger.debug(f"Queued change: {file_path}")
+
+        # Schedule debounced processing
+        if self._loop and self._loop.is_running():
+            self._loop.call_soon_threadsafe(self._schedule_debounce)
+
+    def _schedule_debounce(self) -> None:
+        """Schedule the debounced callback."""
+        if self._debounce_task and not self._debounce_task.done():
+            self._debounce_task.cancel()
+
+        self._debounce_task = asyncio.create_task(self._debounced_callback())
+
+    async def _debounced_callback(self) -> None:
+        """Wait for debounce period then trigger callback."""
+        try:
+            await asyncio.sleep(self._config.debounce_seconds)
+
+            # Get pending changes
+            with self._lock:
+                if not self._pending_changes:
+                    return
+
+                changes = self._pending_changes.copy()
+                self._pending_changes.clear()
+
+            logger.info(f"Processing {len(changes)} file changes")
+
+            # Trigger callback
+            if self._on_change:
+                try:
+                    self._on_change(changes)
+                except Exception as e:
+                    logger.error(f"Error in change callback: {e}")
+
+        except asyncio.CancelledError:
+            # Debounce was reset, ignore
+            pass
+        except Exception as e:
+            logger.error(f"Error in debounced callback: {e}")
+
+    def stop(self) -> None:
+        """Stop file monitoring."""
+        if not self._running:
+            return
+
+        if self._observer:
+            self._observer.stop()
+            self._observer.join(timeout=5)
+            self._observer = None
+
+        if self._debounce_task and not self._debounce_task.done():
+            self._debounce_task.cancel()
+
+        self._running = False
+        self._pending_changes.clear()
+
+        logger.info("Watchdog stopped")
+
+    @property
+    def is_running(self) -> bool:
+        """Check if watchdog is currently running."""
+        return self._running
+
+    @property
+    def pending_count(self) -> int:
+        """Get number of pending changes."""
+        with self._lock:
+            return len(self._pending_changes)
+
+
+class SimpleWatchdog:
+    """
+    Simplified watchdog for polling-based monitoring.
+
+    Use this when the watchdog library is not available
+    or for environments where filesystem events are unreliable.
+    """
+
+    def __init__(
+        self,
+        patterns: list[str] = None,
+        check_interval: float = 10.0,
+        on_change_callback: Optional[Callable[[Set[Path]], None]] = None,
+    ):
+        self._patterns = patterns or ["*.md", "*.py", "*.txt"]
+        self._check_interval = check_interval
+        self._on_change = on_change_callback
+        self._running = False
+        self._file_mtimes: dict[Path, float] = {}
+        self._check_task: Optional[asyncio.Task] = None
+
+    async def start(self, watch_path: Path) -> bool:
+        """Start polling-based monitoring."""
+        if self._running:
+            return True
+
+        if not watch_path.exists():
+            logger.warning(f"Watch path does not exist: {watch_path}")
+            return False
+
+        self._running = True
+        self._check_task = asyncio.create_task(self._poll_loop(watch_path))
+
+        logger.info(f"SimpleWatchdog started polling: {watch_path}")
+        return True
+
+    async def _poll_loop(self, watch_path: Path) -> None:
+        """Main polling loop."""
+        # Initial scan
+        self._scan_directory(watch_path)
+
+        while self._running:
+            try:
+                await asyncio.sleep(self._check_interval)
+
+                if not self._running:
+                    break
+
+                # Check for changes
+                changes = self._check_changes(watch_path)
+
+                if changes and self._on_change:
+                    try:
+                        self._on_change(changes)
+                    except Exception as e:
+                        logger.error(f"Error in change callback: {e}")
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Error in poll loop: {e}")
+
+    def _scan_directory(self, watch_path: Path) -> None:
+        """Scan directory and record file mtimes."""
+        for pattern in self._patterns:
+            for file_path in watch_path.rglob(pattern.lstrip("*")):
+                if file_path.is_file():
+                    try:
+                        self._file_mtimes[file_path] = file_path.stat().st_mtime
+                    except OSError:
+                        pass
+
+    def _check_changes(self, watch_path: Path) -> Set[Path]:
+        """Check for file changes since last scan."""
+        changes = set()
+        current_files: Set[Path] = set()
+
+        # Check existing files
+        for pattern in self._patterns:
+            suffix = pattern.lstrip("*")
+            for file_path in watch_path.rglob(f"*{suffix}"):
+                if not file_path.is_file():
+                    continue
+
+                current_files.add(file_path)
+
+                try:
+                    mtime = file_path.stat().st_mtime
+                except OSError:
+                    continue
+
+                # Check if new or modified
+                if file_path not in self._file_mtimes:
+                    changes.add(file_path)
+                    self._file_mtimes[file_path] = mtime
+                elif self._file_mtimes[file_path] != mtime:
+                    changes.add(file_path)
+                    self._file_mtimes[file_path] = mtime
+
+        # Check for deleted files
+        deleted = set(self._file_mtimes.keys()) - current_files
+        for file_path in deleted:
+            del self._file_mtimes[file_path]
+            # Note: We don't add deleted files to changes
+            # The RAG system should handle missing files gracefully
+
+        return changes
+
+    async def stop(self) -> None:
+        """Stop polling."""
+        self._running = False
+
+        if self._check_task and not self._check_task.done():
+            self._check_task.cancel()
+            try:
+                await self._check_task
+            except asyncio.CancelledError:
+                pass
+
+        self._file_mtimes.clear()
+        logger.info("SimpleWatchdog stopped")
+
+    @property
+    def is_running(self) -> bool:
+        """Check if watchdog is currently running."""
+        return self._running