codebuff-agent 2.1.74__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codebuff_agent-2.1.74.data/data/etc/jupyter/jupyter_server_config.d/codebuff_agent.json +7 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/package.json +157 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/104.2b8f20f9d1ac2c9db01b.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/150.39fa98117cc8ca590465.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/166.386ff9b2c37ac865fdee.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/216.25cf81131bf7274dabf4.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/353.581bd1cb00746399b62e.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/367.575f9621b63c9a641aa9.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/421.11dac223a2c2da1b09f5.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/432.8513b4c6cc1d1232d165.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/432.8513b4c6cc1d1232d165.js.LICENSE.txt +19 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/439.ba5b5ef70ec6de214b8c.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/439.ba5b5ef70ec6de214b8c.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/481.cc2dec9fa869d66f1407.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/487.0796e931d6e651482c14.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/49.d326f554b6f9ccde6580.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/541.26897ea32845659e36be.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/579.e569d3edd8391fd8fbb3.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/58.eb7cc679f7535cc8a2f1.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/58.eb7cc679f7535cc8a2f1.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/666.d9bfdb23f98489d0a188.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/677.d266c39dbed33bc388b3.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/677.d266c39dbed33bc388b3.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/68.26b6f59c8454adb17f20.js +204 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/694.4ac443814260057e68f7.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/791.61affb8d3480d3d60e0d.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/848.4d2db09cfbb09374b299.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/848.4d2db09cfbb09374b299.js.LICENSE.txt +9 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/868.5cfe7e1a348ec5af1b8c.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/97.ea559a5b8a9a09f17c6b.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/remoteEntry.5292fc9c6df7e7920cc8.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/style.js +4 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/third-party-licenses.json +724 -0
- codebuff_agent-2.1.74.dist-info/METADATA +142 -0
- codebuff_agent-2.1.74.dist-info/RECORD +217 -0
- codebuff_agent-2.1.74.dist-info/WHEEL +4 -0
- codebuff_agent-2.1.74.dist-info/licenses/LICENSE +21 -0
- codebuff_core/__init__.py +153 -0
- codebuff_core/codebuff_config_defaults.json +64 -0
- codebuff_core/core/__init__.py +35 -0
- codebuff_core/core/code_validator.py +1238 -0
- codebuff_core/core/error_classifier.py +579 -0
- codebuff_core/core/exceptions.py +49 -0
- codebuff_core/data/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 +100256 -0
- codebuff_core/directory_policy.py +32 -0
- codebuff_core/factory.py +155 -0
- codebuff_core/interfaces.py +197 -0
- codebuff_core/knowledge/__init__.py +31 -0
- codebuff_core/knowledge/chunking.py +375 -0
- codebuff_core/knowledge/loader.py +339 -0
- codebuff_core/knowledge/watchdog_service.py +354 -0
- codebuff_core/legacy_paths.py +69 -0
- codebuff_core/llm/__init__.py +13 -0
- codebuff_core/llm/service.py +407 -0
- codebuff_core/managers/__init__.py +22 -0
- codebuff_core/managers/config_manager.py +690 -0
- codebuff_core/managers/remote_config.py +673 -0
- codebuff_core/managers/remote_skills.py +539 -0
- codebuff_core/managers/session_manager.py +255 -0
- codebuff_core/middleware/__init__.py +3 -0
- codebuff_core/middleware/error_recovery.py +394 -0
- codebuff_core/middleware/logging_utils.py +33 -0
- codebuff_core/models/__init__.py +111 -0
- codebuff_core/models/agent.py +306 -0
- codebuff_core/models/chat.py +57 -0
- codebuff_core/models/common.py +82 -0
- codebuff_core/models/rag.py +300 -0
- codebuff_core/platform_rules.py +127 -0
- codebuff_core/prompts/__init__.py +44 -0
- codebuff_core/prompts/auto_agent_prompts.py +1278 -0
- codebuff_core/services/__init__.py +30 -0
- codebuff_core/services/agent_service.py +345 -0
- codebuff_core/services/analytics_aggregate.py +837 -0
- codebuff_core/services/chat_service.py +898 -0
- codebuff_core/services/context_condenser.py +306 -0
- codebuff_core/services/pii_guardrail.py +393 -0
- codebuff_core/services/rag_chat_agent.py +2993 -0
- codebuff_core/services/rag_collections.py +2740 -0
- codebuff_core/services/rag_manager.py +890 -0
- codebuff_core/services/rag_service.py +145 -0
- codebuff_core/services/retriever.py +260 -0
- codebuff_core/services/vllm_embedding_service.py +290 -0
- codebuff_core/tests/__init__.py +1 -0
- codebuff_core/tests/conftest.py +81 -0
- codebuff_core/tests/test_error_recovery.py +647 -0
- codebuff_core/tests/test_factory.py +135 -0
- codebuff_core/tests/test_services.py +150 -0
- jupyter_ext/__init__.py +266 -0
- jupyter_ext/_version.py +4 -0
- jupyter_ext/agent/__init__.py +49 -0
- jupyter_ext/agent/agent_factory.py +2270 -0
- jupyter_ext/agent/ast_tools.py +451 -0
- jupyter_ext/agent/autofix_middleware.py +481 -0
- jupyter_ext/agent/background_procs.py +372 -0
- jupyter_ext/agent/code_history_middleware.py +211 -0
- jupyter_ext/agent/codex_autofix.py +160 -0
- jupyter_ext/agent/codex_bridge.py +3339 -0
- jupyter_ext/agent/codex_hooks.py +320 -0
- jupyter_ext/agent/codex_slash.py +685 -0
- jupyter_ext/agent/codex_threads.py +114 -0
- jupyter_ext/agent/content_injection.py +377 -0
- jupyter_ext/agent/continuation_middleware.py +548 -0
- jupyter_ext/agent/custom_skills.py +335 -0
- jupyter_ext/agent/file_diffs.py +99 -0
- jupyter_ext/agent/git_history.py +188 -0
- jupyter_ext/agent/guardrail_middleware.py +421 -0
- jupyter_ext/agent/json_repair_utils.py +54 -0
- jupyter_ext/agent/langchain_tools.py +2673 -0
- jupyter_ext/agent/mermaid_fixer.py +289 -0
- jupyter_ext/agent/mermaid_middleware.py +135 -0
- jupyter_ext/agent/middleware.py +359 -0
- jupyter_ext/agent/middleware_prompts.py +53 -0
- jupyter_ext/agent/observability.py +334 -0
- jupyter_ext/agent/patch_tool_calls.py +57 -0
- jupyter_ext/agent/prompts.py +112 -0
- jupyter_ext/agent/skill_middleware.py +257 -0
- jupyter_ext/agent/skills/airflow_dag_development.md +36 -0
- jupyter_ext/agent/skills/anomaly_detection.md +12 -0
- jupyter_ext/agent/skills/athena_sql.md +156 -0
- jupyter_ext/agent/skills/bdp_table_search.md +33 -0
- jupyter_ext/agent/skills/bigquery.md +65 -0
- jupyter_ext/agent/skills/bigquery_oracle.md +231 -0
- jupyter_ext/agent/skills/bigquery_tuning.md +307 -0
- jupyter_ext/agent/skills/classification_evaluation.md +11 -0
- jupyter_ext/agent/skills/clustering_analysis.md +12 -0
- jupyter_ext/agent/skills/comprehensive_eda.md +11 -0
- jupyter_ext/agent/skills/data_analysis.md +236 -0
- jupyter_ext/agent/skills/data_cleaning_pipeline.md +12 -0
- jupyter_ext/agent/skills/data_loading.md +158 -0
- jupyter_ext/agent/skills/data_quality_report.md +13 -0
- jupyter_ext/agent/skills/dimensionality_reduction.md +12 -0
- jupyter_ext/agent/skills/emr_script_development.md +17 -0
- jupyter_ext/agent/skills/experiment_tracking.md +13 -0
- jupyter_ext/agent/skills/feature_importance_analysis.md +10 -0
- jupyter_ext/agent/skills/hyperparameter_tuning.md +12 -0
- jupyter_ext/agent/skills/hypothesis_testing.md +12 -0
- jupyter_ext/agent/skills/imbalanced_data_handling.md +12 -0
- jupyter_ext/agent/skills/inference.md +392 -0
- jupyter_ext/agent/skills/lambda_function_development.md +19 -0
- jupyter_ext/agent/skills/model_comparison.md +11 -0
- jupyter_ext/agent/skills/model_explainability.md +12 -0
- jupyter_ext/agent/skills/model_training.md +318 -0
- jupyter_ext/agent/skills/pyspark.md +352 -0
- jupyter_ext/agent/skills/python_code_quality.md +35 -0
- jupyter_ext/agent/skills/regression_evaluation.md +12 -0
- jupyter_ext/agent/skills/spring_java_development.md +35 -0
- jupyter_ext/agent/skills/text_preprocessing.md +12 -0
- jupyter_ext/agent/skills/time_series_analysis.md +13 -0
- jupyter_ext/agent/skills/unit_testing.md +203 -0
- jupyter_ext/agent/todo_prompts.py +57 -0
- jupyter_ext/agent/tool_report.py +45 -0
- jupyter_ext/analytics_ingest.py +599 -0
- jupyter_ext/etc/jupyter/jupyter_server_config.d/codebuff_agent.json +7 -0
- jupyter_ext/handlers/__init__.py +8 -0
- jupyter_ext/handlers/analytics_handler.py +170 -0
- jupyter_ext/handlers/benchmark_handler.py +794 -0
- jupyter_ext/handlers/eda_benchmark_handler.py +1328 -0
- jupyter_ext/handlers/rag_admin_handler.py +689 -0
- jupyter_ext/handlers/websocket.py +497 -0
- jupyter_ext/http_handlers.py +8744 -0
- jupyter_ext/idle_monitor.py +346 -0
- jupyter_ext/labextension/package.json +157 -0
- jupyter_ext/labextension/static/104.2b8f20f9d1ac2c9db01b.js +1 -0
- jupyter_ext/labextension/static/150.39fa98117cc8ca590465.js +1 -0
- jupyter_ext/labextension/static/166.386ff9b2c37ac865fdee.js +1 -0
- jupyter_ext/labextension/static/216.25cf81131bf7274dabf4.js +1 -0
- jupyter_ext/labextension/static/353.581bd1cb00746399b62e.js +1 -0
- jupyter_ext/labextension/static/367.575f9621b63c9a641aa9.js +1 -0
- jupyter_ext/labextension/static/421.11dac223a2c2da1b09f5.js +1 -0
- jupyter_ext/labextension/static/432.8513b4c6cc1d1232d165.js +2 -0
- jupyter_ext/labextension/static/432.8513b4c6cc1d1232d165.js.LICENSE.txt +19 -0
- jupyter_ext/labextension/static/439.ba5b5ef70ec6de214b8c.js +2 -0
- jupyter_ext/labextension/static/439.ba5b5ef70ec6de214b8c.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/481.cc2dec9fa869d66f1407.js +1 -0
- jupyter_ext/labextension/static/487.0796e931d6e651482c14.js +1 -0
- jupyter_ext/labextension/static/49.d326f554b6f9ccde6580.js +1 -0
- jupyter_ext/labextension/static/541.26897ea32845659e36be.js +1 -0
- jupyter_ext/labextension/static/579.e569d3edd8391fd8fbb3.js +1 -0
- jupyter_ext/labextension/static/58.eb7cc679f7535cc8a2f1.js +2 -0
- jupyter_ext/labextension/static/58.eb7cc679f7535cc8a2f1.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/666.d9bfdb23f98489d0a188.js +1 -0
- jupyter_ext/labextension/static/677.d266c39dbed33bc388b3.js +2 -0
- jupyter_ext/labextension/static/677.d266c39dbed33bc388b3.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/68.26b6f59c8454adb17f20.js +204 -0
- jupyter_ext/labextension/static/694.4ac443814260057e68f7.js +1 -0
- jupyter_ext/labextension/static/791.61affb8d3480d3d60e0d.js +1 -0
- jupyter_ext/labextension/static/848.4d2db09cfbb09374b299.js +2 -0
- jupyter_ext/labextension/static/848.4d2db09cfbb09374b299.js.LICENSE.txt +9 -0
- jupyter_ext/labextension/static/868.5cfe7e1a348ec5af1b8c.js +1 -0
- jupyter_ext/labextension/static/97.ea559a5b8a9a09f17c6b.js +1 -0
- jupyter_ext/labextension/static/remoteEntry.5292fc9c6df7e7920cc8.js +1 -0
- jupyter_ext/labextension/static/style.js +4 -0
- jupyter_ext/labextension/static/third-party-licenses.json +724 -0
- jupyter_ext/log_paths.py +113 -0
- jupyter_ext/resource_usage.py +519 -0
- jupyter_ext/secrets_bootstrap.py +116 -0
- jupyter_ext/team_access.py +105 -0
- jupyter_ext/tests/__init__.py +1 -0
- jupyter_ext/tests/test_context.py +234 -0
- jupyter_ext/tests/test_factory.py +563 -0
- jupyter_ext/tests/test_file_finder.py +139 -0
- jupyter_ext/tests/test_handlers.py +58 -0
- jupyter_ext/tests/test_mcp_selftest.py +204 -0
- jupyter_ext/tests/test_mermaid_fixer.py +272 -0
- jupyter_ext/tests/test_resource_tiers.py +298 -0
- jupyter_ext/tools/__init__.py +15 -0
- jupyter_ext/tools/base.py +90 -0
- jupyter_ext/tools/context.py +268 -0
- jupyter_ext/tools/dir_complete.py +86 -0
- jupyter_ext/tools/file_finder.py +303 -0
- jupyter_ext/tools/file_tools.py +244 -0
- jupyter_ext/tools/jupyter_tools.py +296 -0
- jupyter_ext/tools/registry.py +159 -0
- jupyter_ext/tools/search_tools.py +230 -0
- jupyter_ext/tools/security_tools.py +657 -0
- jupyter_ext/tools/shell_tools.py +109 -0
- jupyter_ext/tools/user_tools.py +132 -0
|
@@ -0,0 +1,890 @@
|
|
|
1
|
+
"""
|
|
2
|
+
RAG Manager - Main orchestrator for the Local RAG system.
|
|
3
|
+
|
|
4
|
+
Responsibilities:
|
|
5
|
+
- Qdrant client lifecycle management
|
|
6
|
+
- Collection creation and schema management
|
|
7
|
+
- Document indexing orchestration
|
|
8
|
+
- Search coordination
|
|
9
|
+
- Watchdog service coordination
|
|
10
|
+
|
|
11
|
+
Design Patterns:
|
|
12
|
+
- Singleton (consistent state across handlers)
|
|
13
|
+
- Lazy initialization (defer heavy operations)
|
|
14
|
+
- Graceful degradation (fallback to keyword search if RAG fails)
|
|
15
|
+
|
|
16
|
+
Multi-Collection Architecture (Mini RAG):
|
|
17
|
+
- hdsp_catalog: Collection metadata for Main Agent routing
|
|
18
|
+
- Domain collections: DS기술가이드, AWS_가이드, K8S_가이드, 테이블메타가이드
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import hashlib
|
|
22
|
+
import logging
|
|
23
|
+
import uuid
|
|
24
|
+
from datetime import datetime
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
27
|
+
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
from codebuff_core.models.rag import RAGConfig
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class RAGManager:
|
|
35
|
+
"""
|
|
36
|
+
Central manager for RAG operations.
|
|
37
|
+
|
|
38
|
+
Coordinates all RAG components:
|
|
39
|
+
- Qdrant vector database
|
|
40
|
+
- Embedding service
|
|
41
|
+
- Retriever (hybrid search)
|
|
42
|
+
- Watchdog (file monitoring)
|
|
43
|
+
|
|
44
|
+
Usage:
|
|
45
|
+
manager = get_rag_manager()
|
|
46
|
+
await manager.initialize()
|
|
47
|
+
results = await manager.search("query")
|
|
48
|
+
context = manager.get_context_for_query("query")
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
_instance: Optional["RAGManager"] = None
|
|
52
|
+
_initialized: bool = False
|
|
53
|
+
|
|
54
|
+
def __new__(cls, *args, **kwargs):
|
|
55
|
+
if cls._instance is None:
|
|
56
|
+
cls._instance = super().__new__(cls)
|
|
57
|
+
return cls._instance
|
|
58
|
+
|
|
59
|
+
def __init__(self, config: Optional["RAGConfig"] = None):
|
|
60
|
+
if self._initialized:
|
|
61
|
+
return
|
|
62
|
+
self._initialized = True
|
|
63
|
+
|
|
64
|
+
from codebuff_core.models.rag import get_default_rag_config
|
|
65
|
+
|
|
66
|
+
self._config = config or get_default_rag_config()
|
|
67
|
+
self._client = None
|
|
68
|
+
self._embedding_service = None
|
|
69
|
+
self._retriever = None
|
|
70
|
+
self._watchdog = None
|
|
71
|
+
self._ready = False
|
|
72
|
+
self._index_stats = {
|
|
73
|
+
"total_documents": 0,
|
|
74
|
+
"total_chunks": 0,
|
|
75
|
+
"last_updated": None,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async def initialize(self) -> bool:
|
|
79
|
+
"""
|
|
80
|
+
Initialize RAG system components.
|
|
81
|
+
Called during FastAPI lifespan startup.
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
True if initialization successful
|
|
85
|
+
"""
|
|
86
|
+
if not self._config.is_enabled():
|
|
87
|
+
logger.info("RAG system disabled in configuration")
|
|
88
|
+
return False
|
|
89
|
+
|
|
90
|
+
try:
|
|
91
|
+
# 1. Initialize Qdrant client
|
|
92
|
+
self._client = self._create_qdrant_client()
|
|
93
|
+
logger.info("Qdrant client initialized")
|
|
94
|
+
|
|
95
|
+
# 2. Initialize remote embedding service (vLLM / OpenAI-compatible)
|
|
96
|
+
from codebuff_core.services.vllm_embedding_service import (
|
|
97
|
+
get_vllm_embedding_service,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
self._embedding_service = get_vllm_embedding_service(self._config.embedding)
|
|
101
|
+
logger.info(
|
|
102
|
+
f"Remote Embedding service initialized (dim={self._embedding_service.dimension})"
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# 3. Initialize retriever (컬렉션은 RAG admin UI에서 적재 시 생성)
|
|
106
|
+
from codebuff_core.services.retriever import Retriever
|
|
107
|
+
|
|
108
|
+
self._retriever = Retriever(
|
|
109
|
+
client=self._client,
|
|
110
|
+
embedding_service=self._embedding_service,
|
|
111
|
+
config=self._config,
|
|
112
|
+
)
|
|
113
|
+
logger.info("Retriever initialized")
|
|
114
|
+
|
|
115
|
+
self._ready = True
|
|
116
|
+
logger.info("RAG system initialized successfully")
|
|
117
|
+
return True
|
|
118
|
+
|
|
119
|
+
except Exception as e:
|
|
120
|
+
logger.error(f"RAG initialization failed: {e}", exc_info=True)
|
|
121
|
+
self._ready = False
|
|
122
|
+
return False
|
|
123
|
+
|
|
124
|
+
async def shutdown(self) -> None:
|
|
125
|
+
"""
|
|
126
|
+
Graceful shutdown of RAG components.
|
|
127
|
+
Called during FastAPI lifespan shutdown.
|
|
128
|
+
"""
|
|
129
|
+
logger.info("Shutting down RAG system...")
|
|
130
|
+
|
|
131
|
+
if self._watchdog:
|
|
132
|
+
self._watchdog.stop()
|
|
133
|
+
logger.info("Watchdog stopped")
|
|
134
|
+
|
|
135
|
+
# Qdrant client doesn't need explicit cleanup for local mode
|
|
136
|
+
self._ready = False
|
|
137
|
+
logger.info("RAG system shutdown complete")
|
|
138
|
+
|
|
139
|
+
def _create_qdrant_client(self):
|
|
140
|
+
"""Create Qdrant client based on configuration mode."""
|
|
141
|
+
try:
|
|
142
|
+
from qdrant_client import QdrantClient
|
|
143
|
+
except ImportError:
|
|
144
|
+
raise ImportError(
|
|
145
|
+
"qdrant-client is required for RAG. "
|
|
146
|
+
"Install with: pip install qdrant-client"
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
cfg = self._config.qdrant
|
|
150
|
+
mode = cfg.get_mode() # Use get_mode() for env override
|
|
151
|
+
|
|
152
|
+
if mode == "local":
|
|
153
|
+
# Local file-based storage
|
|
154
|
+
local_path = cfg.get_local_path()
|
|
155
|
+
Path(local_path).mkdir(parents=True, exist_ok=True)
|
|
156
|
+
logger.info(f"Initializing Qdrant in local mode: {local_path}")
|
|
157
|
+
return QdrantClient(path=local_path)
|
|
158
|
+
|
|
159
|
+
elif mode == "server":
|
|
160
|
+
# Docker or external server
|
|
161
|
+
url = cfg.get_url() # Use get_url() for env override
|
|
162
|
+
logger.info(f"Connecting to Qdrant server: {url}")
|
|
163
|
+
return QdrantClient(url=url)
|
|
164
|
+
|
|
165
|
+
elif mode == "cloud":
|
|
166
|
+
# Qdrant Cloud
|
|
167
|
+
url = cfg.get_url() # Use get_url() for env override
|
|
168
|
+
logger.info("Connecting to Qdrant Cloud")
|
|
169
|
+
return QdrantClient(url=url, api_key=cfg.api_key)
|
|
170
|
+
|
|
171
|
+
else:
|
|
172
|
+
raise ValueError(f"Unknown Qdrant mode: {mode}")
|
|
173
|
+
|
|
174
|
+
async def _ensure_collection(self) -> None:
|
|
175
|
+
"""Create default collection if it doesn't exist (backward compatibility)."""
|
|
176
|
+
await self._ensure_single_collection(self._config.qdrant.collection_name)
|
|
177
|
+
|
|
178
|
+
async def _ensure_collections(self) -> None:
|
|
179
|
+
"""Create all Mini RAG collections (catalog + domain collections)."""
|
|
180
|
+
from codebuff_core.models.rag import CollectionConfig
|
|
181
|
+
|
|
182
|
+
# Create catalog collection
|
|
183
|
+
await self._ensure_single_collection(CollectionConfig.CATALOG_COLLECTION)
|
|
184
|
+
|
|
185
|
+
# Create domain collections
|
|
186
|
+
for collection_name in CollectionConfig.COLLECTIONS.keys():
|
|
187
|
+
await self._ensure_single_collection(collection_name)
|
|
188
|
+
|
|
189
|
+
# Also ensure default collection for backward compatibility
|
|
190
|
+
await self._ensure_single_collection(CollectionConfig.DEFAULT_COLLECTION)
|
|
191
|
+
|
|
192
|
+
logger.info(
|
|
193
|
+
f"All collections ensured: {CollectionConfig.get_all_collection_names()}"
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
async def _ensure_single_collection(self, collection_name: str) -> None:
|
|
197
|
+
"""Create a single collection if it doesn't exist."""
|
|
198
|
+
from qdrant_client.models import Distance, VectorParams
|
|
199
|
+
|
|
200
|
+
try:
|
|
201
|
+
collections = self._client.get_collections().collections
|
|
202
|
+
exists = any(c.name == collection_name for c in collections)
|
|
203
|
+
|
|
204
|
+
if not exists:
|
|
205
|
+
logger.info(f"Creating collection: {collection_name}")
|
|
206
|
+
self._client.create_collection(
|
|
207
|
+
collection_name=collection_name,
|
|
208
|
+
vectors_config=VectorParams(
|
|
209
|
+
size=self._embedding_service.dimension, distance=Distance.COSINE
|
|
210
|
+
),
|
|
211
|
+
)
|
|
212
|
+
logger.info(
|
|
213
|
+
f"Collection created with dimension {self._embedding_service.dimension}"
|
|
214
|
+
)
|
|
215
|
+
else:
|
|
216
|
+
logger.debug(f"Collection exists: {collection_name}")
|
|
217
|
+
except Exception as e:
|
|
218
|
+
logger.error(f"Failed to ensure collection {collection_name}: {e}")
|
|
219
|
+
raise
|
|
220
|
+
|
|
221
|
+
async def _index_collection_catalog(self) -> None:
|
|
222
|
+
"""Index collection metadata to catalog collection for Main Agent routing."""
|
|
223
|
+
from codebuff_core.models.rag import CollectionConfig
|
|
224
|
+
from qdrant_client.models import PointStruct
|
|
225
|
+
|
|
226
|
+
catalog_entries = []
|
|
227
|
+
for name, summary in CollectionConfig.COLLECTIONS.items():
|
|
228
|
+
# Extract categories from summary (simple word extraction)
|
|
229
|
+
categories = self._extract_categories(summary)
|
|
230
|
+
catalog_entries.append(
|
|
231
|
+
{
|
|
232
|
+
"collection_name": name,
|
|
233
|
+
"summary": summary,
|
|
234
|
+
"categories": categories,
|
|
235
|
+
}
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
if not catalog_entries:
|
|
239
|
+
logger.warning("No catalog entries to index")
|
|
240
|
+
return
|
|
241
|
+
|
|
242
|
+
# Generate embeddings for summaries
|
|
243
|
+
texts = [entry["summary"] for entry in catalog_entries]
|
|
244
|
+
embeddings = await self._embedding_service.embed_texts(texts)
|
|
245
|
+
|
|
246
|
+
# Create points
|
|
247
|
+
points = []
|
|
248
|
+
for entry, embedding in zip(catalog_entries, embeddings):
|
|
249
|
+
point_id = str(uuid.uuid4())
|
|
250
|
+
payload = {
|
|
251
|
+
"collection_name": entry["collection_name"],
|
|
252
|
+
"summary": entry["summary"],
|
|
253
|
+
"categories": entry["categories"],
|
|
254
|
+
"indexed_at": datetime.now().isoformat(),
|
|
255
|
+
}
|
|
256
|
+
points.append(PointStruct(id=point_id, vector=embedding, payload=payload))
|
|
257
|
+
|
|
258
|
+
# Upsert to catalog collection
|
|
259
|
+
self._client.upsert(
|
|
260
|
+
collection_name=CollectionConfig.CATALOG_COLLECTION, points=points
|
|
261
|
+
)
|
|
262
|
+
logger.info(f"Indexed {len(points)} catalog entries")
|
|
263
|
+
|
|
264
|
+
def _extract_categories(self, summary: str) -> List[str]:
|
|
265
|
+
"""Extract category keywords from summary text."""
|
|
266
|
+
# Simple extraction: split by common delimiters and filter
|
|
267
|
+
import re
|
|
268
|
+
|
|
269
|
+
# Split by common delimiters
|
|
270
|
+
words = re.split(r"[,\-\s]+", summary.lower())
|
|
271
|
+
# Filter out common words and keep meaningful terms
|
|
272
|
+
stopwords = {
|
|
273
|
+
"및",
|
|
274
|
+
"등",
|
|
275
|
+
"를",
|
|
276
|
+
"을",
|
|
277
|
+
"의",
|
|
278
|
+
"가",
|
|
279
|
+
"이",
|
|
280
|
+
"에",
|
|
281
|
+
"서비스",
|
|
282
|
+
"사용",
|
|
283
|
+
"가이드",
|
|
284
|
+
"문서",
|
|
285
|
+
"정보",
|
|
286
|
+
}
|
|
287
|
+
categories = [w for w in words if len(w) > 1 and w not in stopwords]
|
|
288
|
+
return categories[:10] # Limit to 10 categories
|
|
289
|
+
|
|
290
|
+
def _determine_collection(self, file_path: Path) -> str:
|
|
291
|
+
"""Determine target collection based on file path."""
|
|
292
|
+
from codebuff_core.models.rag import CollectionConfig
|
|
293
|
+
|
|
294
|
+
path_str = str(file_path).lower()
|
|
295
|
+
|
|
296
|
+
# Path-based routing rules
|
|
297
|
+
if "ds기술" in path_str or "datascience" in path_str or "pandas" in path_str:
|
|
298
|
+
return "DS기술가이드"
|
|
299
|
+
elif "aws" in path_str or "bdp" in path_str:
|
|
300
|
+
return "AWS_가이드"
|
|
301
|
+
elif "k8s" in path_str or "kubernetes" in path_str or "engine" in path_str:
|
|
302
|
+
return "K8S_가이드"
|
|
303
|
+
elif "테이블" in path_str or "table" in path_str or "meta" in path_str:
|
|
304
|
+
return "테이블메타가이드"
|
|
305
|
+
else:
|
|
306
|
+
return CollectionConfig.DEFAULT_COLLECTION
|
|
307
|
+
|
|
308
|
+
async def _start_watchdog(self) -> None:
|
|
309
|
+
"""Start file system monitoring."""
|
|
310
|
+
try:
|
|
311
|
+
from codebuff_core.knowledge.watchdog_service import WatchdogService
|
|
312
|
+
|
|
313
|
+
knowledge_path = self._get_knowledge_path()
|
|
314
|
+
if not knowledge_path or not knowledge_path.exists():
|
|
315
|
+
logger.warning(
|
|
316
|
+
f"Knowledge base path not configured or missing: {knowledge_path}"
|
|
317
|
+
)
|
|
318
|
+
return
|
|
319
|
+
|
|
320
|
+
self._watchdog = WatchdogService(
|
|
321
|
+
watch_path=knowledge_path,
|
|
322
|
+
config=self._config.watchdog,
|
|
323
|
+
on_change_callback=self._on_file_change,
|
|
324
|
+
)
|
|
325
|
+
self._watchdog.start()
|
|
326
|
+
logger.info(f"Watchdog started monitoring: {knowledge_path}")
|
|
327
|
+
except ImportError:
|
|
328
|
+
logger.warning("Watchdog service not available")
|
|
329
|
+
except Exception as e:
|
|
330
|
+
logger.warning(f"Failed to start watchdog: {e}")
|
|
331
|
+
|
|
332
|
+
def _get_knowledge_path(self) -> Optional[Path]:
|
|
333
|
+
"""Get knowledge base directory path. None if not configured."""
|
|
334
|
+
custom_path = self._config.get_knowledge_base_path()
|
|
335
|
+
if custom_path:
|
|
336
|
+
return Path(custom_path)
|
|
337
|
+
return None
|
|
338
|
+
|
|
339
|
+
async def _index_knowledge_base(self) -> Dict[str, Any]:
|
|
340
|
+
"""Index all documents in the knowledge base."""
|
|
341
|
+
from codebuff_core.knowledge.chunking import DocumentChunker
|
|
342
|
+
|
|
343
|
+
knowledge_path = self._get_knowledge_path()
|
|
344
|
+
if not knowledge_path or not knowledge_path.exists():
|
|
345
|
+
logger.warning(
|
|
346
|
+
f"Knowledge base path not configured or missing: {knowledge_path}"
|
|
347
|
+
)
|
|
348
|
+
return {"indexed": 0, "skipped": 0, "errors": []}
|
|
349
|
+
|
|
350
|
+
chunker = DocumentChunker(self._config.chunking)
|
|
351
|
+
|
|
352
|
+
indexed = 0
|
|
353
|
+
skipped = 0
|
|
354
|
+
errors = []
|
|
355
|
+
|
|
356
|
+
# Get all files matching patterns
|
|
357
|
+
files = []
|
|
358
|
+
for pattern in self._config.watchdog.patterns:
|
|
359
|
+
files.extend(knowledge_path.glob(f"**/{pattern}"))
|
|
360
|
+
|
|
361
|
+
# Filter out ignored patterns
|
|
362
|
+
files = [f for f in files if not self._should_ignore(f)]
|
|
363
|
+
|
|
364
|
+
logger.info(f"Found {len(files)} files to index in {knowledge_path}")
|
|
365
|
+
|
|
366
|
+
for file_path in files:
|
|
367
|
+
try:
|
|
368
|
+
# Check if file needs re-indexing
|
|
369
|
+
if self._is_file_indexed(file_path):
|
|
370
|
+
skipped += 1
|
|
371
|
+
continue
|
|
372
|
+
|
|
373
|
+
# Read and chunk document
|
|
374
|
+
content = file_path.read_text(encoding="utf-8")
|
|
375
|
+
chunks = chunker.chunk_document(
|
|
376
|
+
content=content,
|
|
377
|
+
metadata={
|
|
378
|
+
"source": str(file_path.relative_to(knowledge_path)),
|
|
379
|
+
"source_type": self._infer_source_type(file_path),
|
|
380
|
+
"file_path": str(file_path),
|
|
381
|
+
"indexed_at": datetime.now().isoformat(),
|
|
382
|
+
},
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
if chunks:
|
|
386
|
+
await self._index_chunks(chunks, file_path)
|
|
387
|
+
indexed += 1
|
|
388
|
+
self._index_stats["total_documents"] += 1
|
|
389
|
+
self._index_stats["total_chunks"] += len(chunks)
|
|
390
|
+
|
|
391
|
+
except Exception as e:
|
|
392
|
+
errors.append({"file": str(file_path), "error": str(e)})
|
|
393
|
+
logger.error(f"Failed to index {file_path}: {e}")
|
|
394
|
+
|
|
395
|
+
self._index_stats["last_updated"] = datetime.now().isoformat()
|
|
396
|
+
result = {"indexed": indexed, "skipped": skipped, "errors": errors}
|
|
397
|
+
logger.info(
|
|
398
|
+
f"Indexing complete: {indexed} indexed, {skipped} skipped, {len(errors)} errors"
|
|
399
|
+
)
|
|
400
|
+
return result
|
|
401
|
+
|
|
402
|
+
def _should_ignore(self, file_path: Path) -> bool:
|
|
403
|
+
"""Check if file should be ignored."""
|
|
404
|
+
path_str = str(file_path)
|
|
405
|
+
for pattern in self._config.watchdog.ignore_patterns:
|
|
406
|
+
if pattern.startswith("."):
|
|
407
|
+
# Hidden files/dirs
|
|
408
|
+
if any(part.startswith(".") for part in file_path.parts):
|
|
409
|
+
return True
|
|
410
|
+
elif pattern in path_str:
|
|
411
|
+
return True
|
|
412
|
+
return False
|
|
413
|
+
|
|
414
|
+
def _is_file_indexed(self, file_path: Path) -> bool:
|
|
415
|
+
"""Check if file is already indexed with current content hash."""
|
|
416
|
+
file_hash = self._compute_file_hash(file_path)
|
|
417
|
+
|
|
418
|
+
try:
|
|
419
|
+
results = self._client.scroll(
|
|
420
|
+
collection_name=self._config.qdrant.collection_name,
|
|
421
|
+
scroll_filter={
|
|
422
|
+
"must": [{"key": "file_path", "match": {"value": str(file_path)}}]
|
|
423
|
+
},
|
|
424
|
+
limit=1,
|
|
425
|
+
with_payload=True,
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
if results[0]: # Has existing points
|
|
429
|
+
existing_hash = results[0][0].payload.get("content_hash")
|
|
430
|
+
return existing_hash == file_hash
|
|
431
|
+
|
|
432
|
+
except Exception as e:
|
|
433
|
+
logger.debug(f"Error checking indexed status: {e}")
|
|
434
|
+
|
|
435
|
+
return False
|
|
436
|
+
|
|
437
|
+
def _compute_file_hash(self, file_path: Path) -> str:
|
|
438
|
+
"""Compute content hash for change detection."""
|
|
439
|
+
content = file_path.read_bytes()
|
|
440
|
+
return hashlib.sha256(content).hexdigest()[:16]
|
|
441
|
+
|
|
442
|
+
def _infer_source_type(self, file_path: Path) -> str:
|
|
443
|
+
"""Infer document source type from path."""
|
|
444
|
+
path_str = str(file_path).lower()
|
|
445
|
+
|
|
446
|
+
if "libraries" in path_str:
|
|
447
|
+
return "library_guide"
|
|
448
|
+
elif "incidents" in path_str:
|
|
449
|
+
return "incident"
|
|
450
|
+
elif "tutorials" in path_str:
|
|
451
|
+
return "tutorial"
|
|
452
|
+
elif "infrastructure" in path_str:
|
|
453
|
+
return "infrastructure"
|
|
454
|
+
else:
|
|
455
|
+
return "general"
|
|
456
|
+
|
|
457
|
+
async def _index_chunks(self, chunks: List[Dict], file_path: Path) -> None:
|
|
458
|
+
"""Index document chunks to Qdrant."""
|
|
459
|
+
from qdrant_client.models import PointStruct
|
|
460
|
+
|
|
461
|
+
# Generate embeddings
|
|
462
|
+
texts = [c["content"] for c in chunks]
|
|
463
|
+
embeddings = await self._embedding_service.embed_texts(texts)
|
|
464
|
+
|
|
465
|
+
# Add content hash to all chunks
|
|
466
|
+
file_hash = self._compute_file_hash(file_path)
|
|
467
|
+
|
|
468
|
+
# Create points
|
|
469
|
+
points = []
|
|
470
|
+
for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
|
|
471
|
+
point_id = str(uuid.uuid4())
|
|
472
|
+
payload = {
|
|
473
|
+
**chunk["metadata"],
|
|
474
|
+
"content": chunk["content"],
|
|
475
|
+
"content_hash": file_hash,
|
|
476
|
+
"chunk_index": i,
|
|
477
|
+
}
|
|
478
|
+
points.append(PointStruct(id=point_id, vector=embedding, payload=payload))
|
|
479
|
+
|
|
480
|
+
# Upsert to Qdrant
|
|
481
|
+
self._client.upsert(
|
|
482
|
+
collection_name=self._config.qdrant.collection_name, points=points
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
logger.debug(f"Indexed {len(points)} chunks from {file_path.name}")
|
|
486
|
+
|
|
487
|
+
async def _on_file_change(self, event_type: str, file_path: Path) -> None:
|
|
488
|
+
"""Handle file change events from watchdog."""
|
|
489
|
+
logger.info(f"File change detected: {event_type} - {file_path}")
|
|
490
|
+
|
|
491
|
+
if event_type == "deleted":
|
|
492
|
+
self._remove_file_from_index(file_path)
|
|
493
|
+
else:
|
|
494
|
+
# Modified or created - re-index
|
|
495
|
+
await self._reindex_file(file_path)
|
|
496
|
+
|
|
497
|
+
def _remove_file_from_index(self, file_path: Path) -> None:
|
|
498
|
+
"""Remove file's chunks from index."""
|
|
499
|
+
try:
|
|
500
|
+
self._client.delete(
|
|
501
|
+
collection_name=self._config.qdrant.collection_name,
|
|
502
|
+
points_selector={
|
|
503
|
+
"filter": {
|
|
504
|
+
"must": [
|
|
505
|
+
{"key": "file_path", "match": {"value": str(file_path)}}
|
|
506
|
+
]
|
|
507
|
+
}
|
|
508
|
+
},
|
|
509
|
+
)
|
|
510
|
+
logger.info(f"Removed from index: {file_path}")
|
|
511
|
+
except Exception as e:
|
|
512
|
+
logger.error(f"Failed to remove from index: {e}")
|
|
513
|
+
|
|
514
|
+
async def _reindex_file(self, file_path: Path) -> None:
|
|
515
|
+
"""Re-index a single file."""
|
|
516
|
+
# Remove old chunks
|
|
517
|
+
self._remove_file_from_index(file_path)
|
|
518
|
+
|
|
519
|
+
if not file_path.exists():
|
|
520
|
+
return
|
|
521
|
+
|
|
522
|
+
# Index new content
|
|
523
|
+
from codebuff_core.knowledge.chunking import DocumentChunker
|
|
524
|
+
|
|
525
|
+
chunker = DocumentChunker(self._config.chunking)
|
|
526
|
+
|
|
527
|
+
try:
|
|
528
|
+
content = file_path.read_text(encoding="utf-8")
|
|
529
|
+
knowledge_path = self._get_knowledge_path() or file_path.parent
|
|
530
|
+
|
|
531
|
+
chunks = chunker.chunk_document(
|
|
532
|
+
content=content,
|
|
533
|
+
metadata={
|
|
534
|
+
"source": str(file_path.relative_to(knowledge_path)),
|
|
535
|
+
"source_type": self._infer_source_type(file_path),
|
|
536
|
+
"file_path": str(file_path),
|
|
537
|
+
"indexed_at": datetime.now().isoformat(),
|
|
538
|
+
},
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
if chunks:
|
|
542
|
+
await self._index_chunks(chunks, file_path)
|
|
543
|
+
logger.info(f"Reindexed: {file_path}")
|
|
544
|
+
except Exception as e:
|
|
545
|
+
logger.error(f"Failed to reindex {file_path}: {e}")
|
|
546
|
+
|
|
547
|
+
# ========== Public Search API ==========
|
|
548
|
+
|
|
549
|
+
async def search(
|
|
550
|
+
self,
|
|
551
|
+
query: str,
|
|
552
|
+
collection_name: Optional[str] = None,
|
|
553
|
+
top_k: Optional[int] = None,
|
|
554
|
+
filters: Optional[Dict] = None,
|
|
555
|
+
) -> List[Dict]:
|
|
556
|
+
"""
|
|
557
|
+
Search the knowledge base.
|
|
558
|
+
|
|
559
|
+
Args:
|
|
560
|
+
query: Search query text
|
|
561
|
+
collection_name: Target collection (if None, uses default)
|
|
562
|
+
top_k: Number of results (default from config)
|
|
563
|
+
filters: Metadata filters
|
|
564
|
+
|
|
565
|
+
Returns:
|
|
566
|
+
List of search results with content, score, metadata
|
|
567
|
+
"""
|
|
568
|
+
if not self._ready:
|
|
569
|
+
logger.warning("RAG system not ready, returning empty results")
|
|
570
|
+
return []
|
|
571
|
+
|
|
572
|
+
return await self._retriever.search(
|
|
573
|
+
query=query,
|
|
574
|
+
collection_name=collection_name,
|
|
575
|
+
top_k=top_k or self._config.top_k,
|
|
576
|
+
filters=filters,
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
async def search_catalog(self, query: str, top_k: int = 3) -> List[Dict[str, Any]]:
|
|
580
|
+
"""
|
|
581
|
+
Search catalog collection for Main Agent routing.
|
|
582
|
+
|
|
583
|
+
This helps the Main Agent determine which domain collection
|
|
584
|
+
to route the query to.
|
|
585
|
+
|
|
586
|
+
Args:
|
|
587
|
+
query: User query to match against collection summaries
|
|
588
|
+
top_k: Number of collection suggestions (default 3)
|
|
589
|
+
|
|
590
|
+
Returns:
|
|
591
|
+
List of collection metadata with relevance scores
|
|
592
|
+
"""
|
|
593
|
+
from codebuff_core.models.rag import CollectionConfig
|
|
594
|
+
|
|
595
|
+
if not self._ready:
|
|
596
|
+
logger.warning("RAG system not ready, returning empty catalog results")
|
|
597
|
+
return []
|
|
598
|
+
|
|
599
|
+
try:
|
|
600
|
+
results = await self._retriever.search(
|
|
601
|
+
query=query,
|
|
602
|
+
collection_name=CollectionConfig.CATALOG_COLLECTION,
|
|
603
|
+
top_k=top_k,
|
|
604
|
+
)
|
|
605
|
+
return results
|
|
606
|
+
except Exception as e:
|
|
607
|
+
logger.error(f"Catalog search failed: {e}")
|
|
608
|
+
return []
|
|
609
|
+
|
|
610
|
+
async def get_context_for_query(
|
|
611
|
+
self,
|
|
612
|
+
query: str,
|
|
613
|
+
detected_libraries: Optional[List[str]] = None,
|
|
614
|
+
max_tokens: Optional[int] = None,
|
|
615
|
+
) -> str:
|
|
616
|
+
"""
|
|
617
|
+
Get formatted RAG context for LLM prompt injection.
|
|
618
|
+
|
|
619
|
+
Args:
|
|
620
|
+
query: User query to find relevant context
|
|
621
|
+
detected_libraries: Libraries detected by LibraryDetector (prioritized in search)
|
|
622
|
+
max_tokens: Approximate token limit for context
|
|
623
|
+
|
|
624
|
+
Returns:
|
|
625
|
+
Formatted context string for prompt injection
|
|
626
|
+
"""
|
|
627
|
+
if not self._ready:
|
|
628
|
+
return ""
|
|
629
|
+
|
|
630
|
+
effective_max_tokens = max_tokens or self._config.max_context_tokens
|
|
631
|
+
results = []
|
|
632
|
+
|
|
633
|
+
try:
|
|
634
|
+
# Strategy: If libraries detected, search with library filter first
|
|
635
|
+
# Then supplement with general search if needed
|
|
636
|
+
if detected_libraries:
|
|
637
|
+
# Search within detected library files first
|
|
638
|
+
for lib in detected_libraries:
|
|
639
|
+
lib_filter = {"source": f"{lib}.md"}
|
|
640
|
+
lib_results = await self.search(
|
|
641
|
+
query=query,
|
|
642
|
+
top_k=3, # Get top 3 from each library
|
|
643
|
+
filters=lib_filter,
|
|
644
|
+
)
|
|
645
|
+
results.extend(lib_results)
|
|
646
|
+
logger.info(
|
|
647
|
+
f"RAG library search [{lib}]: {len(lib_results)} results"
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
# If not enough results, do general search
|
|
651
|
+
if len(results) < self._config.top_k:
|
|
652
|
+
remaining = self._config.top_k - len(results)
|
|
653
|
+
general_results = await self.search(query=query, top_k=remaining)
|
|
654
|
+
# Avoid duplicates
|
|
655
|
+
existing_ids = {r.get("id") for r in results if r.get("id")}
|
|
656
|
+
for r in general_results:
|
|
657
|
+
if r.get("id") not in existing_ids:
|
|
658
|
+
results.append(r)
|
|
659
|
+
|
|
660
|
+
# Sort by score
|
|
661
|
+
results.sort(key=lambda x: x.get("score", 0), reverse=True)
|
|
662
|
+
|
|
663
|
+
except Exception as e:
|
|
664
|
+
logger.error(f"Failed to get RAG context: {e}")
|
|
665
|
+
return ""
|
|
666
|
+
|
|
667
|
+
if not results:
|
|
668
|
+
return ""
|
|
669
|
+
|
|
670
|
+
# Format results as context
|
|
671
|
+
context_parts = []
|
|
672
|
+
char_count = 0
|
|
673
|
+
char_limit = effective_max_tokens * 4 # Rough char-to-token ratio
|
|
674
|
+
|
|
675
|
+
for result in results:
|
|
676
|
+
source = result["metadata"].get("source", "unknown")
|
|
677
|
+
section = result["metadata"].get("section", "")
|
|
678
|
+
score = result["score"]
|
|
679
|
+
|
|
680
|
+
# Format chunk with source info
|
|
681
|
+
chunk_text = f"[Source: {source}"
|
|
682
|
+
if section:
|
|
683
|
+
chunk_text += f" > {section}"
|
|
684
|
+
chunk_text += f" (relevance: {score:.2f})]\n{result['content']}\n"
|
|
685
|
+
|
|
686
|
+
if char_count + len(chunk_text) > char_limit:
|
|
687
|
+
break
|
|
688
|
+
context_parts.append(chunk_text)
|
|
689
|
+
char_count += len(chunk_text)
|
|
690
|
+
|
|
691
|
+
if not context_parts:
|
|
692
|
+
return ""
|
|
693
|
+
|
|
694
|
+
header = "## 📚 라이브러리 API 참조 (RAG Retrieved)\n\n"
|
|
695
|
+
header += "아래 가이드의 API 사용법을 **반드시** 따르세요.\n\n"
|
|
696
|
+
return header + "\n---\n".join(context_parts)
|
|
697
|
+
|
|
698
|
+
def get_status(self) -> Dict[str, Any]:
|
|
699
|
+
"""Get RAG system status."""
|
|
700
|
+
return {
|
|
701
|
+
"ready": self._ready,
|
|
702
|
+
"enabled": self._config.is_enabled(),
|
|
703
|
+
"total_documents": self._index_stats["total_documents"],
|
|
704
|
+
"total_chunks": self._index_stats["total_chunks"],
|
|
705
|
+
"last_updated": self._index_stats["last_updated"],
|
|
706
|
+
"knowledge_base_path": str(self._get_knowledge_path() or "not configured"),
|
|
707
|
+
"qdrant_mode": self._config.qdrant.mode,
|
|
708
|
+
"embedding_model": self._config.embedding.get_model_name(),
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
@property
|
|
712
|
+
def is_ready(self) -> bool:
|
|
713
|
+
"""Check if RAG system is operational."""
|
|
714
|
+
return self._ready
|
|
715
|
+
|
|
716
|
+
async def debug_search(
|
|
717
|
+
self,
|
|
718
|
+
query: str,
|
|
719
|
+
imported_libraries: Optional[List[str]] = None,
|
|
720
|
+
top_k: Optional[int] = None,
|
|
721
|
+
include_full_content: bool = False,
|
|
722
|
+
simulate_plan_context: bool = True,
|
|
723
|
+
) -> Dict[str, Any]:
|
|
724
|
+
"""
|
|
725
|
+
전체 파이프라인 리니지를 포함한 디버그 검색.
|
|
726
|
+
|
|
727
|
+
반환 정보:
|
|
728
|
+
- 라이브러리 감지 결과
|
|
729
|
+
- 청크별 점수 (dense, BM25, fused)
|
|
730
|
+
- 최종 포맷된 컨텍스트
|
|
731
|
+
|
|
732
|
+
Args:
|
|
733
|
+
query: Search query
|
|
734
|
+
imported_libraries: List of imported libraries
|
|
735
|
+
top_k: Override default top_k
|
|
736
|
+
include_full_content: Include full content instead of preview
|
|
737
|
+
simulate_plan_context: Generate formatted context as in plan generation
|
|
738
|
+
|
|
739
|
+
Returns:
|
|
740
|
+
Dict with comprehensive debug information
|
|
741
|
+
"""
|
|
742
|
+
if not self._ready:
|
|
743
|
+
return {"error": "RAG system not ready"}
|
|
744
|
+
|
|
745
|
+
# 1. 라이브러리 감지 (agent.py와 동일 로직)
|
|
746
|
+
try:
|
|
747
|
+
from codebuff_core.knowledge.loader import (
|
|
748
|
+
get_knowledge_base,
|
|
749
|
+
get_library_detector,
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
knowledge_base = get_knowledge_base()
|
|
753
|
+
library_detector = get_library_detector()
|
|
754
|
+
available = (
|
|
755
|
+
knowledge_base.list_available_libraries() if knowledge_base else []
|
|
756
|
+
)
|
|
757
|
+
detected_libraries = (
|
|
758
|
+
library_detector.detect(
|
|
759
|
+
request=query,
|
|
760
|
+
available_libraries=available,
|
|
761
|
+
imported_libraries=imported_libraries or [],
|
|
762
|
+
)
|
|
763
|
+
if library_detector
|
|
764
|
+
else []
|
|
765
|
+
)
|
|
766
|
+
detection_method = "deterministic"
|
|
767
|
+
except Exception as e:
|
|
768
|
+
logger.warning(f"Library detection failed: {e}")
|
|
769
|
+
available = []
|
|
770
|
+
detected_libraries = []
|
|
771
|
+
detection_method = "fallback (detection failed)"
|
|
772
|
+
|
|
773
|
+
library_detection_info = {
|
|
774
|
+
"input_query": query,
|
|
775
|
+
"imported_libraries": imported_libraries or [],
|
|
776
|
+
"available_libraries": available or [],
|
|
777
|
+
"detected_libraries": detected_libraries,
|
|
778
|
+
"detection_method": detection_method,
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
# 2. 설정 정보
|
|
782
|
+
config_info = {
|
|
783
|
+
"top_k": top_k or self._config.top_k,
|
|
784
|
+
"score_threshold": self._config.score_threshold,
|
|
785
|
+
"max_context_tokens": self._config.max_context_tokens,
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
# 3. 디버그 검색 수행
|
|
789
|
+
debug_result = await self._retriever.search_with_debug(
|
|
790
|
+
query=query, top_k=top_k or self._config.top_k
|
|
791
|
+
)
|
|
792
|
+
|
|
793
|
+
# 4. 청크 디버그 정보 구성
|
|
794
|
+
chunks_info = []
|
|
795
|
+
for chunk in debug_result.chunks:
|
|
796
|
+
content_preview = chunk.content
|
|
797
|
+
if not include_full_content and len(chunk.content) > 200:
|
|
798
|
+
content_preview = chunk.content[:200] + "..."
|
|
799
|
+
|
|
800
|
+
chunks_info.append(
|
|
801
|
+
{
|
|
802
|
+
"chunk_id": chunk.chunk_id,
|
|
803
|
+
"content_preview": content_preview,
|
|
804
|
+
"score": chunk.score,
|
|
805
|
+
"rank": chunk.rank,
|
|
806
|
+
"metadata": chunk.metadata,
|
|
807
|
+
"passed_threshold": chunk.passed_threshold,
|
|
808
|
+
}
|
|
809
|
+
)
|
|
810
|
+
|
|
811
|
+
# 5. 포맷된 컨텍스트 생성 (get_context_for_query와 동일)
|
|
812
|
+
formatted_context = ""
|
|
813
|
+
if simulate_plan_context:
|
|
814
|
+
passed_chunks = [c for c in debug_result.chunks if c.passed_threshold]
|
|
815
|
+
|
|
816
|
+
if passed_chunks:
|
|
817
|
+
context_parts = []
|
|
818
|
+
char_count = 0
|
|
819
|
+
char_limit = (
|
|
820
|
+
self._config.max_context_tokens * 4
|
|
821
|
+
) # Rough char-to-token ratio
|
|
822
|
+
|
|
823
|
+
for chunk in passed_chunks:
|
|
824
|
+
source = chunk.metadata.get("source", "unknown")
|
|
825
|
+
section = chunk.metadata.get("section", "")
|
|
826
|
+
score = chunk.score
|
|
827
|
+
|
|
828
|
+
# Format chunk with source info
|
|
829
|
+
chunk_text = f"[Source: {source}"
|
|
830
|
+
if section:
|
|
831
|
+
chunk_text += f" > {section}"
|
|
832
|
+
chunk_text += f" (relevance: {score:.2f})]\n{chunk.content}\n"
|
|
833
|
+
|
|
834
|
+
if char_count + len(chunk_text) > char_limit:
|
|
835
|
+
break
|
|
836
|
+
context_parts.append(chunk_text)
|
|
837
|
+
char_count += len(chunk_text)
|
|
838
|
+
|
|
839
|
+
if context_parts:
|
|
840
|
+
header = "## 📚 라이브러리 API 참조 (RAG Retrieved)\n\n"
|
|
841
|
+
header += "아래 가이드의 API 사용법을 **반드시** 따르세요.\n\n"
|
|
842
|
+
formatted_context = header + "\n---\n".join(context_parts)
|
|
843
|
+
|
|
844
|
+
return {
|
|
845
|
+
"library_detection": library_detection_info,
|
|
846
|
+
"config": config_info,
|
|
847
|
+
"chunks": chunks_info,
|
|
848
|
+
"total_candidates": debug_result.total_candidates,
|
|
849
|
+
"total_passed_threshold": sum(
|
|
850
|
+
1 for c in debug_result.chunks if c.passed_threshold
|
|
851
|
+
),
|
|
852
|
+
"search_ms": debug_result.search_ms,
|
|
853
|
+
"formatted_context": formatted_context,
|
|
854
|
+
"context_char_count": len(formatted_context),
|
|
855
|
+
"estimated_context_tokens": len(formatted_context) // 4,
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
# ============ Singleton Accessor ============
|
|
860
|
+
|
|
861
|
+
_rag_manager: Optional[RAGManager] = None
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def get_rag_manager(config: Optional["RAGConfig"] = None) -> RAGManager:
|
|
865
|
+
"""
|
|
866
|
+
Get the singleton RAGManager instance.
|
|
867
|
+
|
|
868
|
+
Args:
|
|
869
|
+
config: Optional RAGConfig (only used on first call)
|
|
870
|
+
|
|
871
|
+
Returns:
|
|
872
|
+
RAGManager singleton instance
|
|
873
|
+
"""
|
|
874
|
+
global _rag_manager
|
|
875
|
+
if _rag_manager is None:
|
|
876
|
+
_rag_manager = RAGManager(config)
|
|
877
|
+
return _rag_manager
|
|
878
|
+
|
|
879
|
+
|
|
880
|
+
def reset_rag_manager() -> None:
|
|
881
|
+
"""
|
|
882
|
+
Reset the singleton instance (for testing purposes).
|
|
883
|
+
"""
|
|
884
|
+
global _rag_manager
|
|
885
|
+
if _rag_manager is not None:
|
|
886
|
+
_rag_manager._initialized = False
|
|
887
|
+
_rag_manager._ready = False
|
|
888
|
+
_rag_manager = None
|
|
889
|
+
RAGManager._instance = None
|
|
890
|
+
RAGManager._initialized = False
|