codebuff-agent 2.1.74__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codebuff_agent-2.1.74.data/data/etc/jupyter/jupyter_server_config.d/codebuff_agent.json +7 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/package.json +157 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/104.2b8f20f9d1ac2c9db01b.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/150.39fa98117cc8ca590465.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/166.386ff9b2c37ac865fdee.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/216.25cf81131bf7274dabf4.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/353.581bd1cb00746399b62e.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/367.575f9621b63c9a641aa9.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/421.11dac223a2c2da1b09f5.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/432.8513b4c6cc1d1232d165.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/432.8513b4c6cc1d1232d165.js.LICENSE.txt +19 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/439.ba5b5ef70ec6de214b8c.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/439.ba5b5ef70ec6de214b8c.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/481.cc2dec9fa869d66f1407.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/487.0796e931d6e651482c14.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/49.d326f554b6f9ccde6580.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/541.26897ea32845659e36be.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/579.e569d3edd8391fd8fbb3.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/58.eb7cc679f7535cc8a2f1.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/58.eb7cc679f7535cc8a2f1.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/666.d9bfdb23f98489d0a188.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/677.d266c39dbed33bc388b3.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/677.d266c39dbed33bc388b3.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/68.26b6f59c8454adb17f20.js +204 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/694.4ac443814260057e68f7.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/791.61affb8d3480d3d60e0d.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/848.4d2db09cfbb09374b299.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/848.4d2db09cfbb09374b299.js.LICENSE.txt +9 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/868.5cfe7e1a348ec5af1b8c.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/97.ea559a5b8a9a09f17c6b.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/remoteEntry.5292fc9c6df7e7920cc8.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/style.js +4 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/third-party-licenses.json +724 -0
- codebuff_agent-2.1.74.dist-info/METADATA +142 -0
- codebuff_agent-2.1.74.dist-info/RECORD +217 -0
- codebuff_agent-2.1.74.dist-info/WHEEL +4 -0
- codebuff_agent-2.1.74.dist-info/licenses/LICENSE +21 -0
- codebuff_core/__init__.py +153 -0
- codebuff_core/codebuff_config_defaults.json +64 -0
- codebuff_core/core/__init__.py +35 -0
- codebuff_core/core/code_validator.py +1238 -0
- codebuff_core/core/error_classifier.py +579 -0
- codebuff_core/core/exceptions.py +49 -0
- codebuff_core/data/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 +100256 -0
- codebuff_core/directory_policy.py +32 -0
- codebuff_core/factory.py +155 -0
- codebuff_core/interfaces.py +197 -0
- codebuff_core/knowledge/__init__.py +31 -0
- codebuff_core/knowledge/chunking.py +375 -0
- codebuff_core/knowledge/loader.py +339 -0
- codebuff_core/knowledge/watchdog_service.py +354 -0
- codebuff_core/legacy_paths.py +69 -0
- codebuff_core/llm/__init__.py +13 -0
- codebuff_core/llm/service.py +407 -0
- codebuff_core/managers/__init__.py +22 -0
- codebuff_core/managers/config_manager.py +690 -0
- codebuff_core/managers/remote_config.py +673 -0
- codebuff_core/managers/remote_skills.py +539 -0
- codebuff_core/managers/session_manager.py +255 -0
- codebuff_core/middleware/__init__.py +3 -0
- codebuff_core/middleware/error_recovery.py +394 -0
- codebuff_core/middleware/logging_utils.py +33 -0
- codebuff_core/models/__init__.py +111 -0
- codebuff_core/models/agent.py +306 -0
- codebuff_core/models/chat.py +57 -0
- codebuff_core/models/common.py +82 -0
- codebuff_core/models/rag.py +300 -0
- codebuff_core/platform_rules.py +127 -0
- codebuff_core/prompts/__init__.py +44 -0
- codebuff_core/prompts/auto_agent_prompts.py +1278 -0
- codebuff_core/services/__init__.py +30 -0
- codebuff_core/services/agent_service.py +345 -0
- codebuff_core/services/analytics_aggregate.py +837 -0
- codebuff_core/services/chat_service.py +898 -0
- codebuff_core/services/context_condenser.py +306 -0
- codebuff_core/services/pii_guardrail.py +393 -0
- codebuff_core/services/rag_chat_agent.py +2993 -0
- codebuff_core/services/rag_collections.py +2740 -0
- codebuff_core/services/rag_manager.py +890 -0
- codebuff_core/services/rag_service.py +145 -0
- codebuff_core/services/retriever.py +260 -0
- codebuff_core/services/vllm_embedding_service.py +290 -0
- codebuff_core/tests/__init__.py +1 -0
- codebuff_core/tests/conftest.py +81 -0
- codebuff_core/tests/test_error_recovery.py +647 -0
- codebuff_core/tests/test_factory.py +135 -0
- codebuff_core/tests/test_services.py +150 -0
- jupyter_ext/__init__.py +266 -0
- jupyter_ext/_version.py +4 -0
- jupyter_ext/agent/__init__.py +49 -0
- jupyter_ext/agent/agent_factory.py +2270 -0
- jupyter_ext/agent/ast_tools.py +451 -0
- jupyter_ext/agent/autofix_middleware.py +481 -0
- jupyter_ext/agent/background_procs.py +372 -0
- jupyter_ext/agent/code_history_middleware.py +211 -0
- jupyter_ext/agent/codex_autofix.py +160 -0
- jupyter_ext/agent/codex_bridge.py +3339 -0
- jupyter_ext/agent/codex_hooks.py +320 -0
- jupyter_ext/agent/codex_slash.py +685 -0
- jupyter_ext/agent/codex_threads.py +114 -0
- jupyter_ext/agent/content_injection.py +377 -0
- jupyter_ext/agent/continuation_middleware.py +548 -0
- jupyter_ext/agent/custom_skills.py +335 -0
- jupyter_ext/agent/file_diffs.py +99 -0
- jupyter_ext/agent/git_history.py +188 -0
- jupyter_ext/agent/guardrail_middleware.py +421 -0
- jupyter_ext/agent/json_repair_utils.py +54 -0
- jupyter_ext/agent/langchain_tools.py +2673 -0
- jupyter_ext/agent/mermaid_fixer.py +289 -0
- jupyter_ext/agent/mermaid_middleware.py +135 -0
- jupyter_ext/agent/middleware.py +359 -0
- jupyter_ext/agent/middleware_prompts.py +53 -0
- jupyter_ext/agent/observability.py +334 -0
- jupyter_ext/agent/patch_tool_calls.py +57 -0
- jupyter_ext/agent/prompts.py +112 -0
- jupyter_ext/agent/skill_middleware.py +257 -0
- jupyter_ext/agent/skills/airflow_dag_development.md +36 -0
- jupyter_ext/agent/skills/anomaly_detection.md +12 -0
- jupyter_ext/agent/skills/athena_sql.md +156 -0
- jupyter_ext/agent/skills/bdp_table_search.md +33 -0
- jupyter_ext/agent/skills/bigquery.md +65 -0
- jupyter_ext/agent/skills/bigquery_oracle.md +231 -0
- jupyter_ext/agent/skills/bigquery_tuning.md +307 -0
- jupyter_ext/agent/skills/classification_evaluation.md +11 -0
- jupyter_ext/agent/skills/clustering_analysis.md +12 -0
- jupyter_ext/agent/skills/comprehensive_eda.md +11 -0
- jupyter_ext/agent/skills/data_analysis.md +236 -0
- jupyter_ext/agent/skills/data_cleaning_pipeline.md +12 -0
- jupyter_ext/agent/skills/data_loading.md +158 -0
- jupyter_ext/agent/skills/data_quality_report.md +13 -0
- jupyter_ext/agent/skills/dimensionality_reduction.md +12 -0
- jupyter_ext/agent/skills/emr_script_development.md +17 -0
- jupyter_ext/agent/skills/experiment_tracking.md +13 -0
- jupyter_ext/agent/skills/feature_importance_analysis.md +10 -0
- jupyter_ext/agent/skills/hyperparameter_tuning.md +12 -0
- jupyter_ext/agent/skills/hypothesis_testing.md +12 -0
- jupyter_ext/agent/skills/imbalanced_data_handling.md +12 -0
- jupyter_ext/agent/skills/inference.md +392 -0
- jupyter_ext/agent/skills/lambda_function_development.md +19 -0
- jupyter_ext/agent/skills/model_comparison.md +11 -0
- jupyter_ext/agent/skills/model_explainability.md +12 -0
- jupyter_ext/agent/skills/model_training.md +318 -0
- jupyter_ext/agent/skills/pyspark.md +352 -0
- jupyter_ext/agent/skills/python_code_quality.md +35 -0
- jupyter_ext/agent/skills/regression_evaluation.md +12 -0
- jupyter_ext/agent/skills/spring_java_development.md +35 -0
- jupyter_ext/agent/skills/text_preprocessing.md +12 -0
- jupyter_ext/agent/skills/time_series_analysis.md +13 -0
- jupyter_ext/agent/skills/unit_testing.md +203 -0
- jupyter_ext/agent/todo_prompts.py +57 -0
- jupyter_ext/agent/tool_report.py +45 -0
- jupyter_ext/analytics_ingest.py +599 -0
- jupyter_ext/etc/jupyter/jupyter_server_config.d/codebuff_agent.json +7 -0
- jupyter_ext/handlers/__init__.py +8 -0
- jupyter_ext/handlers/analytics_handler.py +170 -0
- jupyter_ext/handlers/benchmark_handler.py +794 -0
- jupyter_ext/handlers/eda_benchmark_handler.py +1328 -0
- jupyter_ext/handlers/rag_admin_handler.py +689 -0
- jupyter_ext/handlers/websocket.py +497 -0
- jupyter_ext/http_handlers.py +8744 -0
- jupyter_ext/idle_monitor.py +346 -0
- jupyter_ext/labextension/package.json +157 -0
- jupyter_ext/labextension/static/104.2b8f20f9d1ac2c9db01b.js +1 -0
- jupyter_ext/labextension/static/150.39fa98117cc8ca590465.js +1 -0
- jupyter_ext/labextension/static/166.386ff9b2c37ac865fdee.js +1 -0
- jupyter_ext/labextension/static/216.25cf81131bf7274dabf4.js +1 -0
- jupyter_ext/labextension/static/353.581bd1cb00746399b62e.js +1 -0
- jupyter_ext/labextension/static/367.575f9621b63c9a641aa9.js +1 -0
- jupyter_ext/labextension/static/421.11dac223a2c2da1b09f5.js +1 -0
- jupyter_ext/labextension/static/432.8513b4c6cc1d1232d165.js +2 -0
- jupyter_ext/labextension/static/432.8513b4c6cc1d1232d165.js.LICENSE.txt +19 -0
- jupyter_ext/labextension/static/439.ba5b5ef70ec6de214b8c.js +2 -0
- jupyter_ext/labextension/static/439.ba5b5ef70ec6de214b8c.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/481.cc2dec9fa869d66f1407.js +1 -0
- jupyter_ext/labextension/static/487.0796e931d6e651482c14.js +1 -0
- jupyter_ext/labextension/static/49.d326f554b6f9ccde6580.js +1 -0
- jupyter_ext/labextension/static/541.26897ea32845659e36be.js +1 -0
- jupyter_ext/labextension/static/579.e569d3edd8391fd8fbb3.js +1 -0
- jupyter_ext/labextension/static/58.eb7cc679f7535cc8a2f1.js +2 -0
- jupyter_ext/labextension/static/58.eb7cc679f7535cc8a2f1.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/666.d9bfdb23f98489d0a188.js +1 -0
- jupyter_ext/labextension/static/677.d266c39dbed33bc388b3.js +2 -0
- jupyter_ext/labextension/static/677.d266c39dbed33bc388b3.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/68.26b6f59c8454adb17f20.js +204 -0
- jupyter_ext/labextension/static/694.4ac443814260057e68f7.js +1 -0
- jupyter_ext/labextension/static/791.61affb8d3480d3d60e0d.js +1 -0
- jupyter_ext/labextension/static/848.4d2db09cfbb09374b299.js +2 -0
- jupyter_ext/labextension/static/848.4d2db09cfbb09374b299.js.LICENSE.txt +9 -0
- jupyter_ext/labextension/static/868.5cfe7e1a348ec5af1b8c.js +1 -0
- jupyter_ext/labextension/static/97.ea559a5b8a9a09f17c6b.js +1 -0
- jupyter_ext/labextension/static/remoteEntry.5292fc9c6df7e7920cc8.js +1 -0
- jupyter_ext/labextension/static/style.js +4 -0
- jupyter_ext/labextension/static/third-party-licenses.json +724 -0
- jupyter_ext/log_paths.py +113 -0
- jupyter_ext/resource_usage.py +519 -0
- jupyter_ext/secrets_bootstrap.py +116 -0
- jupyter_ext/team_access.py +105 -0
- jupyter_ext/tests/__init__.py +1 -0
- jupyter_ext/tests/test_context.py +234 -0
- jupyter_ext/tests/test_factory.py +563 -0
- jupyter_ext/tests/test_file_finder.py +139 -0
- jupyter_ext/tests/test_handlers.py +58 -0
- jupyter_ext/tests/test_mcp_selftest.py +204 -0
- jupyter_ext/tests/test_mermaid_fixer.py +272 -0
- jupyter_ext/tests/test_resource_tiers.py +298 -0
- jupyter_ext/tools/__init__.py +15 -0
- jupyter_ext/tools/base.py +90 -0
- jupyter_ext/tools/context.py +268 -0
- jupyter_ext/tools/dir_complete.py +86 -0
- jupyter_ext/tools/file_finder.py +303 -0
- jupyter_ext/tools/file_tools.py +244 -0
- jupyter_ext/tools/jupyter_tools.py +296 -0
- jupyter_ext/tools/registry.py +159 -0
- jupyter_ext/tools/search_tools.py +230 -0
- jupyter_ext/tools/security_tools.py +657 -0
- jupyter_ext/tools/shell_tools.py +109 -0
- jupyter_ext/tools/user_tools.py +132 -0
|
@@ -0,0 +1,1328 @@
|
|
|
1
|
+
"""EDA Agent Benchmark Handler — 테스트 케이스 관리, 벤치마크 실행, 결과 조회."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import time
|
|
8
|
+
import uuid
|
|
9
|
+
from collections import defaultdict
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from typing import Any, Dict, List, Optional
|
|
12
|
+
|
|
13
|
+
from jupyter_server.base.handlers import APIHandler
|
|
14
|
+
|
|
15
|
+
from codebuff_core.legacy_paths import resolve_home
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
# 벤치마크 데이터 저장 경로
|
|
20
|
+
_EDA_BENCHMARK_DIR = resolve_home("eda_benchmark")
|
|
21
|
+
_TEST_CASES_FILE = _EDA_BENCHMARK_DIR / "test_cases.json"
|
|
22
|
+
_RESULTS_DIR = _EDA_BENCHMARK_DIR / "results"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _ensure_dirs():
|
|
26
|
+
_EDA_BENCHMARK_DIR.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
_RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _load_test_cases() -> List[Dict[str, Any]]:
|
|
31
|
+
_ensure_dirs()
|
|
32
|
+
if _TEST_CASES_FILE.exists():
|
|
33
|
+
try:
|
|
34
|
+
return json.loads(_TEST_CASES_FILE.read_text(encoding="utf-8"))
|
|
35
|
+
except Exception:
|
|
36
|
+
return []
|
|
37
|
+
return []
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _save_test_cases(cases: List[Dict[str, Any]]):
|
|
41
|
+
_ensure_dirs()
|
|
42
|
+
_TEST_CASES_FILE.write_text(
|
|
43
|
+
json.dumps(cases, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _load_results() -> List[Dict[str, Any]]:
|
|
48
|
+
_ensure_dirs()
|
|
49
|
+
results = []
|
|
50
|
+
for f in sorted(_RESULTS_DIR.glob("*.json"), reverse=True):
|
|
51
|
+
try:
|
|
52
|
+
data = json.loads(f.read_text(encoding="utf-8"))
|
|
53
|
+
results.append(
|
|
54
|
+
{
|
|
55
|
+
"id": f.stem,
|
|
56
|
+
"date": data.get("date", ""),
|
|
57
|
+
"memo": data.get("memo", ""),
|
|
58
|
+
"total_cases": data.get("total_cases", 0),
|
|
59
|
+
"summary": data.get("summary", {}),
|
|
60
|
+
}
|
|
61
|
+
)
|
|
62
|
+
except Exception:
|
|
63
|
+
continue
|
|
64
|
+
return results
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _load_result_detail(result_id: str) -> Optional[Dict[str, Any]]:
|
|
68
|
+
path = _RESULTS_DIR / f"{result_id}.json"
|
|
69
|
+
if path.exists():
|
|
70
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _save_result(result: Dict[str, Any]) -> str:
|
|
75
|
+
_ensure_dirs()
|
|
76
|
+
result_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
|
77
|
+
path = _RESULTS_DIR / f"{result_id}.json"
|
|
78
|
+
result["id"] = result_id
|
|
79
|
+
path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
80
|
+
return result_id
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ─── Test Cases CRUD ───
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class EdaBenchmarkTestCasesHandler(APIHandler):
|
|
87
|
+
"""GET/POST /codebuff/eda-benchmark/test-cases"""
|
|
88
|
+
|
|
89
|
+
async def get(self):
|
|
90
|
+
self.write(json.dumps(_load_test_cases(), ensure_ascii=False))
|
|
91
|
+
|
|
92
|
+
async def post(self):
|
|
93
|
+
body = (
|
|
94
|
+
json.loads(self.request.body.decode("utf-8")) if self.request.body else {}
|
|
95
|
+
)
|
|
96
|
+
action = body.get("action", "save_all")
|
|
97
|
+
|
|
98
|
+
if action == "save_all":
|
|
99
|
+
cases = body.get("test_cases", [])
|
|
100
|
+
_save_test_cases(cases)
|
|
101
|
+
self.write({"status": "ok", "count": len(cases)})
|
|
102
|
+
elif action == "add":
|
|
103
|
+
cases = _load_test_cases()
|
|
104
|
+
new_case = body.get("test_case", {})
|
|
105
|
+
if not new_case.get("id"):
|
|
106
|
+
new_case["id"] = f"eda_{uuid.uuid4().hex[:8]}"
|
|
107
|
+
cases.append(new_case)
|
|
108
|
+
_save_test_cases(cases)
|
|
109
|
+
self.write({"status": "ok", "id": new_case["id"]})
|
|
110
|
+
elif action == "delete":
|
|
111
|
+
tc_id = body.get("id")
|
|
112
|
+
cases = [c for c in _load_test_cases() if c.get("id") != tc_id]
|
|
113
|
+
_save_test_cases(cases)
|
|
114
|
+
self.write({"status": "ok"})
|
|
115
|
+
elif action == "update":
|
|
116
|
+
tc_id = body.get("id")
|
|
117
|
+
updated = body.get("test_case", {})
|
|
118
|
+
cases = _load_test_cases()
|
|
119
|
+
for i, c in enumerate(cases):
|
|
120
|
+
if c.get("id") == tc_id:
|
|
121
|
+
cases[i] = {**c, **updated}
|
|
122
|
+
break
|
|
123
|
+
_save_test_cases(cases)
|
|
124
|
+
self.write({"status": "ok"})
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# ─── Sources (bdp_tables 목록) ───
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class EdaBenchmarkSourcesHandler(APIHandler):
|
|
131
|
+
"""GET /codebuff/eda-benchmark/sources — bdp_tables 테이블 목록."""
|
|
132
|
+
|
|
133
|
+
async def get(self):
|
|
134
|
+
try:
|
|
135
|
+
from .rag_admin_handler import _ensure_manager
|
|
136
|
+
|
|
137
|
+
mgr = _ensure_manager()
|
|
138
|
+
if mgr is None:
|
|
139
|
+
self.write(json.dumps({"tables": []}))
|
|
140
|
+
return
|
|
141
|
+
|
|
142
|
+
client = mgr._client
|
|
143
|
+
tables: List[Dict] = []
|
|
144
|
+
scroll_offset = None
|
|
145
|
+
while True:
|
|
146
|
+
pts, scroll_offset = client.scroll(
|
|
147
|
+
collection_name="bdp_tables",
|
|
148
|
+
limit=100,
|
|
149
|
+
with_payload=True,
|
|
150
|
+
with_vectors=False,
|
|
151
|
+
offset=scroll_offset,
|
|
152
|
+
)
|
|
153
|
+
for pt in pts:
|
|
154
|
+
p = pt.payload or {}
|
|
155
|
+
tname = p.get("table_name", p.get("테이블명", ""))
|
|
156
|
+
kname = p.get("korean_name", p.get("테이블한글명", ""))
|
|
157
|
+
cols = p.get("columns", [])
|
|
158
|
+
if tname and len(cols) >= 5:
|
|
159
|
+
col_names = []
|
|
160
|
+
for c in cols:
|
|
161
|
+
cn = (
|
|
162
|
+
(c.get("col_name", "") or c.get("컬럼명", ""))
|
|
163
|
+
if isinstance(c, dict)
|
|
164
|
+
else ""
|
|
165
|
+
)
|
|
166
|
+
if cn:
|
|
167
|
+
col_names.append(cn)
|
|
168
|
+
tables.append(
|
|
169
|
+
{
|
|
170
|
+
"table_name": tname,
|
|
171
|
+
"korean_name": kname,
|
|
172
|
+
"column_count": len(cols),
|
|
173
|
+
"column_names": col_names,
|
|
174
|
+
"description": p.get("description", p.get("설명", "")),
|
|
175
|
+
}
|
|
176
|
+
)
|
|
177
|
+
if scroll_offset is None:
|
|
178
|
+
break
|
|
179
|
+
|
|
180
|
+
self.write(
|
|
181
|
+
json.dumps({"tables": tables, "total": len(tables)}, ensure_ascii=False)
|
|
182
|
+
)
|
|
183
|
+
except Exception as e:
|
|
184
|
+
self.write(json.dumps({"tables": [], "error": str(e)}))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
# ─── Auto-generate test cases ───
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class EdaBenchmarkGenerateHandler(APIHandler):
|
|
191
|
+
"""POST /codebuff/eda-benchmark/generate — 테이블 조합 선정 + 질문 자동 생성."""
|
|
192
|
+
|
|
193
|
+
async def post(self):
|
|
194
|
+
body = (
|
|
195
|
+
json.loads(self.request.body.decode("utf-8")) if self.request.body else {}
|
|
196
|
+
)
|
|
197
|
+
selected_tables = body.get("tables", []) # 사용자가 선택한 테이블 목록
|
|
198
|
+
per_difficulty = body.get("per_difficulty", 3) # easy/medium/hard 각 N개
|
|
199
|
+
|
|
200
|
+
try:
|
|
201
|
+
from .rag_admin_handler import _ensure_manager
|
|
202
|
+
|
|
203
|
+
mgr = _ensure_manager()
|
|
204
|
+
if mgr is None:
|
|
205
|
+
self.set_status(500)
|
|
206
|
+
self.write({"error": "RAG 매니저 미초기화"})
|
|
207
|
+
return
|
|
208
|
+
|
|
209
|
+
client = mgr._client
|
|
210
|
+
|
|
211
|
+
# 1. 전체 테이블 메타 수집
|
|
212
|
+
all_tables: Dict[str, Dict] = {}
|
|
213
|
+
scroll_offset = None
|
|
214
|
+
while True:
|
|
215
|
+
pts, scroll_offset = client.scroll(
|
|
216
|
+
collection_name="bdp_tables",
|
|
217
|
+
limit=100,
|
|
218
|
+
with_payload=True,
|
|
219
|
+
with_vectors=False,
|
|
220
|
+
offset=scroll_offset,
|
|
221
|
+
)
|
|
222
|
+
for pt in pts:
|
|
223
|
+
p = pt.payload or {}
|
|
224
|
+
tname = p.get("table_name", p.get("테이블명", ""))
|
|
225
|
+
if not tname:
|
|
226
|
+
continue
|
|
227
|
+
cols = p.get("columns", [])
|
|
228
|
+
if len(cols) < 5:
|
|
229
|
+
continue
|
|
230
|
+
kname = p.get("korean_name", p.get("테이블한글명", ""))
|
|
231
|
+
if not kname:
|
|
232
|
+
continue
|
|
233
|
+
# 선택된 테이블만 사용 (선택 안 했으면 전체)
|
|
234
|
+
if selected_tables and tname not in selected_tables:
|
|
235
|
+
continue
|
|
236
|
+
col_names = []
|
|
237
|
+
col_details = []
|
|
238
|
+
for c in cols[:50]: # 최대 50개 컬럼
|
|
239
|
+
if isinstance(c, dict):
|
|
240
|
+
cn = c.get("col_name", "") or c.get("컬럼명", "")
|
|
241
|
+
ck = c.get("col_korean_name", "") or c.get("컬럼한글명", "")
|
|
242
|
+
ct = c.get("col_type", "") or c.get("컬럼타입", "")
|
|
243
|
+
if cn:
|
|
244
|
+
col_names.append(cn)
|
|
245
|
+
col_details.append(
|
|
246
|
+
{"name": cn, "korean": ck, "type": ct}
|
|
247
|
+
)
|
|
248
|
+
if tname not in all_tables:
|
|
249
|
+
all_tables[tname] = {
|
|
250
|
+
"table_name": tname,
|
|
251
|
+
"korean_name": kname,
|
|
252
|
+
"description": p.get("description", p.get("설명", "")),
|
|
253
|
+
"column_names": col_names,
|
|
254
|
+
"column_details": col_details,
|
|
255
|
+
}
|
|
256
|
+
else:
|
|
257
|
+
# 청크 병합: 컬럼 추가
|
|
258
|
+
existing = all_tables[tname]
|
|
259
|
+
existing_names = set(existing["column_names"])
|
|
260
|
+
for cn, cd in zip(col_names, col_details):
|
|
261
|
+
if cn not in existing_names:
|
|
262
|
+
existing["column_names"].append(cn)
|
|
263
|
+
existing["column_details"].append(cd)
|
|
264
|
+
existing_names.add(cn)
|
|
265
|
+
if scroll_offset is None:
|
|
266
|
+
break
|
|
267
|
+
|
|
268
|
+
if not all_tables:
|
|
269
|
+
self.write(
|
|
270
|
+
json.dumps(
|
|
271
|
+
{"test_cases": [], "message": "테이블을 찾을 수 없습니다."}
|
|
272
|
+
)
|
|
273
|
+
)
|
|
274
|
+
return
|
|
275
|
+
|
|
276
|
+
# 2. 컬럼 빈도 집계 → 흔한 컬럼 제외
|
|
277
|
+
col_frequency: Dict[str, int] = defaultdict(int)
|
|
278
|
+
for t in all_tables.values():
|
|
279
|
+
for cn in t["column_names"]:
|
|
280
|
+
col_frequency[cn] += 1
|
|
281
|
+
|
|
282
|
+
total_tables = len(all_tables)
|
|
283
|
+
threshold = max(total_tables * 0.2, 5)
|
|
284
|
+
common_cols = {c for c, freq in col_frequency.items() if freq >= threshold}
|
|
285
|
+
|
|
286
|
+
print(
|
|
287
|
+
f"[EDA-Bench] Tables: {total_tables}, "
|
|
288
|
+
f"Common columns excluded ({len(common_cols)}): "
|
|
289
|
+
f"{sorted(common_cols)[:10]}",
|
|
290
|
+
flush=True,
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
# 3. 특화 컬럼으로 JOIN 가능 테이블 쌍 찾기
|
|
294
|
+
table_names = list(all_tables.keys())
|
|
295
|
+
table_specialized: Dict[str, set] = {}
|
|
296
|
+
for tname, tdata in all_tables.items():
|
|
297
|
+
specialized = set(tdata["column_names"]) - common_cols
|
|
298
|
+
table_specialized[tname] = specialized
|
|
299
|
+
|
|
300
|
+
joinable_pairs: List[tuple] = []
|
|
301
|
+
for i, t1 in enumerate(table_names):
|
|
302
|
+
for t2 in table_names[i + 1 :]:
|
|
303
|
+
shared = table_specialized[t1] & table_specialized[t2]
|
|
304
|
+
if shared:
|
|
305
|
+
joinable_pairs.append((t1, t2, shared))
|
|
306
|
+
|
|
307
|
+
print(f"[EDA-Bench] Joinable pairs: {len(joinable_pairs)}", flush=True)
|
|
308
|
+
|
|
309
|
+
# 4. 테이블 조합 선정
|
|
310
|
+
import random
|
|
311
|
+
|
|
312
|
+
random.shuffle(table_names)
|
|
313
|
+
random.shuffle(joinable_pairs)
|
|
314
|
+
|
|
315
|
+
# simple: 단일 테이블
|
|
316
|
+
simple_tables = table_names[: per_difficulty * 2] # 여유 있게
|
|
317
|
+
|
|
318
|
+
# medium: 2개 테이블 쌍
|
|
319
|
+
medium_pairs = joinable_pairs[: per_difficulty * 2]
|
|
320
|
+
|
|
321
|
+
# hard: 3개 테이블 체인 (A-B, B-C 공통 컬럼)
|
|
322
|
+
hard_chains: List[tuple] = []
|
|
323
|
+
pair_by_table: Dict[str, List[tuple]] = defaultdict(list)
|
|
324
|
+
for t1, t2, shared in joinable_pairs:
|
|
325
|
+
pair_by_table[t1].append((t2, shared))
|
|
326
|
+
pair_by_table[t2].append((t1, shared))
|
|
327
|
+
|
|
328
|
+
used_chains: set = set()
|
|
329
|
+
for t1, t2, shared_ab in joinable_pairs:
|
|
330
|
+
if len(hard_chains) >= per_difficulty * 2:
|
|
331
|
+
break
|
|
332
|
+
for t3, shared_bc in pair_by_table.get(t2, []):
|
|
333
|
+
if t3 != t1:
|
|
334
|
+
chain_key = tuple(sorted([t1, t2, t3]))
|
|
335
|
+
if chain_key not in used_chains:
|
|
336
|
+
used_chains.add(chain_key)
|
|
337
|
+
hard_chains.append((t1, t2, t3, shared_ab, shared_bc))
|
|
338
|
+
break
|
|
339
|
+
|
|
340
|
+
# SSE 헤더
|
|
341
|
+
self.set_header("Content-Type", "text/event-stream")
|
|
342
|
+
self.set_header("Cache-Control", "no-cache")
|
|
343
|
+
self.set_header("Connection", "keep-alive")
|
|
344
|
+
|
|
345
|
+
# 5. LLM으로 질문 생성
|
|
346
|
+
from codebuff_core.llm.service import LLMService
|
|
347
|
+
from codebuff_core.managers.config_manager import get_config_manager
|
|
348
|
+
|
|
349
|
+
cfg = get_config_manager().get_config()
|
|
350
|
+
llm_config = cfg.get("vllm", {})
|
|
351
|
+
if not llm_config:
|
|
352
|
+
self.write(
|
|
353
|
+
f"data: {json.dumps({'type': 'error', 'error': 'LLM 설정 없음'})}\n\n"
|
|
354
|
+
)
|
|
355
|
+
self.finish()
|
|
356
|
+
return
|
|
357
|
+
|
|
358
|
+
llm = LLMService({"provider": "vllm", "vllm": llm_config})
|
|
359
|
+
generated: List[Dict] = []
|
|
360
|
+
|
|
361
|
+
# 생성 태스크 준비
|
|
362
|
+
tasks: List[Dict] = []
|
|
363
|
+
|
|
364
|
+
# simple (generate)
|
|
365
|
+
for tname in simple_tables[:per_difficulty]:
|
|
366
|
+
t = all_tables[tname]
|
|
367
|
+
cols_str = ", ".join(
|
|
368
|
+
f"{c['name']}({c['korean']})" for c in t["column_details"][:20]
|
|
369
|
+
)
|
|
370
|
+
tasks.append(
|
|
371
|
+
{
|
|
372
|
+
"type": "generate",
|
|
373
|
+
"difficulty": "easy",
|
|
374
|
+
"tables": [tname],
|
|
375
|
+
"prompt_context": (
|
|
376
|
+
f"테이블: {tname} ({t['korean_name']})\n"
|
|
377
|
+
f"설명: {t['description']}\n"
|
|
378
|
+
f"컬럼: {cols_str}"
|
|
379
|
+
),
|
|
380
|
+
}
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
# tune — joinable pair로 복잡한 비효율 SQL 생성
|
|
384
|
+
tune_pairs = joinable_pairs[
|
|
385
|
+
per_difficulty * 2 : per_difficulty * 2 + per_difficulty
|
|
386
|
+
]
|
|
387
|
+
# pair가 부족하면 단일 테이블로 fallback
|
|
388
|
+
tune_single = simple_tables[per_difficulty : per_difficulty * 2]
|
|
389
|
+
tune_idx = 0
|
|
390
|
+
for _ in range(per_difficulty):
|
|
391
|
+
if tune_idx < len(tune_pairs):
|
|
392
|
+
t1, t2, shared = tune_pairs[tune_idx]
|
|
393
|
+
tune_idx += 1
|
|
394
|
+
d1, d2 = all_tables[t1], all_tables[t2]
|
|
395
|
+
cols1 = ", ".join(
|
|
396
|
+
f"{c['name']}({c['korean']}, {c['type']})"
|
|
397
|
+
for c in d1["column_details"][:20]
|
|
398
|
+
)
|
|
399
|
+
cols2 = ", ".join(
|
|
400
|
+
f"{c['name']}({c['korean']}, {c['type']})"
|
|
401
|
+
for c in d2["column_details"][:20]
|
|
402
|
+
)
|
|
403
|
+
tasks.append(
|
|
404
|
+
{
|
|
405
|
+
"type": "tune",
|
|
406
|
+
"difficulty": "medium",
|
|
407
|
+
"tables": [t1, t2],
|
|
408
|
+
"shared_columns": list(shared),
|
|
409
|
+
"prompt_context": (
|
|
410
|
+
f"테이블1: {t1} ({d1['korean_name']})\n"
|
|
411
|
+
f"컬럼: {cols1}\n\n"
|
|
412
|
+
f"테이블2: {t2} ({d2['korean_name']})\n"
|
|
413
|
+
f"컬럼: {cols2}\n\n"
|
|
414
|
+
f"공통 컬럼(JOIN 키): {', '.join(shared)}"
|
|
415
|
+
),
|
|
416
|
+
}
|
|
417
|
+
)
|
|
418
|
+
elif tune_idx - len(tune_pairs) < len(tune_single):
|
|
419
|
+
tname = tune_single[tune_idx - len(tune_pairs)]
|
|
420
|
+
tune_idx += 1
|
|
421
|
+
t = all_tables.get(tname)
|
|
422
|
+
if not t:
|
|
423
|
+
continue
|
|
424
|
+
cols_str = ", ".join(
|
|
425
|
+
f"{c['name']}({c['korean']}, {c['type']})"
|
|
426
|
+
for c in t["column_details"][:20]
|
|
427
|
+
)
|
|
428
|
+
tasks.append(
|
|
429
|
+
{
|
|
430
|
+
"type": "tune",
|
|
431
|
+
"difficulty": "easy",
|
|
432
|
+
"tables": [tname],
|
|
433
|
+
"prompt_context": (
|
|
434
|
+
f"테이블: {tname} ({t['korean_name']})\n"
|
|
435
|
+
f"설명: {t['description']}\n"
|
|
436
|
+
f"컬럼: {cols_str}"
|
|
437
|
+
),
|
|
438
|
+
}
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
# medium (generate)
|
|
442
|
+
for t1, t2, shared in medium_pairs[:per_difficulty]:
|
|
443
|
+
d1, d2 = all_tables[t1], all_tables[t2]
|
|
444
|
+
cols1 = ", ".join(
|
|
445
|
+
f"{c['name']}({c['korean']})" for c in d1["column_details"][:15]
|
|
446
|
+
)
|
|
447
|
+
cols2 = ", ".join(
|
|
448
|
+
f"{c['name']}({c['korean']})" for c in d2["column_details"][:15]
|
|
449
|
+
)
|
|
450
|
+
tasks.append(
|
|
451
|
+
{
|
|
452
|
+
"type": "generate",
|
|
453
|
+
"difficulty": "medium",
|
|
454
|
+
"tables": [t1, t2],
|
|
455
|
+
"shared_columns": list(shared),
|
|
456
|
+
"prompt_context": (
|
|
457
|
+
f"테이블1: {t1} ({d1['korean_name']})\n"
|
|
458
|
+
f"컬럼: {cols1}\n\n"
|
|
459
|
+
f"테이블2: {t2} ({d2['korean_name']})\n"
|
|
460
|
+
f"컬럼: {cols2}\n\n"
|
|
461
|
+
f"공통 컬럼: {', '.join(shared)}"
|
|
462
|
+
),
|
|
463
|
+
}
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
# hard (generate)
|
|
467
|
+
for chain in hard_chains[:per_difficulty]:
|
|
468
|
+
t1, t2, t3 = chain[0], chain[1], chain[2]
|
|
469
|
+
d1, d2, d3 = all_tables[t1], all_tables[t2], all_tables[t3]
|
|
470
|
+
cols1 = ", ".join(
|
|
471
|
+
f"{c['name']}({c['korean']})" for c in d1["column_details"][:10]
|
|
472
|
+
)
|
|
473
|
+
cols2 = ", ".join(
|
|
474
|
+
f"{c['name']}({c['korean']})" for c in d2["column_details"][:10]
|
|
475
|
+
)
|
|
476
|
+
cols3 = ", ".join(
|
|
477
|
+
f"{c['name']}({c['korean']})" for c in d3["column_details"][:10]
|
|
478
|
+
)
|
|
479
|
+
tasks.append(
|
|
480
|
+
{
|
|
481
|
+
"type": "generate",
|
|
482
|
+
"difficulty": "hard",
|
|
483
|
+
"tables": [t1, t2, t3],
|
|
484
|
+
"prompt_context": (
|
|
485
|
+
f"테이블1: {t1} ({d1['korean_name']})\n컬럼: {cols1}\n\n"
|
|
486
|
+
f"테이블2: {t2} ({d2['korean_name']})\n컬럼: {cols2}\n\n"
|
|
487
|
+
f"테이블3: {t3} ({d3['korean_name']})\n컬럼: {cols3}"
|
|
488
|
+
),
|
|
489
|
+
}
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
total_tasks = len(tasks)
|
|
493
|
+
self.write(
|
|
494
|
+
f"data: {json.dumps({'type': 'progress', 'current': 0, 'total': total_tasks}, ensure_ascii=False)}\n\n"
|
|
495
|
+
)
|
|
496
|
+
await self.flush()
|
|
497
|
+
|
|
498
|
+
# 병렬 LLM 호출
|
|
499
|
+
_PARALLEL = 5
|
|
500
|
+
_completed = 0
|
|
501
|
+
|
|
502
|
+
async def _generate_one(task: Dict) -> Optional[Dict]:
|
|
503
|
+
task_type = task["type"]
|
|
504
|
+
if task_type == "generate":
|
|
505
|
+
prompt = (
|
|
506
|
+
"아래 테이블 정보를 읽고 SQL 쿼리 생성 요청을 만들어주세요.\n\n"
|
|
507
|
+
"규칙:\n"
|
|
508
|
+
"- 실제 분석가가 할 법한 자연어 요청으로 만드세요.\n"
|
|
509
|
+
"- 반드시 '쿼리를 작성해줘', '조회하는 SQL을 만들어줘' 형태로 끝나야 합니다.\n"
|
|
510
|
+
"- expected_columns에는 SQL에 반드시 포함되어야 할 핵심 컬럼명을 넣으세요.\n\n"
|
|
511
|
+
"반드시 아래 JSON 형식으로만 응답하세요:\n"
|
|
512
|
+
'{"question": "자연어 쿼리 생성 요청", '
|
|
513
|
+
'"expected_columns": ["핵심 컬럼명"]}\n\n'
|
|
514
|
+
f"{task['prompt_context']}"
|
|
515
|
+
)
|
|
516
|
+
else: # tune
|
|
517
|
+
prompt = (
|
|
518
|
+
"아래 테이블 정보를 읽고 두 가지를 만들어주세요:\n"
|
|
519
|
+
"1. 이 테이블들을 사용하는 **복잡하고 비효율적인** Athena SQL\n"
|
|
520
|
+
"2. 그 SQL에 대한 튜닝 요청 자연어\n\n"
|
|
521
|
+
"SQL 작성 규칙 (매우 중요):\n"
|
|
522
|
+
"- **최소 50줄 이상**의 복잡한 SQL이어야 합니다.\n"
|
|
523
|
+
"- 반드시 다음 요소를 포함하세요:\n"
|
|
524
|
+
" * 여러 CTE (WITH절) 또는 서브쿼리\n"
|
|
525
|
+
" * JOIN (가능하면 여러 테이블)\n"
|
|
526
|
+
" * GROUP BY + 집계함수 (SUM, COUNT, AVG 등)\n"
|
|
527
|
+
" * CASE WHEN 조건 분기\n"
|
|
528
|
+
" * ORDER BY\n"
|
|
529
|
+
"- 비효율적 요소를 포함하세요:\n"
|
|
530
|
+
" * SELECT * 사용\n"
|
|
531
|
+
" * 파티션 컬럼에 함수 적용 (WHERE YEAR(date_col) = 2024)\n"
|
|
532
|
+
" * 불필요한 서브쿼리 중첩\n"
|
|
533
|
+
" * 큰 테이��이 RIGHT JOIN의 오른쪽에 위치\n"
|
|
534
|
+
" * 타입 불일치 JOIN (문자열 vs 숫자)\n"
|
|
535
|
+
"- SQL은 Athena(Presto/Trino) 문법이어야 합니다.\n"
|
|
536
|
+
"- 테이블명과 컬럼명은 아래 제공된 것만 사용하세요.\n\n"
|
|
537
|
+
"튜닝 요청 규칙:\n"
|
|
538
|
+
"- '이 쿼리의 용도를 분석하고 최적화해줘' 형태로.\n"
|
|
539
|
+
"- expected_improvements에는 기대하는 개선사항을 넣으세요.\n\n"
|
|
540
|
+
"반드시 아래 JSON 형식으로만 응답하세요:\n"
|
|
541
|
+
'{"question": "튜닝 요청 자연어", '
|
|
542
|
+
'"existing_sql": "복잡한 비��율적 SQL (50줄 이상)", '
|
|
543
|
+
'"expected_improvements": ["파티션 필터 개선", "컬럼 축소", '
|
|
544
|
+
'"JOIN 순서 최적화", "서브쿼리 제거"]}\n\n'
|
|
545
|
+
f"{task['prompt_context']}"
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
try:
|
|
549
|
+
resp = await llm.generate_response(prompt)
|
|
550
|
+
resp = resp.strip()
|
|
551
|
+
if resp.startswith("```"):
|
|
552
|
+
resp = resp.split("```")[1]
|
|
553
|
+
if resp.startswith("json"):
|
|
554
|
+
resp = resp[4:]
|
|
555
|
+
parsed = json.loads(resp)
|
|
556
|
+
|
|
557
|
+
tc = {
|
|
558
|
+
"id": f"eda_{uuid.uuid4().hex[:8]}",
|
|
559
|
+
"type": task_type,
|
|
560
|
+
"question": parsed.get("question", ""),
|
|
561
|
+
"difficulty": task["difficulty"],
|
|
562
|
+
"expected_tables": task["tables"],
|
|
563
|
+
"auto_generated": True,
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
if task_type == "generate":
|
|
567
|
+
tc["existing_sql"] = None
|
|
568
|
+
tc["expected_columns"] = parsed.get("expected_columns", [])
|
|
569
|
+
else:
|
|
570
|
+
tc["existing_sql"] = parsed.get("existing_sql", "")
|
|
571
|
+
tc["expected_improvements"] = parsed.get(
|
|
572
|
+
"expected_improvements", []
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
return tc
|
|
576
|
+
except Exception as e:
|
|
577
|
+
logger.warning(f"EDA test case generation failed: {e}")
|
|
578
|
+
return None
|
|
579
|
+
|
|
580
|
+
import asyncio as _aio
|
|
581
|
+
|
|
582
|
+
sem = _aio.Semaphore(_PARALLEL)
|
|
583
|
+
|
|
584
|
+
async def _gen_with_sem(task: Dict):
|
|
585
|
+
nonlocal _completed
|
|
586
|
+
async with sem:
|
|
587
|
+
r = await _generate_one(task)
|
|
588
|
+
_completed += 1
|
|
589
|
+
if r is not None:
|
|
590
|
+
generated.append(r)
|
|
591
|
+
self.write(
|
|
592
|
+
f"data: {json.dumps({'type': 'case', 'test_case': r}, ensure_ascii=False)}\n\n"
|
|
593
|
+
)
|
|
594
|
+
self.write(
|
|
595
|
+
f"data: {json.dumps({'type': 'progress', 'current': _completed, 'total': total_tasks}, ensure_ascii=False)}\n\n"
|
|
596
|
+
)
|
|
597
|
+
await self.flush()
|
|
598
|
+
return r
|
|
599
|
+
|
|
600
|
+
await _aio.gather(*[_gen_with_sem(t) for t in tasks])
|
|
601
|
+
|
|
602
|
+
self.write(
|
|
603
|
+
f"data: {json.dumps({'type': 'complete', 'total': len(generated)})}\n\n"
|
|
604
|
+
)
|
|
605
|
+
await self.flush()
|
|
606
|
+
self.finish()
|
|
607
|
+
|
|
608
|
+
except Exception as e:
|
|
609
|
+
logger.error(f"EDA benchmark generate failed: {e}", exc_info=True)
|
|
610
|
+
self.set_status(500)
|
|
611
|
+
self.write({"error": str(e)})
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
# ─── Run benchmark ───
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
class EdaBenchmarkRunHandler(APIHandler):
|
|
618
|
+
"""POST /codebuff/eda-benchmark/run — EDA 벤치마크 실행 (SSE)."""
|
|
619
|
+
|
|
620
|
+
async def post(self):
|
|
621
|
+
body = (
|
|
622
|
+
json.loads(self.request.body.decode("utf-8")) if self.request.body else {}
|
|
623
|
+
)
|
|
624
|
+
test_case_ids = body.get("test_case_ids")
|
|
625
|
+
memo = body.get("memo", "")
|
|
626
|
+
|
|
627
|
+
all_cases = _load_test_cases()
|
|
628
|
+
if test_case_ids:
|
|
629
|
+
cases = [c for c in all_cases if c.get("id") in test_case_ids]
|
|
630
|
+
else:
|
|
631
|
+
cases = all_cases
|
|
632
|
+
|
|
633
|
+
if not cases:
|
|
634
|
+
self.set_status(400)
|
|
635
|
+
self.write({"error": "테스트 케이스가 없습니다."})
|
|
636
|
+
return
|
|
637
|
+
|
|
638
|
+
self.set_header("Content-Type", "text/event-stream")
|
|
639
|
+
self.set_header("Cache-Control", "no-cache")
|
|
640
|
+
self.set_header("Connection", "keep-alive")
|
|
641
|
+
|
|
642
|
+
from codebuff_core.managers.config_manager import get_config_manager
|
|
643
|
+
|
|
644
|
+
cfg = get_config_manager().get_config()
|
|
645
|
+
|
|
646
|
+
results: List[Dict] = []
|
|
647
|
+
total = len(cases)
|
|
648
|
+
_completed = 0
|
|
649
|
+
_cancelled = False
|
|
650
|
+
_PARALLEL = 3 # EDA는 EXPLAIN 호출이 있어서 낮게
|
|
651
|
+
|
|
652
|
+
import asyncio as _aio
|
|
653
|
+
|
|
654
|
+
sem = _aio.Semaphore(_PARALLEL)
|
|
655
|
+
|
|
656
|
+
async def _run_one(tc):
|
|
657
|
+
nonlocal _completed, _cancelled
|
|
658
|
+
if _cancelled:
|
|
659
|
+
return None
|
|
660
|
+
async with sem:
|
|
661
|
+
if _cancelled:
|
|
662
|
+
return None
|
|
663
|
+
try:
|
|
664
|
+
r = await self._run_single_case(tc, cfg)
|
|
665
|
+
except Exception as e:
|
|
666
|
+
logger.error(f"EDA benchmark case {tc['id']} failed: {e}")
|
|
667
|
+
r = {"case_id": tc["id"], "error": str(e), "scores": {}}
|
|
668
|
+
_completed += 1
|
|
669
|
+
results.append(r)
|
|
670
|
+
try:
|
|
671
|
+
self.write(
|
|
672
|
+
f"data: {json.dumps({'type': 'case_result', 'result': r}, ensure_ascii=False, default=str)}\n\n"
|
|
673
|
+
)
|
|
674
|
+
self.write(
|
|
675
|
+
f"data: {json.dumps({'type': 'progress', 'current': _completed, 'total': total})}\n\n"
|
|
676
|
+
)
|
|
677
|
+
await self.flush()
|
|
678
|
+
except Exception:
|
|
679
|
+
_cancelled = True
|
|
680
|
+
return r
|
|
681
|
+
|
|
682
|
+
try:
|
|
683
|
+
await _aio.gather(*[_run_one(tc) for tc in cases])
|
|
684
|
+
except Exception:
|
|
685
|
+
_cancelled = True
|
|
686
|
+
|
|
687
|
+
valid_results = [r for r in results if r is not None]
|
|
688
|
+
summary = self._calc_summary(valid_results)
|
|
689
|
+
|
|
690
|
+
full_result = {
|
|
691
|
+
"date": datetime.now().isoformat(),
|
|
692
|
+
"memo": memo + (" (중단됨)" if _cancelled else ""),
|
|
693
|
+
"total_cases": len(valid_results),
|
|
694
|
+
"summary": summary,
|
|
695
|
+
"results": valid_results,
|
|
696
|
+
}
|
|
697
|
+
result_id = _save_result(full_result)
|
|
698
|
+
|
|
699
|
+
try:
|
|
700
|
+
self.write(
|
|
701
|
+
f"data: {json.dumps({'type': 'complete', 'result_id': result_id, 'summary': summary, 'cancelled': _cancelled}, ensure_ascii=False, default=str)}\n\n"
|
|
702
|
+
)
|
|
703
|
+
await self.flush()
|
|
704
|
+
except Exception:
|
|
705
|
+
pass
|
|
706
|
+
self.finish()
|
|
707
|
+
|
|
708
|
+
async def _run_single_case(
|
|
709
|
+
self, tc: Dict[str, Any], cfg: Dict[str, Any]
|
|
710
|
+
) -> Dict[str, Any]:
|
|
711
|
+
"""단일 EDA 테스트 케이스 실행."""
|
|
712
|
+
from codebuff_core.services.rag_chat_agent import (
|
|
713
|
+
RAGChatAgent,
|
|
714
|
+
TableConfirmationNeeded,
|
|
715
|
+
)
|
|
716
|
+
|
|
717
|
+
llm_cfg = {"provider": "vllm", "vllm": cfg.get("vllm", {})}
|
|
718
|
+
tc_type = tc.get("type", "generate")
|
|
719
|
+
question = tc.get("question", "")
|
|
720
|
+
existing_sql = tc.get("existing_sql")
|
|
721
|
+
expected_tables = tc.get("expected_tables", [])
|
|
722
|
+
|
|
723
|
+
start_time = time.time()
|
|
724
|
+
result: Dict[str, Any] = {
|
|
725
|
+
"case_id": tc["id"],
|
|
726
|
+
"type": tc_type,
|
|
727
|
+
"question": question,
|
|
728
|
+
"difficulty": tc.get("difficulty", ""),
|
|
729
|
+
"expected_tables": expected_tables,
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
# ── Phase 1: SQL 생성 ──
|
|
733
|
+
from langgraph.checkpoint.memory import MemorySaver
|
|
734
|
+
|
|
735
|
+
# EDA Agent 시스템 프롬프트 사용 (JSON 응답 형식 강제). 벤치마크는 관리자 값이 아니라
|
|
736
|
+
# 기본값을 잰다 — 스냅샷 `prompts.athena_query_prompt` (이슈 #218)
|
|
737
|
+
from codebuff_core.managers.remote_config import snapshot_prompt
|
|
738
|
+
from jupyter_ext.http_handlers import _read_skill_body
|
|
739
|
+
|
|
740
|
+
# RAG 규칙은 bdp_table_search 스킬로 옮겨졌다 — 전과 같은 프롬프트를 재도록 붙인다 (이슈 #220)
|
|
741
|
+
_bench_prompt = (
|
|
742
|
+
snapshot_prompt("athena_query_prompt")
|
|
743
|
+
+ "\n\n"
|
|
744
|
+
+ _read_skill_body("bdp_table_search", None)
|
|
745
|
+
)
|
|
746
|
+
|
|
747
|
+
thread_id = f"eda-bench-{uuid.uuid4().hex[:8]}"
|
|
748
|
+
_eda_grounding = (
|
|
749
|
+
"위 검색 결과의 테이블/컬럼 정보를 활용하여 "
|
|
750
|
+
"사용자 요청에 맞는 응답을 생성하세요.\n"
|
|
751
|
+
"반드시 JSON 형식(title, description, sql)으로 응답하세요."
|
|
752
|
+
)
|
|
753
|
+
from jupyter_ext.agent.langchain_tools import set_eda_only_bdp
|
|
754
|
+
|
|
755
|
+
set_eda_only_bdp(True)
|
|
756
|
+
agent = RAGChatAgent(
|
|
757
|
+
llm_cfg,
|
|
758
|
+
system_prompt_override=_bench_prompt,
|
|
759
|
+
checkpointer=MemorySaver(),
|
|
760
|
+
table_score_threshold=0.7,
|
|
761
|
+
grounding_constraint=_eda_grounding,
|
|
762
|
+
)
|
|
763
|
+
|
|
764
|
+
response_text = ""
|
|
765
|
+
selected_tables = []
|
|
766
|
+
confirmed_tables = []
|
|
767
|
+
path_used = "B" if existing_sql else "A"
|
|
768
|
+
|
|
769
|
+
if tc_type == "tune" and existing_sql:
|
|
770
|
+
# 튜닝: existingSql과 함께 요청
|
|
771
|
+
user_msg = f"[기존 SQL]\n```sql\n{existing_sql}\n```\n\n{question}"
|
|
772
|
+
path_used = "direct"
|
|
773
|
+
try:
|
|
774
|
+
async for chunk in agent.stream_response(
|
|
775
|
+
user_message=user_msg, thread_id=thread_id
|
|
776
|
+
):
|
|
777
|
+
if isinstance(chunk, dict) and chunk.get("content"):
|
|
778
|
+
response_text += chunk["content"]
|
|
779
|
+
except TableConfirmationNeeded as e:
|
|
780
|
+
# tune에서도 테이블 확인 필요 시 top-1 자동 선택 후 재실행
|
|
781
|
+
for _q, candidates in e.candidates.items():
|
|
782
|
+
if candidates:
|
|
783
|
+
selected_tables.append(candidates[0])
|
|
784
|
+
confirmed_tables = e.confirmed_tables or []
|
|
785
|
+
all_selected = selected_tables + confirmed_tables
|
|
786
|
+
table_ctx = json.dumps(all_selected, ensure_ascii=False, default=str)
|
|
787
|
+
user_msg2 = (
|
|
788
|
+
f"[선택된 테이블 정보]\n{table_ctx}\n\n"
|
|
789
|
+
f"[기존 SQL]\n```sql\n{existing_sql}\n```\n\n{question}"
|
|
790
|
+
)
|
|
791
|
+
thread_id2 = f"eda-bench-{uuid.uuid4().hex[:8]}"
|
|
792
|
+
agent2 = RAGChatAgent(
|
|
793
|
+
llm_cfg,
|
|
794
|
+
system_prompt_override=_bench_prompt,
|
|
795
|
+
checkpointer=MemorySaver(),
|
|
796
|
+
grounding_constraint=_eda_grounding,
|
|
797
|
+
)
|
|
798
|
+
try:
|
|
799
|
+
async for chunk in agent2.stream_response(
|
|
800
|
+
user_message=user_msg2, thread_id=thread_id2
|
|
801
|
+
):
|
|
802
|
+
if isinstance(chunk, dict) and chunk.get("content"):
|
|
803
|
+
response_text += chunk["content"]
|
|
804
|
+
except Exception as e2:
|
|
805
|
+
result["error"] = f"Tune Path B failed: {e2}"
|
|
806
|
+
result["scores"] = {}
|
|
807
|
+
return result
|
|
808
|
+
except Exception as e:
|
|
809
|
+
result["error"] = f"SQL generation failed: {e}"
|
|
810
|
+
result["scores"] = {}
|
|
811
|
+
return result
|
|
812
|
+
else:
|
|
813
|
+
# 생성: RAG 경로 (Path A)
|
|
814
|
+
try:
|
|
815
|
+
async for chunk in agent.stream_response(
|
|
816
|
+
user_message=question, thread_id=thread_id
|
|
817
|
+
):
|
|
818
|
+
if isinstance(chunk, dict) and chunk.get("content"):
|
|
819
|
+
response_text += chunk["content"]
|
|
820
|
+
except TableConfirmationNeeded as e:
|
|
821
|
+
# 자동 테이블 선택: 각 그룹의 top-1
|
|
822
|
+
for _q, candidates in e.candidates.items():
|
|
823
|
+
if candidates:
|
|
824
|
+
selected_tables.append(candidates[0])
|
|
825
|
+
confirmed_tables = e.confirmed_tables or []
|
|
826
|
+
|
|
827
|
+
# Path B로 재실행
|
|
828
|
+
path_used = "A→B"
|
|
829
|
+
response_text = ""
|
|
830
|
+
thread_id2 = f"eda-bench-{uuid.uuid4().hex[:8]}"
|
|
831
|
+
agent2 = RAGChatAgent(
|
|
832
|
+
llm_cfg,
|
|
833
|
+
system_prompt_override=_bench_prompt,
|
|
834
|
+
checkpointer=MemorySaver(),
|
|
835
|
+
grounding_constraint=_eda_grounding,
|
|
836
|
+
)
|
|
837
|
+
# selectedTables 형태로 전달
|
|
838
|
+
all_selected = selected_tables + confirmed_tables
|
|
839
|
+
table_ctx = json.dumps(all_selected, ensure_ascii=False, default=str)
|
|
840
|
+
user_msg2 = f"[선택된 테이블 정보]\n{table_ctx}\n\n{question}"
|
|
841
|
+
try:
|
|
842
|
+
async for chunk in agent2.stream_response(
|
|
843
|
+
user_message=user_msg2, thread_id=thread_id2
|
|
844
|
+
):
|
|
845
|
+
if isinstance(chunk, dict) and chunk.get("content"):
|
|
846
|
+
response_text += chunk["content"]
|
|
847
|
+
except Exception as e2:
|
|
848
|
+
result["error"] = f"Path B failed: {e2}"
|
|
849
|
+
result["scores"] = {}
|
|
850
|
+
return result
|
|
851
|
+
except Exception as e:
|
|
852
|
+
result["error"] = f"Path A failed: {e}"
|
|
853
|
+
result["scores"] = {}
|
|
854
|
+
return result
|
|
855
|
+
|
|
856
|
+
# title / description / sql 추출
|
|
857
|
+
sql = ""
|
|
858
|
+
title = ""
|
|
859
|
+
description = ""
|
|
860
|
+
try:
|
|
861
|
+
import re
|
|
862
|
+
|
|
863
|
+
# JSON 파싱 시도 (EDA Agent는 {title, description, sql} 또는 {title, description} JSON 반환)
|
|
864
|
+
json_match = re.search(
|
|
865
|
+
r"\{.*(?:\"sql\"|\"title\")\s*:", response_text, re.DOTALL
|
|
866
|
+
)
|
|
867
|
+
if json_match:
|
|
868
|
+
# JSON 시작점부터 끝까지 추출
|
|
869
|
+
json_start = json_match.start()
|
|
870
|
+
# 중첩 브레이스 처리
|
|
871
|
+
depth = 0
|
|
872
|
+
json_end = json_start
|
|
873
|
+
for ci, ch in enumerate(response_text[json_start:], json_start):
|
|
874
|
+
if ch == "{":
|
|
875
|
+
depth += 1
|
|
876
|
+
elif ch == "}":
|
|
877
|
+
depth -= 1
|
|
878
|
+
if depth == 0:
|
|
879
|
+
json_end = ci + 1
|
|
880
|
+
break
|
|
881
|
+
try:
|
|
882
|
+
parsed = json.loads(response_text[json_start:json_end])
|
|
883
|
+
sql = parsed.get("sql", "")
|
|
884
|
+
title = parsed.get("title", "")
|
|
885
|
+
description = parsed.get("description", "")
|
|
886
|
+
except Exception:
|
|
887
|
+
pass
|
|
888
|
+
if not sql:
|
|
889
|
+
# 코드 블록에서 SQL 추출
|
|
890
|
+
sql_match = re.search(
|
|
891
|
+
r"```(?:sql)?\s*\n(.*?)\n```", response_text, re.DOTALL
|
|
892
|
+
)
|
|
893
|
+
if sql_match:
|
|
894
|
+
sql = sql_match.group(1).strip()
|
|
895
|
+
except Exception:
|
|
896
|
+
pass
|
|
897
|
+
|
|
898
|
+
result["response"] = response_text[:2000]
|
|
899
|
+
result["sql"] = sql
|
|
900
|
+
result["title"] = title
|
|
901
|
+
result["description"] = description
|
|
902
|
+
result["path_used"] = path_used
|
|
903
|
+
result["selected_tables"] = [
|
|
904
|
+
t.get("table_name", "") if isinstance(t, dict) else str(t)
|
|
905
|
+
for t in selected_tables + confirmed_tables
|
|
906
|
+
]
|
|
907
|
+
|
|
908
|
+
# ── Phase 2: EXPLAIN 검증 ──
|
|
909
|
+
explain_success = False
|
|
910
|
+
explain_error = ""
|
|
911
|
+
fix_count = 0
|
|
912
|
+
|
|
913
|
+
if sql:
|
|
914
|
+
from jupyter_ext.http_handlers import _run_explain
|
|
915
|
+
|
|
916
|
+
explain_result = await _run_explain(sql)
|
|
917
|
+
explain_success = explain_result.get("success", False)
|
|
918
|
+
explain_error = explain_result.get("error", "")
|
|
919
|
+
|
|
920
|
+
# Fix loop (최대 3회 — 벤치마크에서는 간소화)
|
|
921
|
+
current_sql = sql
|
|
922
|
+
max_fixes = 3
|
|
923
|
+
while not explain_success and fix_count < max_fixes:
|
|
924
|
+
fix_count += 1
|
|
925
|
+
# LLM에게 fix 요청
|
|
926
|
+
from codebuff_core.llm.service import LLMService
|
|
927
|
+
|
|
928
|
+
fix_llm = LLMService({"provider": "vllm", "vllm": cfg.get("vllm", {})})
|
|
929
|
+
fix_prompt = (
|
|
930
|
+
f"아래 SQL에서 EXPLAIN 실행 시 오류가 발생했습니다. 오류를 수정하세요.\n\n"
|
|
931
|
+
f"[SQL]\n{current_sql}\n\n"
|
|
932
|
+
f"[오류]\n{explain_error}\n\n"
|
|
933
|
+
'수정된 SQL만 JSON으로 응답하세요: {{"sql": "수정된 SQL"}}'
|
|
934
|
+
)
|
|
935
|
+
try:
|
|
936
|
+
fix_resp = await fix_llm.generate_response(fix_prompt)
|
|
937
|
+
fix_resp = fix_resp.strip()
|
|
938
|
+
if fix_resp.startswith("```"):
|
|
939
|
+
fix_resp = fix_resp.split("```")[1]
|
|
940
|
+
if fix_resp.startswith("json"):
|
|
941
|
+
fix_resp = fix_resp[4:]
|
|
942
|
+
fix_parsed = json.loads(fix_resp)
|
|
943
|
+
fixed_sql = fix_parsed.get("sql", "")
|
|
944
|
+
if fixed_sql:
|
|
945
|
+
current_sql = fixed_sql
|
|
946
|
+
explain_result = await _run_explain(current_sql)
|
|
947
|
+
explain_success = explain_result.get("success", False)
|
|
948
|
+
explain_error = explain_result.get("error", "")
|
|
949
|
+
except Exception:
|
|
950
|
+
break
|
|
951
|
+
|
|
952
|
+
result["final_sql"] = current_sql
|
|
953
|
+
result["explain_success"] = explain_success
|
|
954
|
+
result["explain_error"] = explain_error if not explain_success else ""
|
|
955
|
+
result["fix_count"] = fix_count
|
|
956
|
+
|
|
957
|
+
# ── Phase 3: 쿼리플랜 튜닝 + REASON 주석 ──
|
|
958
|
+
plan_tuned = False
|
|
959
|
+
tuned_sql = ""
|
|
960
|
+
tuning_description = ""
|
|
961
|
+
tune_explain_success = False
|
|
962
|
+
reason_count = 0
|
|
963
|
+
|
|
964
|
+
if explain_success and (existing_sql or explain_result.get("plan")):
|
|
965
|
+
explain_plan = explain_result.get("plan", "")
|
|
966
|
+
try:
|
|
967
|
+
from codebuff_core.llm.service import LLMService
|
|
968
|
+
|
|
969
|
+
tune_llm = LLMService(
|
|
970
|
+
{"provider": "vllm", "vllm": cfg.get("vllm", {})}
|
|
971
|
+
)
|
|
972
|
+
tune_system = (
|
|
973
|
+
_bench_prompt
|
|
974
|
+
+ "\n\n\n"
|
|
975
|
+
+ snapshot_prompt("athena_plan_tuning_prompt")
|
|
976
|
+
)
|
|
977
|
+
_plan_block = (
|
|
978
|
+
f"\n\n[쿼리 플랜]\n{explain_plan[:3000]}"
|
|
979
|
+
if explain_plan
|
|
980
|
+
else ""
|
|
981
|
+
)
|
|
982
|
+
_origin_block = (
|
|
983
|
+
f"\n\n[원본 SQL]\n{existing_sql}" if existing_sql else ""
|
|
984
|
+
)
|
|
985
|
+
_desc_block = (
|
|
986
|
+
f"\n\n[변경 설명]\n{description}"
|
|
987
|
+
if existing_sql and description
|
|
988
|
+
else ""
|
|
989
|
+
)
|
|
990
|
+
tune_prompt = (
|
|
991
|
+
f"[SQL]\n{current_sql}{_plan_block}{_origin_block}{_desc_block}"
|
|
992
|
+
)
|
|
993
|
+
tune_resp = await tune_llm.generate_response(
|
|
994
|
+
tune_prompt, system_prompt=tune_system
|
|
995
|
+
)
|
|
996
|
+
tune_resp = tune_resp.strip()
|
|
997
|
+
if tune_resp.startswith("```"):
|
|
998
|
+
tune_resp = tune_resp.split("```")[1]
|
|
999
|
+
if tune_resp.startswith("json"):
|
|
1000
|
+
tune_resp = tune_resp[4:]
|
|
1001
|
+
|
|
1002
|
+
# JSON 파싱
|
|
1003
|
+
import re as _re
|
|
1004
|
+
|
|
1005
|
+
_tm = _re.search(
|
|
1006
|
+
r"\{.*(?:\"sql\"|\"title\")\s*:", tune_resp, _re.DOTALL
|
|
1007
|
+
)
|
|
1008
|
+
if _tm:
|
|
1009
|
+
_ts = _tm.start()
|
|
1010
|
+
_depth = 0
|
|
1011
|
+
_te = _ts
|
|
1012
|
+
for _ci, _ch in enumerate(tune_resp[_ts:], _ts):
|
|
1013
|
+
if _ch == "{":
|
|
1014
|
+
_depth += 1
|
|
1015
|
+
elif _ch == "}":
|
|
1016
|
+
_depth -= 1
|
|
1017
|
+
if _depth == 0:
|
|
1018
|
+
_te = _ci + 1
|
|
1019
|
+
break
|
|
1020
|
+
tune_parsed = json.loads(tune_resp[_ts:_te])
|
|
1021
|
+
tuned_sql = tune_parsed.get("sql", "")
|
|
1022
|
+
tuning_description = tune_parsed.get(
|
|
1023
|
+
"tuning_info", ""
|
|
1024
|
+
) or tune_parsed.get("description", "")
|
|
1025
|
+
|
|
1026
|
+
if tuned_sql and tuned_sql.strip() != current_sql.strip():
|
|
1027
|
+
plan_tuned = True
|
|
1028
|
+
# Count REASON comments
|
|
1029
|
+
reason_count = len(
|
|
1030
|
+
_re.findall(
|
|
1031
|
+
r"^\s*--\s*\[REASON\]",
|
|
1032
|
+
tuned_sql,
|
|
1033
|
+
_re.MULTILINE | _re.IGNORECASE,
|
|
1034
|
+
)
|
|
1035
|
+
)
|
|
1036
|
+
|
|
1037
|
+
# Phase 4: 튜닝 후 EXPLAIN 재검증
|
|
1038
|
+
tune_explain = await _run_explain(tuned_sql)
|
|
1039
|
+
tune_explain_success = tune_explain.get("success", False)
|
|
1040
|
+
if tune_explain_success:
|
|
1041
|
+
current_sql = tuned_sql
|
|
1042
|
+
result["final_sql"] = current_sql
|
|
1043
|
+
|
|
1044
|
+
except Exception as e:
|
|
1045
|
+
logger.warning(f"Plan tuning failed: {e}")
|
|
1046
|
+
|
|
1047
|
+
result["plan_tuned"] = plan_tuned
|
|
1048
|
+
result["tuned_sql"] = tuned_sql if plan_tuned else ""
|
|
1049
|
+
result["tuning_description"] = tuning_description
|
|
1050
|
+
result["tune_explain_success"] = tune_explain_success
|
|
1051
|
+
result["reason_count"] = reason_count
|
|
1052
|
+
|
|
1053
|
+
else:
|
|
1054
|
+
result["final_sql"] = ""
|
|
1055
|
+
result["explain_success"] = False
|
|
1056
|
+
result["explain_error"] = "SQL 생성 실패"
|
|
1057
|
+
result["fix_count"] = 0
|
|
1058
|
+
result["plan_tuned"] = False
|
|
1059
|
+
result["tuned_sql"] = ""
|
|
1060
|
+
result["tuning_description"] = ""
|
|
1061
|
+
result["tune_explain_success"] = False
|
|
1062
|
+
result["reason_count"] = 0
|
|
1063
|
+
|
|
1064
|
+
elapsed = round(time.time() - start_time, 1)
|
|
1065
|
+
|
|
1066
|
+
# ── 평가 ──
|
|
1067
|
+
scores: Dict[str, Any] = {
|
|
1068
|
+
"response_time": elapsed,
|
|
1069
|
+
"explain_first_try": explain_success and fix_count == 0,
|
|
1070
|
+
"explain_final": explain_success,
|
|
1071
|
+
"fix_count": fix_count,
|
|
1072
|
+
"path_used": path_used,
|
|
1073
|
+
"plan_tuned": result.get("plan_tuned", False),
|
|
1074
|
+
"tune_explain_success": result.get("tune_explain_success", False),
|
|
1075
|
+
"reason_count": result.get("reason_count", 0),
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
# 컬럼 정확성 (기본 문자열 매칭)
|
|
1079
|
+
|
|
1080
|
+
# 컬럼 정확성
|
|
1081
|
+
expected_cols = tc.get("expected_columns", [])
|
|
1082
|
+
if expected_cols and sql:
|
|
1083
|
+
sql_lower = sql.lower()
|
|
1084
|
+
col_hits = sum(1 for c in expected_cols if c.lower() in sql_lower)
|
|
1085
|
+
scores["column_accuracy"] = round(col_hits / len(expected_cols) * 100, 1)
|
|
1086
|
+
|
|
1087
|
+
# LLM 평가 (title/description/sql 각각)
|
|
1088
|
+
if sql or title or description:
|
|
1089
|
+
try:
|
|
1090
|
+
from codebuff_core.llm.service import LLMService
|
|
1091
|
+
|
|
1092
|
+
eval_llm = LLMService({"provider": "vllm", "vllm": cfg.get("vllm", {})})
|
|
1093
|
+
expected_tables_str = (
|
|
1094
|
+
", ".join(expected_tables) if expected_tables else "(없음)"
|
|
1095
|
+
)
|
|
1096
|
+
eval_prompt = (
|
|
1097
|
+
"아래 EDA Agent의 응답을 평가하세요.\n\n"
|
|
1098
|
+
f"[사용자 질문]\n{question}\n\n"
|
|
1099
|
+
f"[기대 테이블]\n{expected_tables_str}\n\n"
|
|
1100
|
+
f"[Title]\n{title or '(없음)'}\n\n"
|
|
1101
|
+
f"[Description]\n{description or '(없음)'}\n\n"
|
|
1102
|
+
f"[SQL]\n{sql or '(없음)'}\n\n"
|
|
1103
|
+
"평가 항목 (각 1~10점):\n"
|
|
1104
|
+
"- title_quality: 제목이 요청의 핵심을 한 줄로 잘 요약했는지\n"
|
|
1105
|
+
"- description_quality: 설명이 SQL의 목적·로직·사용 테이블을 "
|
|
1106
|
+
"명확히 설명하는지\n"
|
|
1107
|
+
"- semantic_accuracy: SQL이 질문의 의도를 정확히 반영하는지\n"
|
|
1108
|
+
"- sql_quality: SQL 문법·구조·효율성이 적절한지\n\n"
|
|
1109
|
+
"테이블 사용 분석:\n"
|
|
1110
|
+
"- SQL에서 FROM/JOIN에 사용된 테이블명을 모두 추출하세요.\n"
|
|
1111
|
+
"- [기대 테이블]과 비교하여 올바른 테이블을 사용했는지 확인하세요.\n"
|
|
1112
|
+
"- used_tables: SQL에서 실제 사용한 테이블명 배열\n\n"
|
|
1113
|
+
"파티션 컬럼 사용 여부:\n"
|
|
1114
|
+
"- SQL에서 사용하는 테이블에 날짜/시간 기반 파티션 컬럼이 "
|
|
1115
|
+
"있을 가능성이 높습니다.\n"
|
|
1116
|
+
"- WHERE절에 파티션 컬럼(날짜, dt, date, yyyymmdd 등)을 "
|
|
1117
|
+
"사용하고 있는지 확인하세요.\n"
|
|
1118
|
+
"- partition_missing: 파티션 컬럼이 있을 가능성이 높은데 "
|
|
1119
|
+
"WHERE절에서 사용하지 않았으면 true, 그 외 false\n\n"
|
|
1120
|
+
"반드시 아래 JSON만 출력:\n"
|
|
1121
|
+
'{"title_quality": N, "description_quality": N, '
|
|
1122
|
+
'"semantic_accuracy": N, "sql_quality": N, '
|
|
1123
|
+
'"used_tables": ["테이블명1", "테이블명2"], '
|
|
1124
|
+
'"partition_missing": true/false, '
|
|
1125
|
+
'"comment": "종합 평가 의견"}'
|
|
1126
|
+
)
|
|
1127
|
+
eval_resp = await eval_llm.generate_response(eval_prompt)
|
|
1128
|
+
eval_resp = eval_resp.strip()
|
|
1129
|
+
if eval_resp.startswith("```"):
|
|
1130
|
+
eval_resp = eval_resp.split("```")[1]
|
|
1131
|
+
if eval_resp.startswith("json"):
|
|
1132
|
+
eval_resp = eval_resp[4:]
|
|
1133
|
+
import re as _re
|
|
1134
|
+
|
|
1135
|
+
# JSON 추출 (배열 포함 가능)
|
|
1136
|
+
json_match = _re.search(r"\{[^{}]*(?:\[[^\[\]]*\])?[^{}]*\}", eval_resp)
|
|
1137
|
+
if json_match:
|
|
1138
|
+
eval_resp = json_match.group(0)
|
|
1139
|
+
eval_parsed = json.loads(eval_resp)
|
|
1140
|
+
scores["title_quality"] = eval_parsed.get("title_quality", 5)
|
|
1141
|
+
scores["description_quality"] = eval_parsed.get(
|
|
1142
|
+
"description_quality", 5
|
|
1143
|
+
)
|
|
1144
|
+
scores["semantic_accuracy"] = eval_parsed.get("semantic_accuracy", 5)
|
|
1145
|
+
scores["sql_quality"] = eval_parsed.get("sql_quality", 5)
|
|
1146
|
+
scores["partition_missing"] = eval_parsed.get(
|
|
1147
|
+
"partition_missing", False
|
|
1148
|
+
)
|
|
1149
|
+
scores["eval_comment"] = eval_parsed.get("comment", "")
|
|
1150
|
+
|
|
1151
|
+
# LLM이 추출한 테이블로 F1 계산
|
|
1152
|
+
used_tables = eval_parsed.get("used_tables", [])
|
|
1153
|
+
if used_tables and expected_tables:
|
|
1154
|
+
used_set = set(t.lower() for t in used_tables)
|
|
1155
|
+
exp_set = set(t.lower() for t in expected_tables)
|
|
1156
|
+
if used_set:
|
|
1157
|
+
precision = len(used_set & exp_set) / len(used_set)
|
|
1158
|
+
recall = len(used_set & exp_set) / len(exp_set)
|
|
1159
|
+
f1 = (
|
|
1160
|
+
2 * precision * recall / (precision + recall)
|
|
1161
|
+
if (precision + recall) > 0
|
|
1162
|
+
else 0
|
|
1163
|
+
)
|
|
1164
|
+
scores["table_precision"] = round(precision * 100, 1)
|
|
1165
|
+
scores["table_recall"] = round(recall * 100, 1)
|
|
1166
|
+
scores["table_f1"] = round(f1 * 100, 1)
|
|
1167
|
+
scores["sql_tables"] = sorted(used_set)
|
|
1168
|
+
except Exception as e:
|
|
1169
|
+
logger.warning(f"EDA eval failed: {e}")
|
|
1170
|
+
scores["title_quality"] = 5
|
|
1171
|
+
scores["description_quality"] = 5
|
|
1172
|
+
scores["semantic_accuracy"] = 5
|
|
1173
|
+
scores["sql_quality"] = 5
|
|
1174
|
+
scores["partition_missing"] = False
|
|
1175
|
+
scores["eval_comment"] = f"평가 실패: {e}"
|
|
1176
|
+
|
|
1177
|
+
# 종합 점수
|
|
1178
|
+
overall_parts = []
|
|
1179
|
+
if scores.get("title_quality"):
|
|
1180
|
+
overall_parts.append(scores["title_quality"])
|
|
1181
|
+
if scores.get("description_quality"):
|
|
1182
|
+
overall_parts.append(scores["description_quality"])
|
|
1183
|
+
if scores.get("semantic_accuracy"):
|
|
1184
|
+
overall_parts.append(scores["semantic_accuracy"])
|
|
1185
|
+
if scores.get("sql_quality"):
|
|
1186
|
+
overall_parts.append(scores["sql_quality"])
|
|
1187
|
+
# EXPLAIN 보너스/페널티
|
|
1188
|
+
if explain_success:
|
|
1189
|
+
explain_bonus = {0: 15, 1: 5, 2: 0}.get(fix_count, -5)
|
|
1190
|
+
else:
|
|
1191
|
+
explain_bonus = -15
|
|
1192
|
+
scores["explain_bonus"] = explain_bonus
|
|
1193
|
+
|
|
1194
|
+
# 파티션 미사용 페널티
|
|
1195
|
+
partition_penalty = -10 if scores.get("partition_missing") else 0
|
|
1196
|
+
scores["partition_penalty"] = partition_penalty
|
|
1197
|
+
|
|
1198
|
+
# 쿼리플랜 튜닝 보너스
|
|
1199
|
+
tune_bonus = 0
|
|
1200
|
+
if result.get("plan_tuned"):
|
|
1201
|
+
if result.get("tune_explain_success"):
|
|
1202
|
+
tune_bonus = 5 # 튜닝 적용 + EXPLAIN 성공
|
|
1203
|
+
else:
|
|
1204
|
+
tune_bonus = -5 # 튜닝 적용했지만 EXPLAIN 실패
|
|
1205
|
+
scores["tune_bonus"] = tune_bonus
|
|
1206
|
+
|
|
1207
|
+
# REASON 주석 보너스 (existing_sql이 있는 tune 케이스에서만)
|
|
1208
|
+
reason_bonus = 0
|
|
1209
|
+
rc = scores.get("reason_count", 0)
|
|
1210
|
+
if existing_sql and rc > 0:
|
|
1211
|
+
reason_bonus = min(rc, 5) # 최대 +5
|
|
1212
|
+
scores["reason_bonus"] = reason_bonus
|
|
1213
|
+
|
|
1214
|
+
scores["overall"] = (
|
|
1215
|
+
round(
|
|
1216
|
+
sum(overall_parts) / len(overall_parts) * 10
|
|
1217
|
+
+ explain_bonus
|
|
1218
|
+
+ partition_penalty
|
|
1219
|
+
+ tune_bonus
|
|
1220
|
+
+ reason_bonus,
|
|
1221
|
+
1,
|
|
1222
|
+
)
|
|
1223
|
+
if overall_parts
|
|
1224
|
+
else 0
|
|
1225
|
+
)
|
|
1226
|
+
|
|
1227
|
+
result["scores"] = scores
|
|
1228
|
+
set_eda_only_bdp(False)
|
|
1229
|
+
return result
|
|
1230
|
+
|
|
1231
|
+
def _calc_summary(self, results: List[Dict]) -> Dict[str, Any]:
|
|
1232
|
+
"""결과 요약."""
|
|
1233
|
+
if not results:
|
|
1234
|
+
return {}
|
|
1235
|
+
|
|
1236
|
+
valid = [r for r in results if r.get("scores")]
|
|
1237
|
+
|
|
1238
|
+
# 유형별 분류
|
|
1239
|
+
gen_a = [
|
|
1240
|
+
r
|
|
1241
|
+
for r in valid
|
|
1242
|
+
if r.get("type") == "generate" and "A" in r.get("path_used", "")
|
|
1243
|
+
]
|
|
1244
|
+
gen_b = [
|
|
1245
|
+
r
|
|
1246
|
+
for r in valid
|
|
1247
|
+
if r.get("type") == "generate" and r.get("path_used") == "B"
|
|
1248
|
+
]
|
|
1249
|
+
tune = [r for r in valid if r.get("type") == "tune"]
|
|
1250
|
+
|
|
1251
|
+
def _group_summary(group, label):
|
|
1252
|
+
if not group:
|
|
1253
|
+
return {}
|
|
1254
|
+
success = sum(1 for r in group if r["scores"].get("explain_final"))
|
|
1255
|
+
avg_fix = sum(r["scores"].get("fix_count", 0) for r in group) / len(group)
|
|
1256
|
+
avg_time = sum(r["scores"].get("response_time", 0) for r in group) / len(
|
|
1257
|
+
group
|
|
1258
|
+
)
|
|
1259
|
+
avg_overall = sum(r["scores"].get("overall", 0) for r in group) / len(group)
|
|
1260
|
+
s = {
|
|
1261
|
+
"label": label,
|
|
1262
|
+
"count": len(group),
|
|
1263
|
+
"success_rate": round(success / len(group) * 100, 1),
|
|
1264
|
+
"avg_fix_count": round(avg_fix, 2),
|
|
1265
|
+
"avg_response_time": round(avg_time, 1),
|
|
1266
|
+
"avg_overall": round(avg_overall, 1),
|
|
1267
|
+
}
|
|
1268
|
+
# 테이블 F1 (generate만)
|
|
1269
|
+
f1_vals = [
|
|
1270
|
+
r["scores"]["table_f1"] for r in group if "table_f1" in r["scores"]
|
|
1271
|
+
]
|
|
1272
|
+
if f1_vals:
|
|
1273
|
+
s["avg_table_f1"] = round(sum(f1_vals) / len(f1_vals), 1)
|
|
1274
|
+
return s
|
|
1275
|
+
|
|
1276
|
+
return {
|
|
1277
|
+
"total_cases": len(valid),
|
|
1278
|
+
"generate_path_a": _group_summary(gen_a, "쿼리 생성 (RAG)"),
|
|
1279
|
+
"generate_path_b": _group_summary(gen_b, "쿼리 생성 (직접)"),
|
|
1280
|
+
"tune": _group_summary(tune, "SQL 튜닝"),
|
|
1281
|
+
"overall_success_rate": round(
|
|
1282
|
+
sum(1 for r in valid if r["scores"].get("explain_final"))
|
|
1283
|
+
/ len(valid)
|
|
1284
|
+
* 100,
|
|
1285
|
+
1,
|
|
1286
|
+
)
|
|
1287
|
+
if valid
|
|
1288
|
+
else 0,
|
|
1289
|
+
"overall_avg_fix": round(
|
|
1290
|
+
sum(r["scores"].get("fix_count", 0) for r in valid) / len(valid), 2
|
|
1291
|
+
)
|
|
1292
|
+
if valid
|
|
1293
|
+
else 0,
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
|
|
1297
|
+
# ─── Results ───
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
class EdaBenchmarkResultsHandler(APIHandler):
|
|
1301
|
+
"""GET/POST /codebuff/eda-benchmark/results"""
|
|
1302
|
+
|
|
1303
|
+
async def get(self):
|
|
1304
|
+
result_id = self.get_argument("id", None)
|
|
1305
|
+
if result_id:
|
|
1306
|
+
detail = _load_result_detail(result_id)
|
|
1307
|
+
if detail:
|
|
1308
|
+
self.write(json.dumps(detail, ensure_ascii=False, default=str))
|
|
1309
|
+
else:
|
|
1310
|
+
self.set_status(404)
|
|
1311
|
+
self.write({"error": "결과를 찾을 수 없습니다."})
|
|
1312
|
+
else:
|
|
1313
|
+
self.write(json.dumps(_load_results(), ensure_ascii=False, default=str))
|
|
1314
|
+
|
|
1315
|
+
async def post(self):
|
|
1316
|
+
body = (
|
|
1317
|
+
json.loads(self.request.body.decode("utf-8")) if self.request.body else {}
|
|
1318
|
+
)
|
|
1319
|
+
action = body.get("action")
|
|
1320
|
+
if action == "delete":
|
|
1321
|
+
ids = body.get("ids", [])
|
|
1322
|
+
deleted = 0
|
|
1323
|
+
for rid in ids:
|
|
1324
|
+
path = _RESULTS_DIR / f"{rid}.json"
|
|
1325
|
+
if path.exists():
|
|
1326
|
+
path.unlink()
|
|
1327
|
+
deleted += 1
|
|
1328
|
+
self.write({"status": "ok", "deleted": deleted})
|