sqlas 1.1.0__py3-none-any.whl

sqlas/correctness.py

@@ -0,0 +1,289 @@
+"""
+Core SQL Correctness Metrics.
+- Execution Accuracy (output + structure + efficiency)
+- Syntax Validity (sqlglot parse)
+- Semantic Equivalence (LLM-as-Judge)
+- Result Set Similarity (Jaccard on normalized result sets)
+
+Author: SQLAS Contributors
+"""
+
+import logging
+import time
+import sqlite3
+
+import sqlglot
+
+from sqlas.core import LLMJudge, _parse_score
+
+logger = logging.getLogger(__name__)
+
+# Default SQL execution timeout in seconds
+_DEFAULT_TIMEOUT_S = 30
+_PROGRESS_INTERVAL = 1_000_000  # check timeout every N SQLite VM instructions
+
+
+def _connect_readonly(db_path: str, timeout_s: int = _DEFAULT_TIMEOUT_S) -> sqlite3.Connection:
+    """Open a read-only SQLite connection with a timeout guard."""
+    conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=timeout_s)
+    _start = time.monotonic()
+
+    def _progress_handler():
+        if time.monotonic() - _start > timeout_s:
+            return 1  # non-zero → abort query
+        return 0
+
+    conn.set_progress_handler(_progress_handler, _PROGRESS_INTERVAL)
+    return conn
+
+
+# ── Helpers ─────────────────────────────────────────────────────────────────
+
+def _extract_row_numbers(row) -> list[float]:
+    return sorted([round(float(v), 2) for v in row if isinstance(v, (int, float)) and v is not None])
+
+
+def _values_found_in(needle: list[float], haystack: list[float], tol: float = 0.5) -> float:
+    if not needle:
+        return 1.0
+    remaining = list(haystack)
+    matched = 0
+    for nv in needle:
+        best_idx, best_diff = -1, float("inf")
+        for i, hv in enumerate(remaining):
+            diff = abs(nv - hv)
+            if diff < best_diff:
+                best_diff, best_idx = diff, i
+        if best_idx >= 0 and best_diff <= tol:
+            remaining.pop(best_idx)
+            matched += 1
+    return matched / len(needle)
+
+
+def _row_values_match(pred_nums: list[float], gold_nums: list[float], tol: float = 0.5) -> float:
+    if not gold_nums and not pred_nums:
+        return 1.0
+    if not gold_nums:
+        return 0.8
+    if not pred_nums:
+        return 0.0
+    if len(pred_nums) < len(gold_nums):
+        subset_score = _values_found_in(pred_nums, gold_nums, tol)
+        return 1.0 if subset_score >= 0.99 else subset_score
+    return _values_found_in(gold_nums, pred_nums, tol)
+
+
+def _match_result_sets(pred_rows: list, gold_rows: list) -> float:
+    if not gold_rows:
+        return 1.0 if not pred_rows else 0.5
+    pred_nums_list = [_extract_row_numbers(r) for r in pred_rows]
+    gold_nums_list = [_extract_row_numbers(r) for r in gold_rows]
+    used_pred = set()
+    total_score = 0.0
+    for gn in gold_nums_list:
+        best_score, best_pi = 0.0, -1
+        for pi, pn in enumerate(pred_nums_list):
+            if pi in used_pred:
+                continue
+            score = _row_values_match(pn, gn)
+            if score > best_score:
+                best_score, best_pi = score, pi
+        if best_pi >= 0:
+            used_pred.add(best_pi)
+        total_score += best_score
+    return total_score / len(gold_rows)
+
+
+# ── Public API ──────────────────────────────────────────────────────────────
+
+def execution_accuracy(generated_sql: str, gold_sql: str, db_path: str) -> tuple[float, dict]:
+    """
+    Semantic execution accuracy.
+
+    Formula: 60% Output Match + 20% Structure Match + 20% Efficiency
+
+    Output Match:  Row-by-row numeric comparison. Ignores label differences
+                   (0 vs 'Male'), tolerates ROUND, handles extra columns.
+    Structure:     Same row count.
+    Efficiency:    Generated query speed vs gold query speed.
+
+    Args:
+        generated_sql: SQL produced by the agent
+        gold_sql: Ground-truth SQL
+        db_path: Path to SQLite database (or any sqlite3-compatible path)
+
+    Returns:
+        (score, details) where score is 0.0–1.0
+    """
+    try:
+        conn = _connect_readonly(db_path)
+    except Exception as e:
+        return 0.0, {"error": f"db_connect_failed: {e}"}
+    try:
+        start = time.perf_counter()
+        gold_result = conn.execute(gold_sql).fetchall()
+        gold_time = max((time.perf_counter() - start) * 1000, 0.01)
+
+        start = time.perf_counter()
+        pred_result = conn.execute(generated_sql).fetchall()
+        pred_time = max((time.perf_counter() - start) * 1000, 0.01)
+    except Exception as e:
+        return 0.0, {"error": str(e)}
+    finally:
+        conn.close()
+
+    output_score = _match_result_sets(pred_result, gold_result)
+
+    struct_score = 0.0
+    if len(pred_result) == len(gold_result):
+        struct_score = 1.0
+    elif pred_result and gold_result:
+        struct_score = min(len(pred_result), len(gold_result)) / max(len(pred_result), len(gold_result))
+
+    time_ratio = gold_time / pred_time if pred_time > 0 else 1.0
+    efficiency = min(time_ratio, 1.0)
+
+    final = round(0.6 * output_score + 0.2 * struct_score + 0.2 * efficiency, 4)
+
+    return final, {
+        "output_score": round(output_score, 4),
+        "structural_score": round(struct_score, 4),
+        "efficiency_score": round(efficiency, 4),
+        "predicted_rows": len(pred_result),
+        "gold_rows": len(gold_result),
+    }
+
+
+def syntax_valid(sql: str, dialect: str = "sqlite") -> float:
+    """Check if SQL parses without errors. Returns 1.0 or 0.0."""
+    try:
+        results = sqlglot.parse(sql, dialect=dialect)
+        return 1.0 if results and results[0] is not None else 0.0
+    except Exception:
+        return 0.0
+
+
+def semantic_equivalence(
+    question: str,
+    generated_sql: str,
+    llm_judge: LLMJudge,
+    gold_sql: str | None = None,
+) -> tuple[float, dict]:
+    """
+    LLM judges whether the SQL correctly answers the user's question.
+    Handles alias differences, join variations, CASE WHEN labels.
+
+    Args:
+        question: User's natural language question
+        generated_sql: SQL produced by the agent
+        llm_judge: Function (prompt: str) -> str
+        gold_sql: Optional reference SQL for comparison
+
+    Returns:
+        (score, details) where score is 0.0–1.0
+    """
+    gold_section = f"\n**Reference SQL:**\n```sql\n{gold_sql}\n```" if gold_sql else ""
+
+    prompt = f"""You are a SQL expert judge. Evaluate if the Generated SQL correctly answers the User Question.
+{gold_section}
+
+**User Question:** {question}
+
+**Generated SQL:**
+```sql
+{generated_sql}
+```
+
+Evaluate:
+1. Does the SQL retrieve the correct data to answer the question?
+2. Are the right tables, columns, and filters used?
+3. Are aggregations applied correctly?
+4. Are JOINs correct and necessary?
+
+Score 0.0 to 1.0:
+- 1.0: Perfectly answers the question
+- 0.7-0.9: Minor issues not affecting the core answer
+- 0.4-0.6: Partially correct, missing key elements
+- 0.0-0.3: Wrong approach or major errors
+
+Respond EXACTLY:
+Semantic_Score: [score]
+Reasoning: [one sentence]"""
+
+    try:
+        result = llm_judge(prompt)
+    except Exception as e:
+        logger.warning("LLM judge failed in semantic_equivalence: %s", e)
+        return 0.0, {"error": str(e)}
+
+    score, reasoning = _parse_score(result, "Semantic_Score")
+    return score, {"reasoning": reasoning}
+
+
+def result_set_similarity(
+    generated_sql: str,
+    gold_sql: str,
+    db_path: str,
+) -> tuple[float, dict]:
+    """
+    RAGAS Answer Similarity for SQL agents.
+
+    Computes Jaccard similarity on normalized result sets between
+    generated and gold SQL execution outputs.
+
+    Args:
+        generated_sql: SQL produced by the agent
+        gold_sql: Ground-truth SQL
+        db_path: Path to SQLite database
+
+    Returns:
+        (similarity score 0.0–1.0, details dict)
+    """
+    try:
+        conn = _connect_readonly(db_path)
+    except Exception as e:
+        return 0.0, {"error": f"db_connect_failed: {e}"}
+    try:
+        gold_rows = conn.execute(gold_sql).fetchall()
+        gold_desc = conn.execute(gold_sql).description
+        pred_rows = conn.execute(generated_sql).fetchall()
+        pred_desc = conn.execute(generated_sql).description
+    except Exception as e:
+        return 0.0, {"error": str(e)}
+    finally:
+        conn.close()
+
+    def _normalize_row(row):
+        cells = []
+        for v in row:
+            if isinstance(v, float):
+                cells.append(round(v, 2))
+            elif isinstance(v, str):
+                cells.append(v.strip().lower())
+            else:
+                cells.append(v)
+        return tuple(cells)
+
+    gold_set = {_normalize_row(r) for r in gold_rows}
+    pred_set = {_normalize_row(r) for r in pred_rows}
+
+    union = gold_set | pred_set
+    intersection = gold_set & pred_set
+
+    jaccard = len(intersection) / len(union) if union else 1.0
+
+    # Column count match
+    gold_cols = len(gold_desc) if gold_desc else 0
+    pred_cols = len(pred_desc) if pred_desc else 0
+    col_match = 1.0 if gold_cols == pred_cols else min(gold_cols, pred_cols) / max(gold_cols, pred_cols) if max(gold_cols, pred_cols) > 0 else 1.0
+
+    score = round(0.8 * jaccard + 0.2 * col_match, 4)
+
+    return score, {
+        "jaccard": round(jaccard, 4),
+        "column_match": round(col_match, 4),
+        "generated_row_count": len(pred_rows),
+        "gold_row_count": len(gold_rows),
+        "intersection_size": len(intersection),
+        "union_size": len(union),
+    }

sqlas/evaluate.py

@@ -0,0 +1,218 @@
+"""
+Main evaluation API — single query and batch evaluation.
+
+Author: SQLAS Contributors
+"""
+
+import logging
+import os
+
+from sqlas.core import SQLASScores, TestCase, LLMJudge, WEIGHTS, compute_composite_score
+from sqlas.correctness import execution_accuracy, syntax_valid, semantic_equivalence, result_set_similarity
+from sqlas.quality import sql_quality, schema_compliance, complexity_match
+from sqlas.production import data_scan_efficiency, execution_result
+from sqlas.response import faithfulness, answer_relevance, answer_completeness, fluency
+from sqlas.safety import safety_score, read_only_compliance
+from sqlas.context import context_precision, context_recall, entity_recall, noise_robustness
+
+logger = logging.getLogger(__name__)
+
+
+def evaluate(
+    question: str,
+    generated_sql: str,
+    llm_judge: LLMJudge,
+    gold_sql: str | None = None,
+    db_path: str | None = None,
+    response: str | None = None,
+    result_data: dict | None = None,
+    valid_tables: set[str] | None = None,
+    valid_columns: dict[str, set[str]] | None = None,
+    schema_context: str = "",
+    expected_nonempty: bool = True,
+    pii_columns: list[str] | None = None,
+    weights: dict | None = None,
+) -> SQLASScores:
+    """
+    Evaluate a single SQL agent query across all SQLAS metrics.
+
+    Args:
+        question:        User's natural language question
+        generated_sql:   SQL produced by the agent
+        llm_judge:       Function (prompt: str) -> str for LLM-as-judge metrics
+        gold_sql:        Ground-truth SQL (optional, enables execution accuracy & context metrics)
+        db_path:         Path to SQLite database (required for execution accuracy)
+        response:        Agent's natural language response (optional, enables faithfulness/relevance)
+        result_data:     Query result dict: {columns, rows, row_count, execution_time_ms}
+        valid_tables:    Set of valid table names (enables schema compliance)
+        valid_columns:   Dict of {table: {col1, col2}} (enables schema compliance)
+        schema_context:  Brief schema text for SQL quality judge
+        expected_nonempty: Whether non-empty result is expected
+        pii_columns:     Custom PII column names for safety check
+        weights:         Custom weight dict (defaults to SQLAS production weights)
+
+    Returns:
+        SQLASScores with all metrics and overall_score
+    """
+    # ── Input validation ────────────────────────────────────────────────
+    if not generated_sql or not isinstance(generated_sql, str):
+        logger.error("generated_sql must be a non-empty string")
+        scores = SQLASScores()
+        scores.details["error"] = "generated_sql is empty or invalid"
+        return scores
+
+    if db_path and not os.path.exists(db_path):
+        logger.error("db_path does not exist: %s", db_path)
+        scores = SQLASScores()
+        scores.details["error"] = f"db_path not found: {db_path}"
+        return scores
+
+    if weights:
+        weight_sum = sum(weights.values())
+        if abs(weight_sum - 1.0) > 0.01:
+            logger.warning("Custom weights sum to %.4f (expected ~1.0)", weight_sum)
+
+    scores = SQLASScores()
+
+    # ── 1. Core Correctness ─────────────────────────────────────────────
+    scores.syntax_valid = syntax_valid(generated_sql)
+
+    if gold_sql and db_path:
+        ex_acc, ex_details = execution_accuracy(generated_sql, gold_sql, db_path)
+        scores.execution_accuracy = ex_acc
+        scores.details["execution_accuracy"] = ex_details
+
+        # Efficiency (VES) — reuse timing from execution_accuracy
+        scores.efficiency_score = ex_details.get("efficiency_score", 0.0)
+    else:
+        scores.execution_accuracy = 1.0 if result_data else 0.0
+        scores.efficiency_score = 1.0 if result_data else 0.0
+
+    sem, sem_details = semantic_equivalence(question, generated_sql, llm_judge, gold_sql)
+    scores.semantic_equivalence = sem
+    scores.details["semantic_equivalence"] = sem_details
+
+    # ── 2. Context Quality (RAGAS-mapped) ───────────────────────────────
+    if gold_sql:
+        cp, cp_details = context_precision(generated_sql, gold_sql)
+        scores.context_precision = cp
+        scores.details["context_precision"] = cp_details
+
+        cr, cr_details = context_recall(generated_sql, gold_sql)
+        scores.context_recall = cr
+        scores.details["context_recall"] = cr_details
+
+        er, er_details = entity_recall(generated_sql, gold_sql)
+        scores.entity_recall = er
+        scores.details["entity_recall"] = er_details
+
+        nr, nr_details = noise_robustness(generated_sql, gold_sql, valid_tables, valid_columns)
+        scores.noise_robustness = nr
+        scores.details["noise_robustness"] = nr_details
+
+        if db_path:
+            rs, rs_details = result_set_similarity(generated_sql, gold_sql, db_path)
+            scores.result_set_similarity = rs
+            scores.details["result_set_similarity"] = rs_details
+
+    # ── 3. SQL Quality ──────────────────────────────────────────────────
+    if valid_tables and valid_columns:
+        sc, sc_details = schema_compliance(generated_sql, valid_tables, valid_columns)
+        scores.schema_compliance = sc
+        scores.details["schema_compliance"] = sc_details
+    else:
+        scores.schema_compliance = 1.0  # can't check without schema
+
+    sq, sq_details = sql_quality(question, generated_sql, llm_judge, schema_context)
+    scores.sql_quality = sq
+    scores.details["sql_quality"] = sq_details
+
+    cm, cm_details = complexity_match(question, generated_sql, llm_judge)
+    scores.complexity_match = cm
+    scores.details["complexity_match"] = cm_details
+
+    # ── 4. Production Execution ─────────────────────────────────────────
+    exec_eval = execution_result(result_data, expected_nonempty)
+    scores.execution_success = exec_eval["execution_success"]
+    scores.execution_time_ms = exec_eval["execution_time_ms"]
+    scores.result_row_count = exec_eval["result_row_count"]
+    scores.empty_result_penalty = exec_eval["empty_result_penalty"]
+    scores.row_explosion_detected = exec_eval["row_explosion_detected"]
+
+    scan, scan_details = data_scan_efficiency(generated_sql, scores.result_row_count)
+    scores.data_scan_efficiency = scan
+    scores.details["data_scan"] = scan_details
+
+    # ── 5. Response Quality ─────────────────────────────────────────────
+    if response and result_data:
+        result_preview = f"Columns: {result_data.get('columns', [])}\n"
+        for row in result_data.get("rows", [])[:15]:
+            result_preview += f"{row}\n"
+
+        f_score, f_details = faithfulness(question, response, result_preview, llm_judge)
+        scores.faithfulness = f_score
+        scores.details["faithfulness"] = f_details
+
+        r_score, r_details = answer_relevance(question, response, llm_judge)
+        scores.answer_relevance = r_score
+        scores.details["answer_relevance"] = r_details
+
+        c_score, c_details = answer_completeness(question, response, result_preview, llm_judge)
+        scores.answer_completeness = c_score
+        scores.details["answer_completeness"] = c_details
+
+        fl_score, fl_details = fluency(response, llm_judge)
+        scores.fluency = fl_score
+        scores.details["fluency"] = fl_details
+
+    # ── 6. Safety ───────────────────────────────────────────────────────
+    scores.read_only_compliance = read_only_compliance(generated_sql)
+
+    safety, safety_details = safety_score(generated_sql, response or "", pii_columns)
+    scores.safety_score = safety
+    scores.details["safety"] = safety_details
+
+    # ── Composite ───────────────────────────────────────────────────────
+    scores.overall_score = compute_composite_score(scores, weights)
+
+    return scores
+
+
+def evaluate_batch(
+    test_cases: list[dict],
+    llm_judge: LLMJudge,
+    db_path: str | None = None,
+    valid_tables: set[str] | None = None,
+    valid_columns: dict[str, set[str]] | None = None,
+    schema_context: str = "",
+    pii_columns: list[str] | None = None,
+    weights: dict | None = None,
+) -> list[SQLASScores]:
+    """
+    Evaluate a batch of test cases.
+
+    Each dict in test_cases should have:
+        question, generated_sql, and optionally:
+        gold_sql, response, result_data, expected_nonempty
+
+    Returns list of SQLASScores.
+    """
+    results = []
+    for tc in test_cases:
+        scores = evaluate(
+            question=tc["question"],
+            generated_sql=tc["generated_sql"],
+            llm_judge=llm_judge,
+            gold_sql=tc.get("gold_sql"),
+            db_path=db_path,
+            response=tc.get("response"),
+            result_data=tc.get("result_data"),
+            valid_tables=valid_tables,
+            valid_columns=valid_columns,
+            schema_context=tc.get("schema_context", schema_context),
+            expected_nonempty=tc.get("expected_nonempty", True),
+            pii_columns=pii_columns,
+            weights=weights,
+        )
+        results.append(scores)
+    return results

sqlas/production.py

@@ -0,0 +1,74 @@
+"""
+Production Execution Metrics.
+- Data Scan Efficiency (full scan detection)
+- Execution Result (success, empty result, row explosion)
+
+Author: SQLAS Contributors
+"""
+
+import re
+
+
+def data_scan_efficiency(sql: str, result_row_count: int = 0) -> tuple[float, dict]:
+    """
+    Detect inefficient data access patterns:
+    - SELECT * without WHERE
+    - Missing filters on large queries
+    - Cartesian products from bad JOINs
+    - No LIMIT on detail queries
+    """
+    upper = sql.upper()
+    issues = []
+    score = 1.0
+
+    if "SELECT *" in upper or "SELECT  *" in upper:
+        issues.append("SELECT * — should specify columns")
+        score -= 0.2
+
+    has_where = "WHERE" in upper
+    has_group = "GROUP BY" in upper
+    has_limit = "LIMIT" in upper
+
+    if not has_where and not has_group and not has_limit:
+        issues.append("No WHERE, GROUP BY, or LIMIT — potential full scan")
+        score -= 0.3
+
+    if result_row_count > 10000 and "JOIN" in upper:
+        issues.append(f"Large result ({result_row_count} rows) from JOIN — possible cartesian product")
+        score -= 0.3
+
+    if not has_group and not has_limit and result_row_count > 100:
+        issues.append("No LIMIT on detail query returning many rows")
+        score -= 0.1
+
+    return max(score, 0.0), {"issues": issues or ["none"]}
+
+
+def execution_result(
+    data: dict | None,
+    expected_nonempty: bool = True,
+) -> dict:
+    """
+    Evaluate execution outcome.
+
+    Args:
+        data: Query result dict with keys: row_count, execution_time_ms, truncated
+        expected_nonempty: Whether non-empty result is expected
+    """
+    if data is None:
+        return {
+            "execution_success": 0.0,
+            "empty_result_penalty": 0.0,
+            "row_explosion_detected": False,
+            "execution_time_ms": 0,
+            "result_row_count": 0,
+        }
+
+    row_count = data.get("row_count", 0)
+    return {
+        "execution_success": 1.0,
+        "execution_time_ms": data.get("execution_time_ms", 0),
+        "result_row_count": row_count,
+        "empty_result_penalty": 0.0 if (expected_nonempty and row_count == 0) else 1.0,
+        "row_explosion_detected": row_count > 50000,
+    }