sqlas 1.1.0__py3-none-any.whl

sqlas/__init__.py

@@ -0,0 +1,69 @@
+"""
+SQLAS — SQL Agent Scoring Framework
+A RAGAS-equivalent evaluation library for Text-to-SQL and SQL AI agents.
+
+Author: SQLAS Contributors
+
+Usage:
+    from sqlas import evaluate, SQLASScores, TestCase, WEIGHTS
+
+    scores = evaluate(
+        question="How many users are active?",
+        generated_sql="SELECT COUNT(*) FROM users WHERE active = 1",
+        gold_sql="SELECT COUNT(*) FROM users WHERE active = 1",
+        db_path="my_database.db",
+        llm_judge=my_llm_function,
+    )
+    print(scores.overall_score)
+"""
+
+from sqlas.core import SQLASScores, TestCase, WEIGHTS, WEIGHTS_V2, compute_composite_score
+from sqlas.evaluate import evaluate, evaluate_batch
+from sqlas.correctness import execution_accuracy, syntax_valid, semantic_equivalence, result_set_similarity
+from sqlas.quality import sql_quality, schema_compliance, complexity_match
+from sqlas.production import data_scan_efficiency, execution_result
+from sqlas.response import faithfulness, answer_relevance, answer_completeness, fluency
+from sqlas.safety import safety_score, read_only_compliance
+from sqlas.context import context_precision, context_recall, entity_recall, noise_robustness
+from sqlas.runner import run_suite
+
+__version__ = "1.1.0"
+__author__ = "SQLAS Contributors"
+
+__all__ = [
+    # Core
+    "SQLASScores",
+    "TestCase",
+    "WEIGHTS",
+    "WEIGHTS_V2",
+    "compute_composite_score",
+    # Top-level API
+    "evaluate",
+    "evaluate_batch",
+    "run_suite",
+    # Correctness metrics
+    "execution_accuracy",
+    "syntax_valid",
+    "semantic_equivalence",
+    "result_set_similarity",
+    # Quality metrics
+    "sql_quality",
+    "schema_compliance",
+    "complexity_match",
+    # Production metrics
+    "data_scan_efficiency",
+    "execution_result",
+    # Response metrics
+    "faithfulness",
+    "answer_relevance",
+    "answer_completeness",
+    "fluency",
+    # Safety metrics
+    "safety_score",
+    "read_only_compliance",
+    # Context metrics (RAGAS-mapped)
+    "context_precision",
+    "context_recall",
+    "entity_recall",
+    "noise_robustness",
+]

sqlas/context.py

@@ -0,0 +1,268 @@
+"""
+Context Quality Metrics (RAGAS-mapped for SQL agents).
+- Context Precision (schema element precision)
+- Context Recall (schema element recall)
+- Entity Recall (strict entity-level recall)
+- Noise Robustness (irrelevant schema resistance)
+
+Author: SQLAS Contributors
+"""
+
+import sqlglot
+
+
+# ── Shared AST Extraction ──────────────────────────────────────────────────
+
+def _extract_sql_elements(sql: str, dialect: str = "sqlite") -> dict:
+    """Extract tables, columns, literals, and functions from SQL AST.
+
+    Returns:
+        {
+            "tables": set of lowered table names,
+            "columns": set of lowered column names,
+            "table_columns": set of (table, column) tuples,
+            "literals": set of string representations of literal values,
+            "functions": set of lowered function names,
+        }
+    """
+    try:
+        parsed = sqlglot.parse_one(sql, dialect=dialect)
+    except Exception:
+        return {
+            "tables": set(),
+            "columns": set(),
+            "table_columns": set(),
+            "literals": set(),
+            "functions": set(),
+        }
+
+    tables = set()
+    for table in parsed.find_all(sqlglot.exp.Table):
+        if table.name:
+            tables.add(table.name.lower())
+
+    columns = set()
+    table_columns = set()
+    for col in parsed.find_all(sqlglot.exp.Column):
+        if col.name:
+            col_name = col.name.lower()
+            columns.add(col_name)
+            tbl = col.table.lower() if col.table else ""
+            if tbl:
+                table_columns.add((tbl, col_name))
+            else:
+                table_columns.add(("", col_name))
+
+    literals = set()
+    for lit in parsed.find_all(sqlglot.exp.Literal):
+        literals.add(str(lit.this).lower())
+
+    functions = set()
+    # Built-in aggregate / scalar expressions
+    for cls in (sqlglot.exp.Count, sqlglot.exp.Sum, sqlglot.exp.Avg,
+                sqlglot.exp.Min, sqlglot.exp.Max):
+        for _ in parsed.find_all(cls):
+            functions.add(cls.key.lower())
+    # Named functions (e.g. ROUND, COALESCE, custom UDFs)
+    for func in parsed.find_all(sqlglot.exp.Anonymous):
+        if func.name:
+            functions.add(func.name.lower())
+
+    return {
+        "tables": tables,
+        "columns": columns,
+        "table_columns": table_columns,
+        "literals": literals,
+        "functions": functions,
+    }
+
+
+# ── Public API ─────────────────────────────────────────────────────────────
+
+def context_precision(
+    generated_sql: str,
+    gold_sql: str,
+    dialect: str = "sqlite",
+) -> tuple[float, dict]:
+    """
+    RAGAS Context Precision for SQL agents.
+
+    Of all schema elements (tables + columns) referenced in the generated SQL,
+    what fraction are also referenced in the gold SQL? Penalizes referencing
+    unnecessary schema elements.
+
+    Args:
+        generated_sql: SQL produced by the agent
+        gold_sql: Ground-truth SQL
+        dialect: SQL dialect for parsing
+
+    Returns:
+        (precision score 0.0–1.0, details dict)
+    """
+    gen = _extract_sql_elements(generated_sql, dialect)
+    gold = _extract_sql_elements(gold_sql, dialect)
+
+    gen_elements = gen["tables"] | gen["columns"]
+    gold_elements = gold["tables"] | gold["columns"]
+
+    if not gen_elements:
+        return 1.0, {"generated_elements": [], "gold_elements": list(gold_elements),
+                      "extra_elements": [], "precision": 1.0}
+
+    overlap = gen_elements & gold_elements
+    extra = gen_elements - gold_elements
+    precision = len(overlap) / len(gen_elements)
+
+    return round(precision, 4), {
+        "generated_elements": sorted(gen_elements),
+        "gold_elements": sorted(gold_elements),
+        "extra_elements": sorted(extra),
+        "precision": round(precision, 4),
+    }
+
+
+def context_recall(
+    generated_sql: str,
+    gold_sql: str,
+    dialect: str = "sqlite",
+) -> tuple[float, dict]:
+    """
+    RAGAS Context Recall for SQL agents.
+
+    Of all schema elements (tables + columns) required by the gold SQL,
+    what fraction does the generated SQL also reference? Penalizes missing
+    necessary elements.
+
+    Args:
+        generated_sql: SQL produced by the agent
+        gold_sql: Ground-truth SQL
+        dialect: SQL dialect for parsing
+
+    Returns:
+        (recall score 0.0–1.0, details dict)
+    """
+    gen = _extract_sql_elements(generated_sql, dialect)
+    gold = _extract_sql_elements(gold_sql, dialect)
+
+    gen_elements = gen["tables"] | gen["columns"]
+    gold_elements = gold["tables"] | gold["columns"]
+
+    if not gold_elements:
+        return 1.0, {"missing_elements": [], "recall": 1.0}
+
+    overlap = gen_elements & gold_elements
+    missing = gold_elements - gen_elements
+    recall = len(overlap) / len(gold_elements)
+
+    return round(recall, 4), {
+        "missing_elements": sorted(missing),
+        "matched_elements": sorted(overlap),
+        "total_gold_elements": len(gold_elements),
+        "recall": round(recall, 4),
+    }
+
+
+def entity_recall(
+    generated_sql: str,
+    gold_sql: str,
+    dialect: str = "sqlite",
+) -> tuple[float, dict]:
+    """
+    RAGAS Context Entity Recall for SQL agents.
+
+    Strict entity-level check: are all entities (tables, columns, literals,
+    functions) from the gold SQL present in the generated SQL?
+
+    This is stricter than context_recall because it also checks literal values
+    (e.g. WHERE status = 'active') and function usage (COUNT, SUM, etc.).
+
+    Args:
+        generated_sql: SQL produced by the agent
+        gold_sql: Ground-truth SQL
+        dialect: SQL dialect for parsing
+
+    Returns:
+        (recall score 0.0–1.0, details dict)
+    """
+    gen = _extract_sql_elements(generated_sql, dialect)
+    gold = _extract_sql_elements(gold_sql, dialect)
+
+    gold_entities = gold["tables"] | gold["columns"] | gold["literals"] | gold["functions"]
+    gen_entities = gen["tables"] | gen["columns"] | gen["literals"] | gen["functions"]
+
+    if not gold_entities:
+        return 1.0, {"missing_entities": [], "matched_entities": [],
+                      "total_gold_entities": 0}
+
+    matched = gold_entities & gen_entities
+    missing = gold_entities - gen_entities
+    recall = len(matched) / len(gold_entities)
+
+    return round(recall, 4), {
+        "missing_entities": sorted(missing),
+        "matched_entities": sorted(matched),
+        "total_gold_entities": len(gold_entities),
+    }
+
+
+def noise_robustness(
+    generated_sql: str,
+    gold_sql: str,
+    valid_tables: set[str] | None = None,
+    valid_columns: dict[str, set[str]] | None = None,
+    dialect: str = "sqlite",
+) -> tuple[float, dict]:
+    """
+    RAGAS Noise Sensitivity for SQL agents.
+
+    Does the agent avoid pulling in irrelevant schema elements? Measures
+    resilience to a large, noisy schema by checking if the generated SQL
+    references tables/columns that exist in the full schema but are NOT
+    needed by the gold SQL.
+
+    Args:
+        generated_sql: SQL produced by the agent
+        gold_sql: Ground-truth SQL
+        valid_tables: Full set of available table names (optional)
+        valid_columns: Full dict of {table: {col1, col2, ...}} (optional)
+        dialect: SQL dialect for parsing
+
+    Returns:
+        (robustness score 0.0–1.0, details dict)
+    """
+    gen = _extract_sql_elements(generated_sql, dialect)
+    gold = _extract_sql_elements(gold_sql, dialect)
+
+    gold_elements = gold["tables"] | gold["columns"]
+    gen_elements = gen["tables"] | gen["columns"]
+
+    if not gen_elements:
+        return 1.0, {"noise_tables": [], "noise_columns": [], "noise_count": 0}
+
+    # Elements in generated SQL that are NOT in gold SQL
+    extra = gen_elements - gold_elements
+
+    # If full schema is provided, only count extras that exist in the schema
+    # (i.e. real noise, not hallucinated tables/columns)
+    if valid_tables or valid_columns:
+        all_schema = set()
+        if valid_tables:
+            all_schema.update(t.lower() for t in valid_tables)
+        if valid_columns:
+            for cols in valid_columns.values():
+                all_schema.update(c.lower() for c in cols)
+        noise = extra & all_schema
+    else:
+        noise = extra
+
+    noise_tables = noise & gen["tables"]
+    noise_columns = noise & gen["columns"]
+    noise_count = len(noise)
+
+    score = max(0.0, 1.0 - (noise_count / len(gen_elements)))
+
+    return round(score, 4), {
+        "noise_tables": sorted(noise_tables),
+        "noise_columns": sorted(noise_columns),
+        "noise_count": noise_count,
+    }

sqlas/core.py

@@ -0,0 +1,208 @@
+"""
+Core data structures and composite scoring for SQLAS.
+
+Author: SQLAS Contributors
+"""
+
+import re
+from dataclasses import dataclass, field
+from typing import Callable
+
+
+# ── Production Composite Weights (v1 — 15 metrics) ────────────────────────
+# Aligned with industry-standard SQL agent evaluation:
+#   40% Execution Accuracy — does the SQL return correct results?
+#   15% Semantic Correctness — does the SQL answer the user's intent?
+#   15% Cost Efficiency — is the query efficient?
+#   10% Execution Quality — does the query execute successfully?
+#   10% Task Success — does the user get a correct, complete answer?
+#   10% Safety — is the query safe?
+# ────────────────────────────────────────────────────────────────────────────
+
+WEIGHTS = {
+    # 1. Execution Accuracy (40%)
+    "execution_accuracy": 0.40,
+    # 2. Semantic Correctness (15%)
+    "semantic_equivalence": 0.15,
+    # 3. Cost Efficiency (15%)
+    "efficiency_score": 0.05,
+    "data_scan_efficiency": 0.05,
+    "sql_quality": 0.03,
+    "schema_compliance": 0.02,
+    # 4. Execution Quality (10%)
+    "execution_success": 0.05,
+    "complexity_match": 0.03,
+    "empty_result_penalty": 0.02,
+    # 5. Task Success (10%)
+    "faithfulness": 0.04,
+    "answer_relevance": 0.03,
+    "answer_completeness": 0.02,
+    "fluency": 0.01,
+    # 6. Safety (10%)
+    "read_only_compliance": 0.05,
+    "safety_score": 0.05,
+}
+
+
+# ── Production Composite Weights (v2 — 20 metrics with context quality) ───
+# Adds RAGAS-mapped context metrics for SQL agents.
+# ────────────────────────────────────────────────────────────────────────────
+
+WEIGHTS_V2 = {
+    # 1. Execution Accuracy (35%)
+    "execution_accuracy": 0.35,
+    # 2. Semantic Correctness (13%)
+    "semantic_equivalence": 0.13,
+    # 3. Context Quality (10%) — RAGAS-mapped
+    "context_precision": 0.03,
+    "context_recall": 0.03,
+    "entity_recall": 0.02,
+    "noise_robustness": 0.02,
+    # 4. Cost Efficiency (12%)
+    "efficiency_score": 0.04,
+    "data_scan_efficiency": 0.04,
+    "sql_quality": 0.02,
+    "schema_compliance": 0.02,
+    # 5. Execution Quality (8%)
+    "execution_success": 0.04,
+    "complexity_match": 0.02,
+    "empty_result_penalty": 0.02,
+    # 6. Task Success (8%)
+    "faithfulness": 0.03,
+    "answer_relevance": 0.02,
+    "answer_completeness": 0.02,
+    "fluency": 0.01,
+    # 7. Result Similarity (4%)
+    "result_set_similarity": 0.04,
+    # 8. Safety (10%)
+    "read_only_compliance": 0.05,
+    "safety_score": 0.05,
+}
+
+
+@dataclass
+class TestCase:
+    """A single evaluation test case."""
+    question: str
+    gold_sql: str | None = None
+    expected_tables: list[str] | None = None
+    expects_join: bool = False
+    expected_nonempty: bool = True
+    category: str = "general"
+
+
+@dataclass
+class SQLASScores:
+    """Complete production-grade evaluation scores for a single query."""
+
+    # 1. Core SQL Correctness
+    execution_accuracy: float = 0.0
+    syntax_valid: float = 0.0
+    semantic_equivalence: float = 0.0
+
+    # 2. SQL Quality & Structure
+    schema_compliance: float = 0.0
+    sql_quality: float = 0.0
+    complexity_match: float = 0.0
+
+    # 3. Production Execution
+    execution_success: float = 0.0
+    execution_time_ms: float = 0.0
+    efficiency_score: float = 0.0
+    data_scan_efficiency: float = 0.0
+    result_row_count: int = 0
+    empty_result_penalty: float = 0.0
+    row_explosion_detected: bool = False
+
+    # 4. Response Quality
+    faithfulness: float = 0.0
+    answer_relevance: float = 0.0
+    answer_completeness: float = 0.0
+    fluency: float = 0.0
+
+    # 5. Safety & Governance
+    read_only_compliance: float = 0.0
+    safety_score: float = 0.0
+
+    # 6. Context Quality (RAGAS-mapped)
+    context_precision: float = 0.0
+    context_recall: float = 0.0
+    entity_recall: float = 0.0
+    noise_robustness: float = 0.0
+    result_set_similarity: float = 0.0
+
+    # Composite
+    overall_score: float = 0.0
+    details: dict = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        """Export all scores as a flat dictionary."""
+        all_keys = set(WEIGHTS.keys()) | set(WEIGHTS_V2.keys())
+        d = {}
+        for key in all_keys:
+            d[key] = getattr(self, key, 0.0)
+        d["overall_score"] = self.overall_score
+        d["syntax_valid"] = self.syntax_valid
+        d["execution_time_ms"] = self.execution_time_ms
+        d["result_row_count"] = self.result_row_count
+        d["row_explosion_detected"] = self.row_explosion_detected
+        return d
+
+    def summary(self) -> str:
+        """Human-readable summary."""
+        lines = [f"SQLAS Score: {self.overall_score:.4f} / 1.0"]
+        cats = {
+            "Execution Accuracy": [("execution_accuracy", self.execution_accuracy)],
+            "Semantic Correctness": [("semantic_equivalence", self.semantic_equivalence)],
+            "Context Quality": [("context_precision", self.context_precision), ("context_recall", self.context_recall), ("entity_recall", self.entity_recall), ("noise_robustness", self.noise_robustness), ("result_similarity", self.result_set_similarity)],
+            "Cost Efficiency": [("efficiency", self.efficiency_score), ("data_scan", self.data_scan_efficiency), ("sql_quality", self.sql_quality), ("schema", self.schema_compliance)],
+            "Execution Quality": [("exec_success", self.execution_success), ("complexity", self.complexity_match), ("empty_result", self.empty_result_penalty)],
+            "Task Success": [("faithfulness", self.faithfulness), ("relevance", self.answer_relevance), ("completeness", self.answer_completeness), ("fluency", self.fluency)],
+            "Safety": [("read_only", self.read_only_compliance), ("safety", self.safety_score)],
+        }
+        for cat, metrics in cats.items():
+            lines.append(f"  {cat}")
+            for name, val in metrics:
+                lines.append(f"    {name}: {val:.4f}")
+        return "\n".join(lines)
+
+
+# ── LLM Judge type ──────────────────────────────────────────────────────────
+# Users provide their own LLM function: (prompt: str) -> str
+LLMJudge = Callable[[str], str]
+
+
+def _parse_score(result: str, key: str) -> tuple[float, str]:
+    """Shared helper to extract a score and reasoning from LLM judge output.
+
+    Looks for lines like 'Key: 0.85' and 'Reasoning: ...' in the result text.
+
+    Args:
+        result: Raw LLM judge output
+        key: The score key to look for (e.g. 'Faithfulness', 'Relevance')
+
+    Returns:
+        (score clamped to 0.0–1.0, reasoning string)
+    """
+    score, reasoning = 0.0, ""
+    for line in result.strip().split("\n"):
+        if line.startswith(key + ":"):
+            try:
+                score = float(re.search(r"[\d.]+", line.split(":")[-1]).group())
+            except Exception:
+                pass
+        if line.startswith("Reasoning:"):
+            reasoning = line.split(":", 1)[-1].strip()
+    return min(score, 1.0), reasoning
+
+
+def compute_composite_score(scores: SQLASScores, weights: dict | None = None) -> float:
+    """Compute weighted overall SQLAS score."""
+    w = weights or WEIGHTS
+    total = 0.0
+    for metric, weight in w.items():
+        val = getattr(scores, metric, 0.0)
+        if isinstance(val, bool):
+            val = 1.0 if val else 0.0
+        total += val * weight
+    return round(total, 4)