plm-sql 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plm_sql/__init__.py +1 -0
- plm_sql/claims.py +100 -0
- plm_sql/controller.py +175 -0
- plm_sql/flags.py +58 -0
- plm_sql/levels.py +56 -0
- plm_sql/manifest.py +11 -0
- plm_sql/sandbox.py +76 -0
- plm_sql/sqlgen.py +46 -0
- plm_sql-0.1.0.dist-info/METADATA +20 -0
- plm_sql-0.1.0.dist-info/RECORD +13 -0
- plm_sql-0.1.0.dist-info/WHEEL +4 -0
- plm_sql-0.1.0.dist-info/entry_points.txt +2 -0
- plm_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
plm_sql/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""SQL pack: SQLite queries written by a model, checked by running them on the task's own schema."""
|
plm_sql/claims.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Which tasks the SQL pack claims: a schema to query, or a request for a query against a table.
|
|
2
|
+
|
|
3
|
+
Certain (inf) on either, so SQL wins over the numeric packs, whose term counts a
|
|
4
|
+
schema's column names can trip. Software's certain claim on "write a ... function"
|
|
5
|
+
ties it, and software's higher priority keeps a Python function that happens to use
|
|
6
|
+
SQLite. Questions about SQL that ask for no query ("What is SQL injection?",
|
|
7
|
+
"INNER vs LEFT JOIN") claim nothing and go to the general pack.
|
|
8
|
+
|
|
9
|
+
With a database set (PLM_SQL_DB, as the server reads it), a question with no schema
|
|
10
|
+
in it can still be about that data: "How many singers do we have?" The pack then
|
|
11
|
+
claims a data question (one starting what, which, how many, list, show ...; none that
|
|
12
|
+
asks for prose or advice: write, explain, why, how do I ...) that names a table of the
|
|
13
|
+
database, two of its columns, or one column that isn't a generic word like "name" or
|
|
14
|
+
"year". Names match word by word, plurals either way ("Pets" ~ "pet", "movies" ~ "movie").
|
|
15
|
+
The claim is the number of names used minus a half, so a numeric problem whose own
|
|
16
|
+
pack counts as many terms ("A stadium speaker draws 8 A at 230 V") stays with that pack.
|
|
17
|
+
Designed on Spider train questions and databases; judged once on held-out cases by
|
|
18
|
+
evals/test_sqldb_routing.py (SQLDB_RULE, 2026-11-04): 115/120 Spider dev questions and
|
|
19
|
+
18/20 casual ones to sql; 30/34 look-alikes and 9855/9860 other routing texts unmoved.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import math
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import sqlite3
|
|
26
|
+
from functools import lru_cache
|
|
27
|
+
|
|
28
|
+
SCHEMA = re.compile(r"\bcreate\s+table\b", re.IGNORECASE)
|
|
29
|
+
QUERY = re.compile(r"\b(?:quer(?:y|ies)|select\s+statement)\b|\bselect\b.{1,80}?\bfrom\b", re.IGNORECASE | re.DOTALL)
|
|
30
|
+
TABLE = re.compile(r"\bsql(?:ite)?\b|\btables?\b|\b\w+\(\s*\w+(?:\s*,\s*\w+)+\s*\)", re.IGNORECASE)
|
|
31
|
+
|
|
32
|
+
WORD = re.compile(r"[A-Za-z]+")
|
|
33
|
+
PART = re.compile(r"[A-Z]+(?![a-z])|[A-Z]?[a-z]+") # "AirportName" -> Airport, Name; "MPG" -> MPG
|
|
34
|
+
DATA = re.compile(r"^\s*(?:please\s+)?(?:what|which|who|whose|whom|when|where|how\s+(?:many|much|old|long)|list|show|find|"
|
|
35
|
+
r"give|return|count|tell\s+me|sort|order|rank|top|for\s+each|select|get|display|compute|calculate|"
|
|
36
|
+
r"number\s+of|average|total|names?\s+of)\b", re.IGNORECASE)
|
|
37
|
+
PROSE = re.compile(r"\b(?:write|explain|why|recommend|suggest|plan|draft|summari[sz]e|translate|describe|compose|"
|
|
38
|
+
r"how\s+(?:do|can|should|would|could)\s+(?:i|you|we)|what\s+(?:makes|should|does|do)\b|"
|
|
39
|
+
r"what\s+(?:is|are)\s+(?:a|an|the\s+(?:best|history|rules?|difference))\b)", re.IGNORECASE)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
GENERIC = {"id", "name", "year", "date", "age", "type", "level", "code", "day", "time", "number", "title", "price",
|
|
43
|
+
"amount", "weight", "height", "period", "rank", "result", "status", "note", "details", "description", "location",
|
|
44
|
+
"city", "country", "state", "address", "first", "last", "count", "total", "average", "size", "color", "sex",
|
|
45
|
+
"gender", "place", "position", "area", "value", "rating", "score", "share"}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def forms(word):
|
|
49
|
+
"""The word and each singular it could be: "movies" -> movies, movie, movy; "wines" -> wines, wine, win."""
|
|
50
|
+
w = word.lower()
|
|
51
|
+
out = {w}
|
|
52
|
+
if len(w) > 3 and w.endswith("s") and not w.endswith("ss"):
|
|
53
|
+
out.add(w[:-1])
|
|
54
|
+
if w.endswith("es"):
|
|
55
|
+
out.add(w[:-2])
|
|
56
|
+
if w.endswith("ies"):
|
|
57
|
+
out.add(w[:-3] + "y")
|
|
58
|
+
return out
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def parts(name):
|
|
62
|
+
return tuple(p.lower() for w in WORD.findall(name) for p in PART.findall(w))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@lru_cache(maxsize=8)
|
|
66
|
+
def schema_names(path, mtime):
|
|
67
|
+
"""(table names, other column names) of the database, each name as its lowercase parts (mtime only keys the cache)."""
|
|
68
|
+
con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
69
|
+
try:
|
|
70
|
+
tables, columns = set(), set()
|
|
71
|
+
for (table,) in con.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"):
|
|
72
|
+
tables.add(parts(table))
|
|
73
|
+
quoted = table.replace('"', '""')
|
|
74
|
+
columns.update(parts(r[1]) for r in con.execute(f'PRAGMA table_info("{quoted}")'))
|
|
75
|
+
return frozenset(n for n in tables if n), frozenset(n for n in columns - tables if n)
|
|
76
|
+
finally:
|
|
77
|
+
con.close()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def db_claim(text, path):
|
|
81
|
+
"""How many of the database's names a data question uses (0 when it isn't one, or names none)."""
|
|
82
|
+
if not DATA.search(text) or PROSE.search(text):
|
|
83
|
+
return 0
|
|
84
|
+
try:
|
|
85
|
+
names = schema_names(path, os.path.getmtime(path))
|
|
86
|
+
except (OSError, sqlite3.Error): # unreadable or not SQLite: the server checked it exists, nothing more
|
|
87
|
+
return 0
|
|
88
|
+
words = set().union(*(forms(w) for w in WORD.findall(text)))
|
|
89
|
+
hit = [[n for n in group if all(forms(p) & words for p in n)] for group in names]
|
|
90
|
+
tables, columns = len(hit[0]), len(hit[1])
|
|
91
|
+
distinctive = any(len(n) > 1 or n[0] not in GENERIC for n in hit[1])
|
|
92
|
+
# half a point down, so a pack whose own terms count as many wins the tie (SQL's priority would)
|
|
93
|
+
return tables + columns - 0.5 if tables or columns >= 2 or distinctive else 0
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def claims(text):
|
|
97
|
+
if SCHEMA.search(text) or (QUERY.search(text) and TABLE.search(text)):
|
|
98
|
+
return math.inf
|
|
99
|
+
db = os.environ.get("PLM_SQL_DB")
|
|
100
|
+
return db_claim(text, os.path.expanduser(db)) if db else 0
|
plm_sql/controller.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""SQL pack: a question about tables becomes a SQLite query, checked by running it on the task's schema.
|
|
2
|
+
|
|
3
|
+
The task's CREATE TABLE statements (and any INSERTs) are loaded into a throwaway
|
|
4
|
+
read-only database (plm_sql.sandbox) and the query must run there: that catches
|
|
5
|
+
syntax errors, misspelt tables and columns, and non-SQLite dialect. It can't see a
|
|
6
|
+
query that runs but answers a different question; with no schema in the task
|
|
7
|
+
there is nothing to run, and the answer is reported unchecked.
|
|
8
|
+
|
|
9
|
+
Policies and rules were fixed before each run (evals/run_sql.py, evals/bench_sql.py). Our own
|
|
10
|
+
sets (45 eval tasks, then 48 harder fresh ones) settled on always 8B, with the coder tied (42/48).
|
|
11
|
+
The public benchmarks, written by others and larger, disagreed:
|
|
12
|
+
|
|
13
|
+
Spider (299 dev) BIRD mini-dev (496) compute vs always-coder
|
|
14
|
+
always coder 14B 228 (76%, 72-81%) 254 (51%, 47-56%) 100%
|
|
15
|
+
cascade 8B -> coder 209 (70%) 207 (42%) 70-80%
|
|
16
|
+
always 8B 201 (67%, 62-73%) 172 (35%, 30-39%) 55-66%
|
|
17
|
+
|
|
18
|
+
BENCH_RULE capped extra compute at +10 points, so it kept always 8B; the coder beats it on both
|
|
19
|
+
benchmarks with paired 95% intervals well above 0 (+15 to +40, +62 to +105). On 2026-10-24 the user
|
|
20
|
+
chose accuracy over compute for SQL: every level goes to the coder alone, about 1.5-1.8x the 8B's
|
|
21
|
+
compute. A query that doesn't run is reported, not escalated (the check caught 38 of 242 of the
|
|
22
|
+
coder's wrong BIRD queries). Budgets are 2.9-3.7x the coder's longest answer at each level over
|
|
23
|
+
every set and benchmark (268, 321 and 210 tokens; none hit a cap).
|
|
24
|
+
|
|
25
|
+
With the user's database (`plm --sql-db <file>`, or PLM_SQL_DB for the server; opened read-only, and
|
|
26
|
+
its schema is read from the file when the task gives none), the query also runs on the real data and
|
|
27
|
+
plm_sql.flags reads the result. When the coder's query is flagged, the 4B answers too, and its query
|
|
28
|
+
is used iff it raises no flag; otherwise the coder's is kept and reported with its flags. FLAGS_RULE
|
|
29
|
+
(evals/bench_sql_check.py), fixed before, adopted this on 2880 unseen Spider questions on 2026-10-29:
|
|
30
|
+
2230 vs 2212 for the coder alone (+7 to +30, paired 95%), at 105% compute. Without a database the
|
|
31
|
+
pack is as above.
|
|
32
|
+
|
|
33
|
+
Model names can be overridden with PLM_CODER_MODEL and PLM_SMALL_MODEL.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
import os
|
|
37
|
+
import sqlite3
|
|
38
|
+
import time
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
|
|
41
|
+
from plm.ollama import ask
|
|
42
|
+
|
|
43
|
+
from plm_sql.flags import HUGE_ROWS, named_tables, result_flags
|
|
44
|
+
from plm_sql.levels import classify
|
|
45
|
+
from plm_sql.sandbox import run_query
|
|
46
|
+
from plm_sql.sqlgen import build_prompt, extract_sql, split_task
|
|
47
|
+
|
|
48
|
+
MODELS = {"coder": os.environ.get("PLM_CODER_MODEL", "qwen2.5-coder:14b"),
|
|
49
|
+
"small": os.environ.get("PLM_SMALL_MODEL", "hf.co/bartowski/Qwen_Qwen3-4B-GGUF:Q4_K_M")}
|
|
50
|
+
SECOND_OPINION = "small" # asked only when the coder's query is flagged on the user's database
|
|
51
|
+
LEVELS = {
|
|
52
|
+
"calm": {"num_predict": 768, "chain": ["coder"]},
|
|
53
|
+
"alert": {"num_predict": 1024, "chain": ["coder"]},
|
|
54
|
+
"intensive": {"num_predict": 768, "chain": ["coder"]},
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def check(task, query):
|
|
59
|
+
"""Issues with a query: none if it runs on the task's schema. (issues, run result or None)."""
|
|
60
|
+
if query is None:
|
|
61
|
+
return ["no SQL query in the reply"], None
|
|
62
|
+
if not task["setup"]:
|
|
63
|
+
return [], None
|
|
64
|
+
ran = run_query(task["setup"], query)
|
|
65
|
+
if ran["setup_error"]:
|
|
66
|
+
return [], None
|
|
67
|
+
return ([f"doesn't run on the schema: {ran['error']}"] if ran["error"] else []), ran
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def open_db(path):
|
|
71
|
+
"""(resolved path, schema SQL, table names) of a database file, read-only. Raises ValueError if it can't be read."""
|
|
72
|
+
path = Path(path).expanduser().resolve()
|
|
73
|
+
if not path.is_file():
|
|
74
|
+
raise ValueError(f"no database file at {path}")
|
|
75
|
+
try:
|
|
76
|
+
con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
77
|
+
try:
|
|
78
|
+
rows = con.execute("SELECT name, sql FROM sqlite_master WHERE type = 'table' AND sql IS NOT NULL "
|
|
79
|
+
"AND name NOT LIKE 'sqlite_%' ORDER BY rowid").fetchall()
|
|
80
|
+
finally:
|
|
81
|
+
con.close()
|
|
82
|
+
except sqlite3.Error as e:
|
|
83
|
+
raise ValueError(f"{path} isn't a readable SQLite database: {e}") from None
|
|
84
|
+
if not rows:
|
|
85
|
+
raise ValueError(f"{path} has no tables")
|
|
86
|
+
return str(path), ";\n".join(r[1].strip() for r in rows) + ";", [r[0] for r in rows]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
FLAG_TEXT = {"empty": "returns no rows on the database", "huge": f"returns more than {HUGE_ROWS} rows on the database",
|
|
90
|
+
"nulls": "returns only NULLs on the database"}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def db_issues(task, query, db):
|
|
94
|
+
"""Flags for a query on the user's database, as issues, and its run there (or None)."""
|
|
95
|
+
path, _, tables = db
|
|
96
|
+
ran = run_query("", query, seconds=30, db_path=path) if query else None
|
|
97
|
+
issues = []
|
|
98
|
+
for f in result_flags(task["question"], query, ran, tables):
|
|
99
|
+
if f == "error":
|
|
100
|
+
issues.append(f"doesn't run on the database: {ran['error']}" if ran else "no SQL query in the reply")
|
|
101
|
+
elif f == "tables":
|
|
102
|
+
issues.append("uses none of the tables the question names (" + ", ".join(named_tables(task["question"], tables)) + ")")
|
|
103
|
+
else:
|
|
104
|
+
issues.append(FLAG_TEXT[f])
|
|
105
|
+
return issues, ran
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def attempt(task, text, model, num_predict, timeout, db=None):
|
|
109
|
+
start = time.time()
|
|
110
|
+
r = ask(model, build_prompt(text), False, num_predict, timeout)
|
|
111
|
+
query = extract_sql(r["response"])
|
|
112
|
+
issues, ran = check(task, query)
|
|
113
|
+
on_db = None
|
|
114
|
+
if db: # the run on the real data replaces the run on the bare schema, as in the benchmark that judged it
|
|
115
|
+
issues, ran_db = db_issues(task, query, db)
|
|
116
|
+
on_db = ran_db and {k: ran_db[k] for k in ("columns", "n_rows", "error")}
|
|
117
|
+
return {"model": model, "tokens": r["eval_tokens"], "prompt_tokens": r["prompt_tokens"],
|
|
118
|
+
"model_text": r["response"], "capped": r["done_reason"] == "length", "code": query,
|
|
119
|
+
"ran": ran and {k: ran[k] for k in ("columns", "n_rows", "error")}, "on_db": on_db, "issues": issues,
|
|
120
|
+
"seconds": time.time() - start}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def solve(text, level=None, timeout=300, db=None):
|
|
124
|
+
"""Write one query. Returns it plus a record of every model call.
|
|
125
|
+
|
|
126
|
+
db: a SQLite file to check the query on (read-only); defaults to PLM_SQL_DB. Its schema is put
|
|
127
|
+
in front of the task when the task has no CREATE TABLE statements of its own.
|
|
128
|
+
"""
|
|
129
|
+
start = time.time()
|
|
130
|
+
db = db or os.environ.get("PLM_SQL_DB") or None
|
|
131
|
+
db = open_db(db) if db else None
|
|
132
|
+
asked = text
|
|
133
|
+
if db and not split_task(text)["setup"]:
|
|
134
|
+
text = f"{db[1]}\n\n{text.strip()}"
|
|
135
|
+
task = split_task(text)
|
|
136
|
+
if level:
|
|
137
|
+
reason = "set by caller"
|
|
138
|
+
else:
|
|
139
|
+
level, reason = classify(text)
|
|
140
|
+
plan = LEVELS[level]
|
|
141
|
+
steps = []
|
|
142
|
+
for model_key in plan["chain"]:
|
|
143
|
+
steps.append(attempt(task, text, MODELS[model_key], plan["num_predict"], timeout, db))
|
|
144
|
+
if not steps[-1]["issues"]:
|
|
145
|
+
break
|
|
146
|
+
final = steps[-1]
|
|
147
|
+
if db and final["issues"]:
|
|
148
|
+
# FLAGS_RULE's policy: a second opinion, used only if it raises no flag
|
|
149
|
+
steps.append(attempt(task, text, MODELS[SECOND_OPINION], plan["num_predict"], timeout, db))
|
|
150
|
+
if not steps[-1]["issues"]:
|
|
151
|
+
final = steps[-1]
|
|
152
|
+
if db:
|
|
153
|
+
unchecked = None
|
|
154
|
+
elif not task["setup"]:
|
|
155
|
+
unchecked = "no CREATE TABLE statements given, so the query couldn't be run"
|
|
156
|
+
elif final["code"] is not None and final["ran"] is None:
|
|
157
|
+
unchecked = "the schema given doesn't load in SQLite, so the query couldn't be run"
|
|
158
|
+
else:
|
|
159
|
+
unchecked = None
|
|
160
|
+
return {
|
|
161
|
+
"pack": "sql",
|
|
162
|
+
"task": asked,
|
|
163
|
+
"level": level,
|
|
164
|
+
"reason": reason,
|
|
165
|
+
"answer": {"code": final["code"], "language": "sql"} if final["code"] else None,
|
|
166
|
+
"checked": unchecked is None,
|
|
167
|
+
"unchecked_reason": unchecked,
|
|
168
|
+
"trusted": unchecked is None and not final["issues"],
|
|
169
|
+
"final_step": steps.index(final) + 1,
|
|
170
|
+
"database": db and db[0],
|
|
171
|
+
"chosen_note": "the second opinion was flagged too, so the coder's query is kept" if db else None,
|
|
172
|
+
"steps": steps,
|
|
173
|
+
"tokens": sum(s["tokens"] for s in steps),
|
|
174
|
+
"seconds": time.time() - start,
|
|
175
|
+
}
|
plm_sql/flags.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Signs that a query which runs is still probably wrong, read from its result on a real database.
|
|
2
|
+
|
|
3
|
+
The schema-only check (plm_sql.controller.check) catches a query that doesn't run; on empty
|
|
4
|
+
tables it can't see one that runs and answers something else. With the user's database (opened
|
|
5
|
+
read-only), the result itself says more:
|
|
6
|
+
|
|
7
|
+
error the query doesn't run (or there is no query)
|
|
8
|
+
empty it returns no rows
|
|
9
|
+
huge it returns more than HUGE_ROWS rows
|
|
10
|
+
nulls every value it returns is NULL
|
|
11
|
+
tables the question names a table (every word of its name, plurals folded) and the query
|
|
12
|
+
uses none of the tables the question names
|
|
13
|
+
|
|
14
|
+
Fixed on 2026-10-27 from the saved Spider (300) and BIRD mini-dev (496) answers, before any model
|
|
15
|
+
answered the sets that judge them (evals/bench_sql_check.py, FLAGS_RULE).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
|
|
20
|
+
HUGE_ROWS = 100
|
|
21
|
+
WORD = re.compile(r"[A-Za-z]+")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def stem(word):
|
|
25
|
+
"""Lowercase, with a plural ending folded: "singers" -> "singer", "countries" -> "country"."""
|
|
26
|
+
w = word.lower()
|
|
27
|
+
for suffix in ("ies", "es", "s"):
|
|
28
|
+
if w.endswith(suffix) and len(w) > len(suffix) + 2:
|
|
29
|
+
return w[: -len(suffix)] + ("y" if suffix == "ies" else "")
|
|
30
|
+
return w
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def named_tables(question, tables):
|
|
34
|
+
"""The tables whose every name part (split on non-letters) appears in the question."""
|
|
35
|
+
words = {stem(w) for w in WORD.findall(question)}
|
|
36
|
+
return [t for t in tables if WORD.findall(t) and all(stem(p) in words for p in WORD.findall(t))]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def used_tables(query, tables):
|
|
40
|
+
return [t for t in tables if re.search(rf"(?<![\w.]){re.escape(t)}(?![\w])", query, re.IGNORECASE)]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def result_flags(question, query, ran, tables):
|
|
44
|
+
"""Flags for one query: `ran` is plm_sql.sandbox.run_query's result on the real database (or None
|
|
45
|
+
when there was no query), `tables` the database's table names. An empty list means nothing looks wrong."""
|
|
46
|
+
if query is None or ran is None or ran["error"]:
|
|
47
|
+
return ["error"]
|
|
48
|
+
flags = []
|
|
49
|
+
if ran["n_rows"] == 0:
|
|
50
|
+
flags.append("empty")
|
|
51
|
+
if ran["n_rows"] > HUGE_ROWS:
|
|
52
|
+
flags.append("huge")
|
|
53
|
+
if ran["n_rows"] and all(v is None for row in ran["rows"] for v in row):
|
|
54
|
+
flags.append("nulls")
|
|
55
|
+
named = named_tables(question, tables)
|
|
56
|
+
if named and not set(named) & set(used_tables(query, tables)):
|
|
57
|
+
flags.append("tables")
|
|
58
|
+
return flags
|
plm_sql/levels.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Adrenaline level for a SQL task, from its question and the tables in its schema: rules only, no model.
|
|
2
|
+
|
|
3
|
+
calm one table: filters, sorting, a single aggregate
|
|
4
|
+
alert one join or one GROUP BY: the question names two of the schema's tables, or asks per group
|
|
5
|
+
intensive subqueries, window functions, rows that are missing, comparisons against an aggregate
|
|
6
|
+
or another row, the top row of each group, counts thresholded per group
|
|
7
|
+
|
|
8
|
+
The cue families are written from those definitions (evals/sql/build_tasks.py), not fitted per task.
|
|
9
|
+
Evaluate them with: .venv/bin/python evals/eval_levels.py --pack sql
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
|
|
14
|
+
from plm_sql.sqlgen import split_task
|
|
15
|
+
|
|
16
|
+
INTENSIVE_CUES = {
|
|
17
|
+
"missing": r"\bnever\b|\bha(?:s|ve)n't\b|\bha(?:s|ve) no\b|\bnot (?:\w+ ){0,2}(?:to )?any\b|\bwithout (?:any|an?)\b",
|
|
18
|
+
"versus aggregate": r"\b(?:more|less|higher|lower|above|below|greater)\b.{0,30}\bthan (?:the |their |its )?"
|
|
19
|
+
r"(?:average|overall|mean|manager|budget)\b",
|
|
20
|
+
"top of group": r"\b(?:which|who)\b.{0,60}\b(?:the most|most often|the highest|the lowest|the fewest|the largest"
|
|
21
|
+
r"|most expensive|highest-paid)\b|\bfor each\b.{0,40}\b(?:most|highest|lowest)\b",
|
|
22
|
+
"all of a set": r"\bboth\b.{0,40}\band\b|\bby every\b|\bevery \w+ who\b",
|
|
23
|
+
"window": r"\branks?\b|\branking\b|\bprevious\b|\brunning (?:total|sum)\b|\bcumulative\b",
|
|
24
|
+
"group threshold": r"\bwith (?:at least|more than|fewer than) \d+\b",
|
|
25
|
+
"recursive": r"\brecursive\b|\beveryone under\b",
|
|
26
|
+
}
|
|
27
|
+
GROUP_CUE = re.compile(r"\beach\b|\bper\b", re.IGNORECASE)
|
|
28
|
+
TABLE_NAME = re.compile(r"create\s+table\s+(?:if\s+not\s+exists\s+)?[\"`\[]?(\w+)", re.IGNORECASE)
|
|
29
|
+
SORT_PHRASE = re.compile(r"\bin (?:\w+ )?order\b|\border(?:ed)? by\b", re.IGNORECASE)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def stem(table):
|
|
33
|
+
"""How a table is named in prose: the last word of its name, singular, cut to 6 letters ("enroll")."""
|
|
34
|
+
word = table.lower().rsplit("_", 1)[-1]
|
|
35
|
+
word = word[:-3] + "y" if word.endswith("ies") else word[:-1] if word.endswith("s") else word
|
|
36
|
+
return word[:6]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def tables_named(question, setup):
|
|
40
|
+
prose = SORT_PHRASE.sub(" ", question.lower())
|
|
41
|
+
return sorted({t for t in TABLE_NAME.findall(setup) if re.search(rf"\b{stem(t)}", prose)})
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def classify(text):
|
|
45
|
+
"""Return (level, reason)."""
|
|
46
|
+
task = split_task(text)
|
|
47
|
+
q = task["question"]
|
|
48
|
+
intensive = sorted(n for n, p in INTENSIVE_CUES.items() if re.search(p, q, re.IGNORECASE))
|
|
49
|
+
if intensive:
|
|
50
|
+
return "intensive", "cues: " + ", ".join(intensive)
|
|
51
|
+
named = tables_named(q, task["setup"])
|
|
52
|
+
if len(named) >= 2:
|
|
53
|
+
return "alert", "joins " + ", ".join(named)
|
|
54
|
+
if GROUP_CUE.search(q):
|
|
55
|
+
return "alert", "per-group cue"
|
|
56
|
+
return "calm", "one table, no group" if task["setup"] else "no schema, no join or group cues"
|
plm_sql/manifest.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""The SQL pack's manifest: what the router needs to know about it."""
|
|
2
|
+
|
|
3
|
+
from plm.pack import Pack
|
|
4
|
+
|
|
5
|
+
from plm_sql import controller
|
|
6
|
+
from plm_sql.claims import claims
|
|
7
|
+
|
|
8
|
+
# Priority below software's 100: both claim with certainty a Python function that runs SQL, and that
|
|
9
|
+
# is a software task; above the numeric packs, which a schema's column names can trip.
|
|
10
|
+
PACK = Pack(name="sql", version="0.1.0", claims=claims, priority=90, solve=controller.solve,
|
|
11
|
+
levels=controller.LEVELS, models=controller.MODELS, eval_set="evals/sql/tasks.jsonl")
|
plm_sql/sandbox.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Run one model-written query on a throwaway SQLite database, in an isolated process.
|
|
2
|
+
|
|
3
|
+
The database is built in memory from setup SQL (the task's CREATE TABLE statements,
|
|
4
|
+
and any INSERTs it gives), then made read-only: `PRAGMA query_only`, plus an
|
|
5
|
+
authorizer that allows only reads, so ATTACH, PRAGMA and writes are refused. A
|
|
6
|
+
progress handler stops a query that runs too long. It runs in a separate
|
|
7
|
+
`python -I -S` process in an empty temp directory, like plm/numeric/calc.py; the
|
|
8
|
+
point is to keep a runaway or hostile query away from this process and its files.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import tempfile
|
|
16
|
+
|
|
17
|
+
MAX_ROWS = 1000 # rows returned to the caller; the count is exact either way
|
|
18
|
+
|
|
19
|
+
HARNESS = r"""
|
|
20
|
+
import json, sqlite3, sys, time
|
|
21
|
+
|
|
22
|
+
p = json.loads(sys.stdin.read())
|
|
23
|
+
|
|
24
|
+
def out(**kw):
|
|
25
|
+
print("\n" + json.dumps(kw))
|
|
26
|
+
sys.exit(0)
|
|
27
|
+
|
|
28
|
+
if p.get("db_path"): # an existing database file, opened read-only
|
|
29
|
+
db = sqlite3.connect(f"file:{p['db_path']}?mode=ro", uri=True)
|
|
30
|
+
else:
|
|
31
|
+
db = sqlite3.connect(":memory:")
|
|
32
|
+
try:
|
|
33
|
+
db.executescript(p["setup"])
|
|
34
|
+
except sqlite3.Error as e:
|
|
35
|
+
out(setup_error=f"{type(e).__name__}: {e}"[:300])
|
|
36
|
+
db.execute("PRAGMA query_only = ON")
|
|
37
|
+
READS = {sqlite3.SQLITE_SELECT, sqlite3.SQLITE_READ, sqlite3.SQLITE_FUNCTION, getattr(sqlite3, "SQLITE_RECURSIVE", 33)}
|
|
38
|
+
db.set_authorizer(lambda action, *args: sqlite3.SQLITE_OK if action in READS else sqlite3.SQLITE_DENY)
|
|
39
|
+
start = time.time()
|
|
40
|
+
db.set_progress_handler(lambda: int(time.time() - start > p["seconds"]), 10000)
|
|
41
|
+
try:
|
|
42
|
+
cur = db.execute(p["query"])
|
|
43
|
+
columns = [d[0] for d in cur.description or []]
|
|
44
|
+
rows = cur.fetchall()
|
|
45
|
+
except Exception as e:
|
|
46
|
+
out(error=f"{type(e).__name__}: {e}"[:300])
|
|
47
|
+
out(columns=columns, rows=[list(r) for r in rows[:p["max_rows"]]], n_rows=len(rows))
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
READ_ONLY = re.compile(r"^\s*(?:--[^\n]*\n\s*)*(?:select|with|values)\b", re.IGNORECASE)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def run_query(setup, query, seconds=5.0, db_path=None):
|
|
54
|
+
"""{"error": str|None, "columns": [...], "rows": [[...]], "n_rows": int}; setup errors are reported as such.
|
|
55
|
+
|
|
56
|
+
With db_path, the query runs on that database file (opened read-only) instead of on `setup`.
|
|
57
|
+
"""
|
|
58
|
+
result = {"error": None, "setup_error": None, "columns": [], "rows": [], "n_rows": 0}
|
|
59
|
+
if not READ_ONLY.match(query or ""):
|
|
60
|
+
return result | {"error": "refused: not a single read-only query (SELECT or WITH)"}
|
|
61
|
+
payload = json.dumps({"setup": setup, "query": query, "seconds": seconds, "max_rows": MAX_ROWS,
|
|
62
|
+
"db_path": str(db_path) if db_path else None})
|
|
63
|
+
with tempfile.TemporaryDirectory() as cwd:
|
|
64
|
+
try:
|
|
65
|
+
proc = subprocess.run([sys.executable, "-I", "-S", "-c", HARNESS], input=payload, capture_output=True,
|
|
66
|
+
text=True, cwd=cwd, timeout=seconds + 10)
|
|
67
|
+
except subprocess.TimeoutExpired:
|
|
68
|
+
return result | {"error": "query process timed out"}
|
|
69
|
+
last = proc.stdout.strip().rsplit("\n", 1)[-1] if proc.stdout.strip() else ""
|
|
70
|
+
try:
|
|
71
|
+
got = json.loads(last)
|
|
72
|
+
except json.JSONDecodeError:
|
|
73
|
+
return result | {"error": f"query process failed: {proc.stderr.strip()[-200:]}"}
|
|
74
|
+
if got.get("error") == "OperationalError: interrupted":
|
|
75
|
+
got["error"] = f"query ran longer than {seconds:g} s"
|
|
76
|
+
return result | got
|
plm_sql/sqlgen.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Prompting for SQL tasks, splitting the schema out of a task, and pulling the query out of a reply."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import sqlite3
|
|
5
|
+
|
|
6
|
+
PROMPT_VERSION = 1
|
|
7
|
+
SQL_FORMAT = (
|
|
8
|
+
"Reply with one ```sql code block containing a single SQLite query (SELECT, optionally with WITH). "
|
|
9
|
+
"Return exactly the columns the question asks for, in the order it names them, and nothing else."
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
STATEMENT_START = re.compile(r"^\s*(?:create\s+(?:table|view|index|unique\s+index)|insert\s+into)\b", re.IGNORECASE)
|
|
13
|
+
SQL_RE = re.compile(r"```(?:sql|sqlite)?[ \t]*\n(.*?)```", re.DOTALL | re.IGNORECASE)
|
|
14
|
+
BARE_QUERY = re.compile(r"^\s*(?:select|with)\b.*", re.IGNORECASE | re.DOTALL)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def split_task(text):
|
|
18
|
+
"""{"setup": the CREATE/INSERT statements in the text, "question": the rest, "has_rows": any INSERT}."""
|
|
19
|
+
setup, rest, buf = [], [], []
|
|
20
|
+
for line in text.splitlines():
|
|
21
|
+
if buf or STATEMENT_START.match(line):
|
|
22
|
+
buf.append(line)
|
|
23
|
+
if sqlite3.complete_statement("\n".join(buf)):
|
|
24
|
+
setup.append("\n".join(buf))
|
|
25
|
+
buf = []
|
|
26
|
+
else:
|
|
27
|
+
rest.append(line)
|
|
28
|
+
rest += buf # an unterminated statement is left in the question
|
|
29
|
+
return {"setup": "\n".join(setup), "question": "\n".join(rest).strip(),
|
|
30
|
+
"has_rows": any(s.lstrip().lower().startswith("insert") for s in setup)}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def build_prompt(text):
|
|
34
|
+
return f"{text.strip()}\n\n{SQL_FORMAT}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def extract_sql(text):
|
|
38
|
+
"""The last fenced block; else the reply itself if it is a bare query. Trailing semicolons dropped."""
|
|
39
|
+
blocks = SQL_RE.findall(text or "")
|
|
40
|
+
if blocks:
|
|
41
|
+
query = blocks[-1]
|
|
42
|
+
elif BARE_QUERY.match(text or ""):
|
|
43
|
+
query = text
|
|
44
|
+
else:
|
|
45
|
+
return None
|
|
46
|
+
return query.strip().rstrip(";").strip() or None
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: plm-sql
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: plm pack: SQLite queries, checked by running them on the schema given with the task
|
|
5
|
+
Project-URL: Homepage, https://github.com/ro-rbl/plm-orchestrator
|
|
6
|
+
Project-URL: Source, https://github.com/ro-rbl/plm-orchestrator
|
|
7
|
+
Project-URL: Issues, https://github.com/ro-rbl/plm-orchestrator/issues
|
|
8
|
+
Author: ro-rbl
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Python: >=3.13
|
|
12
|
+
Requires-Dist: plm-core<0.2,>=0.1
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
# plm-sql
|
|
16
|
+
|
|
17
|
+
Plm pack: SQLite queries, checked by running them on the schema given with the task.
|
|
18
|
+
|
|
19
|
+
A domain pack for [plm-core](https://github.com/ro-rbl/plm-orchestrator): install it with `plm packs install sql`, which also pulls the
|
|
20
|
+
models it needs into Ollama. Documentation, benchmark results and caveats: https://github.com/ro-rbl/plm-orchestrator
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
plm_sql/__init__.py,sha256=OO0TIq7ZcyMFlMTS9WMqg97879UshYPnzu6z1PqfWbE,101
|
|
2
|
+
plm_sql/claims.py,sha256=Vs1U0_u0D1E-RU94x_j7Zd0dZrAQIr4QXObrz0TEMYQ,5297
|
|
3
|
+
plm_sql/controller.py,sha256=oKKm3t0OUOEhlsVB809yBoumJyHbki7fj5NeGPcfXJw,8246
|
|
4
|
+
plm_sql/flags.py,sha256=UfV_vCm0qnlE-vWrClIqGnmD095oESd0UGeDLA1sJ5U,2384
|
|
5
|
+
plm_sql/levels.py,sha256=27t8vhIeZRVoaYATv_BcZ0x2hXoaK4lOsCU79Evf8ks,2817
|
|
6
|
+
plm_sql/manifest.py,sha256=jsibYtMPK4Os28TlYPVeFzM2OUkp66nTjEZ5O09ofuA,544
|
|
7
|
+
plm_sql/sandbox.py,sha256=AakoiiQ5gB-A8rIxhSW5UVxsiqmAFpUWvrGXRq_C2FM,3323
|
|
8
|
+
plm_sql/sqlgen.py,sha256=hlQtR_G7nE5mNquiTRLVGbG6HT2Xus5A5T12uD6hzRs,1767
|
|
9
|
+
plm_sql-0.1.0.dist-info/METADATA,sha256=C5ubEspNnptvhlaLwGqI1ABFbTe5WvnJoArb9D4L3bo,856
|
|
10
|
+
plm_sql-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
11
|
+
plm_sql-0.1.0.dist-info/entry_points.txt,sha256=Ex3pTwYwh1o7mBLxO0RVlC5qWPgxasaHR4gxrzjx4Dg,40
|
|
12
|
+
plm_sql-0.1.0.dist-info/licenses/LICENSE,sha256=KcrxZgcJO8eG0MOaibWc_3Pxvh6U_KruTKPaGwV6-vA,1063
|
|
13
|
+
plm_sql-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ro-rbl
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|