sql-harness 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sql_harness/SKILL.md +61 -0
- sql_harness/__init__.py +27 -0
- sql_harness/agent_loader.py +55 -0
- sql_harness/cli.py +869 -0
- sql_harness/config.py +241 -0
- sql_harness/drivers/__init__.py +59 -0
- sql_harness/drivers/mysql.py +43 -0
- sql_harness/drivers/postgres.py +55 -0
- sql_harness/drivers/redis.py +33 -0
- sql_harness/drivers/sqlite.py +39 -0
- sql_harness/drivers/ssh.py +293 -0
- sql_harness/helpers.py +789 -0
- sql_harness/manager.py +122 -0
- sql_harness/paths.py +112 -0
- sql_harness/run.py +85 -0
- sql_harness-0.1.0.dist-info/METADATA +137 -0
- sql_harness-0.1.0.dist-info/RECORD +19 -0
- sql_harness-0.1.0.dist-info/WHEEL +4 -0
- sql_harness-0.1.0.dist-info/entry_points.txt +2 -0
sql_harness/helpers.py
ADDED
|
@@ -0,0 +1,789 @@
|
|
|
1
|
+
"""Heredoc helpers — auto-imported into the sql-harness exec namespace.
|
|
2
|
+
|
|
3
|
+
These are thin wrappers over the active workspace's engine. They expect
|
|
4
|
+
`set_active(...)` to have been called by `run.py` (or `use_workspace(name)`
|
|
5
|
+
within the heredoc).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import importlib.util
|
|
11
|
+
import re
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from sqlalchemy import text
|
|
17
|
+
from sqlalchemy.engine import Connection
|
|
18
|
+
|
|
19
|
+
from . import paths
|
|
20
|
+
from .config import ConnectionConfig
|
|
21
|
+
from .manager import SqlHarness, Workspace
|
|
22
|
+
|
|
23
|
+
# --- Active-harness state (set once per process by run.py / cli.py) ---------
|
|
24
|
+
|
|
25
|
+
_active: SqlHarness | None = None
|
|
26
|
+
_current_workspace: str = ""
|
|
27
|
+
_active_connection: str = "" # the DSN zone (mirrors browser-harness domain)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def set_active(harness: SqlHarness) -> None:
|
|
31
|
+
"""Called by run.py after construction. Not for heredoc use.
|
|
32
|
+
|
|
33
|
+
Resets the global current-workspace pointer so a fresh process starts
|
|
34
|
+
cleanly (relevant for tests).
|
|
35
|
+
"""
|
|
36
|
+
global _active, _current_workspace, _active_connection
|
|
37
|
+
_active = harness
|
|
38
|
+
_current_workspace = ""
|
|
39
|
+
_active_connection = ""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def active() -> SqlHarness:
|
|
43
|
+
if _active is None:
|
|
44
|
+
raise RuntimeError(
|
|
45
|
+
"sql-harness is not initialized; this should be unreachable "
|
|
46
|
+
"from a properly-launched sql-harness process."
|
|
47
|
+
)
|
|
48
|
+
return _active
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# --- Connection / zone resolution ------------------------------------------
|
|
52
|
+
|
|
53
|
+
def active_connection() -> str:
|
|
54
|
+
"""Return the currently-active connection name (the DSN zone)."""
|
|
55
|
+
return _active_connection
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _require_connection() -> str:
|
|
59
|
+
if not _active_connection:
|
|
60
|
+
raise RuntimeError(
|
|
61
|
+
"no connection is active; call use_workspace(name) first to set "
|
|
62
|
+
"the DSN zone for scripts/skills/helpers"
|
|
63
|
+
)
|
|
64
|
+
return _active_connection
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _load_zone_helpers(connection: str, target_globals: dict) -> int:
|
|
68
|
+
"""Merge a connection's per-zone helpers.py over the current namespace.
|
|
69
|
+
|
|
70
|
+
Mirrors browser-harness's auto-load of agent_helpers.py, but scoped per
|
|
71
|
+
DSN. Returns the number of public names merged (0 if no file).
|
|
72
|
+
"""
|
|
73
|
+
path = paths.zone_helpers_file(connection)
|
|
74
|
+
if not path.is_file():
|
|
75
|
+
return 0
|
|
76
|
+
spec = importlib.util.spec_from_file_location(
|
|
77
|
+
f"sql_harness_zone_helpers_{connection}", path
|
|
78
|
+
)
|
|
79
|
+
if spec is None or spec.loader is None:
|
|
80
|
+
return 0
|
|
81
|
+
module = importlib.util.module_from_spec(spec)
|
|
82
|
+
# Pre-seed with current namespace so zone helpers can call query()/etc.
|
|
83
|
+
for n, v in target_globals.items():
|
|
84
|
+
if not n.startswith("__"):
|
|
85
|
+
setattr(module, n, v)
|
|
86
|
+
spec.loader.exec_module(module)
|
|
87
|
+
merged = 0
|
|
88
|
+
for name, value in vars(module).items():
|
|
89
|
+
if name.startswith("_"):
|
|
90
|
+
continue
|
|
91
|
+
target_globals[name] = value
|
|
92
|
+
merged += 1
|
|
93
|
+
return merged
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# --- Workspace management ---------------------------------------------------
|
|
97
|
+
|
|
98
|
+
def use_workspace(name: str, _globals: dict | None = None) -> "Workspace | dict":
|
|
99
|
+
"""Switch the active workspace (opens engine lazily) + activate its zone.
|
|
100
|
+
|
|
101
|
+
Mirrors browser-harness's goto_url: switching context also activates the
|
|
102
|
+
per-DSN isolation zone. Any per-connection `helpers.py` is merged over
|
|
103
|
+
the namespace (overriding the shared base), so its functions become
|
|
104
|
+
callable as bare names in heredoc / run mode.
|
|
105
|
+
|
|
106
|
+
Return value (mirrors browser-harness's domain-skills pattern):
|
|
107
|
+
- Default: returns the Workspace object (no info dump).
|
|
108
|
+
- When $BH_SQL_ZONE_SKILLS=1: returns a dict with the Workspace
|
|
109
|
+
+ auto-surfaced zone skills + scripts. Agent should read each
|
|
110
|
+
surfaced skill BEFORE touching tables.
|
|
111
|
+
|
|
112
|
+
`_globals` is normally left None: per-zone helpers are merged into this
|
|
113
|
+
helpers module's own namespace, which run.py shares as the exec namespace.
|
|
114
|
+
"""
|
|
115
|
+
import os as _os
|
|
116
|
+
global _current_workspace, _active_connection
|
|
117
|
+
h = active()
|
|
118
|
+
ws = h.workspace(name)
|
|
119
|
+
_current_workspace = name
|
|
120
|
+
_active_connection = name
|
|
121
|
+
|
|
122
|
+
# Merge per-connection helpers over the namespace (zone helpers win).
|
|
123
|
+
# Default target = this module's globals (the exec namespace run.py uses).
|
|
124
|
+
target = _globals if _globals is not None else globals()
|
|
125
|
+
_load_zone_helpers(name, target)
|
|
126
|
+
|
|
127
|
+
# Optional auto-surface: when $BH_SQL_ZONE_SKILLS=1, return a dict that
|
|
128
|
+
# includes the zone's skills + scripts so the agent can read them before
|
|
129
|
+
# touching the tables. Mirrors browser-harness's BH_DOMAIN_SKILLS=1 +
|
|
130
|
+
# goto_url returning domain_skills.
|
|
131
|
+
if _os.environ.get("BH_SQL_ZONE_SKILLS") == "1":
|
|
132
|
+
return {
|
|
133
|
+
"workspace": ws,
|
|
134
|
+
"connection": name,
|
|
135
|
+
"driver": ws.driver.name,
|
|
136
|
+
"zone_skills": _list_zone_skills(name, limit=10),
|
|
137
|
+
"zone_scripts": _list_zone_scripts(name, limit=10),
|
|
138
|
+
"hint": (
|
|
139
|
+
"Set $BH_SQL_ZONE_SKILLS=1 to auto-surface these. "
|
|
140
|
+
"Read each zone_skill before querying — it captures non-obvious "
|
|
141
|
+
"schema knowledge. When the flag is off, you get the bare "
|
|
142
|
+
"Workspace object instead."
|
|
143
|
+
),
|
|
144
|
+
}
|
|
145
|
+
return ws
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _list_zone_skills(connection: str, limit: int = 10) -> list[str]:
|
|
149
|
+
"""List skill filenames in the connection's zone (cap at `limit`)."""
|
|
150
|
+
d = paths.zone_skills_dir(connection)
|
|
151
|
+
if not d.is_dir():
|
|
152
|
+
return []
|
|
153
|
+
return sorted(p.name for p in d.glob("*.md"))[:limit]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _list_zone_scripts(connection: str, limit: int = 10) -> list[str]:
|
|
157
|
+
"""List saved script filenames in the connection's zone (cap at `limit`)."""
|
|
158
|
+
d = paths.zone_scripts_dir(connection)
|
|
159
|
+
if not d.is_dir():
|
|
160
|
+
return []
|
|
161
|
+
return sorted(p.stem for p in d.glob("*.py"))[:limit]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def use_workspace_info(name: str, _globals: dict | None = None) -> dict:
|
|
165
|
+
"""Always returns a dict describing the activated zone (forces the surface).
|
|
166
|
+
|
|
167
|
+
Use from heredocs when you want to see what skills/scripts exist for the
|
|
168
|
+
DSN you just switched to, regardless of the BH_SQL_ZONE_SKILLS env var.
|
|
169
|
+
Mirrors browser-harness's goto_url returning domain skills.
|
|
170
|
+
"""
|
|
171
|
+
use_workspace(name, _globals=_globals)
|
|
172
|
+
return {
|
|
173
|
+
"connection": name,
|
|
174
|
+
"driver": current_workspace().driver.name,
|
|
175
|
+
"skills": list_skills(),
|
|
176
|
+
"scripts": list_scripts(),
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def current_workspace() -> Workspace:
|
|
181
|
+
"""Return the currently-active workspace (must already be opened)."""
|
|
182
|
+
h = active()
|
|
183
|
+
if not _current_workspace:
|
|
184
|
+
raise RuntimeError(
|
|
185
|
+
"no workspace is active; call use_workspace(name) first "
|
|
186
|
+
"(config has default_workspace=" + repr(h.default_workspace_name) + ")"
|
|
187
|
+
)
|
|
188
|
+
ws = h.get_workspace(_current_workspace)
|
|
189
|
+
if ws is None:
|
|
190
|
+
# Stale pointer — reopen.
|
|
191
|
+
return use_workspace(_current_workspace)
|
|
192
|
+
return ws
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def workspaces() -> list[str]:
|
|
196
|
+
"""List names of all open workspaces."""
|
|
197
|
+
return active().list_workspaces()
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def connection_info() -> dict:
|
|
201
|
+
"""Return dict: name, driver, url (masked), pool settings, status."""
|
|
202
|
+
ws = current_workspace()
|
|
203
|
+
pool = ws.engine.pool
|
|
204
|
+
# SQLAlchemy's pool.size is a property; NullPool has size() as int attr.
|
|
205
|
+
size_attr = getattr(pool, "size", None)
|
|
206
|
+
size = size_attr() if callable(size_attr) else size_attr
|
|
207
|
+
checked = getattr(pool, "checkedout", lambda: None)()
|
|
208
|
+
overflow = getattr(pool, "overflow", lambda: None)()
|
|
209
|
+
return {
|
|
210
|
+
"name": ws.config.connection.name,
|
|
211
|
+
"driver": ws.driver.name,
|
|
212
|
+
"url": ws.config.connection.masked_url(),
|
|
213
|
+
"application_name": ws.config.connection.application_name,
|
|
214
|
+
"pool": {
|
|
215
|
+
"size": size,
|
|
216
|
+
"checked_out": checked,
|
|
217
|
+
"overflow": overflow,
|
|
218
|
+
},
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def dispose() -> None:
|
|
223
|
+
"""Close every engine."""
|
|
224
|
+
active().close_all()
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
# --- Query helpers ----------------------------------------------------------
|
|
228
|
+
|
|
229
|
+
def query(sql: str, params: dict | list | None = None) -> list[dict]:
|
|
230
|
+
"""Run a SELECT and return rows as list[dict] (column name -> value)."""
|
|
231
|
+
ws = current_workspace()
|
|
232
|
+
with ws.engine.connect() as conn:
|
|
233
|
+
result = conn.execute(text(sql), params or {})
|
|
234
|
+
cols = result.keys()
|
|
235
|
+
return [dict(zip(cols, row)) for row in result.fetchall()]
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def execute(sql: str, params: dict | list | None = None) -> dict:
|
|
239
|
+
"""Run a write statement; return rowcount + lastrowid/oid info."""
|
|
240
|
+
ws = current_workspace()
|
|
241
|
+
with ws.engine.begin() as conn:
|
|
242
|
+
result = conn.execute(text(sql), params or {})
|
|
243
|
+
info: dict[str, Any] = {"rowcount": result.rowcount}
|
|
244
|
+
# INSERTs expose lastrowid (MySQL) / inserted_primary_key (PG via RETURNING).
|
|
245
|
+
# Both can raise on non-INSERT statements or on drivers that lack the
|
|
246
|
+
# attribute (psycopg3 cursor has no `lastrowid`), so probe defensively.
|
|
247
|
+
try:
|
|
248
|
+
lrid = result.lastrowid
|
|
249
|
+
except Exception:
|
|
250
|
+
lrid = None
|
|
251
|
+
if lrid is not None:
|
|
252
|
+
info["lastrowid"] = lrid
|
|
253
|
+
try:
|
|
254
|
+
ipk = result.inserted_primary_key
|
|
255
|
+
except Exception:
|
|
256
|
+
ipk = None
|
|
257
|
+
if ipk:
|
|
258
|
+
info["inserted_primary_key"] = list(ipk)
|
|
259
|
+
return info
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def table(name: str, schema: str | None = None, limit: int | None = None) -> list[dict]:
|
|
263
|
+
"""SELECT * FROM <name> (small tables only — use streaming for big ones)."""
|
|
264
|
+
ws = current_workspace()
|
|
265
|
+
qid = ws.driver.quote_ident(name)
|
|
266
|
+
sql = f"SELECT * FROM {qid}"
|
|
267
|
+
if schema:
|
|
268
|
+
sql = f"SELECT * FROM {ws.driver.quote_ident(schema)}.{qid}"
|
|
269
|
+
if limit is not None:
|
|
270
|
+
sql += f" LIMIT {int(limit)}"
|
|
271
|
+
return query(sql)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def list_tables(schema: str | None = None) -> list[str]:
|
|
275
|
+
"""List all tables and views in the active workspace."""
|
|
276
|
+
return current_workspace().driver.list_tables(current_workspace().engine, schema)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def describe(table: str, schema: str | None = None) -> list[dict]:
|
|
280
|
+
"""Return column metadata for the given table."""
|
|
281
|
+
return current_workspace().driver.describe(
|
|
282
|
+
current_workspace().engine, table, schema
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def explain(sql: str) -> list[dict]:
|
|
287
|
+
"""Run EXPLAIN on the given SQL and return the plan rows."""
|
|
288
|
+
ws = current_workspace()
|
|
289
|
+
if ws.driver.name in ("postgres", "mysql", "sqlite"):
|
|
290
|
+
prefix = "EXPLAIN"
|
|
291
|
+
else:
|
|
292
|
+
raise NotImplementedError(f"explain not supported for driver {ws.driver.name}")
|
|
293
|
+
return query(f"{prefix} {sql}")
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
@contextmanager
|
|
297
|
+
def with_transaction():
|
|
298
|
+
"""Run statements inside a transaction with auto-commit/rollback.
|
|
299
|
+
|
|
300
|
+
Use the yielded connection for raw control:
|
|
301
|
+
|
|
302
|
+
with with_transaction() as conn:
|
|
303
|
+
conn.execute(text("..."))
|
|
304
|
+
"""
|
|
305
|
+
ws = current_workspace()
|
|
306
|
+
with ws.engine.begin() as conn:
|
|
307
|
+
yield conn
|
|
308
|
+
# `engine.begin()` auto-commits on clean exit; auto-rollback on exception.
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
# --- Skill / script helpers (per-DSN zone) ---------------------------------
|
|
312
|
+
|
|
313
|
+
def apply_skill(name: str) -> str:
|
|
314
|
+
"""Read a skill markdown from the ACTIVE connection's zone (or global fallback).
|
|
315
|
+
|
|
316
|
+
Mirrors browser-harness's per-domain skill lookup. Zone wins if present;
|
|
317
|
+
falls back to the global agent-workspace/skills/ directory so shipped
|
|
318
|
+
skills (pool.md, workspace.md) remain reachable.
|
|
319
|
+
"""
|
|
320
|
+
conn = _require_connection()
|
|
321
|
+
zone_path = paths.zone_skills_dir(conn) / f"{name}.md"
|
|
322
|
+
if zone_path.is_file():
|
|
323
|
+
return zone_path.read_text(encoding="utf-8")
|
|
324
|
+
# Fallback to global skills dir (shipped examples).
|
|
325
|
+
global_path = paths.workspace_dir() / "skills" / f"{name}.md"
|
|
326
|
+
if global_path.is_file():
|
|
327
|
+
return global_path.read_text(encoding="utf-8")
|
|
328
|
+
raise FileNotFoundError(
|
|
329
|
+
f"no skill named {name!r} in zone {conn!r} or global {paths.workspace_dir() / 'skills'}"
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def list_skills() -> list[str]:
|
|
334
|
+
"""List skills in the ACTIVE connection's zone UNION the global skills dir."""
|
|
335
|
+
zone = set()
|
|
336
|
+
d = paths.zone_skills_dir(_require_connection())
|
|
337
|
+
if d.is_dir():
|
|
338
|
+
zone.update(p.stem for p in d.glob("*.md"))
|
|
339
|
+
global_d = paths.workspace_dir() / "skills"
|
|
340
|
+
if global_d.is_dir():
|
|
341
|
+
zone.update(p.stem for p in global_d.glob("*.md"))
|
|
342
|
+
return sorted(zone)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def list_scripts() -> list[str]:
|
|
346
|
+
"""List saved-script names in the ACTIVE connection's zone."""
|
|
347
|
+
conn = _require_connection()
|
|
348
|
+
d = paths.zone_scripts_dir(conn)
|
|
349
|
+
return sorted(p.stem for p in d.glob("*.py")) if d.is_dir() else []
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
# --- Server version (handy diagnostic) -------------------------------------
|
|
353
|
+
|
|
354
|
+
def server_version() -> str:
|
|
355
|
+
"""Return the backend's reported version string."""
|
|
356
|
+
ws = current_workspace()
|
|
357
|
+
fn = getattr(ws.driver, "server_version", None)
|
|
358
|
+
if fn is None:
|
|
359
|
+
return f"{ws.driver.name} (no server_version helper)"
|
|
360
|
+
return fn(ws.engine)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
# --- PostgreSQL performance helpers -----------------------------------------
|
|
364
|
+
#
|
|
365
|
+
# Wrap PostgreSQL's statistics catalogs (pg_stat_user_tables,
|
|
366
|
+
# pg_stat_user_indexes, pg_constraint, pg_stat_statements) and EXPLAIN ANALYZE.
|
|
367
|
+
# Each guards on `ws.driver.name == "postgres"` and raises NotImplementedError
|
|
368
|
+
# otherwise — same pattern as explain(). Mined from the upstream
|
|
369
|
+
# "PostgreSQL Performance Essentials" guide; see interaction-skills/postgres/.
|
|
370
|
+
|
|
371
|
+
def explain_analyze(sql: str, buffers: bool = True, format: str = "text") -> list[dict]:
|
|
372
|
+
"""Run EXPLAIN (ANALYZE) on ``sql`` and return the plan rows.
|
|
373
|
+
|
|
374
|
+
WARNING: ANALYZE *executes* the statement. Pass only read-only SELECTs,
|
|
375
|
+
or wrap writes in a transaction you roll back::
|
|
376
|
+
|
|
377
|
+
with with_transaction() as conn:
|
|
378
|
+
explain_analyze("DELETE FROM accounts WHERE ...") # then rollback
|
|
379
|
+
|
|
380
|
+
``buffers`` adds Buffer hit/miss detail. ``format`` is one of
|
|
381
|
+
text|json|xml|yaml. Output mirrors :func:`explain`: a list of row dicts;
|
|
382
|
+
for ``format='text'`` each row is ``{'QUERY PLAN': '<line>'}``.
|
|
383
|
+
"""
|
|
384
|
+
ws = current_workspace()
|
|
385
|
+
if ws.driver.name != "postgres":
|
|
386
|
+
raise NotImplementedError(
|
|
387
|
+
f"explain_analyze not supported for driver {ws.driver.name}"
|
|
388
|
+
)
|
|
389
|
+
fmt = format.upper()
|
|
390
|
+
if fmt not in ("TEXT", "JSON", "XML", "YAML"):
|
|
391
|
+
raise ValueError(f"unsupported EXPLAIN format: {format!r}")
|
|
392
|
+
opts = ["ANALYZE"]
|
|
393
|
+
if buffers:
|
|
394
|
+
opts.append("BUFFERS")
|
|
395
|
+
opts.append(f"FORMAT {fmt}") # whitelisted; safe to interpolate
|
|
396
|
+
return query(f"EXPLAIN ({', '.join(opts)}) {sql}")
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def table_stats(name: str, schema: str | None = None) -> list[dict]:
|
|
400
|
+
"""Return size + live/dead row estimates for one table (pg_stat_user_tables).
|
|
401
|
+
|
|
402
|
+
Columns: schemaname, table_name, live_rows, dead_rows, total_size_bytes,
|
|
403
|
+
total_size, table_size, last_vacuum, last_analyze, vacuum_count,
|
|
404
|
+
analyze_count.
|
|
405
|
+
"""
|
|
406
|
+
ws = current_workspace()
|
|
407
|
+
if ws.driver.name != "postgres":
|
|
408
|
+
raise NotImplementedError(
|
|
409
|
+
f"table_stats not supported for driver {ws.driver.name}"
|
|
410
|
+
)
|
|
411
|
+
return query(
|
|
412
|
+
"""
|
|
413
|
+
SELECT s.schemaname,
|
|
414
|
+
s.relname AS table_name,
|
|
415
|
+
s.n_live_tup AS live_rows,
|
|
416
|
+
s.n_dead_tup AS dead_rows,
|
|
417
|
+
pg_total_relation_size(s.relid) AS total_size_bytes,
|
|
418
|
+
pg_size_pretty(pg_total_relation_size(s.relid)) AS total_size,
|
|
419
|
+
pg_size_pretty(pg_relation_size(s.relid)) AS table_size,
|
|
420
|
+
s.last_vacuum,
|
|
421
|
+
s.last_analyze,
|
|
422
|
+
s.vacuum_count,
|
|
423
|
+
s.analyze_count
|
|
424
|
+
FROM pg_stat_user_tables s
|
|
425
|
+
WHERE s.relname = :name
|
|
426
|
+
AND (:schema IS NULL OR s.schemaname = :schema)
|
|
427
|
+
""",
|
|
428
|
+
{"name": name, "schema": schema},
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def index_usage_stats(table: str | None = None, schema: str | None = None) -> list[dict]:
|
|
433
|
+
"""Per-index scan counts from pg_stat_user_indexes.
|
|
434
|
+
|
|
435
|
+
Columns: schemaname, table_name, index_name, idx_scan, idx_tup_read,
|
|
436
|
+
idx_tup_fetch. Rows with ``idx_scan = 0`` are drop candidates
|
|
437
|
+
(see :func:`unused_indexes`).
|
|
438
|
+
"""
|
|
439
|
+
ws = current_workspace()
|
|
440
|
+
if ws.driver.name != "postgres":
|
|
441
|
+
raise NotImplementedError(
|
|
442
|
+
f"index_usage_stats not supported for driver {ws.driver.name}"
|
|
443
|
+
)
|
|
444
|
+
return query(
|
|
445
|
+
"""
|
|
446
|
+
SELECT s.schemaname,
|
|
447
|
+
s.relname AS table_name,
|
|
448
|
+
s.indexrelname AS index_name,
|
|
449
|
+
s.idx_scan,
|
|
450
|
+
s.idx_tup_read,
|
|
451
|
+
s.idx_tup_fetch
|
|
452
|
+
FROM pg_stat_user_indexes s
|
|
453
|
+
WHERE (:table IS NULL OR s.relname = :table)
|
|
454
|
+
AND (:schema IS NULL OR s.schemaname = :schema)
|
|
455
|
+
ORDER BY s.schemaname, s.relname, s.indexrelname
|
|
456
|
+
""",
|
|
457
|
+
{"table": table, "schema": schema},
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def unused_indexes(table: str | None = None, schema: str | None = None) -> list[dict]:
|
|
462
|
+
"""Indexes never scanned (``idx_scan = 0``) — drop candidates.
|
|
463
|
+
|
|
464
|
+
Excludes unique indexes (needed for constraints) and expression indexes
|
|
465
|
+
(``indkey`` contains 0), which may legitimately show zero scans.
|
|
466
|
+
Columns: schemaname, table_name, index_name, times_used,
|
|
467
|
+
index_size_bytes, index_size, index_ddl.
|
|
468
|
+
"""
|
|
469
|
+
ws = current_workspace()
|
|
470
|
+
if ws.driver.name != "postgres":
|
|
471
|
+
raise NotImplementedError(
|
|
472
|
+
f"unused_indexes not supported for driver {ws.driver.name}"
|
|
473
|
+
)
|
|
474
|
+
return query(
|
|
475
|
+
"""
|
|
476
|
+
SELECT s.schemaname,
|
|
477
|
+
s.relname AS table_name,
|
|
478
|
+
s.indexrelname AS index_name,
|
|
479
|
+
s.idx_scan AS times_used,
|
|
480
|
+
pg_relation_size(s.indexrelid) AS index_size_bytes,
|
|
481
|
+
pg_size_pretty(pg_relation_size(s.indexrelid)) AS index_size,
|
|
482
|
+
idx.indexdef AS index_ddl
|
|
483
|
+
FROM pg_stat_user_indexes s
|
|
484
|
+
JOIN pg_index i ON i.indexrelid = s.indexrelid
|
|
485
|
+
JOIN pg_indexes idx ON idx.schemaname = s.schemaname
|
|
486
|
+
AND idx.indexname = s.indexrelname
|
|
487
|
+
WHERE s.idx_scan = 0
|
|
488
|
+
AND 0 <> ALL(i.indkey)
|
|
489
|
+
AND NOT i.indisunique
|
|
490
|
+
AND (:table IS NULL OR s.relname = :table)
|
|
491
|
+
AND (:schema IS NULL OR s.schemaname = :schema)
|
|
492
|
+
ORDER BY pg_relation_size(s.indexrelid) DESC
|
|
493
|
+
""",
|
|
494
|
+
{"table": table, "schema": schema},
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def seq_scan_heavy(table: str | None = None, schema: str | None = None) -> list[dict]:
|
|
499
|
+
"""Tables being full-scanned a lot, ranked by rows touched (pg_stat_user_tables).
|
|
500
|
+
|
|
501
|
+
High ``seq_tup_read`` with ``seq_scan > 0`` hints a missing index.
|
|
502
|
+
Columns: schemaname, table_name, seq_scan, seq_tup_read, idx_scan,
|
|
503
|
+
idx_tup_fetch.
|
|
504
|
+
"""
|
|
505
|
+
ws = current_workspace()
|
|
506
|
+
if ws.driver.name != "postgres":
|
|
507
|
+
raise NotImplementedError(
|
|
508
|
+
f"seq_scan_heavy not supported for driver {ws.driver.name}"
|
|
509
|
+
)
|
|
510
|
+
return query(
|
|
511
|
+
"""
|
|
512
|
+
SELECT s.schemaname,
|
|
513
|
+
s.relname AS table_name,
|
|
514
|
+
s.seq_scan,
|
|
515
|
+
s.seq_tup_read,
|
|
516
|
+
s.idx_scan,
|
|
517
|
+
s.idx_tup_fetch
|
|
518
|
+
FROM pg_stat_user_tables s
|
|
519
|
+
WHERE s.seq_scan > 0
|
|
520
|
+
AND (:table IS NULL OR s.relname = :table)
|
|
521
|
+
AND (:schema IS NULL OR s.schemaname = :schema)
|
|
522
|
+
ORDER BY s.seq_tup_read DESC
|
|
523
|
+
""",
|
|
524
|
+
{"table": table, "schema": schema},
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def slow_queries(limit: int = 10) -> list[dict]:
|
|
529
|
+
"""Top-N slow queries from pg_stat_statements by mean exec time.
|
|
530
|
+
|
|
531
|
+
Requires the ``pg_stat_statements`` extension *and*
|
|
532
|
+
``shared_preload_libraries='pg_stat_statements'``; if absent the backend
|
|
533
|
+
raises. Columns: short_query, calls, total_exec_time_ms, mean_exec_time_ms,
|
|
534
|
+
percent_of_total. (PG 13+ column names.)
|
|
535
|
+
"""
|
|
536
|
+
ws = current_workspace()
|
|
537
|
+
if ws.driver.name != "postgres":
|
|
538
|
+
raise NotImplementedError(
|
|
539
|
+
f"slow_queries not supported for driver {ws.driver.name}"
|
|
540
|
+
)
|
|
541
|
+
return query(
|
|
542
|
+
"""
|
|
543
|
+
SELECT SUBSTRING(query, 1, 80) AS short_query,
|
|
544
|
+
calls,
|
|
545
|
+
ROUND(total_exec_time::numeric, 2) AS total_exec_time_ms,
|
|
546
|
+
ROUND(mean_exec_time::numeric, 2) AS mean_exec_time_ms,
|
|
547
|
+
ROUND((100 * total_exec_time
|
|
548
|
+
/ NULLIF(SUM(total_exec_time) OVER (), 0))::numeric, 2) AS percent_of_total
|
|
549
|
+
FROM pg_stat_statements
|
|
550
|
+
ORDER BY mean_exec_time DESC
|
|
551
|
+
LIMIT :limit
|
|
552
|
+
""",
|
|
553
|
+
{"limit": int(limit)},
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def missing_indexes_hint(table: str | None = None, schema: str | None = None) -> list[dict]:
|
|
558
|
+
"""Foreign-key columns with no covering index — suggests CREATE INDEX.
|
|
559
|
+
|
|
560
|
+
Returns a ready-to-run ``CREATE INDEX`` for each FK whose leading columns
|
|
561
|
+
aren't covered by any valid index. Columns: schema_name, table_name,
|
|
562
|
+
fk_constraint_name, referenced_table, fk_columns, create_sql.
|
|
563
|
+
"""
|
|
564
|
+
ws = current_workspace()
|
|
565
|
+
if ws.driver.name != "postgres":
|
|
566
|
+
raise NotImplementedError(
|
|
567
|
+
f"missing_indexes_hint not supported for driver {ws.driver.name}"
|
|
568
|
+
)
|
|
569
|
+
return query(
|
|
570
|
+
"""
|
|
571
|
+
SELECT n.nspname AS schema_name,
|
|
572
|
+
cl.relname AS table_name,
|
|
573
|
+
c.conname AS fk_constraint_name,
|
|
574
|
+
refcl.relname AS referenced_table,
|
|
575
|
+
ARRAY_AGG(a.attname ORDER BY u.ord) AS fk_columns,
|
|
576
|
+
('CREATE INDEX ON '
|
|
577
|
+
|| quote_ident(n.nspname) || '.' || quote_ident(cl.relname)
|
|
578
|
+
|| ' (' || STRING_AGG(quote_ident(a.attname), ', ' ORDER BY u.ord) || ');'
|
|
579
|
+
) AS create_sql
|
|
580
|
+
FROM pg_constraint c
|
|
581
|
+
JOIN pg_class cl ON cl.oid = c.conrelid
|
|
582
|
+
JOIN pg_namespace n ON n.oid = cl.relnamespace
|
|
583
|
+
JOIN pg_class refcl ON refcl.oid = c.confrelid
|
|
584
|
+
JOIN LATERAL UNNEST(c.conkey) WITH ORDINALITY AS u(attnum, ord) ON TRUE
|
|
585
|
+
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = u.attnum
|
|
586
|
+
WHERE c.contype = 'f'
|
|
587
|
+
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
|
|
588
|
+
AND NOT EXISTS (
|
|
589
|
+
SELECT 1
|
|
590
|
+
FROM pg_index i
|
|
591
|
+
WHERE i.indrelid = c.conrelid
|
|
592
|
+
AND i.indisvalid
|
|
593
|
+
AND (i.indkey::int2[])[1:array_length(c.conkey::int2[], 1)]
|
|
594
|
+
= c.conkey::int2[]
|
|
595
|
+
)
|
|
596
|
+
AND (:table IS NULL
|
|
597
|
+
OR (cl.relname = :table AND (:schema IS NULL OR n.nspname = :schema)))
|
|
598
|
+
GROUP BY n.nspname, cl.relname, c.conname, c.conrelid,
|
|
599
|
+
c.confrelid, c.conkey, refcl.relname
|
|
600
|
+
ORDER BY n.nspname, cl.relname, c.conname
|
|
601
|
+
""",
|
|
602
|
+
{"table": table, "schema": schema},
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
# --- Multi-statement file execution -----------------------------------------
|
|
607
|
+
|
|
608
|
+
def run_sql_file(path: str) -> list[dict]:
|
|
609
|
+
"""Read a ``.sql`` file, execute every statement, return the last result set.
|
|
610
|
+
|
|
611
|
+
Splits on top-level ``;`` (respecting single-quoted strings, double-quoted
|
|
612
|
+
identifiers, ``--`` / ``/* */`` comments, and PG ``$tag$`` dollar-quoting),
|
|
613
|
+
then runs every statement inside one transaction on the active workspace.
|
|
614
|
+
Ideal for loading schema/seed fixtures or replaying a saved practice
|
|
615
|
+
query::
|
|
616
|
+
|
|
617
|
+
run_sql_file("lab/sql_harness/practice/pgexercises/schema.sql")
|
|
618
|
+
run_sql_file("lab/sql_harness/practice/pgexercises/seed.sql")
|
|
619
|
+
rows = run_sql_file(".../basic_01_retrieve_everything.sql")
|
|
620
|
+
|
|
621
|
+
Returns the rows of the final statement if it produced a result set;
|
|
622
|
+
otherwise an empty list.
|
|
623
|
+
"""
|
|
624
|
+
sql_text = Path(path).read_text(encoding="utf-8")
|
|
625
|
+
ws = current_workspace()
|
|
626
|
+
last_rows: list[dict] = []
|
|
627
|
+
with ws.engine.begin() as conn:
|
|
628
|
+
for stmt in _split_sql(sql_text):
|
|
629
|
+
result = conn.execute(text(stmt))
|
|
630
|
+
if result.returns_rows:
|
|
631
|
+
cols = result.keys()
|
|
632
|
+
last_rows = [dict(zip(cols, row)) for row in result.fetchall()]
|
|
633
|
+
else:
|
|
634
|
+
last_rows = []
|
|
635
|
+
return last_rows
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _split_sql(sql: str) -> list[str]:
|
|
639
|
+
"""Split SQL into top-level statements on ``;``.
|
|
640
|
+
|
|
641
|
+
Respects single-quoted strings (``''`` escape), double-quoted identifiers
|
|
642
|
+
(``""`` escape), ``--`` line comments, ``/* */`` block comments, and PG
|
|
643
|
+
``$tag$...$tag$`` dollar-quoting. Whitespace/comment-only chunks are dropped.
|
|
644
|
+
"""
|
|
645
|
+
stmts: list[str] = []
|
|
646
|
+
buf: list[str] = []
|
|
647
|
+
i = 0
|
|
648
|
+
n = len(sql)
|
|
649
|
+
while i < n:
|
|
650
|
+
c = sql[i]
|
|
651
|
+
nxt = sql[i + 1] if i + 1 < n else ""
|
|
652
|
+
# -- line comment: skip to end of line (leave '\n' as whitespace)
|
|
653
|
+
if c == "-" and nxt == "-":
|
|
654
|
+
j = sql.find("\n", i)
|
|
655
|
+
i = n if j == -1 else j
|
|
656
|
+
continue
|
|
657
|
+
# /* block comment */
|
|
658
|
+
if c == "/" and nxt == "*":
|
|
659
|
+
j = sql.find("*/", i + 2)
|
|
660
|
+
i = n if j == -1 else j + 2
|
|
661
|
+
continue
|
|
662
|
+
# 'single-quoted string'
|
|
663
|
+
if c == "'":
|
|
664
|
+
buf.append(c)
|
|
665
|
+
i += 1
|
|
666
|
+
while i < n:
|
|
667
|
+
ch = sql[i]
|
|
668
|
+
buf.append(ch)
|
|
669
|
+
i += 1
|
|
670
|
+
if ch == "'":
|
|
671
|
+
if i < n and sql[i] == "'": # '' escaped quote
|
|
672
|
+
buf.append(sql[i])
|
|
673
|
+
i += 1
|
|
674
|
+
continue
|
|
675
|
+
break
|
|
676
|
+
continue
|
|
677
|
+
# "double-quoted identifier"
|
|
678
|
+
if c == '"':
|
|
679
|
+
buf.append(c)
|
|
680
|
+
i += 1
|
|
681
|
+
while i < n:
|
|
682
|
+
ch = sql[i]
|
|
683
|
+
buf.append(ch)
|
|
684
|
+
i += 1
|
|
685
|
+
if ch == '"':
|
|
686
|
+
if i < n and sql[i] == '"': # "" escaped quote
|
|
687
|
+
buf.append(sql[i])
|
|
688
|
+
i += 1
|
|
689
|
+
continue
|
|
690
|
+
break
|
|
691
|
+
continue
|
|
692
|
+
# $tag$...$tag$ dollar-quoting (PG)
|
|
693
|
+
if c == "$":
|
|
694
|
+
m = re.match(r"\$[A-Za-z_0-9]*\$", sql[i:])
|
|
695
|
+
if m:
|
|
696
|
+
tag = m.group(0)
|
|
697
|
+
buf.append(tag)
|
|
698
|
+
i += len(tag)
|
|
699
|
+
end = sql.find(tag, i)
|
|
700
|
+
if end == -1:
|
|
701
|
+
buf.extend(sql[i:])
|
|
702
|
+
i = n
|
|
703
|
+
else:
|
|
704
|
+
buf.extend(sql[i:end + len(tag)])
|
|
705
|
+
i = end + len(tag)
|
|
706
|
+
continue
|
|
707
|
+
# statement separator
|
|
708
|
+
if c == ";":
|
|
709
|
+
stmt = "".join(buf).strip()
|
|
710
|
+
if stmt:
|
|
711
|
+
stmts.append(stmt)
|
|
712
|
+
buf = []
|
|
713
|
+
i += 1
|
|
714
|
+
continue
|
|
715
|
+
buf.append(c)
|
|
716
|
+
i += 1
|
|
717
|
+
tail = "".join(buf).strip()
|
|
718
|
+
if tail:
|
|
719
|
+
stmts.append(tail)
|
|
720
|
+
return stmts
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
# --- SSH helpers -----------------------------------------------------------
|
|
724
|
+
#
|
|
725
|
+
# Analogous to the SQL helpers above, but for SSH workspaces (driver=ssh).
|
|
726
|
+
# The active workspace's .engine is a `_SshHandle` exposing .client / .sftp /
|
|
727
|
+
# .exec / .upload / .download. We guard on the driver name and raise a clear
|
|
728
|
+
# error if called against a DB workspace.
|
|
729
|
+
|
|
730
|
+
def _ssh_handle():
|
|
731
|
+
"""Return the active workspace's SSH handle, or raise if not an SSH workspace."""
|
|
732
|
+
ws = current_workspace()
|
|
733
|
+
if ws.driver.name != "ssh":
|
|
734
|
+
raise RuntimeError(
|
|
735
|
+
f"workspace {ws.config.connection.name!r} is a {ws.driver.name} workspace, "
|
|
736
|
+
"not an SSH workspace. Call use_workspace('<ssh-name>') first, or add "
|
|
737
|
+
"a connection with driver='ssh' in connections.toml."
|
|
738
|
+
)
|
|
739
|
+
return ws.engine # the _SshHandle
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def ssh_exec(command: str, timeout: float = 30.0) -> dict:
|
|
743
|
+
"""Run a shell command on the active SSH workspace.
|
|
744
|
+
|
|
745
|
+
Returns the same dict shape as `query()`: `{stdout, stderr, exit_code, ok}`.
|
|
746
|
+
"""
|
|
747
|
+
handle = _ssh_handle()
|
|
748
|
+
result = handle.exec(command, timeout=timeout)
|
|
749
|
+
return result.to_dict()
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def ssh_upload(local_path: str, remote_path: str) -> None:
|
|
753
|
+
"""Copy a local file to the active SSH workspace."""
|
|
754
|
+
_ssh_handle().upload(local_path, remote_path)
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
def ssh_download(remote_path: str, local_path: str) -> None:
|
|
758
|
+
"""Copy a file from the active SSH workspace to local disk."""
|
|
759
|
+
_ssh_handle().download(remote_path, local_path)
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def ssh_run_script(
|
|
763
|
+
local_script_path: str,
|
|
764
|
+
remote_dir: str = "/tmp",
|
|
765
|
+
interpreter: str = "bash",
|
|
766
|
+
) -> dict:
|
|
767
|
+
"""Upload a local script and execute it remotely.
|
|
768
|
+
|
|
769
|
+
Convenience: equivalent to `ssh_upload + ssh_exec "bash <remote_path>"`.
|
|
770
|
+
Returns the command result dict.
|
|
771
|
+
"""
|
|
772
|
+
handle = _ssh_handle()
|
|
773
|
+
from .drivers.ssh import run_remote_script
|
|
774
|
+
|
|
775
|
+
result = run_remote_script(handle, local_script_path, remote_dir, interpreter)
|
|
776
|
+
return result.to_dict()
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
def ssh_info() -> dict:
|
|
780
|
+
"""Return basic info about the active SSH workspace (user, host, port, transport)."""
|
|
781
|
+
handle = _ssh_handle()
|
|
782
|
+
transport = handle.client.get_transport()
|
|
783
|
+
return {
|
|
784
|
+
"connection": current_workspace().config.connection.name,
|
|
785
|
+
"user": handle.client.get_transport().get_username() if transport else None,
|
|
786
|
+
"host": transport.sock.getpeername()[0] if transport else None,
|
|
787
|
+
"port": transport.sock.getpeername()[1] if transport else None,
|
|
788
|
+
"sftp_available": handle.sftp is not None,
|
|
789
|
+
}
|