sqlite-hub 1.4.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/bin/sqlite-hub-mcp.js +8 -0
- package/bin/sqlite-hub.js +555 -122
- package/docs/API.md +30 -0
- package/docs/CLI.md +22 -0
- package/docs/CLI_API_PARITY.md +61 -0
- package/docs/MCP.md +85 -0
- package/docs/changelog.md +13 -1
- package/docs/guidelines/AGENTS.md +32 -0
- package/docs/{DESIGN_GUIDELINES.md → guidelines/DESIGN.md} +10 -0
- package/docs/todo.md +1 -14
- package/frontend/js/api.js +49 -0
- package/frontend/js/app.js +202 -2
- package/frontend/js/components/connectionCard.js +3 -1
- package/frontend/js/components/modal.js +356 -15
- package/frontend/js/components/sidebar.js +41 -7
- package/frontend/js/components/topNav.js +2 -14
- package/frontend/js/router.js +10 -0
- package/frontend/js/store.js +483 -7
- package/frontend/js/utils/inputClear.js +36 -0
- package/frontend/js/utils/syntheticData.js +240 -0
- package/frontend/js/views/backups.js +52 -0
- package/frontend/js/views/data.js +16 -0
- package/frontend/js/views/logs.js +339 -0
- package/frontend/js/views/settings.js +266 -30
- package/frontend/js/views/structure.js +6 -0
- package/frontend/js/views/tableAdvisor.js +385 -0
- package/frontend/js/views/tableDesigner.js +6 -0
- package/frontend/styles/components.css +149 -0
- package/frontend/styles/tailwind.generated.css +1 -1
- package/frontend/styles/views.css +31 -0
- package/package.json +4 -2
- package/server/mcp/stdioServer.js +272 -0
- package/server/routes/data.js +45 -0
- package/server/routes/externalApi.js +285 -2
- package/server/routes/logs.js +127 -0
- package/server/routes/settings.js +47 -0
- package/server/server.js +3 -0
- package/server/services/databaseCommandService.js +284 -2
- package/server/services/mcpStatusService.js +140 -0
- package/server/services/mcpToolService.js +274 -0
- package/server/services/sqlite/dataBrowserService.js +36 -0
- package/server/services/sqlite/introspection.js +226 -22
- package/server/services/sqlite/syntheticDataGenerator.js +728 -0
- package/server/services/sqlite/tableAdvisor.js +790 -0
- package/server/services/storage/appStateStore.js +821 -169
|
@@ -0,0 +1,790 @@
|
|
|
1
|
+
const { quoteIdentifier } = require("../../utils/identifier");
|
|
2
|
+
const { getTableDetail } = require("./introspection");
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @typedef {"info" | "warning" | "critical"} TableAdvisorSeverity
|
|
6
|
+
* @typedef {"schema" | "constraints" | "performance" | "data-quality" | "documentation"} TableAdvisorCategory
|
|
7
|
+
* @typedef {"low" | "medium" | "high"} TableAdvisorRisk
|
|
8
|
+
*
|
|
9
|
+
* @typedef {Object} ColumnProfile
|
|
10
|
+
* @property {string} name
|
|
11
|
+
* @property {string} type
|
|
12
|
+
* @property {boolean} notNull
|
|
13
|
+
* @property {string | null} defaultValue
|
|
14
|
+
* @property {boolean} primaryKey
|
|
15
|
+
* @property {number} nullCount
|
|
16
|
+
* @property {number} emptyStringCount
|
|
17
|
+
* @property {number} distinctCount
|
|
18
|
+
* @property {number} totalCount
|
|
19
|
+
* @property {{ value: string | number | null, count: number }[]} topValues
|
|
20
|
+
* @property {string | number | null} [minValue]
|
|
21
|
+
* @property {string | number | null} [maxValue]
|
|
22
|
+
*
|
|
23
|
+
* @typedef {Object} TableAdvisorIssue
|
|
24
|
+
* @property {string} id
|
|
25
|
+
* @property {TableAdvisorSeverity} severity
|
|
26
|
+
* @property {TableAdvisorCategory} category
|
|
27
|
+
* @property {string} title
|
|
28
|
+
* @property {string} explanation
|
|
29
|
+
* @property {string} [evidence]
|
|
30
|
+
* @property {string} recommendation
|
|
31
|
+
* @property {string} [sql]
|
|
32
|
+
* @property {TableAdvisorRisk} risk
|
|
33
|
+
*
|
|
34
|
+
* @typedef {Object} TableAdvisorResult
|
|
35
|
+
* @property {string} tableName
|
|
36
|
+
* @property {number} score
|
|
37
|
+
* @property {TableAdvisorIssue[]} issues
|
|
38
|
+
* @property {string} analyzedAt
|
|
39
|
+
* @property {number} [issueCount]
|
|
40
|
+
* @property {number} [rowCount]
|
|
41
|
+
* @property {ColumnProfile[]} [columnProfiles]
|
|
42
|
+
* @property {{ columnCount: number, indexCount: number, foreignKeyCount: number }} [table]
|
|
43
|
+
*/
|
|
44
|
+
|
|
45
|
+
const LIKELY_UNIQUE_COLUMNS = new Set([
|
|
46
|
+
"email",
|
|
47
|
+
"slug",
|
|
48
|
+
"uuid",
|
|
49
|
+
"username",
|
|
50
|
+
"handle",
|
|
51
|
+
"external_id",
|
|
52
|
+
"externalid",
|
|
53
|
+
]);
|
|
54
|
+
|
|
55
|
+
const ENUM_LIKE_COLUMNS = new Set([
|
|
56
|
+
"status",
|
|
57
|
+
"state",
|
|
58
|
+
"type",
|
|
59
|
+
"kind",
|
|
60
|
+
"category",
|
|
61
|
+
]);
|
|
62
|
+
|
|
63
|
+
const GENERIC_COLUMN_NAMES = new Set([
|
|
64
|
+
"data",
|
|
65
|
+
"value",
|
|
66
|
+
"text",
|
|
67
|
+
"temp",
|
|
68
|
+
"misc",
|
|
69
|
+
"payload",
|
|
70
|
+
"json",
|
|
71
|
+
"field1",
|
|
72
|
+
"field2",
|
|
73
|
+
]);
|
|
74
|
+
|
|
75
|
+
const CREATED_AT_COLUMNS = new Set(["created_at", "createdat", "inserted_at", "created"]);
|
|
76
|
+
const UPDATED_AT_COLUMNS = new Set(["updated_at", "updatedat"]);
|
|
77
|
+
const LARGE_TABLE_DUPLICATE_CHECK_LIMIT = 100000;
|
|
78
|
+
|
|
79
|
+
function normalizeColumnName(name) {
|
|
80
|
+
return String(name ?? "").trim().toLowerCase();
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function normalizeIdentifierPart(value) {
|
|
84
|
+
return String(value ?? "")
|
|
85
|
+
.trim()
|
|
86
|
+
.replace(/[^A-Za-z0-9_]+/g, "_")
|
|
87
|
+
.replace(/^_+|_+$/g, "")
|
|
88
|
+
.toLowerCase() || "column";
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function escapeSqlString(value) {
|
|
92
|
+
return String(value ?? "").replaceAll("'", "''");
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function isSimpleSqlIdentifier(value) {
|
|
96
|
+
return /^[A-Za-z_][A-Za-z0-9_]*$/.test(String(value ?? ""));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function formatColumnDefinitionName(columnName) {
|
|
100
|
+
return isSimpleSqlIdentifier(columnName) ? columnName : quoteIdentifier(columnName);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function getColumnSqlType(column) {
|
|
104
|
+
return column.declaredType || column.affinity || "ANY";
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function getProfileType(column) {
|
|
108
|
+
return column.declaredType || column.affinity || "ANY";
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function isPrimaryKey(column) {
|
|
112
|
+
return Number(column.primaryKeyPosition ?? 0) > 0;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function isTextLikeColumn(column) {
|
|
116
|
+
const type = getColumnSqlType(column).toUpperCase();
|
|
117
|
+
return column.affinity === "TEXT" || /CHAR|CLOB|TEXT|VARCHAR|STRING|JSON/i.test(type);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function isNumericLikeColumn(column) {
|
|
121
|
+
return ["INTEGER", "REAL", "NUMERIC"].includes(column.affinity);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function isDateLikeColumn(column) {
|
|
125
|
+
const name = normalizeColumnName(column.name);
|
|
126
|
+
const type = getColumnSqlType(column).toUpperCase();
|
|
127
|
+
|
|
128
|
+
return (
|
|
129
|
+
/DATE|TIME/i.test(type) ||
|
|
130
|
+
name.endsWith("_at") ||
|
|
131
|
+
name.endsWith("at") ||
|
|
132
|
+
name.includes("date") ||
|
|
133
|
+
name.includes("time")
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function isLikelyUniqueColumn(column) {
|
|
138
|
+
return LIKELY_UNIQUE_COLUMNS.has(normalizeColumnName(column.name));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function isEnumLikeColumn(column) {
|
|
142
|
+
return ENUM_LIKE_COLUMNS.has(normalizeColumnName(column.name));
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function isGenericColumnName(column) {
|
|
146
|
+
return GENERIC_COLUMN_NAMES.has(normalizeColumnName(column.name));
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function isCreatedAtColumn(column) {
|
|
150
|
+
return CREATED_AT_COLUMNS.has(normalizeColumnName(column.name));
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function isUpdatedAtColumn(column) {
|
|
154
|
+
return UPDATED_AT_COLUMNS.has(normalizeColumnName(column.name));
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function isForeignKeyLikeColumn(column) {
|
|
158
|
+
const name = String(column.name ?? "");
|
|
159
|
+
|
|
160
|
+
return !isPrimaryKey(column) && (name.endsWith("_id") || /Id$/.test(name));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function hasCurrentTimestampDefault(defaultValue) {
|
|
164
|
+
return /\bCURRENT_TIMESTAMP\b|\bCURRENT_DATE\b|\bdatetime\s*\(/i.test(String(defaultValue ?? ""));
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function hasExactUniqueIndex(tableDetail, columnName) {
|
|
168
|
+
return (tableDetail.indexes ?? []).some((index) => {
|
|
169
|
+
if (!index.unique || index.partial) {
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const columns = (index.columns ?? [])
|
|
174
|
+
.filter((column) => column.name)
|
|
175
|
+
.map((column) => column.name);
|
|
176
|
+
|
|
177
|
+
return columns.length === 1 && columns[0] === columnName;
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function hasIndexOnColumn(tableDetail, columnName) {
|
|
182
|
+
return (tableDetail.indexes ?? []).some((index) => {
|
|
183
|
+
const columns = (index.columns ?? [])
|
|
184
|
+
.filter((column) => column.name)
|
|
185
|
+
.map((column) => column.name);
|
|
186
|
+
|
|
187
|
+
return columns[0] === columnName;
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function hasForeignKeyOnColumn(tableDetail, columnName) {
|
|
192
|
+
return (tableDetail.foreignKeys ?? []).some((foreignKey) =>
|
|
193
|
+
(foreignKey.mappings ?? []).some((mapping) => mapping.from === columnName)
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function countRows(db, tableName, whereClause = "", params = []) {
|
|
198
|
+
const sql = [
|
|
199
|
+
"SELECT COUNT(*) AS count FROM",
|
|
200
|
+
quoteIdentifier(tableName),
|
|
201
|
+
whereClause ? `WHERE ${whereClause}` : "",
|
|
202
|
+
]
|
|
203
|
+
.filter(Boolean)
|
|
204
|
+
.join(" ");
|
|
205
|
+
|
|
206
|
+
return Number(db.prepare(sql).get(...params)?.count ?? 0);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function getDistinctCount(db, tableName, columnName) {
|
|
210
|
+
const sql = [
|
|
211
|
+
"SELECT COUNT(DISTINCT",
|
|
212
|
+
quoteIdentifier(columnName),
|
|
213
|
+
") AS count FROM",
|
|
214
|
+
quoteIdentifier(tableName),
|
|
215
|
+
].join(" ");
|
|
216
|
+
|
|
217
|
+
return Number(db.prepare(sql).get()?.count ?? 0);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function normalizeProfileValue(value) {
|
|
221
|
+
if (value === null || value === undefined) {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
if (typeof value === "number" || typeof value === "string") {
|
|
226
|
+
return value;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (Buffer.isBuffer(value)) {
|
|
230
|
+
return `BLOB ${value.length} bytes`;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
return String(value);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function getTopValues(db, tableName, columnName) {
|
|
237
|
+
const columnSql = quoteIdentifier(columnName);
|
|
238
|
+
const sql = [
|
|
239
|
+
"SELECT",
|
|
240
|
+
columnSql,
|
|
241
|
+
"AS value, COUNT(*) AS count FROM",
|
|
242
|
+
quoteIdentifier(tableName),
|
|
243
|
+
"GROUP BY",
|
|
244
|
+
columnSql,
|
|
245
|
+
"ORDER BY count DESC LIMIT 10",
|
|
246
|
+
].join(" ");
|
|
247
|
+
|
|
248
|
+
return db.prepare(sql).all().map((row) => ({
|
|
249
|
+
value: normalizeProfileValue(row.value),
|
|
250
|
+
count: Number(row.count ?? 0),
|
|
251
|
+
}));
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function getMinMax(db, tableName, columnName) {
|
|
255
|
+
const columnSql = quoteIdentifier(columnName);
|
|
256
|
+
const row = db
|
|
257
|
+
.prepare(
|
|
258
|
+
[
|
|
259
|
+
"SELECT MIN(",
|
|
260
|
+
columnSql,
|
|
261
|
+
") AS minValue, MAX(",
|
|
262
|
+
columnSql,
|
|
263
|
+
") AS maxValue FROM",
|
|
264
|
+
quoteIdentifier(tableName),
|
|
265
|
+
].join(" ")
|
|
266
|
+
)
|
|
267
|
+
.get();
|
|
268
|
+
|
|
269
|
+
return {
|
|
270
|
+
minValue: normalizeProfileValue(row?.minValue),
|
|
271
|
+
maxValue: normalizeProfileValue(row?.maxValue),
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function getDuplicateValues(db, tableName, columnName) {
|
|
276
|
+
const columnSql = quoteIdentifier(columnName);
|
|
277
|
+
const sql = [
|
|
278
|
+
"SELECT",
|
|
279
|
+
columnSql,
|
|
280
|
+
"AS value, COUNT(*) AS count FROM",
|
|
281
|
+
quoteIdentifier(tableName),
|
|
282
|
+
"WHERE",
|
|
283
|
+
columnSql,
|
|
284
|
+
"IS NOT NULL",
|
|
285
|
+
"GROUP BY",
|
|
286
|
+
columnSql,
|
|
287
|
+
"HAVING COUNT(*) > 1",
|
|
288
|
+
"ORDER BY count DESC LIMIT 10",
|
|
289
|
+
].join(" ");
|
|
290
|
+
|
|
291
|
+
return db.prepare(sql).all().map((row) => ({
|
|
292
|
+
value: normalizeProfileValue(row.value),
|
|
293
|
+
count: Number(row.count ?? 0),
|
|
294
|
+
}));
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function profileColumn(db, tableDetail, column) {
|
|
298
|
+
const tableName = tableDetail.name;
|
|
299
|
+
const columnSql = quoteIdentifier(column.name);
|
|
300
|
+
const profile = {
|
|
301
|
+
name: column.name,
|
|
302
|
+
type: getProfileType(column),
|
|
303
|
+
affinity: column.affinity,
|
|
304
|
+
notNull: Boolean(column.notNull),
|
|
305
|
+
defaultValue: column.defaultValue ?? null,
|
|
306
|
+
primaryKey: isPrimaryKey(column),
|
|
307
|
+
nullCount: countRows(db, tableName, `${columnSql} IS NULL`),
|
|
308
|
+
emptyStringCount: 0,
|
|
309
|
+
distinctCount: getDistinctCount(db, tableName, column.name),
|
|
310
|
+
totalCount: Number(tableDetail.rowCount ?? 0),
|
|
311
|
+
topValues: getTopValues(db, tableName, column.name),
|
|
312
|
+
duplicateValues: [],
|
|
313
|
+
};
|
|
314
|
+
|
|
315
|
+
if (isTextLikeColumn(column)) {
|
|
316
|
+
profile.emptyStringCount = countRows(
|
|
317
|
+
db,
|
|
318
|
+
tableName,
|
|
319
|
+
`${columnSql} IS NOT NULL AND TRIM(CAST(${columnSql} AS TEXT)) = ''`
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
if (isNumericLikeColumn(column) || isDateLikeColumn(column)) {
|
|
324
|
+
const range = getMinMax(db, tableName, column.name);
|
|
325
|
+
profile.minValue = range.minValue;
|
|
326
|
+
profile.maxValue = range.maxValue;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (
|
|
330
|
+
isLikelyUniqueColumn(column) &&
|
|
331
|
+
Number(tableDetail.rowCount ?? 0) <= LARGE_TABLE_DUPLICATE_CHECK_LIMIT
|
|
332
|
+
) {
|
|
333
|
+
profile.duplicateValues = getDuplicateValues(db, tableName, column.name);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
return profile;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function profileTable(db, tableDetail) {
|
|
340
|
+
return (tableDetail.columns ?? [])
|
|
341
|
+
.filter((column) => column.visible && !column.generated)
|
|
342
|
+
.map((column) => profileColumn(db, tableDetail, column));
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
function formatEvidenceValue(value) {
|
|
346
|
+
if (value === null) {
|
|
347
|
+
return "NULL";
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
if (value === undefined) {
|
|
351
|
+
return "undefined";
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
if (typeof value === "object") {
|
|
355
|
+
return JSON.stringify(value);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
return String(value);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function formatTopValueList(values = []) {
|
|
362
|
+
return values
|
|
363
|
+
.slice(0, 5)
|
|
364
|
+
.map((item) => `${formatEvidenceValue(item.value)} (${item.count})`)
|
|
365
|
+
.join(", ");
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function buildIndexName(tableName, columnName, suffix = "") {
|
|
369
|
+
return normalizeIdentifierPart(["idx", tableName, columnName, suffix].filter(Boolean).join("_"));
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function buildCreateIndexSql({ tableName, columnName, unique = false }) {
|
|
373
|
+
const indexName = buildIndexName(tableName, columnName, unique ? "unique" : "");
|
|
374
|
+
const createKeyword = unique ? "CREATE UNIQUE INDEX" : "CREATE INDEX";
|
|
375
|
+
|
|
376
|
+
return `${createKeyword} IF NOT EXISTS ${quoteIdentifier(indexName)} ON ${quoteIdentifier(tableName)}(${quoteIdentifier(columnName)});`;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
function buildCreatedAtColumnSql(column) {
|
|
380
|
+
return `${formatColumnDefinitionName(column.name)} TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP`;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function formatSqlLiteral(value) {
|
|
384
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
385
|
+
return String(value);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
return `'${escapeSqlString(value)}'`;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function buildCheckColumnSql(column, values = []) {
|
|
392
|
+
const name = formatColumnDefinitionName(column.name);
|
|
393
|
+
const valueList = values.map((value) => formatSqlLiteral(value)).join(", ");
|
|
394
|
+
|
|
395
|
+
return `${name} ${getColumnSqlType(column)} CHECK(${name} IN (${valueList}))`;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
function buildEmptyStringCleanupSql(tableName, columnName) {
|
|
399
|
+
const columnSql = quoteIdentifier(columnName);
|
|
400
|
+
|
|
401
|
+
return `UPDATE ${quoteIdentifier(tableName)} SET ${columnSql} = NULL WHERE TRIM(CAST(${columnSql} AS TEXT)) = '';`;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function buildUpdatedAtTriggerSql(tableDetail, columnName) {
|
|
405
|
+
const triggerName = normalizeIdentifierPart(["trg", tableDetail.name, columnName].join("_"));
|
|
406
|
+
const columnSql = quoteIdentifier(columnName);
|
|
407
|
+
let whereClause = "";
|
|
408
|
+
|
|
409
|
+
if (tableDetail.identityStrategy?.type === "primaryKey") {
|
|
410
|
+
whereClause = tableDetail.identityStrategy.columns
|
|
411
|
+
.map((primaryKeyColumn) => {
|
|
412
|
+
const quoted = quoteIdentifier(primaryKeyColumn);
|
|
413
|
+
return `${quoted} IS NEW.${quoted}`;
|
|
414
|
+
})
|
|
415
|
+
.join(" AND ");
|
|
416
|
+
} else if (tableDetail.identityStrategy?.type === "rowid") {
|
|
417
|
+
whereClause = "rowid = NEW.rowid";
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
if (!whereClause) {
|
|
421
|
+
return null;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
return [
|
|
425
|
+
`CREATE TRIGGER IF NOT EXISTS ${quoteIdentifier(triggerName)}`,
|
|
426
|
+
`AFTER UPDATE ON ${quoteIdentifier(tableDetail.name)}`,
|
|
427
|
+
"FOR EACH ROW",
|
|
428
|
+
`WHEN OLD.${columnSql} IS NEW.${columnSql}`,
|
|
429
|
+
"BEGIN",
|
|
430
|
+
` UPDATE ${quoteIdentifier(tableDetail.name)}`,
|
|
431
|
+
` SET ${columnSql} = CURRENT_TIMESTAMP`,
|
|
432
|
+
` WHERE ${whereClause};`,
|
|
433
|
+
"END;",
|
|
434
|
+
].join("\n");
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function makeIssue(issue) {
|
|
438
|
+
return { ...issue };
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function getProfile(columnProfiles, columnName) {
|
|
442
|
+
return columnProfiles.find((profile) => profile.name === columnName);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function generateMissingPrimaryKeyIssue(tableDetail) {
|
|
446
|
+
const hasPrimaryKey = (tableDetail.columns ?? []).some((column) => isPrimaryKey(column));
|
|
447
|
+
|
|
448
|
+
if (hasPrimaryKey) {
|
|
449
|
+
return null;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
return makeIssue({
|
|
453
|
+
id: "schema:missing-primary-key",
|
|
454
|
+
severity: "critical",
|
|
455
|
+
category: "schema",
|
|
456
|
+
title: "Table has no primary key",
|
|
457
|
+
explanation:
|
|
458
|
+
"Rows in this table are not uniquely identifiable. This can make updates, relations, exports and synchronization harder.",
|
|
459
|
+
evidence: `Table ${tableDetail.name} has 0 primary key columns.`,
|
|
460
|
+
recommendation: "Add an INTEGER PRIMARY KEY column if this table stores entities.",
|
|
461
|
+
risk: "high",
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
function generateColumnIssues(tableDetail, columnProfiles) {
|
|
466
|
+
const issues = [];
|
|
467
|
+
const rowCount = Number(tableDetail.rowCount ?? 0);
|
|
468
|
+
|
|
469
|
+
(tableDetail.columns ?? [])
|
|
470
|
+
.filter((column) => column.visible && !column.generated)
|
|
471
|
+
.forEach((column) => {
|
|
472
|
+
const profile = getProfile(columnProfiles, column.name);
|
|
473
|
+
|
|
474
|
+
if (!profile) {
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
const nonNullCount = Math.max(0, rowCount - profile.nullCount);
|
|
479
|
+
const nullRatio = rowCount > 0 ? profile.nullCount / rowCount : 0;
|
|
480
|
+
const duplicateValues = profile.duplicateValues ?? [];
|
|
481
|
+
|
|
482
|
+
if (isCreatedAtColumn(column) && !hasCurrentTimestampDefault(column.defaultValue)) {
|
|
483
|
+
issues.push(
|
|
484
|
+
makeIssue({
|
|
485
|
+
id: `schema:${column.name}:missing-created-default`,
|
|
486
|
+
severity: "warning",
|
|
487
|
+
category: "schema",
|
|
488
|
+
title: `${column.name} has no timestamp default`,
|
|
489
|
+
explanation:
|
|
490
|
+
"Creation timestamp columns are more reliable when SQLite assigns the value consistently at insert time.",
|
|
491
|
+
evidence: `${column.name} default is ${column.defaultValue ?? "NULL"}.`,
|
|
492
|
+
recommendation: "Use DEFAULT CURRENT_TIMESTAMP when rebuilding or adding this column.",
|
|
493
|
+
sql: buildCreatedAtColumnSql(column),
|
|
494
|
+
risk: "medium",
|
|
495
|
+
})
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
if (isUpdatedAtColumn(column)) {
|
|
500
|
+
const triggerSql = buildUpdatedAtTriggerSql(tableDetail, column.name);
|
|
501
|
+
|
|
502
|
+
issues.push(
|
|
503
|
+
makeIssue({
|
|
504
|
+
id: `schema:${column.name}:updated-at-maintenance`,
|
|
505
|
+
severity: "info",
|
|
506
|
+
category: "schema",
|
|
507
|
+
title: `${column.name} needs maintenance logic`,
|
|
508
|
+
explanation:
|
|
509
|
+
"SQLite does not update timestamp columns automatically. Use application logic or a trigger if this column should track changes.",
|
|
510
|
+
evidence: `${column.name} exists on ${tableDetail.name}.`,
|
|
511
|
+
recommendation: "Verify writes always refresh this column before relying on it for recency.",
|
|
512
|
+
...(triggerSql ? { sql: triggerSql } : {}),
|
|
513
|
+
risk: "medium",
|
|
514
|
+
})
|
|
515
|
+
);
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
if (isLikelyUniqueColumn(column) && duplicateValues.length) {
|
|
519
|
+
const duplicateRows = duplicateValues.reduce((total, item) => total + Number(item.count ?? 0), 0);
|
|
520
|
+
issues.push(
|
|
521
|
+
makeIssue({
|
|
522
|
+
id: `data-quality:${column.name}:duplicate-values`,
|
|
523
|
+
severity: duplicateRows > Math.max(10, rowCount * 0.05) ? "critical" : "warning",
|
|
524
|
+
category: "data-quality",
|
|
525
|
+
title: `${column.name} contains duplicates`,
|
|
526
|
+
explanation:
|
|
527
|
+
"This column looks like a natural identifier, but duplicate values would make a UNIQUE constraint fail.",
|
|
528
|
+
evidence: `Duplicate examples: ${formatTopValueList(duplicateValues)}.`,
|
|
529
|
+
recommendation: "Resolve duplicate values before adding a UNIQUE index.",
|
|
530
|
+
risk: "medium",
|
|
531
|
+
})
|
|
532
|
+
);
|
|
533
|
+
} else if (
|
|
534
|
+
isLikelyUniqueColumn(column) &&
|
|
535
|
+
rowCount > 0 &&
|
|
536
|
+
nonNullCount > 0 &&
|
|
537
|
+
nullRatio <= 0.02 &&
|
|
538
|
+
profile.distinctCount === nonNullCount &&
|
|
539
|
+
!hasExactUniqueIndex(tableDetail, column.name)
|
|
540
|
+
) {
|
|
541
|
+
issues.push(
|
|
542
|
+
makeIssue({
|
|
543
|
+
id: `constraints:${column.name}:missing-unique-index`,
|
|
544
|
+
severity: "warning",
|
|
545
|
+
category: "constraints",
|
|
546
|
+
title: `${column.name} looks unique but is not enforced`,
|
|
547
|
+
explanation:
|
|
548
|
+
"The sampled profile shows unique non-null values, but no single-column UNIQUE index enforces that expectation.",
|
|
549
|
+
evidence: `${formatTopValueList(profile.topValues)}. Distinct non-null values: ${profile.distinctCount}/${nonNullCount}.`,
|
|
550
|
+
recommendation: "Add a UNIQUE index after verifying the column is meant to be unique.",
|
|
551
|
+
sql: buildCreateIndexSql({
|
|
552
|
+
tableName: tableDetail.name,
|
|
553
|
+
columnName: column.name,
|
|
554
|
+
unique: true,
|
|
555
|
+
}),
|
|
556
|
+
risk: "medium",
|
|
557
|
+
})
|
|
558
|
+
);
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
if (
|
|
562
|
+
isEnumLikeColumn(column) &&
|
|
563
|
+
rowCount > 0 &&
|
|
564
|
+
profile.distinctCount >= 2 &&
|
|
565
|
+
profile.distinctCount <= 10 &&
|
|
566
|
+
!(column.allowedValues ?? []).length
|
|
567
|
+
) {
|
|
568
|
+
const values = (profile.topValues ?? [])
|
|
569
|
+
.map((item) => item.value)
|
|
570
|
+
.filter((value) => value !== null && value !== undefined)
|
|
571
|
+
.slice(0, 10);
|
|
572
|
+
|
|
573
|
+
if (values.length >= 2) {
|
|
574
|
+
issues.push(
|
|
575
|
+
makeIssue({
|
|
576
|
+
id: `constraints:${column.name}:check-constraint`,
|
|
577
|
+
severity: "info",
|
|
578
|
+
category: "constraints",
|
|
579
|
+
title: `${column.name} has enum-like values`,
|
|
580
|
+
explanation:
|
|
581
|
+
"The column uses a small set of values. A CHECK constraint can document and enforce the allowed set, but SQLite cannot add it to an existing column without a rebuild.",
|
|
582
|
+
evidence: `Observed values: ${formatTopValueList(profile.topValues)}.`,
|
|
583
|
+
recommendation: "Consider a CHECK constraint in the next schema rebuild.",
|
|
584
|
+
sql: buildCheckColumnSql(column, values),
|
|
585
|
+
risk: "high",
|
|
586
|
+
})
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
if (rowCount > 0 && nullRatio > 0.5) {
|
|
592
|
+
issues.push(
|
|
593
|
+
makeIssue({
|
|
594
|
+
id: `data-quality:${column.name}:mostly-null`,
|
|
595
|
+
severity: "info",
|
|
596
|
+
category: "data-quality",
|
|
597
|
+
title: `${column.name} is mostly NULL`,
|
|
598
|
+
explanation: "Mostly-empty columns can be valid, but they often indicate optional data that should be reviewed.",
|
|
599
|
+
evidence: `${profile.nullCount}/${rowCount} rows are NULL.`,
|
|
600
|
+
recommendation: "Confirm this column is still useful or split optional data into a related table.",
|
|
601
|
+
risk: "low",
|
|
602
|
+
})
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
if (!column.notNull && !isPrimaryKey(column) && rowCount >= 10 && profile.nullCount === 0) {
|
|
607
|
+
issues.push(
|
|
608
|
+
makeIssue({
|
|
609
|
+
id: `constraints:${column.name}:nullable-required`,
|
|
610
|
+
severity: "info",
|
|
611
|
+
category: "constraints",
|
|
612
|
+
title: `${column.name} is nullable but currently always filled`,
|
|
613
|
+
explanation:
|
|
614
|
+
"The data profile suggests the column may be required even though the schema allows NULL.",
|
|
615
|
+
evidence: `0/${rowCount} rows are NULL.`,
|
|
616
|
+
recommendation: "Consider NOT NULL in a future table rebuild if this is a real requirement.",
|
|
617
|
+
risk: "high",
|
|
618
|
+
})
|
|
619
|
+
);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
if (profile.emptyStringCount > 0) {
|
|
623
|
+
issues.push(
|
|
624
|
+
makeIssue({
|
|
625
|
+
id: `data-quality:${column.name}:empty-strings`,
|
|
626
|
+
severity: "warning",
|
|
627
|
+
category: "data-quality",
|
|
628
|
+
title: `${column.name} contains empty strings`,
|
|
629
|
+
explanation:
|
|
630
|
+
"Empty strings and NULL values can represent different states. Mixing both often makes filtering and validation harder.",
|
|
631
|
+
evidence: `${profile.emptyStringCount}/${rowCount} rows are empty strings.`,
|
|
632
|
+
recommendation:
|
|
633
|
+
"Normalize empty strings to NULL or enforce a NOT NULL plus CHECK(length(trim(column)) > 0) rule.",
|
|
634
|
+
sql: buildEmptyStringCleanupSql(tableDetail.name, column.name),
|
|
635
|
+
risk: "medium",
|
|
636
|
+
})
|
|
637
|
+
);
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
if (isForeignKeyLikeColumn(column) && !hasForeignKeyOnColumn(tableDetail, column.name)) {
|
|
641
|
+
issues.push(
|
|
642
|
+
makeIssue({
|
|
643
|
+
id: `schema:${column.name}:missing-foreign-key`,
|
|
644
|
+
severity: "warning",
|
|
645
|
+
category: "schema",
|
|
646
|
+
title: `${column.name} looks like a foreign key`,
|
|
647
|
+
explanation:
|
|
648
|
+
"The column name follows an id-reference pattern, but the schema has no FOREIGN KEY constraint for it.",
|
|
649
|
+
evidence: `${column.name} has no foreign_key_list entry.`,
|
|
650
|
+
recommendation: "Add a FOREIGN KEY in a table rebuild if this column references another table.",
|
|
651
|
+
risk: "high",
|
|
652
|
+
})
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
if (isForeignKeyLikeColumn(column) && !hasIndexOnColumn(tableDetail, column.name)) {
|
|
657
|
+
issues.push(
|
|
658
|
+
makeIssue({
|
|
659
|
+
id: `performance:${column.name}:missing-index`,
|
|
660
|
+
severity: "warning",
|
|
661
|
+
category: "performance",
|
|
662
|
+
title: `${column.name} is not indexed`,
|
|
663
|
+
explanation:
|
|
664
|
+
"Foreign-key-like columns are commonly used in joins and filters. An index usually improves those lookups.",
|
|
665
|
+
evidence: `No index starts with ${column.name}.`,
|
|
666
|
+
recommendation: "Add an index if queries join or filter by this column.",
|
|
667
|
+
sql: buildCreateIndexSql({
|
|
668
|
+
tableName: tableDetail.name,
|
|
669
|
+
columnName: column.name,
|
|
670
|
+
}),
|
|
671
|
+
risk: "low",
|
|
672
|
+
})
|
|
673
|
+
);
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
if (isGenericColumnName(column)) {
|
|
677
|
+
issues.push(
|
|
678
|
+
makeIssue({
|
|
679
|
+
id: `documentation:${column.name}:generic-name`,
|
|
680
|
+
severity: "info",
|
|
681
|
+
category: "documentation",
|
|
682
|
+
title: `${column.name} is a generic column name`,
|
|
683
|
+
explanation:
|
|
684
|
+
"Generic column names hide intent and make downstream queries harder to understand.",
|
|
685
|
+
evidence: `${column.name} is in the generic-name list.`,
|
|
686
|
+
recommendation: "Rename or document the column purpose in the next schema cleanup.",
|
|
687
|
+
risk: "medium",
|
|
688
|
+
})
|
|
689
|
+
);
|
|
690
|
+
}
|
|
691
|
+
});
|
|
692
|
+
|
|
693
|
+
return issues;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
function generateLargeTableIndexIssue(tableDetail) {
|
|
697
|
+
const rowCount = Number(tableDetail.rowCount ?? 0);
|
|
698
|
+
const secondaryIndexes = (tableDetail.indexes ?? []).filter((index) => index.origin !== "pk");
|
|
699
|
+
|
|
700
|
+
if (rowCount <= 1000 || secondaryIndexes.length > 0) {
|
|
701
|
+
return null;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
return makeIssue({
|
|
705
|
+
id: "performance:large-table-without-indexes",
|
|
706
|
+
severity: "warning",
|
|
707
|
+
category: "performance",
|
|
708
|
+
title: "Large table has no secondary indexes",
|
|
709
|
+
explanation:
|
|
710
|
+
"Tables with more than 1,000 rows often need indexes on frequent filter and join columns.",
|
|
711
|
+
evidence: `${tableDetail.name} has ${rowCount} rows and no secondary indexes.`,
|
|
712
|
+
recommendation: "Review common queries and add indexes for WHERE, JOIN, and ORDER BY columns.",
|
|
713
|
+
risk: "low",
|
|
714
|
+
});
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
function generateTableAdvisorIssues(tableDetail, columnProfiles) {
|
|
718
|
+
return [
|
|
719
|
+
generateMissingPrimaryKeyIssue(tableDetail),
|
|
720
|
+
...generateColumnIssues(tableDetail, columnProfiles),
|
|
721
|
+
generateLargeTableIndexIssue(tableDetail),
|
|
722
|
+
].filter(Boolean);
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
function calculateScore(issues = []) {
|
|
726
|
+
const penalties = {
|
|
727
|
+
critical: 20,
|
|
728
|
+
warning: 8,
|
|
729
|
+
info: 2,
|
|
730
|
+
};
|
|
731
|
+
const score = issues.reduce((currentScore, issue) => {
|
|
732
|
+
return currentScore - (penalties[issue.severity] ?? 0);
|
|
733
|
+
}, 100);
|
|
734
|
+
|
|
735
|
+
return Math.max(0, Math.min(100, score));
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
function sortIssues(issues = []) {
|
|
739
|
+
const severityOrder = {
|
|
740
|
+
critical: 0,
|
|
741
|
+
warning: 1,
|
|
742
|
+
info: 2,
|
|
743
|
+
};
|
|
744
|
+
|
|
745
|
+
return [...issues].sort((left, right) => {
|
|
746
|
+
const severityDelta = severityOrder[left.severity] - severityOrder[right.severity];
|
|
747
|
+
|
|
748
|
+
if (severityDelta !== 0) {
|
|
749
|
+
return severityDelta;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
return String(left.id).localeCompare(String(right.id));
|
|
753
|
+
});
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function analyzeTable(db, tableName) {
|
|
757
|
+
const tableDetail = getTableDetail(db, tableName);
|
|
758
|
+
const columnProfiles = profileTable(db, tableDetail);
|
|
759
|
+
const issues = sortIssues(generateTableAdvisorIssues(tableDetail, columnProfiles));
|
|
760
|
+
|
|
761
|
+
return {
|
|
762
|
+
tableName: tableDetail.name,
|
|
763
|
+
score: calculateScore(issues),
|
|
764
|
+
issueCount: issues.length,
|
|
765
|
+
rowCount: Number(tableDetail.rowCount ?? 0),
|
|
766
|
+
issues,
|
|
767
|
+
columnProfiles,
|
|
768
|
+
table: {
|
|
769
|
+
columnCount: (tableDetail.columns ?? []).filter((column) => column.visible && !column.generated).length,
|
|
770
|
+
indexCount: (tableDetail.indexes ?? []).length,
|
|
771
|
+
foreignKeyCount: (tableDetail.foreignKeys ?? []).length,
|
|
772
|
+
},
|
|
773
|
+
analyzedAt: new Date().toISOString(),
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
class TableAdvisorService {
|
|
778
|
+
analyzeTable(db, tableName) {
|
|
779
|
+
return analyzeTable(db, tableName);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
module.exports = {
|
|
784
|
+
TableAdvisorService,
|
|
785
|
+
analyzeTable,
|
|
786
|
+
calculateScore,
|
|
787
|
+
generateTableAdvisorIssues,
|
|
788
|
+
profileTable,
|
|
789
|
+
quoteIdentifier,
|
|
790
|
+
};
|