uql-orm 0.37.1 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/uql-browser.min.js +2 -2
- package/dist/browser/uql-browser.min.js.map +5 -5
- package/dist/cockroachdb/cockroachDialect.d.ts +5 -24
- package/dist/cockroachdb/cockroachDialect.js +3 -29
- package/dist/dialect/abstractSqlDialect.d.ts +42 -12
- package/dist/dialect/abstractSqlDialect.js +117 -48
- package/dist/dialect/aliases.d.ts +6 -0
- package/dist/dialect/aliases.js +6 -0
- package/dist/dialect/index.d.ts +0 -5
- package/dist/dialect/index.js +2 -5
- package/dist/dialect/jsonSql.d.ts +17 -2
- package/dist/dialect/jsonSql.js +15 -0
- package/dist/dialect/mysqlLikeSqlDialect.d.ts +13 -14
- package/dist/dialect/mysqlLikeSqlDialect.js +16 -20
- package/dist/dialect/pgLikeSqlDialect.d.ts +17 -20
- package/dist/dialect/pgLikeSqlDialect.js +30 -62
- package/dist/dialect/vectorCast.d.ts +0 -9
- package/dist/dialect/vectorCast.js +0 -8
- package/dist/dialect/vectorSqlDialect.d.ts +47 -17
- package/dist/dialect/vectorSqlDialect.js +78 -30
- package/dist/entity/decorator/entity.d.ts +1 -1
- package/dist/entity/metadata/definition.d.ts +1 -1
- package/dist/entity/metadata/definition.js +4 -3
- package/dist/http/query.js +3 -2
- package/dist/libsql/libsqlDialect.d.ts +2 -2
- package/dist/libsql/libsqlDialect.js +3 -3
- package/dist/maria/mariaDialect.d.ts +12 -17
- package/dist/maria/mariaDialect.js +21 -36
- package/dist/maria/mariaVectorMetrics.d.ts +8 -0
- package/dist/maria/mariaVectorMetrics.js +10 -0
- package/dist/maria/mariadbQuerier.d.ts +5 -0
- package/dist/maria/mariadbQuerier.js +5 -0
- package/dist/migrate/builder/migrationBuilder.d.ts +8 -17
- package/dist/migrate/builder/migrationBuilder.js +48 -136
- package/dist/migrate/builder/types.d.ts +0 -2
- package/dist/migrate/ddl/index.d.ts +11 -0
- package/dist/migrate/ddl/index.js +34 -0
- package/dist/{dialect/indexSqlDialect.d.ts → migrate/ddl/indexDdl.d.ts} +23 -19
- package/dist/migrate/ddl/indexDdl.js +126 -0
- package/dist/migrate/ddl/mysqlIndexDdl.d.ts +52 -0
- package/dist/migrate/ddl/mysqlIndexDdl.js +125 -0
- package/dist/migrate/ddl/pgIndexDdl.d.ts +36 -0
- package/dist/migrate/ddl/pgIndexDdl.js +87 -0
- package/dist/migrate/drift/driftDetector.js +6 -1
- package/dist/migrate/generator/mongoSchemaGenerator.js +1 -1
- package/dist/migrate/index.d.ts +1 -0
- package/dist/migrate/index.js +2 -0
- package/dist/migrate/introspection/mysqlIntrospector.d.ts +9 -3
- package/dist/migrate/introspection/mysqlIntrospector.js +27 -5
- package/dist/migrate/migrator.js +3 -2
- package/dist/migrate/schemaGenerator.d.ts +6 -6
- package/dist/migrate/schemaGenerator.js +13 -22
- package/dist/mongo/mongoDialect.d.ts +1 -2
- package/dist/mongo/mongoDialect.js +29 -22
- package/dist/mongo/mongodbQuerier.js +1 -1
- package/dist/mysql/mysqlDialect.d.ts +3 -6
- package/dist/mysql/mysqlDialect.js +4 -11
- package/dist/postgres/postgresDialect.d.ts +2 -0
- package/dist/postgres/postgresDialect.js +2 -0
- package/dist/querier/abstractSqlQuerier.d.ts +11 -0
- package/dist/querier/abstractSqlQuerier.js +35 -0
- package/dist/schema/canonicalType.js +35 -36
- package/dist/schema/indexDifferences.d.ts +2 -1
- package/dist/schema/indexDifferences.js +3 -2
- package/dist/sqlite/sqliteDialect.d.ts +2 -2
- package/dist/sqlite/sqliteDialect.js +5 -5
- package/dist/turso/tursoDialect.d.ts +2 -2
- package/dist/turso/tursoDialect.js +4 -4
- package/dist/type/dialect.d.ts +21 -7
- package/dist/type/dialect.js +17 -0
- package/dist/type/entity.d.ts +117 -8
- package/dist/type/entity.js +7 -0
- package/dist/type/query.d.ts +18 -0
- package/dist/type/query.js +6 -0
- package/dist/type/queryWhere.d.ts +46 -2
- package/dist/type/vector.d.ts +45 -7
- package/dist/type/vector.js +16 -1
- package/dist/util/dialect.util.d.ts +20 -1
- package/dist/util/dialect.util.js +49 -4
- package/dist/util/object.util.d.ts +7 -1
- package/dist/util/object.util.js +8 -0
- package/dist/util/relationQuery.util.d.ts +3 -1
- package/dist/util/relationQuery.util.js +8 -3
- package/package.json +1 -1
- package/dist/dialect/indexSqlDialect.js +0 -103
|
@@ -5,6 +5,7 @@ import { buildSchemaAST } from '../schema/schemaASTBuilder.js';
|
|
|
5
5
|
import { getKeys, isAutoIncrement, qualifyName } from '../util/index.js';
|
|
6
6
|
import { derivedForeignKeyName } from '../util/sql.util.js';
|
|
7
7
|
import { formatDefaultValue, SqlExpression } from './builder/expressions.js';
|
|
8
|
+
import { indexDdlFor } from './ddl/index.js';
|
|
8
9
|
import { fullColumnDefinitionToNode, tableDefinitionToNode } from './generator/definitionToNode.js';
|
|
9
10
|
import { indexNodeToSchema } from './generator/indexNodeToSchema.js';
|
|
10
11
|
/**
|
|
@@ -14,9 +15,12 @@ import { indexNodeToSchema } from './generator/indexNodeToSchema.js';
|
|
|
14
15
|
export class SqlSchemaGenerator {
|
|
15
16
|
dialect;
|
|
16
17
|
defaultForeignKeyAction;
|
|
18
|
+
/** `CREATE INDEX` for this dialect: the migrator's, so a runtime import carries none of it. */
|
|
19
|
+
indexDdl;
|
|
17
20
|
constructor(dialect, defaultForeignKeyAction = 'NO ACTION') {
|
|
18
21
|
this.dialect = dialect;
|
|
19
22
|
this.defaultForeignKeyAction = defaultForeignKeyAction;
|
|
23
|
+
this.indexDdl = indexDdlFor(dialect);
|
|
20
24
|
}
|
|
21
25
|
get namingStrategy() {
|
|
22
26
|
return this.dialect.namingStrategy;
|
|
@@ -188,7 +192,7 @@ export class SqlSchemaGenerator {
|
|
|
188
192
|
return statements;
|
|
189
193
|
}
|
|
190
194
|
generateCreateIndex(tableName, index, options = {}) {
|
|
191
|
-
return this.
|
|
195
|
+
return this.indexDdl.getCreateIndexStatement(tableName, index, options);
|
|
192
196
|
}
|
|
193
197
|
/**
|
|
194
198
|
* `schema` is the table's, because that is where its indexes live. MySQL takes it from the table
|
|
@@ -374,23 +378,16 @@ export class SqlSchemaGenerator {
|
|
|
374
378
|
*
|
|
375
379
|
* Only ever additive. An index the entity does not name is left alone: it may well have been
|
|
376
380
|
* created deliberately outside the ORM, and dropping it is a decision for a reviewed migration.
|
|
381
|
+
*
|
|
382
|
+
* A vector index is one of these like any other: MariaDB's `CREATE VECTOR INDEX ... ON t (col)`
|
|
383
|
+
* adds one to a table that already exists, which the inline `CREATE TABLE` form it also has cannot.
|
|
377
384
|
*/
|
|
378
385
|
missingIndexes(entity, currentTable) {
|
|
379
386
|
// Keyed by the qualified name, so a table in a schema finds itself rather than reporting that
|
|
380
387
|
// the entity declares no indexes at all.
|
|
381
388
|
const desired = buildEntityAST(this, [entity]).getTable(qualifyName(currentTable.name, currentTable.schema))?.indexes ?? [];
|
|
382
389
|
const present = new Set(currentTable.indexes.map((index) => index.name));
|
|
383
|
-
return desired
|
|
384
|
-
.filter((index) => !present.has(index.name) && !this.isInlineVectorIndex(index))
|
|
385
|
-
.map(indexNodeToSchema);
|
|
386
|
-
}
|
|
387
|
-
/**
|
|
388
|
-
* A vector index this dialect declares inside `CREATE TABLE` rather than as a statement of its own,
|
|
389
|
-
* which MariaDB is alone in doing. It has no `CREATE INDEX` form, so it can only ever be created
|
|
390
|
-
* with its table, never added to one.
|
|
391
|
-
*/
|
|
392
|
-
isInlineVectorIndex(index) {
|
|
393
|
-
return this.features.inlineVectorIndex && index.type === 'vector';
|
|
390
|
+
return desired.filter((index) => !present.has(index.name)).map(indexNodeToSchema);
|
|
394
391
|
}
|
|
395
392
|
columnNodeToSchema(col) {
|
|
396
393
|
return {
|
|
@@ -483,11 +480,11 @@ export class SqlSchemaGenerator {
|
|
|
483
480
|
generateCreateTableFromNode(table, options = {}) {
|
|
484
481
|
const columns = [];
|
|
485
482
|
const constraints = [];
|
|
486
|
-
const vectorIndexes = table.indexes.filter((index) => this.isInlineVectorIndex(index));
|
|
487
|
-
const regularIndexes = table.indexes.filter((index) => !this.isInlineVectorIndex(index));
|
|
488
483
|
// MariaDB rejects a `VECTOR INDEX` whose column is nullable ("All parts of a VECTOR index must
|
|
489
484
|
// be NOT NULL"), so being indexed decides it rather than the entity's own nullability.
|
|
490
|
-
const indexedVectorColumns = new Set(
|
|
485
|
+
const indexedVectorColumns = new Set(this.features.vectorIndexRequiresNotNull
|
|
486
|
+
? table.indexes.filter((index) => index.type === 'vector').flatMap((idx) => idx.entries.map((e) => e.column))
|
|
487
|
+
: []);
|
|
491
488
|
for (const col of table.columns.values()) {
|
|
492
489
|
const colDef = this.generateColumnFromNode(indexedVectorColumns.has(col.name) ? { ...col, nullable: false } : col);
|
|
493
490
|
columns.push(colDef);
|
|
@@ -512,12 +509,6 @@ export class SqlSchemaGenerator {
|
|
|
512
509
|
createSql += ',\n';
|
|
513
510
|
createSql += constraints.map((c) => ` ${c}`).join(',\n');
|
|
514
511
|
}
|
|
515
|
-
if (vectorIndexes.length > 0) {
|
|
516
|
-
createSql += ',\n';
|
|
517
|
-
createSql += vectorIndexes
|
|
518
|
-
.map((idx) => ` ${this.dialect.getInlineVectorIndexDeclaration(indexNodeToSchema(idx))}`)
|
|
519
|
-
.join(',\n');
|
|
520
|
-
}
|
|
521
512
|
createSql += '\n)';
|
|
522
513
|
if (this.dialect.tableOptions) {
|
|
523
514
|
createSql += ` ${this.dialect.tableOptions}`;
|
|
@@ -531,7 +522,7 @@ export class SqlSchemaGenerator {
|
|
|
531
522
|
}
|
|
532
523
|
}
|
|
533
524
|
statements.push(createSql);
|
|
534
|
-
for (const idx of
|
|
525
|
+
for (const idx of table.indexes) {
|
|
535
526
|
statements.push(this.generateCreateIndexFromNode(idx));
|
|
536
527
|
}
|
|
537
528
|
return statements;
|
|
@@ -18,7 +18,6 @@ export declare class MongoDialect extends AbstractDialect {
|
|
|
18
18
|
readonly dialectName = "mongodb";
|
|
19
19
|
readonly insertIdSource = "returning";
|
|
20
20
|
private static readonly ID_KEY;
|
|
21
|
-
private static readonly VECTOR_INDEX_TYPES;
|
|
22
21
|
/** Atlas rejects a `$vectorSearch` asking for more candidates than this. */
|
|
23
22
|
private static readonly MAX_NUM_CANDIDATES;
|
|
24
23
|
private static readonly AGGREGATE_OP_MAP;
|
|
@@ -223,7 +222,7 @@ export declare class MongoDialect extends AbstractDialect {
|
|
|
223
222
|
* Build a `$vectorSearch` aggregation pipeline stage.
|
|
224
223
|
* Merges `$where` into `$vectorSearch.filter` for optimal pre-filtering.
|
|
225
224
|
*/
|
|
226
|
-
buildVectorSearchStage<E extends Document>(entity: Type<E>, key: string, search: QueryVectorSearch, where: QueryWhere<E> | undefined, limit: number, opts?: QueryOptions): Record<string, unknown>;
|
|
225
|
+
buildVectorSearchStage<E extends Document>(entity: Type<E>, key: string, search: QueryVectorSearch, where: QueryWhere<E> | undefined, limit: number, opts?: QueryOptions, candidates?: number): Record<string, unknown>;
|
|
227
226
|
}
|
|
228
227
|
export type MongoAggregationPipelineEntry<E extends Document> = {
|
|
229
228
|
$lookup?: MongoAggregationLookup<E>;
|
|
@@ -4,7 +4,7 @@ import { COUNT_ALIAS, REL_NESTED_KEY, REL_TEMP_PREFIX, sortCountField } from '..
|
|
|
4
4
|
import { resolveQueryJoins, resolveSortableJoin } from '../dialect/queryJoins.js';
|
|
5
5
|
import { getMeta } from '../entity/index.js';
|
|
6
6
|
import { QueryRaw } from '../type/queryRaw.js';
|
|
7
|
-
import { asSelectMap, assertAggregateColumns, assertNonNegativeInteger, buildQueryWhereAsMap, fillOnFields, filterFieldKeys, getKeys, getRelationRequestSummary, hasKeys, isJsonUpdateOp, isOperatorMap, isOperatorObject, isVectorSearch, normalizeScalarFieldSelection, parseGroupMap, parseRelationSize, parseSortByCount, someKey, } from '../util/index.js';
|
|
7
|
+
import { asSelectMap, assertAggregateColumns, assertNonNegativeInteger, buildQueryWhereAsMap, entityName, fillOnFields, filterFieldKeys, findVectorIndex, findVectorSort, getKeys, getRelationRequestSummary, hasKeys, isJsonUpdateOp, isOperatorMap, isOperatorObject, isVectorSearch, normalizeScalarFieldSelection, parseGroupMap, parseRelationSize, parseSortByCount, someKey, } from '../util/index.js';
|
|
8
8
|
/** Default {@link DialectFeatures} for MongoDB; shared by {@link MongoDialect} and its schema generator. */
|
|
9
9
|
export const mongoDialectFeatures = {
|
|
10
10
|
explicitJsonCast: false,
|
|
@@ -17,7 +17,7 @@ export const mongoDialectFeatures = {
|
|
|
17
17
|
renameColumn: false,
|
|
18
18
|
foreignKeyAlter: false,
|
|
19
19
|
columnComment: false,
|
|
20
|
-
|
|
20
|
+
vectorIndexRequiresNotNull: false,
|
|
21
21
|
vectorSupportsLength: false,
|
|
22
22
|
supportsTimestamptz: false,
|
|
23
23
|
defaultStringAsText: false,
|
|
@@ -28,7 +28,6 @@ export class MongoDialect extends AbstractDialect {
|
|
|
28
28
|
// The MongoDB driver reports the exact `_id` of every inserted document (`insertedIds`).
|
|
29
29
|
insertIdSource = 'returning';
|
|
30
30
|
static ID_KEY = '_id';
|
|
31
|
-
static VECTOR_INDEX_TYPES = new Set(['vectorSearch', 'hnsw', 'ivfflat', 'vector']);
|
|
32
31
|
/** Atlas rejects a `$vectorSearch` asking for more candidates than this. */
|
|
33
32
|
static MAX_NUM_CANDIDATES = 10_000;
|
|
34
33
|
// Direct field aggregates → MongoDB accumulator. `$count` is handled separately (COUNT(*) vs
|
|
@@ -254,7 +253,7 @@ export class MongoDialect extends AbstractDialect {
|
|
|
254
253
|
if (root === MongoDialect.ID_KEY || root === meta.id || meta.fields[root]) {
|
|
255
254
|
return;
|
|
256
255
|
}
|
|
257
|
-
throw new TypeError(`path ${key} does not exist in ${meta
|
|
256
|
+
throw new TypeError(`path ${key} does not exist in ${entityName(meta)}`);
|
|
258
257
|
}
|
|
259
258
|
mapTableNameRow(row) {
|
|
260
259
|
return row.table_name;
|
|
@@ -329,6 +328,16 @@ export class MongoDialect extends AbstractDialect {
|
|
|
329
328
|
case '$text':
|
|
330
329
|
result['$text'] = { $search: val };
|
|
331
330
|
break;
|
|
331
|
+
case '$near':
|
|
332
|
+
// Atlas has no distance operator. The only threshold it offers is a `$match` on
|
|
333
|
+
// `{$meta:'vectorSearchScore'}`, which is a *similarity* on a scale set by the index's own
|
|
334
|
+
// `similarity` - and that lives in the Atlas index definition, which UQL neither emits nor
|
|
335
|
+
// reads (the same reason `$distance` is index-defined for `$text` above). Converting a
|
|
336
|
+
// distance to that scale would mean guessing which metric produced the score, and guessing
|
|
337
|
+
// wrong filters the wrong rows silently. So this refuses, the way an unsupported
|
|
338
|
+
// `DISTANCE=` does rather than defaulting.
|
|
339
|
+
throw new TypeError('$near is not supported on MongoDB: Atlas scores by index-defined similarity, not distance. ' +
|
|
340
|
+
"Project the score with $sort's $project and filter on it instead.");
|
|
332
341
|
default:
|
|
333
342
|
result[op] = val;
|
|
334
343
|
break;
|
|
@@ -912,29 +921,27 @@ export class MongoDialect extends AbstractDialect {
|
|
|
912
921
|
* Returns `undefined` if no vector sort is present.
|
|
913
922
|
*/
|
|
914
923
|
extractVectorSort(sort) {
|
|
915
|
-
|
|
924
|
+
const found = sort && findVectorSort(sort);
|
|
925
|
+
if (!found) {
|
|
916
926
|
return undefined;
|
|
917
|
-
|
|
918
|
-
|
|
927
|
+
}
|
|
928
|
+
// The remaining entries order the rows the vector stage already picked, so they stay a `$sort`.
|
|
929
|
+
// Copied in one pass rather than `entries().filter().fromEntries()`, which walks the map three
|
|
930
|
+
// times over. Every key of the map is optional, so dropping one leaves a valid map - which
|
|
931
|
+
// neither `Omit` nor a computed-key rest can say over a mapped type with no index signature.
|
|
919
932
|
const regularSort = {};
|
|
920
|
-
for (const
|
|
921
|
-
if (
|
|
922
|
-
|
|
923
|
-
vectorSearch = value;
|
|
924
|
-
}
|
|
925
|
-
else {
|
|
926
|
-
regularSort[key] = value;
|
|
933
|
+
for (const key of getKeys(sort)) {
|
|
934
|
+
if (key !== found.key) {
|
|
935
|
+
regularSort[key] = sort[key];
|
|
927
936
|
}
|
|
928
937
|
}
|
|
929
|
-
|
|
930
|
-
return undefined;
|
|
931
|
-
return { vectorKey, vectorSearch, regularSort };
|
|
938
|
+
return { vectorKey: found.key, vectorSearch: found.search, regularSort: regularSort };
|
|
932
939
|
}
|
|
933
940
|
/**
|
|
934
941
|
* Build a `$vectorSearch` aggregation pipeline stage.
|
|
935
942
|
* Merges `$where` into `$vectorSearch.filter` for optimal pre-filtering.
|
|
936
943
|
*/
|
|
937
|
-
buildVectorSearchStage(entity, key, search, where, limit, opts) {
|
|
944
|
+
buildVectorSearchStage(entity, key, search, where, limit, opts, candidates) {
|
|
938
945
|
const meta = getMeta(entity);
|
|
939
946
|
const field = meta.fields[key];
|
|
940
947
|
if (!field) {
|
|
@@ -942,8 +949,7 @@ export class MongoDialect extends AbstractDialect {
|
|
|
942
949
|
}
|
|
943
950
|
const colName = this.resolveColumnName(key, field);
|
|
944
951
|
// Resolve index name from @Index metadata, or fall back to convention
|
|
945
|
-
const
|
|
946
|
-
const indexName = indexMeta?.name ?? `${colName}_index`;
|
|
952
|
+
const indexName = findVectorIndex(meta, key)?.name ?? `${colName}_index`;
|
|
947
953
|
if (!limit) {
|
|
948
954
|
throw new TypeError(`$vectorSearch requires $limit (vector sort on '${key}' of '${meta.name}')`);
|
|
949
955
|
}
|
|
@@ -951,8 +957,9 @@ export class MongoDialect extends AbstractDialect {
|
|
|
951
957
|
index: indexName,
|
|
952
958
|
path: colName,
|
|
953
959
|
queryVector: [...search.$vector],
|
|
954
|
-
//
|
|
955
|
-
|
|
960
|
+
// `$candidates` is the caller's own budget; the fallback is 10x the limit, which is what Atlas
|
|
961
|
+
// suggests as a floor. Either way it is clamped: Atlas rejects a stage asking for more.
|
|
962
|
+
numCandidates: Math.min(candidates ?? limit * 10, MongoDialect.MAX_NUM_CANDIDATES),
|
|
956
963
|
limit,
|
|
957
964
|
};
|
|
958
965
|
// Pre-filter: merge $where into $vectorSearch.filter
|
|
@@ -133,7 +133,7 @@ export class MongodbQuerier extends AbstractQuerier {
|
|
|
133
133
|
buildVectorPipeline(entity, q, vectorSort, opts) {
|
|
134
134
|
const scoreAlias = vectorSort.vectorSearch.$project;
|
|
135
135
|
return [
|
|
136
|
-
this.dialect.buildVectorSearchStage(entity, vectorSort.vectorKey, vectorSort.vectorSearch, q.$where, q.$limit ?? 10, opts),
|
|
136
|
+
this.dialect.buildVectorSearchStage(entity, vectorSort.vectorKey, vectorSort.vectorSearch, q.$where, q.$limit ?? 10, opts, q.$candidates),
|
|
137
137
|
// The score becomes a real field before anything reads it, so the lookups and the projection
|
|
138
138
|
// that follow treat it like any other - and a query with no projection keeps its own columns.
|
|
139
139
|
...(scoreAlias ? [{ $addFields: { [scoreAlias]: { $meta: 'vectorSearchScore' } } }] : []),
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
import { MysqlLikeSqlDialect } from '../dialect/mysqlLikeSqlDialect.js';
|
|
2
|
-
import type { IndexSchema } from '../type/index.js';
|
|
3
2
|
export declare class MySqlDialect extends MysqlLikeSqlDialect {
|
|
4
3
|
readonly dialectName = "mysql";
|
|
5
4
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* syntax either. Verified against MySQL 9.7, which does have `VECTOR` columns and
|
|
9
|
-
* `STRING_TO_VECTOR`, but no distance function outside HeatWave - hence nothing to index for.
|
|
5
|
+
* `VALUES(col)` inside `ON DUPLICATE KEY UPDATE` has been deprecated since MySQL 8.0.20 and is
|
|
6
|
+
* "subject to removal in a future version"; aliasing the inserted row (8.0.19+) is its replacement.
|
|
10
7
|
*/
|
|
11
|
-
protected
|
|
8
|
+
protected readonly upsertNewRowAlias = "_uql_new";
|
|
12
9
|
}
|
|
@@ -1,17 +1,10 @@
|
|
|
1
|
+
import { UPSERT_NEW_ROW_ALIAS } from '../dialect/aliases.js';
|
|
1
2
|
import { MysqlLikeSqlDialect } from '../dialect/mysqlLikeSqlDialect.js';
|
|
2
3
|
export class MySqlDialect extends MysqlLikeSqlDialect {
|
|
3
4
|
dialectName = 'mysql';
|
|
4
5
|
/**
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* syntax either. Verified against MySQL 9.7, which does have `VECTOR` columns and
|
|
8
|
-
* `STRING_TO_VECTOR`, but no distance function outside HeatWave - hence nothing to index for.
|
|
6
|
+
* `VALUES(col)` inside `ON DUPLICATE KEY UPDATE` has been deprecated since MySQL 8.0.20 and is
|
|
7
|
+
* "subject to removal in a future version"; aliasing the inserted row (8.0.19+) is its replacement.
|
|
9
8
|
*/
|
|
10
|
-
|
|
11
|
-
if (index.type === 'vector' || index.type === 'hnsw' || index.type === 'ivfflat') {
|
|
12
|
-
throw new TypeError(`${this.dialectName} has no vector index (index "${index.name}" declares type "${index.type}"). ` +
|
|
13
|
-
'Vector search on MySQL needs HeatWave.');
|
|
14
|
-
}
|
|
15
|
-
return super.indexAccessMethod(index);
|
|
16
|
-
}
|
|
9
|
+
upsertNewRowAlias = UPSERT_NEW_ROW_ALIAS;
|
|
17
10
|
}
|
|
@@ -10,6 +10,8 @@ import type { QueryConflictPaths, QueryContext, SqlDialectName, Type } from '../
|
|
|
10
10
|
export declare class PostgresDialect extends PgLikeSqlDialect {
|
|
11
11
|
readonly dialectName: SqlDialectName;
|
|
12
12
|
readonly vectorExtension: string | undefined;
|
|
13
|
+
/** pgvector is the only engine with `halfvec` and `sparsevec`; every other maps them onto `vector`. */
|
|
14
|
+
protected readonly hasNarrowVectorTypes = true;
|
|
13
15
|
upsert<E>(ctx: QueryContext, entity: Type<E>, conflictPaths: QueryConflictPaths<E>, payload: E | E[]): void;
|
|
14
16
|
/**
|
|
15
17
|
* `to_regclass` rather than a `::regclass` cast: it answers `NULL` for a table that does not exist
|
|
@@ -11,6 +11,8 @@ import { getMeta } from '../entity/index.js';
|
|
|
11
11
|
export class PostgresDialect extends PgLikeSqlDialect {
|
|
12
12
|
dialectName = 'postgres';
|
|
13
13
|
vectorExtension = 'vector';
|
|
14
|
+
/** pgvector is the only engine with `halfvec` and `sparsevec`; every other maps them onto `vector`. */
|
|
15
|
+
hasNarrowVectorTypes = true;
|
|
14
16
|
upsert(ctx, entity, conflictPaths, payload) {
|
|
15
17
|
// The xmax system column is 0 for a newly inserted row and non-zero for an updated one (MVCC).
|
|
16
18
|
super.upsert(ctx, entity, conflictPaths, payload, `, (xmax = 0) AS ${this.escapeId('_created')}`);
|
|
@@ -45,6 +45,17 @@ export declare abstract class AbstractSqlQuerier extends AbstractQuerier impleme
|
|
|
45
45
|
* actionable answer, and on SQLite it is the answer either way.
|
|
46
46
|
*/
|
|
47
47
|
protected assertLockable<E>(entity: Type<E>, q: Query<E>): void;
|
|
48
|
+
/**
|
|
49
|
+
* Run the `SET`s that tune an ANN index for this query, and refuse the ones that would not apply.
|
|
50
|
+
*
|
|
51
|
+
* Same shape as {@link assertLockable} and for the same reason: a `SET LOCAL` outside a transaction
|
|
52
|
+
* is accepted, applies to nothing, and leaves the query running at the engine's default recall -
|
|
53
|
+
* correct SQL, silently untuned. Only the querier knows whether a transaction is open.
|
|
54
|
+
*
|
|
55
|
+
* The statements go through `internalRun`, sharing this querier's single connection with the query
|
|
56
|
+
* they precede; `SET LOCAL` then expires with the transaction, so nothing is left behind.
|
|
57
|
+
*/
|
|
58
|
+
private applyVectorTuning;
|
|
48
59
|
protected internalFindMany<E extends object>(entity: Type<E>, q: Query<E>, opts?: QueryOptions): Promise<E[]>;
|
|
49
60
|
/**
|
|
50
61
|
* One statement for both: the page carries its own unpaged total in an extra column. An empty page
|
|
@@ -76,6 +76,31 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
76
76
|
throw new TypeError('$lock requires an open transaction');
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Run the `SET`s that tune an ANN index for this query, and refuse the ones that would not apply.
|
|
81
|
+
*
|
|
82
|
+
* Same shape as {@link assertLockable} and for the same reason: a `SET LOCAL` outside a transaction
|
|
83
|
+
* is accepted, applies to nothing, and leaves the query running at the engine's default recall -
|
|
84
|
+
* correct SQL, silently untuned. Only the querier knows whether a transaction is open.
|
|
85
|
+
*
|
|
86
|
+
* The statements go through `internalRun`, sharing this querier's single connection with the query
|
|
87
|
+
* they precede; `SET LOCAL` then expires with the transaction, so nothing is left behind.
|
|
88
|
+
*/
|
|
89
|
+
async applyVectorTuning(entity, q) {
|
|
90
|
+
// Resolved before the transaction check, so the refusal fires only where the tuning would have
|
|
91
|
+
// meant something: a query with no vector search, or on a field carrying no ANN index, has
|
|
92
|
+
// nothing to set and no reason to demand a transaction for it.
|
|
93
|
+
const statements = this.dialect.vectorTuningStatements(getMeta(entity), q);
|
|
94
|
+
if (!statements.length) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (this.dialect.vectorTuningNeedsTransaction && !this.hasOpenTransaction) {
|
|
98
|
+
throw new TypeError(`$candidates requires an open transaction on ${this.dialect.dialectName}; run the query inside pool.transaction(...)`);
|
|
99
|
+
}
|
|
100
|
+
for (const statement of statements) {
|
|
101
|
+
await this.internalRun(statement);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
79
104
|
async internalFindMany(entity, q, opts) {
|
|
80
105
|
return this.hydrateRows(entity, q, await this.selectRows(entity, q, opts));
|
|
81
106
|
}
|
|
@@ -119,6 +144,12 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
119
144
|
}
|
|
120
145
|
async selectRows(entity, q, opts, totalAlias) {
|
|
121
146
|
this.assertLockable(entity, q);
|
|
147
|
+
// Guarded rather than awaited unconditionally, here and in the stream below: an `await` on this
|
|
148
|
+
// path defers a microtask on every read, which reorders the two statements `findManyAndCount`
|
|
149
|
+
// issues concurrently. Keep the guard at any new call site.
|
|
150
|
+
if (q.$candidates !== undefined) {
|
|
151
|
+
await this.applyVectorTuning(entity, q);
|
|
152
|
+
}
|
|
122
153
|
const ctx = this.dialect.createContext();
|
|
123
154
|
this.dialect.find(ctx, entity, q, opts, totalAlias);
|
|
124
155
|
return this.all(ctx.sql, ctx.values);
|
|
@@ -130,6 +161,10 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
130
161
|
}
|
|
131
162
|
async *internalFindManyStream(entity, q, opts) {
|
|
132
163
|
this.assertLockable(entity, q);
|
|
164
|
+
// Guarded for the reason `selectRows` above spells out.
|
|
165
|
+
if (q.$candidates !== undefined) {
|
|
166
|
+
await this.applyVectorTuning(entity, q);
|
|
167
|
+
}
|
|
133
168
|
const meta = getMeta(entity);
|
|
134
169
|
const { toManyKeys } = getRelationRequestSummary(meta, q.$populate);
|
|
135
170
|
if (toManyKeys.length) {
|
|
@@ -83,7 +83,7 @@ const SQL_TO_CANONICAL = {
|
|
|
83
83
|
};
|
|
84
84
|
/**
|
|
85
85
|
* pgvector is the only engine with three vector column types, so every other dialect maps all three
|
|
86
|
-
* canonical categories onto the single type it does have (see
|
|
86
|
+
* canonical categories onto the single type it does have (see `hasNarrowVectorTypes` on the dialect, the
|
|
87
87
|
* dialect-side half of the same fact).
|
|
88
88
|
*/
|
|
89
89
|
function withVectorType(scalars, vector) {
|
|
@@ -125,7 +125,7 @@ const CANONICAL_TO_SQL = {
|
|
|
125
125
|
uuid: 'CHAR(36)',
|
|
126
126
|
blob: 'BLOB',
|
|
127
127
|
},
|
|
128
|
-
// MySQL
|
|
128
|
+
// MySQL does have a `VECTOR` type (26.7), but no distance function outside HeatWave and no vector
|
|
129
129
|
// index, so JSON keeps the column queryable with the JSON operators and needs no conversion.
|
|
130
130
|
'JSON'),
|
|
131
131
|
sqlite: withVectorType({
|
|
@@ -186,44 +186,43 @@ const PG_SIZE_MODIFIERS = {
|
|
|
186
186
|
big: 'DOUBLE PRECISION',
|
|
187
187
|
},
|
|
188
188
|
};
|
|
189
|
+
/**
|
|
190
|
+
* MariaDB is a MySQL fork and spells every one of these the same way, so both take the one map.
|
|
191
|
+
* Listed per-dialect, MariaDB's had only `integer`: a `double` column was created `FLOAT` there, four
|
|
192
|
+
* bytes where the entity asked for eight, and a `big` string or blob lost its `LONG` prefix.
|
|
193
|
+
*/
|
|
194
|
+
const MYSQL_SIZE_MODIFIERS = {
|
|
195
|
+
integer: {
|
|
196
|
+
tiny: 'TINYINT',
|
|
197
|
+
small: 'SMALLINT',
|
|
198
|
+
medium: 'MEDIUMINT',
|
|
199
|
+
big: 'BIGINT',
|
|
200
|
+
},
|
|
201
|
+
float: {
|
|
202
|
+
tiny: 'FLOAT',
|
|
203
|
+
small: 'FLOAT',
|
|
204
|
+
medium: 'DOUBLE',
|
|
205
|
+
big: 'DOUBLE',
|
|
206
|
+
},
|
|
207
|
+
string: {
|
|
208
|
+
tiny: 'TINYTEXT',
|
|
209
|
+
small: 'TEXT',
|
|
210
|
+
medium: 'MEDIUMTEXT',
|
|
211
|
+
big: 'LONGTEXT',
|
|
212
|
+
},
|
|
213
|
+
blob: {
|
|
214
|
+
tiny: 'TINYBLOB',
|
|
215
|
+
small: 'BLOB',
|
|
216
|
+
medium: 'MEDIUMBLOB',
|
|
217
|
+
big: 'LONGBLOB',
|
|
218
|
+
},
|
|
219
|
+
};
|
|
189
220
|
const SIZE_MODIFIERS = {
|
|
190
221
|
postgres: PG_SIZE_MODIFIERS,
|
|
191
222
|
cockroachdb: PG_SIZE_MODIFIERS,
|
|
192
|
-
mysql:
|
|
193
|
-
integer: {
|
|
194
|
-
tiny: 'TINYINT',
|
|
195
|
-
small: 'SMALLINT',
|
|
196
|
-
medium: 'MEDIUMINT',
|
|
197
|
-
big: 'BIGINT',
|
|
198
|
-
},
|
|
199
|
-
float: {
|
|
200
|
-
tiny: 'FLOAT',
|
|
201
|
-
small: 'FLOAT',
|
|
202
|
-
medium: 'DOUBLE',
|
|
203
|
-
big: 'DOUBLE',
|
|
204
|
-
},
|
|
205
|
-
string: {
|
|
206
|
-
tiny: 'TINYTEXT',
|
|
207
|
-
small: 'TEXT',
|
|
208
|
-
medium: 'MEDIUMTEXT',
|
|
209
|
-
big: 'LONGTEXT',
|
|
210
|
-
},
|
|
211
|
-
blob: {
|
|
212
|
-
tiny: 'TINYBLOB',
|
|
213
|
-
small: 'BLOB',
|
|
214
|
-
medium: 'MEDIUMBLOB',
|
|
215
|
-
big: 'LONGBLOB',
|
|
216
|
-
},
|
|
217
|
-
},
|
|
223
|
+
mysql: MYSQL_SIZE_MODIFIERS,
|
|
218
224
|
sqlite: {}, // SQLite uses affinity, no size modifiers
|
|
219
|
-
mariadb:
|
|
220
|
-
integer: {
|
|
221
|
-
tiny: 'TINYINT',
|
|
222
|
-
small: 'SMALLINT',
|
|
223
|
-
medium: 'MEDIUMINT',
|
|
224
|
-
big: 'BIGINT',
|
|
225
|
-
},
|
|
226
|
-
},
|
|
225
|
+
mariadb: MYSQL_SIZE_MODIFIERS,
|
|
227
226
|
mongodb: {},
|
|
228
227
|
};
|
|
229
228
|
/**
|
|
@@ -17,6 +17,7 @@ export type IndexFacet = 'order' | 'nulls' | 'opsClass' | 'accessMethod' | 'incl
|
|
|
17
17
|
* `status = ANY (ARRAY['a'::text, 'b'::text])`, `LIKE` as `~~`, and a date literal with its time zone
|
|
18
18
|
* spelled out. Folding that back needs a SQL parser, and every near-miss reports drift that no
|
|
19
19
|
* migration can settle. So a partial index's predicate is never compared, and an index over an
|
|
20
|
-
* expression
|
|
20
|
+
* expression - a `raw()` one, or a JSON path, which the engines report as an expression too - has
|
|
21
|
+
* its entries left alone while the rest of it still compares.
|
|
21
22
|
*/
|
|
22
23
|
export declare function describeIndexDifferences(source: IndexNode, target: IndexNode, facets: ReadonlySet<IndexFacet>): string[];
|
|
@@ -6,11 +6,12 @@
|
|
|
6
6
|
* `status = ANY (ARRAY['a'::text, 'b'::text])`, `LIKE` as `~~`, and a date literal with its time zone
|
|
7
7
|
* spelled out. Folding that back needs a SQL parser, and every near-miss reports drift that no
|
|
8
8
|
* migration can settle. So a partial index's predicate is never compared, and an index over an
|
|
9
|
-
* expression
|
|
9
|
+
* expression - a `raw()` one, or a JSON path, which the engines report as an expression too - has
|
|
10
|
+
* its entries left alone while the rest of it still compares.
|
|
10
11
|
*/
|
|
11
12
|
export function describeIndexDifferences(source, target, facets) {
|
|
12
13
|
const differences = [];
|
|
13
|
-
const comparableEntries = ![...source.entries, ...target.entries].some((entry) => entry.expression);
|
|
14
|
+
const comparableEntries = ![...source.entries, ...target.entries].some((entry) => entry.expression || entry.jsonPath || entry.jsonArray);
|
|
14
15
|
if (comparableEntries) {
|
|
15
16
|
const [sourceColumns, targetColumns] = [source.entries, target.entries].map((entries) => entries.map((entry) => entrySignature(entry, facets)).join(', '));
|
|
16
17
|
if (sourceColumns !== targetColumns) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AbstractSqlDialect } from '../dialect/abstractSqlDialect.js';
|
|
2
|
-
import type { DialectFeatures, EntityMeta, FieldOptions, QueryContext, QueryPager, QuerySizeComparisonOps, QueryTextSearchOptions, Type, VectorDistance } from '../type/index.js';
|
|
2
|
+
import type { DialectFeatures, EntityMeta, FieldOptions, QueryContext, QueryPager, QuerySizeComparisonOps, QueryTextSearchOptions, Type, VectorDistance, VectorMetric } from '../type/index.js';
|
|
3
3
|
export declare class SqliteDialect extends AbstractSqlDialect {
|
|
4
4
|
/** Default {@link DialectFeatures} for SQLite and SQLite-derived dialects. */
|
|
5
5
|
protected readonly featureDefaults: DialectFeatures;
|
|
@@ -21,7 +21,7 @@ export declare class SqliteDialect extends AbstractSqlDialect {
|
|
|
21
21
|
* loaded on the connection (see `Sqlite3QuerierPool`'s `extensions` option). libSQL and Turso ship
|
|
22
22
|
* their own vector functions instead, so `LibsqlDialect` overrides this.
|
|
23
23
|
*/
|
|
24
|
-
|
|
24
|
+
readonly vectorMetrics: ReadonlyMap<VectorDistance, VectorMetric>;
|
|
25
25
|
/**
|
|
26
26
|
* SQLite does not support the `DEFAULT` keyword inside `VALUES`. Inline the metadata default
|
|
27
27
|
* when declared, else `NULL` (which is also how SQLite auto-generates INTEGER PRIMARY KEYs).
|
|
@@ -14,7 +14,7 @@ export class SqliteDialect extends AbstractSqlDialect {
|
|
|
14
14
|
renameColumn: true,
|
|
15
15
|
foreignKeyAlter: false, // SQLite does not support adding FKs to existing tables
|
|
16
16
|
columnComment: false, // SQLite does not support column comments
|
|
17
|
-
|
|
17
|
+
vectorIndexRequiresNotNull: false,
|
|
18
18
|
vectorSupportsLength: false,
|
|
19
19
|
supportsTimestamptz: false,
|
|
20
20
|
defaultStringAsText: true,
|
|
@@ -38,10 +38,10 @@ export class SqliteDialect extends AbstractSqlDialect {
|
|
|
38
38
|
* loaded on the connection (see `Sqlite3QuerierPool`'s `extensions` option). libSQL and Turso ship
|
|
39
39
|
* their own vector functions instead, so `LibsqlDialect` overrides this.
|
|
40
40
|
*/
|
|
41
|
-
|
|
42
|
-
['cosine', 'vec_distance_cosine'],
|
|
43
|
-
['l2', 'vec_distance_L2'],
|
|
44
|
-
['l1', 'vec_distance_L1'],
|
|
41
|
+
vectorMetrics = new Map([
|
|
42
|
+
['cosine', { fn: 'vec_distance_cosine' }],
|
|
43
|
+
['l2', { fn: 'vec_distance_L2' }],
|
|
44
|
+
['l1', { fn: 'vec_distance_L1' }],
|
|
45
45
|
]);
|
|
46
46
|
/**
|
|
47
47
|
* SQLite does not support the `DEFAULT` keyword inside `VALUES`. Inline the metadata default
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { LibsqlDialect } from '../libsql/libsqlDialect.js';
|
|
2
|
-
import type { VectorDistance } from '../type/index.js';
|
|
2
|
+
import type { VectorDistance, VectorMetric } from '../type/index.js';
|
|
3
3
|
/**
|
|
4
4
|
* SQLite Dialect specialization for Turso Database.
|
|
5
5
|
*
|
|
@@ -11,5 +11,5 @@ import type { VectorDistance } from '../type/index.js';
|
|
|
11
11
|
*/
|
|
12
12
|
export declare class TursoDialect extends LibsqlDialect {
|
|
13
13
|
/** The Rust engine adds a dot-product distance to libSQL's cosine and L2. */
|
|
14
|
-
|
|
14
|
+
readonly vectorMetrics: ReadonlyMap<VectorDistance, VectorMetric>;
|
|
15
15
|
}
|
|
@@ -10,9 +10,9 @@ import { LibsqlDialect } from '../libsql/libsqlDialect.js';
|
|
|
10
10
|
*/
|
|
11
11
|
export class TursoDialect extends LibsqlDialect {
|
|
12
12
|
/** The Rust engine adds a dot-product distance to libSQL's cosine and L2. */
|
|
13
|
-
|
|
14
|
-
['cosine', 'vector_distance_cos'],
|
|
15
|
-
['l2', 'vector_distance_l2'],
|
|
16
|
-
['inner', 'vector_distance_dot'],
|
|
13
|
+
vectorMetrics = new Map([
|
|
14
|
+
['cosine', { fn: 'vector_distance_cos' }],
|
|
15
|
+
['l2', { fn: 'vector_distance_l2' }],
|
|
16
|
+
['inner', { fn: 'vector_distance_dot' }],
|
|
17
17
|
]);
|
|
18
18
|
}
|
package/dist/type/dialect.d.ts
CHANGED
|
@@ -94,11 +94,11 @@ export interface EngineFeatures {
|
|
|
94
94
|
/** Whether the dialect supports inline COMMENT on columns (MySQL/MariaDB). */
|
|
95
95
|
readonly columnComment: boolean;
|
|
96
96
|
/**
|
|
97
|
-
* Whether a vector index
|
|
98
|
-
*
|
|
99
|
-
*
|
|
97
|
+
* Whether every column of a vector index has to be `NOT NULL`, which MariaDB 12.3 enforces ("All
|
|
98
|
+
* parts of a VECTOR index must be NOT NULL") and CockroachDB 26.3 does not - so being indexed, not
|
|
99
|
+
* the entity, decides the column's nullability there.
|
|
100
100
|
*/
|
|
101
|
-
readonly
|
|
101
|
+
readonly vectorIndexRequiresNotNull: boolean;
|
|
102
102
|
/** Whether the dialect requires/allows (n) length constraints on vector types. */
|
|
103
103
|
readonly vectorSupportsLength: boolean;
|
|
104
104
|
/** Whether the dialect natively supports the TIMESTAMPTZ alias/type. */
|
|
@@ -222,11 +222,25 @@ export interface SqlQueryDialect extends QueryDialect {
|
|
|
222
222
|
* indexes exist everywhere but MariaDB 12.3 (which needs a generated column); prefix lengths are
|
|
223
223
|
* MySQL-family only and *required* there to index `TEXT`; `NULLS FIRST/LAST` and operator classes are
|
|
224
224
|
* Postgres-only (CockroachDB 26.2 answers "unimplemented"); `INCLUDE` is Postgres-wire only; the
|
|
225
|
-
* MySQL family is alone in having no partial indexes.
|
|
225
|
+
* MySQL family is alone in having no partial indexes. The two JSON ones split the other way: the
|
|
226
|
+
* engines that index a path inside a document are the ones whose planner matches the query's own
|
|
227
|
+
* extraction back to it (Postgres, CockroachDB, SQLite - MySQL matches neither
|
|
228
|
+
* `CAST(col->>'$.x' AS CHAR(n))` nor `JSON_VALUE(... RETURNING ...)`, verified on 26.7, and needs a
|
|
229
|
+
* generated column instead), while MySQL alone has the multi-valued index a JSON array needs.
|
|
226
230
|
*
|
|
227
231
|
* Introspectors reuse the vocabulary for what they can read *back*, which is what diffing may
|
|
228
232
|
* compare. The two sets are deliberately not the same object and must not be unified: Postgres can
|
|
229
233
|
* emit an expression index and read one back, but MySQL emits one it cannot describe afterwards.
|
|
230
234
|
*/
|
|
231
|
-
export
|
|
232
|
-
|
|
235
|
+
export declare const INDEX_FEATURE_LABELS: {
|
|
236
|
+
readonly expression: 'expression indexes';
|
|
237
|
+
readonly partial: 'partial indexes';
|
|
238
|
+
readonly prefixLength: 'index prefix lengths';
|
|
239
|
+
readonly nullsOrder: 'NULLS FIRST/LAST in an index';
|
|
240
|
+
readonly opsClass: 'index operator classes';
|
|
241
|
+
readonly include: 'covering indexes (INCLUDE)';
|
|
242
|
+
readonly jsonPath: 'indexes over a path inside a JSON column';
|
|
243
|
+
readonly jsonArray: 'multi-valued indexes over a JSON array';
|
|
244
|
+
};
|
|
245
|
+
/** Derived from the labels, so a feature cannot be added without the words an error reports it in. */
|
|
246
|
+
export type IndexFeature = keyof typeof INDEX_FEATURE_LABELS;
|