uql-orm 0.70.0 → 0.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/browser/uql-browser.min.js +2 -2
- package/dist/browser/uql-browser.min.js.map +3 -3
- package/dist/cockroachdb/cockroachDialect.d.ts +6 -0
- package/dist/cockroachdb/cockroachDialect.js +8 -0
- package/dist/d1/d1SqliteDialect.d.ts +3 -0
- package/dist/d1/d1SqliteDialect.js +3 -1
- package/dist/dialect/abstractSqlDialect.d.ts +14 -2
- package/dist/dialect/abstractSqlDialect.js +48 -16
- package/dist/dialect/aliases.d.ts +2 -0
- package/dist/dialect/aliases.js +2 -0
- package/dist/dialect/hydrateColumn.d.ts +3 -2
- package/dist/dialect/hydrateColumn.js +10 -1
- package/dist/dialect/mysqlLikeSqlDialect.d.ts +2 -0
- package/dist/dialect/mysqlLikeSqlDialect.js +6 -1
- package/dist/dialect/pgLikeSqlDialect.d.ts +21 -5
- package/dist/dialect/pgLikeSqlDialect.js +48 -15
- package/dist/dialect/queryJoins.d.ts +14 -4
- package/dist/dialect/queryJoins.js +31 -11
- package/dist/dialect/vectorCast.d.ts +2 -0
- package/dist/dialect/vectorCast.js +7 -0
- package/dist/dialect/vectorSqlDialect.d.ts +7 -3
- package/dist/dialect/vectorSqlDialect.js +15 -10
- package/dist/libsql/libsqlDialect.d.ts +1 -1
- package/dist/libsql/libsqlDialect.js +3 -3
- package/dist/maria/mariaDialect.d.ts +3 -9
- package/dist/maria/mariaDialect.js +5 -15
- package/dist/maria/mariadbQuerier.js +9 -3
- package/dist/migrate/ddl/index.d.ts +1 -0
- package/dist/migrate/ddl/index.js +4 -2
- package/dist/migrate/ddl/indexDdl.d.ts +2 -0
- package/dist/migrate/ddl/indexDdl.js +10 -2
- package/dist/migrate/ddl/pgIndexDdl.d.ts +3 -4
- package/dist/migrate/ddl/pgIndexDdl.js +11 -11
- package/dist/migrate/ddl/sqliteIndexDdl.d.ts +11 -0
- package/dist/migrate/ddl/sqliteIndexDdl.js +38 -0
- package/dist/migrate/generator/mongoCommand.d.ts +28 -0
- package/dist/migrate/generator/mongoCommand.js +8 -0
- package/dist/migrate/generator/mongoSchemaGenerator.d.ts +2 -0
- package/dist/migrate/generator/mongoSchemaGenerator.js +50 -6
- package/dist/migrate/introspection/mongoIntrospector.js +33 -7
- package/dist/migrate/introspection/postgresIntrospector.js +19 -0
- package/dist/migrate/introspection/sqliteIntrospector.d.ts +6 -0
- package/dist/migrate/introspection/sqliteIntrospector.js +29 -3
- package/dist/mongo/mongoDialect.d.ts +5 -3
- package/dist/mongo/mongoDialect.js +40 -17
- package/dist/mongo/mongodbQuerier.js +4 -5
- package/dist/mssql/mssqlDialect.js +1 -0
- package/dist/schema/canonicalType.js +10 -4
- package/dist/schema/indexDifferences.d.ts +5 -2
- package/dist/schema/indexDifferences.js +5 -0
- package/dist/schema/schemaASTBuilder.js +3 -2
- package/dist/sqlite/localSqliteQuerierPool.d.ts +10 -11
- package/dist/sqlite/localSqliteQuerierPool.js +8 -11
- package/dist/sqlite/nodeSqliteQuerierPool.d.ts +4 -2
- package/dist/sqlite/nodeSqliteQuerierPool.js +7 -4
- package/dist/sqlite/sqliteDialect.d.ts +11 -1
- package/dist/sqlite/sqliteDialect.js +46 -3
- package/dist/sqlite/sqliteQuerierPool.d.ts +6 -5
- package/dist/sqlite/sqliteQuerierPool.js +10 -7
- package/dist/turso/tursoLocalDialect.d.ts +1 -1
- package/dist/turso/tursoLocalDialect.js +1 -1
- package/dist/turso/tursoLocalQuerierPool.d.ts +4 -5
- package/dist/turso/tursoLocalQuerierPool.js +3 -10
- package/dist/type/dialect.d.ts +5 -0
- package/dist/type/entity.d.ts +27 -13
- package/dist/type/migration.d.ts +7 -9
- package/dist/type/query.d.ts +11 -4
- package/dist/type/queryAggregate.d.ts +5 -16
- package/dist/type/utility.d.ts +13 -0
- package/dist/type/vector.d.ts +17 -0
- package/dist/type/vector.js +6 -0
- package/dist/util/ddlExpression.util.d.ts +2 -0
- package/dist/util/ddlExpression.util.js +5 -0
- package/dist/util/dialect.util.d.ts +21 -2
- package/dist/util/dialect.util.js +42 -2
- package/package.json +4 -3
- package/skills/uql-orm/SKILL.md +146 -0
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
import { unsupportedVectorMetric } from '../../type/vector.js';
|
|
2
2
|
import { IndexDdl } from './indexDdl.js';
|
|
3
|
-
/** `$text` computes its `TO_TSVECTOR` per row, which no index over the raw columns serves. */
|
|
4
|
-
const PG_INDEX_TYPE_HINTS = new Map([
|
|
5
|
-
['fulltext', '. $text needs none there; name the columns it searches with $fields.'],
|
|
6
|
-
]);
|
|
7
3
|
/** `CREATE INDEX ... USING hnsw ("embedding" vector_cosine_ops) WITH (m = ...)`, pgvector's form. */
|
|
8
4
|
export class PgIndexDdl extends IndexDdl {
|
|
9
|
-
/** Postgres 18's `pg_am`, with pgvector's two. */
|
|
5
|
+
/** Postgres 18's `pg_am`, with pgvector's two, and `fulltext`, which builds a `gin` one. */
|
|
10
6
|
indexTypes = new Set([
|
|
11
7
|
'btree',
|
|
12
8
|
'hash',
|
|
@@ -15,8 +11,8 @@ export class PgIndexDdl extends IndexDdl {
|
|
|
15
11
|
'brin',
|
|
16
12
|
'hnsw',
|
|
17
13
|
'ivfflat',
|
|
14
|
+
'fulltext',
|
|
18
15
|
]);
|
|
19
|
-
indexTypeHints = PG_INDEX_TYPE_HINTS;
|
|
20
16
|
indexFeatures = new Set([
|
|
21
17
|
'expression',
|
|
22
18
|
'partial',
|
|
@@ -30,6 +26,9 @@ export class PgIndexDdl extends IndexDdl {
|
|
|
30
26
|
return index.type === 'hnsw' || index.type === 'ivfflat';
|
|
31
27
|
}
|
|
32
28
|
indexAccessMethod(index) {
|
|
29
|
+
if (index.type === 'fulltext') {
|
|
30
|
+
return ' USING gin';
|
|
31
|
+
}
|
|
33
32
|
return index.type ? ` USING ${index.type}` : '';
|
|
34
33
|
}
|
|
35
34
|
/**
|
|
@@ -75,9 +74,8 @@ export class PgIndexDdl extends IndexDdl {
|
|
|
75
74
|
export class CockroachIndexDdl extends PgIndexDdl {
|
|
76
75
|
indexFeatures = new Set(['expression', 'partial', 'include', 'jsonPath']);
|
|
77
76
|
/** v26.3 answers `hash` and `brin` "unimplemented", `ivfflat` "unrecognized"; `hnsw` builds its vector index. */
|
|
78
|
-
indexTypes = new Set(['btree', 'gin', 'gist', 'hnsw', 'vector']);
|
|
77
|
+
indexTypes = new Set(['btree', 'gin', 'gist', 'hnsw', 'vector', 'fulltext']);
|
|
79
78
|
indexTypeHints = new Map([
|
|
80
|
-
...PG_INDEX_TYPE_HINTS,
|
|
81
79
|
['ivfflat', "; declare type: 'vector' instead"],
|
|
82
80
|
]);
|
|
83
81
|
isNativeVectorIndex(index) {
|
|
@@ -92,8 +90,10 @@ export class CockroachIndexDdl extends PgIndexDdl {
|
|
|
92
90
|
indexAccessMethod(index) {
|
|
93
91
|
return this.isNativeVectorIndex(index) ? '' : super.indexAccessMethod(index);
|
|
94
92
|
}
|
|
95
|
-
/**
|
|
96
|
-
indexTuning() {
|
|
97
|
-
return
|
|
93
|
+
/** Its build-time candidate list alone: pgvector's `WITH (m = 16)` answers "invalid storage parameter", `hnsw` included. */
|
|
94
|
+
indexTuning(index) {
|
|
95
|
+
return this.isVectorIndex(index) && index.efConstruction !== undefined
|
|
96
|
+
? ` WITH (build_beam_size = ${index.efConstruction})`
|
|
97
|
+
: '';
|
|
98
98
|
}
|
|
99
99
|
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { IndexColumnSchema, IndexSchema } from '../../type/index.js';
|
|
2
|
+
import { IndexDdl } from './indexDdl.js';
|
|
3
|
+
/**
|
|
4
|
+
* SQLite's `CREATE INDEX`, which names no index type. A vector index is libSQL's DiskANN where the dialect
|
|
5
|
+
* can index its metric, and a plain index on an engine with none, so an entity written for Postgres migrates.
|
|
6
|
+
*/
|
|
7
|
+
export declare class SqliteIndexDdl extends IndexDdl {
|
|
8
|
+
protected indexColumn(entry: IndexColumnSchema, index: IndexSchema): string;
|
|
9
|
+
/** The metric libSQL's index names, or `undefined` where the index is no vector index or the engine has none. */
|
|
10
|
+
private vectorIndexMetric;
|
|
11
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { indexDistance, isVectorIndexType, unsupportedVectorMetric } from '../../type/vector.js';
|
|
2
|
+
import { IndexDdl } from './indexDdl.js';
|
|
3
|
+
/**
|
|
4
|
+
* SQLite's `CREATE INDEX`, which names no index type. A vector index is libSQL's DiskANN where the dialect
|
|
5
|
+
* can index its metric, and a plain index on an engine with none, so an entity written for Postgres migrates.
|
|
6
|
+
*/
|
|
7
|
+
export class SqliteIndexDdl extends IndexDdl {
|
|
8
|
+
indexColumn(entry, index) {
|
|
9
|
+
const metric = this.vectorIndexMetric(index);
|
|
10
|
+
if (metric === undefined) {
|
|
11
|
+
return super.indexColumn(entry, index);
|
|
12
|
+
}
|
|
13
|
+
const options = [
|
|
14
|
+
`metric=${metric}`,
|
|
15
|
+
...(index.m === undefined ? [] : [`max_neighbors=${index.m}`]),
|
|
16
|
+
...(index.efConstruction === undefined ? [] : [`insert_l=${index.efConstruction}`]),
|
|
17
|
+
];
|
|
18
|
+
const args = [this.dialect.escapeId(entry.column), ...options.map((option) => this.dialect.escape(option))];
|
|
19
|
+
return `libsql_vector_idx(${args.join(', ')})`;
|
|
20
|
+
}
|
|
21
|
+
/** The metric libSQL's index names, or `undefined` where the index is no vector index or the engine has none. */
|
|
22
|
+
vectorIndexMetric(index) {
|
|
23
|
+
const { vectorMetrics, dialectName } = this.dialect;
|
|
24
|
+
if (!isVectorIndexType(index.type) || !this.dialect.hasVectorIndex()) {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
const distance = indexDistance(index);
|
|
28
|
+
const metric = vectorMetrics.get(distance)?.index;
|
|
29
|
+
if (!metric) {
|
|
30
|
+
throw unsupportedVectorMetric(dialectName, distance, index.name);
|
|
31
|
+
}
|
|
32
|
+
// Its tables are named after the index, unquoted: any other name fails with "unable to initialize diskann".
|
|
33
|
+
if (!/^\w+$/.test(index.name) || index.entries.length !== 1) {
|
|
34
|
+
throw new TypeError(`libSQL names a vector index only by letters, digits and underscores, over one column (index "${index.name}")`);
|
|
35
|
+
}
|
|
36
|
+
return metric;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -6,6 +6,24 @@ export type MongoIndexOptions = {
|
|
|
6
6
|
readonly name: string;
|
|
7
7
|
readonly partialFilterExpression?: Readonly<Record<string, unknown>>;
|
|
8
8
|
};
|
|
9
|
+
/** A field of an Atlas vector search index: the vector itself, or one its `filter` pre-filters on. */
|
|
10
|
+
export type MongoVectorSearchField = {
|
|
11
|
+
readonly type: 'vector';
|
|
12
|
+
readonly path: string;
|
|
13
|
+
readonly numDimensions: number;
|
|
14
|
+
readonly similarity: string;
|
|
15
|
+
} | {
|
|
16
|
+
readonly type: 'filter';
|
|
17
|
+
readonly path: string;
|
|
18
|
+
};
|
|
19
|
+
/** An Atlas search index as `createSearchIndex` takes one. */
|
|
20
|
+
export type MongoSearchIndex = {
|
|
21
|
+
readonly name: string;
|
|
22
|
+
readonly type: 'vectorSearch';
|
|
23
|
+
readonly definition: {
|
|
24
|
+
readonly fields: readonly MongoVectorSearchField[];
|
|
25
|
+
};
|
|
26
|
+
};
|
|
9
27
|
/** The commands {@link MongoSchemaGenerator} emits as JSON, one per statement. */
|
|
10
28
|
export type MongoCommand = {
|
|
11
29
|
readonly action: 'createCollection';
|
|
@@ -27,6 +45,14 @@ export type MongoCommand = {
|
|
|
27
45
|
readonly action: 'dropIndex';
|
|
28
46
|
readonly collection: string;
|
|
29
47
|
readonly name: string;
|
|
48
|
+
} | {
|
|
49
|
+
readonly action: 'createSearchIndex';
|
|
50
|
+
readonly collection: string;
|
|
51
|
+
readonly index: MongoSearchIndex;
|
|
52
|
+
} | {
|
|
53
|
+
readonly action: 'dropSearchIndex';
|
|
54
|
+
readonly collection: string;
|
|
55
|
+
readonly name: string;
|
|
30
56
|
};
|
|
31
57
|
export declare function serializeMongoCommand(command: MongoCommand): string;
|
|
32
58
|
/**
|
|
@@ -41,6 +67,8 @@ export type MongoCommandTarget = {
|
|
|
41
67
|
drop(): Promise<unknown>;
|
|
42
68
|
createIndex(key: MongoIndexKey, options: MongoIndexOptions): Promise<unknown>;
|
|
43
69
|
dropIndex(name: string): Promise<unknown>;
|
|
70
|
+
createSearchIndex(index: MongoSearchIndex): Promise<unknown>;
|
|
71
|
+
dropSearchIndex(name: string): Promise<unknown>;
|
|
44
72
|
};
|
|
45
73
|
};
|
|
46
74
|
/** Execute one emitted command. */
|
|
@@ -25,6 +25,10 @@ export function runMongoCommand(db, statement) {
|
|
|
25
25
|
return db.collection(command.collection).createIndex(command.key, command.options);
|
|
26
26
|
case 'dropIndex':
|
|
27
27
|
return db.collection(command.collection).dropIndex(command.name);
|
|
28
|
+
case 'createSearchIndex':
|
|
29
|
+
return db.collection(command.collection).createSearchIndex(command.index);
|
|
30
|
+
case 'dropSearchIndex':
|
|
31
|
+
return db.collection(command.collection).dropSearchIndex(command.name);
|
|
28
32
|
default:
|
|
29
33
|
// Unreachable for a command this module produced; a hand-written statement lands here rather
|
|
30
34
|
// than being silently skipped.
|
|
@@ -46,6 +50,10 @@ export function mongoCommandSource(statement, db) {
|
|
|
46
50
|
return `${db}.collection(${literal(command.collection)}).createIndex(${literal(command.key)}, ${literal(command.options)})`;
|
|
47
51
|
case 'dropIndex':
|
|
48
52
|
return `${db}.collection(${literal(command.collection)}).dropIndex(${literal(command.name)})`;
|
|
53
|
+
case 'createSearchIndex':
|
|
54
|
+
return `${db}.collection(${literal(command.collection)}).createSearchIndex(${literal(command.index)})`;
|
|
55
|
+
case 'dropSearchIndex':
|
|
56
|
+
return `${db}.collection(${literal(command.collection)}).dropSearchIndex(${literal(command.name)})`;
|
|
49
57
|
default:
|
|
50
58
|
throw unsupportedMongoCommand(statement);
|
|
51
59
|
}
|
|
@@ -35,6 +35,8 @@ export declare class MongoSchemaGenerator extends MongoDialect implements Schema
|
|
|
35
35
|
generateAlterTableDown(diff: SchemaDiff): string[];
|
|
36
36
|
/** An index as MongoDB's key spec (`-1` descending, `'text'` full-text), refusing the SQL-only options. */
|
|
37
37
|
generateCreateIndex(tableName: string, index: IndexSchema): string;
|
|
38
|
+
/** An Atlas vector search index: its vector field first, then each field a `$vectorSearch` pre-filters on. */
|
|
39
|
+
private generateCreateSearchIndex;
|
|
38
40
|
generateDropIndex(tableName: string, indexName: string): string;
|
|
39
41
|
/** A collection and its indexes, which is all a document store has: a column, a constraint or SQL throws. */
|
|
40
42
|
generateOperation(operation: AnyMigrationOperation): string[];
|
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
import { getMeta } from '../../entity/index.js';
|
|
2
2
|
import { MongoDialect } from '../../mongo/mongoDialect.js';
|
|
3
3
|
import { QueryRaw, } from '../../type/index.js';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { indexDistance, unsupportedVectorMetric } from '../../type/vector.js';
|
|
5
|
+
import { declaredIndexes, declaredIndexName, renderIndexColumn } from '../../util/ddlExpression.util.js';
|
|
6
6
|
import { assertIndexFeatures, assertIndexType } from '../ddl/indexDdl.js';
|
|
7
7
|
import { assertIndexPredicate, refusedIndexPredicate } from '../indexPredicate.js';
|
|
8
8
|
import { renderIndexDefinition } from './definitionToNode.js';
|
|
9
9
|
import { serializeMongoCommand } from './mongoCommand.js';
|
|
10
|
-
/** The index types a key spec can say
|
|
11
|
-
const MONGO_INDEX_TYPES = new Set(['btree', 'fulltext']);
|
|
10
|
+
/** The index types a key spec can say, a plain key or `'text'`, and Atlas's vector search index. */
|
|
11
|
+
const MONGO_INDEX_TYPES = new Set(['btree', 'fulltext', 'vectorSearch']);
|
|
12
12
|
/** A key spec's one feature beyond its keys: a partial filter. */
|
|
13
13
|
const MONGO_INDEX_FEATURES = new Set(['partial']);
|
|
14
|
+
/** Atlas's name for each metric a vector search index scores by. */
|
|
15
|
+
const ATLAS_SIMILARITY = {
|
|
16
|
+
cosine: 'cosine',
|
|
17
|
+
l2: 'euclidean',
|
|
18
|
+
inner: 'dotProduct',
|
|
19
|
+
};
|
|
14
20
|
export class MongoSchemaGenerator extends MongoDialect {
|
|
15
21
|
defaultForeignKeyAction;
|
|
16
22
|
constructor(namingStrategy, defaultForeignKeyAction) {
|
|
@@ -51,7 +57,11 @@ export class MongoSchemaGenerator extends MongoDialect {
|
|
|
51
57
|
const entries = index.columns
|
|
52
58
|
.map((entry) => renderIndexColumn(entry, () => this.compileDdl()))
|
|
53
59
|
.map((entry) => ({ ...entry, column: this.columnOf(meta, entry.column) }));
|
|
54
|
-
const
|
|
60
|
+
const [first] = index.columns;
|
|
61
|
+
const vector = index.type === 'vectorSearch' && typeof first?.column === 'string' ? meta.fields[first.column] : undefined;
|
|
62
|
+
const name = vector
|
|
63
|
+
? this.vectorSearchIndexName(index.name, entries[0].column)
|
|
64
|
+
: declaredIndexName(index.name, collectionName, entries);
|
|
55
65
|
return {
|
|
56
66
|
name,
|
|
57
67
|
entries,
|
|
@@ -59,6 +69,8 @@ export class MongoSchemaGenerator extends MongoDialect {
|
|
|
59
69
|
type: index.type,
|
|
60
70
|
include: index.include,
|
|
61
71
|
where: index.where && this.compileIndexPredicate(index.where, meta.entity, name),
|
|
72
|
+
distance: index.distance ?? vector?.distance,
|
|
73
|
+
dimensions: vector?.dimensions,
|
|
62
74
|
};
|
|
63
75
|
}
|
|
64
76
|
/**
|
|
@@ -94,11 +106,16 @@ export class MongoSchemaGenerator extends MongoDialect {
|
|
|
94
106
|
return (diff.indexesToAdd ?? []).map((index) => this.generateCreateIndex(diff.tableName, index));
|
|
95
107
|
}
|
|
96
108
|
generateAlterTableDown(diff) {
|
|
97
|
-
return (diff.indexesToAdd ?? []).map((index) =>
|
|
109
|
+
return (diff.indexesToAdd ?? []).map((index) => index.type === 'vectorSearch'
|
|
110
|
+
? serializeMongoCommand({ action: 'dropSearchIndex', collection: diff.tableName, name: index.name })
|
|
111
|
+
: this.generateDropIndex(diff.tableName, index.name));
|
|
98
112
|
}
|
|
99
113
|
/** An index as MongoDB's key spec (`-1` descending, `'text'` full-text), refusing the SQL-only options. */
|
|
100
114
|
generateCreateIndex(tableName, index) {
|
|
101
115
|
assertIndexType(index, MONGO_INDEX_TYPES, this.dialectName);
|
|
116
|
+
if (index.type === 'vectorSearch') {
|
|
117
|
+
return this.generateCreateSearchIndex(tableName, index);
|
|
118
|
+
}
|
|
102
119
|
assertIndexFeatures(index, MONGO_INDEX_FEATURES, this.dialectName);
|
|
103
120
|
const key = {};
|
|
104
121
|
for (const entry of index.entries) {
|
|
@@ -116,6 +133,33 @@ export class MongoSchemaGenerator extends MongoDialect {
|
|
|
116
133
|
},
|
|
117
134
|
});
|
|
118
135
|
}
|
|
136
|
+
/** An Atlas vector search index: its vector field first, then each field a `$vectorSearch` pre-filters on. */
|
|
137
|
+
generateCreateSearchIndex(tableName, index) {
|
|
138
|
+
assertIndexFeatures(index, new Set(), this.dialectName);
|
|
139
|
+
const [vector, ...filters] = index.entries;
|
|
140
|
+
if (!vector || index.dimensions === undefined) {
|
|
141
|
+
throw new TypeError(`an Atlas vector search index states its field's dimensions (index "${index.name}")`);
|
|
142
|
+
}
|
|
143
|
+
const distance = indexDistance(index);
|
|
144
|
+
const similarity = ATLAS_SIMILARITY[distance];
|
|
145
|
+
if (!similarity) {
|
|
146
|
+
throw unsupportedVectorMetric(this.dialectName, distance, index.name);
|
|
147
|
+
}
|
|
148
|
+
return serializeMongoCommand({
|
|
149
|
+
action: 'createSearchIndex',
|
|
150
|
+
collection: tableName,
|
|
151
|
+
index: {
|
|
152
|
+
name: index.name,
|
|
153
|
+
type: 'vectorSearch',
|
|
154
|
+
definition: {
|
|
155
|
+
fields: [
|
|
156
|
+
{ type: 'vector', path: vector.column, numDimensions: index.dimensions, similarity },
|
|
157
|
+
...filters.map((entry) => ({ type: 'filter', path: entry.column })),
|
|
158
|
+
],
|
|
159
|
+
},
|
|
160
|
+
},
|
|
161
|
+
});
|
|
162
|
+
}
|
|
119
163
|
generateDropIndex(tableName, indexName) {
|
|
120
164
|
return serializeMongoCommand({ action: 'dropIndex', collection: tableName, name: indexName });
|
|
121
165
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { createTableNode, SchemaAST } from '../../schema/schemaAST.js';
|
|
2
2
|
import { isMongoQuerier, } from '../../type/index.js';
|
|
3
|
+
/** What a server without Atlas Search answers a search index command with. */
|
|
4
|
+
const SEARCH_NOT_ENABLED = 31082;
|
|
3
5
|
/**
|
|
4
6
|
* MongoDB schema introspector.
|
|
5
7
|
* MongoDB doesn't have a fixed schema, so this primarily focuses on collections and indexes.
|
|
@@ -29,15 +31,27 @@ export class MongoSchemaIntrospector {
|
|
|
29
31
|
}
|
|
30
32
|
// Annotated rather than inferred: the driver's `indexes()` is overloaded and resolves to `any` on
|
|
31
33
|
// some versions, which silently made every field below unchecked.
|
|
32
|
-
const
|
|
34
|
+
const collection = db.collection(tableName);
|
|
35
|
+
const indexes = await collection.indexes();
|
|
36
|
+
const searchIndexes = await listSearchIndexes(collection);
|
|
33
37
|
return {
|
|
34
38
|
name: tableName,
|
|
35
39
|
columns: [],
|
|
36
|
-
indexes:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
40
|
+
indexes: [
|
|
41
|
+
...indexes.map((idx) => ({
|
|
42
|
+
name: idx.name ?? Object.keys(idx.key).join('_'),
|
|
43
|
+
entries: Object.keys(idx.key).map((column) => ({ column })),
|
|
44
|
+
unique: !!idx.unique,
|
|
45
|
+
})),
|
|
46
|
+
...searchIndexes
|
|
47
|
+
.filter((idx) => idx.type === 'vectorSearch')
|
|
48
|
+
.map((idx) => ({
|
|
49
|
+
name: idx.name,
|
|
50
|
+
entries: idx.latestDefinition.fields.map(({ path }) => ({ column: path })),
|
|
51
|
+
unique: false,
|
|
52
|
+
type: 'vectorSearch',
|
|
53
|
+
})),
|
|
54
|
+
],
|
|
41
55
|
};
|
|
42
56
|
});
|
|
43
57
|
}
|
|
@@ -61,6 +75,18 @@ export class MongoSchemaIntrospector {
|
|
|
61
75
|
});
|
|
62
76
|
}
|
|
63
77
|
}
|
|
78
|
+
/** A collection's Atlas search indexes, none where the server has no Atlas Search. */
|
|
79
|
+
async function listSearchIndexes(collection) {
|
|
80
|
+
try {
|
|
81
|
+
return await collection.aggregate([{ $listSearchIndexes: {} }]).toArray();
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
if (error instanceof Error && 'code' in error && error.code === SEARCH_NOT_ENABLED) {
|
|
85
|
+
return [];
|
|
86
|
+
}
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
64
90
|
async function hasCollection(db, name) {
|
|
65
91
|
const collections = await db.listCollections({ name, type: 'collection' }, { nameOnly: true }).toArray();
|
|
66
92
|
return collections.length > 0;
|
|
@@ -83,7 +109,7 @@ function buildTable({ name, indexes = [] }) {
|
|
|
83
109
|
});
|
|
84
110
|
}
|
|
85
111
|
}
|
|
86
|
-
table.indexes.push({ name: index.name, table, entries: index.entries, unique: index.unique });
|
|
112
|
+
table.indexes.push({ name: index.name, table, entries: index.entries, unique: index.unique, type: index.type });
|
|
87
113
|
}
|
|
88
114
|
return table;
|
|
89
115
|
}
|
|
@@ -199,6 +199,7 @@ export class PostgresSchemaIntrospector extends AbstractSqlSchemaIntrospector {
|
|
|
199
199
|
type: INDEX_TYPES.find((type) => type === rows[0].method),
|
|
200
200
|
where: rows[0].predicate ?? undefined,
|
|
201
201
|
include: include.length > 0 ? include : undefined,
|
|
202
|
+
...fulltextIndex(rows),
|
|
202
203
|
};
|
|
203
204
|
});
|
|
204
205
|
}
|
|
@@ -267,6 +268,24 @@ const FOREIGN_KEY_ACTION_CODES = {
|
|
|
267
268
|
};
|
|
268
269
|
const NUMBER_DEFAULT = /^\(?(-?\d+(?:\.\d+)?)\)?$/;
|
|
269
270
|
const QUOTED_NUMBER_DEFAULT = /^'(-?\d+(?:\.\d+)?)'::(?:smallint|integer|bigint|numeric|real|double precision)$/;
|
|
271
|
+
/**
|
|
272
|
+
* A `fulltext` index read back as declared, from the document its `GIN` index (`inverted` on CockroachDB)
|
|
273
|
+
* is over as UQL builds it, `to_tsvector('english'::regconfig, COALESCE(title, ''::text) || ...)`: its
|
|
274
|
+
* columns and config, so it compares with the entity. Any other expression stays the expression it is.
|
|
275
|
+
*/
|
|
276
|
+
function fulltextIndex(rows) {
|
|
277
|
+
const [row] = rows;
|
|
278
|
+
const document = rows.length === 1 && TEXT_INDEX_METHODS.has(row.method) && row.is_expression;
|
|
279
|
+
const config = document ? /^\(?to_tsvector\('((?:[^']|'')*)'::/i.exec(row.entry) : null;
|
|
280
|
+
const columns = config
|
|
281
|
+
? [...row.entry.matchAll(/COALESCE\(("(?:[^"]|"")+"|[^,()\s]+),/gi)].map(([, column]) => column.startsWith('"') ? column.slice(1, -1).replaceAll('""', '"') : column)
|
|
282
|
+
: [];
|
|
283
|
+
if (!config || !columns.length) {
|
|
284
|
+
return undefined;
|
|
285
|
+
}
|
|
286
|
+
return { type: 'fulltext', entries: columns.map((column) => ({ column })), config: config[1].replaceAll("''", "'") };
|
|
287
|
+
}
|
|
288
|
+
const TEXT_INDEX_METHODS = new Set(['gin', 'inverted']);
|
|
270
289
|
/**
|
|
271
290
|
* Postgres states every entry in full: a plain column still reports `order: 'asc'`, and only a
|
|
272
291
|
* non-default operator class is named. The diff defaults the entity side to match, so an option
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import type { IndexFacet } from '../../schema/indexDifferences.js';
|
|
1
2
|
import type { ColumnSchema, ForeignKeySchema, IndexSchema } from '../../type/index.js';
|
|
2
3
|
import { AbstractSqlSchemaIntrospector, type TableRowReader } from './abstractSqlSchemaIntrospector.js';
|
|
3
4
|
/**
|
|
4
5
|
* SQLite schema introspector
|
|
5
6
|
*/
|
|
6
7
|
export declare class SqliteSchemaIntrospector extends AbstractSqlSchemaIntrospector {
|
|
8
|
+
/** Whether an index is libSQL's vector index, where the engine has one; elsewhere a declared one is built plain. */
|
|
9
|
+
readonly indexFacets: ReadonlySet<IndexFacet>;
|
|
10
|
+
/** Not SQLite's own tables, nor the ones libSQL keeps a vector index in: its metadata and `<index>_shadow`. */
|
|
7
11
|
protected getTableNamesQuery(): string;
|
|
8
12
|
protected tableExistsQuery(): string;
|
|
9
13
|
protected parseTableExistsResult([row]: SqliteCountRow[]): boolean;
|
|
@@ -29,6 +33,8 @@ export declare class SqliteSchemaIntrospector extends AbstractSqlSchemaIntrospec
|
|
|
29
33
|
protected mapForeignKeysResult(_read: TableRowReader, tableName: string, results: SqliteForeignKeyRow[]): Promise<ForeignKeySchema[]>;
|
|
30
34
|
protected mapPrimaryKeyResult(results: SqliteColumnRow[]): string[] | undefined;
|
|
31
35
|
private getUniqueColumns;
|
|
36
|
+
/** libSQL's `libsql_vector_idx(col, 'metric=...')`, read back from the statement that created it. */
|
|
37
|
+
private getVectorIndex;
|
|
32
38
|
private getIndexColumns;
|
|
33
39
|
protected normalizeType(type: string): string;
|
|
34
40
|
protected extractLength(type: string): number | undefined;
|
|
@@ -4,12 +4,17 @@ import { AbstractSqlSchemaIntrospector } from './abstractSqlSchemaIntrospector.j
|
|
|
4
4
|
* SQLite schema introspector
|
|
5
5
|
*/
|
|
6
6
|
export class SqliteSchemaIntrospector extends AbstractSqlSchemaIntrospector {
|
|
7
|
+
/** Whether an index is libSQL's vector index, where the engine has one; elsewhere a declared one is built plain. */
|
|
8
|
+
indexFacets = new Set(this.dialect.hasVectorIndex() ? ['vector'] : []);
|
|
9
|
+
/** Not SQLite's own tables, nor the ones libSQL keeps a vector index in: its metadata and `<index>_shadow`. */
|
|
7
10
|
getTableNamesQuery() {
|
|
8
11
|
return /*sql*/ `
|
|
9
12
|
SELECT name
|
|
10
13
|
FROM sqlite_master
|
|
11
14
|
WHERE type = 'table'
|
|
12
15
|
AND name NOT LIKE 'sqlite_%'
|
|
16
|
+
AND name <> 'libsql_vector_meta_shadow'
|
|
17
|
+
AND name NOT IN (SELECT name || '_shadow' FROM sqlite_master WHERE type = 'index')
|
|
13
18
|
ORDER BY name
|
|
14
19
|
`;
|
|
15
20
|
}
|
|
@@ -85,15 +90,24 @@ export class SqliteSchemaIntrospector extends AbstractSqlSchemaIntrospector {
|
|
|
85
90
|
const isCompositeUnique = index.origin === 'u' && columns.length > 1;
|
|
86
91
|
// `PRAGMA index_info` names an expression entry `null` (its `cid` is -2), and the expression text
|
|
87
92
|
// lives only in `sqlite_master.sql`. Reporting `{ column: null }` put a column literally named
|
|
88
|
-
// `null` into the diff, so an index UQL cannot describe is left out
|
|
93
|
+
// `null` into the diff, so an index UQL cannot describe is left out, libSQL's vector index aside.
|
|
89
94
|
const named = columns.filter((column) => column.name !== null);
|
|
90
|
-
if (
|
|
95
|
+
if (!isUserCreated && !isCompositeUnique) {
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (named.length === columns.length) {
|
|
91
99
|
indexSchemas.push({
|
|
92
100
|
name: index.name,
|
|
93
101
|
entries: named.map((column) => ({ column: column.name })),
|
|
94
102
|
unique: Boolean(index.unique),
|
|
95
103
|
});
|
|
96
104
|
}
|
|
105
|
+
else {
|
|
106
|
+
const vectorIndex = await this.getVectorIndex(read, index.name);
|
|
107
|
+
if (vectorIndex) {
|
|
108
|
+
indexSchemas.push(vectorIndex);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
97
111
|
}
|
|
98
112
|
return indexSchemas;
|
|
99
113
|
}
|
|
@@ -143,12 +157,24 @@ export class SqliteSchemaIntrospector extends AbstractSqlSchemaIntrospector {
|
|
|
143
157
|
}
|
|
144
158
|
return uniqueColumns;
|
|
145
159
|
}
|
|
160
|
+
/** libSQL's `libsql_vector_idx(col, 'metric=...')`, read back from the statement that created it. */
|
|
161
|
+
async getVectorIndex(read, indexName) {
|
|
162
|
+
const [row] = await read(
|
|
163
|
+
/*sql*/ `SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?`, [indexName]);
|
|
164
|
+
const column = row?.sql?.match(/libsql_vector_idx\s*\(\s*[`"[]?([^`"\],\s)]+)/i)?.[1];
|
|
165
|
+
if (!column) {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
const metric = row.sql?.match(/'metric=(\w+)'/i)?.[1]?.toLowerCase();
|
|
169
|
+
const distances = new Map([...this.dialect.vectorMetrics].map(([distance, { index }]) => [index, distance]));
|
|
170
|
+
return { name: indexName, entries: [{ column }], unique: false, type: 'vector', distance: distances.get(metric) };
|
|
171
|
+
}
|
|
146
172
|
getIndexColumns(read, indexName) {
|
|
147
173
|
return read(/*sql*/ `PRAGMA index_info(${this.escapeId(indexName)})`);
|
|
148
174
|
}
|
|
149
175
|
normalizeType(type) {
|
|
150
176
|
// Extract base type without length/precision
|
|
151
|
-
const match = type.match(/^([A-Za-z]
|
|
177
|
+
const match = type.match(/^([A-Za-z][A-Za-z0-9_]*)/);
|
|
152
178
|
return match ? match[1].toUpperCase() : type.toUpperCase();
|
|
153
179
|
}
|
|
154
180
|
extractLength(type) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Document, type Filter, type Sort, type UpdateFilter } from 'mongodb';
|
|
2
2
|
import { AbstractDialect } from '../dialect/abstractDialect.js';
|
|
3
|
-
import type { DialectFeatures, EntityData, EntityMeta, Query, QueryAggMap, QueryAggregate, QueryExclude, QueryGroupMap, QueryOptions, QueryPager,
|
|
3
|
+
import type { DialectFeatures, EntityData, EntityMeta, Query, QueryAggMap, QueryAggregate, QueryExclude, QueryGroupMap, QueryOptions, QueryPager, QuerySelectValue, QuerySortMap, QueryVectorSearch, QueryWhere, Type } from '../type/index.js';
|
|
4
4
|
import { type CallbackKey } from '../util/index.js';
|
|
5
5
|
/** What a read pipeline contributes to {@link MongoDialect.readStages} beyond the query itself. */
|
|
6
6
|
type MongoReadStages = {
|
|
@@ -122,7 +122,7 @@ export declare class MongoDialect extends AbstractDialect {
|
|
|
122
122
|
* means a *populated* one, at every level of the path: a lookup adds a field to the result, so one
|
|
123
123
|
* added for the sort alone would change what the caller gets back.
|
|
124
124
|
*/
|
|
125
|
-
sort<E extends Document>(entity: Type<E>, sort
|
|
125
|
+
sort<E extends Document>(entity: Type<E>, { $sort: sort, $populate: populate, $where: where }: Query<E>): Sort;
|
|
126
126
|
/** Walks `$sort` against the metadata of the entity each level addresses, as the SQL dialects do. */
|
|
127
127
|
private collectSort;
|
|
128
128
|
/**
|
|
@@ -237,7 +237,7 @@ export declare class MongoDialect extends AbstractDialect {
|
|
|
237
237
|
*/
|
|
238
238
|
getUpdateFilter<E extends Document>(persistable: Partial<E>): UpdateFilter<E> | Document[];
|
|
239
239
|
/**
|
|
240
|
-
*
|
|
240
|
+
* An update as one pipeline, since MongoDB refuses two operators on one path: each path composed as
|
|
241
241
|
* `$pull`, `$set`, `$push`, then `$unset`, as SQL does, with values as `$literal` so `$x` stays data.
|
|
242
242
|
*/
|
|
243
243
|
private getUpdatePipeline;
|
|
@@ -275,6 +275,8 @@ export declare class MongoDialect extends AbstractDialect {
|
|
|
275
275
|
* Returns `undefined` if no vector sort is present.
|
|
276
276
|
*/
|
|
277
277
|
extractVectorSort<E extends Document>(sort: QuerySortMap<E> | undefined): ExtractedVectorSort<E> | undefined;
|
|
278
|
+
/** The Atlas index a `$vectorSearch` over `column` reads, and migrations create: its declared name, else `<column>_index`. */
|
|
279
|
+
protected vectorSearchIndexName(name: string | undefined, column: string): string;
|
|
278
280
|
/**
|
|
279
281
|
* Build a `$vectorSearch` aggregation pipeline stage.
|
|
280
282
|
* Merges `$where` into `$vectorSearch.filter` for optimal pre-filtering.
|