uql-orm 0.70.0 → 0.71.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +2 -0
  2. package/dist/cockroachdb/cockroachDialect.d.ts +6 -0
  3. package/dist/cockroachdb/cockroachDialect.js +8 -0
  4. package/dist/d1/d1SqliteDialect.d.ts +3 -0
  5. package/dist/d1/d1SqliteDialect.js +3 -1
  6. package/dist/dialect/abstractSqlDialect.d.ts +5 -0
  7. package/dist/dialect/abstractSqlDialect.js +12 -5
  8. package/dist/dialect/hydrateColumn.d.ts +3 -2
  9. package/dist/dialect/hydrateColumn.js +10 -1
  10. package/dist/dialect/mysqlLikeSqlDialect.js +2 -1
  11. package/dist/dialect/pgLikeSqlDialect.d.ts +17 -5
  12. package/dist/dialect/pgLikeSqlDialect.js +33 -15
  13. package/dist/dialect/queryJoins.d.ts +14 -3
  14. package/dist/dialect/queryJoins.js +31 -11
  15. package/dist/dialect/vectorCast.d.ts +2 -0
  16. package/dist/dialect/vectorCast.js +7 -0
  17. package/dist/dialect/vectorSqlDialect.d.ts +7 -3
  18. package/dist/dialect/vectorSqlDialect.js +15 -10
  19. package/dist/libsql/libsqlDialect.d.ts +1 -1
  20. package/dist/libsql/libsqlDialect.js +3 -3
  21. package/dist/maria/mariaDialect.d.ts +3 -9
  22. package/dist/maria/mariaDialect.js +4 -13
  23. package/dist/maria/mariadbQuerier.js +9 -3
  24. package/dist/migrate/ddl/index.d.ts +1 -0
  25. package/dist/migrate/ddl/index.js +4 -2
  26. package/dist/migrate/ddl/indexDdl.d.ts +2 -0
  27. package/dist/migrate/ddl/indexDdl.js +10 -2
  28. package/dist/migrate/ddl/pgIndexDdl.d.ts +3 -4
  29. package/dist/migrate/ddl/pgIndexDdl.js +11 -11
  30. package/dist/migrate/ddl/sqliteIndexDdl.d.ts +11 -0
  31. package/dist/migrate/ddl/sqliteIndexDdl.js +38 -0
  32. package/dist/migrate/generator/mongoCommand.d.ts +28 -0
  33. package/dist/migrate/generator/mongoCommand.js +8 -0
  34. package/dist/migrate/generator/mongoSchemaGenerator.d.ts +2 -0
  35. package/dist/migrate/generator/mongoSchemaGenerator.js +50 -6
  36. package/dist/migrate/introspection/mongoIntrospector.js +33 -7
  37. package/dist/migrate/introspection/postgresIntrospector.js +19 -0
  38. package/dist/migrate/introspection/sqliteIntrospector.d.ts +6 -0
  39. package/dist/migrate/introspection/sqliteIntrospector.js +29 -3
  40. package/dist/mongo/mongoDialect.d.ts +2 -0
  41. package/dist/mongo/mongoDialect.js +8 -4
  42. package/dist/mongo/mongodbQuerier.js +2 -2
  43. package/dist/mssql/mssqlDialect.js +1 -0
  44. package/dist/schema/canonicalType.js +10 -4
  45. package/dist/schema/indexDifferences.d.ts +5 -2
  46. package/dist/schema/indexDifferences.js +5 -0
  47. package/dist/schema/schemaASTBuilder.js +3 -2
  48. package/dist/sqlite/localSqliteQuerierPool.d.ts +10 -11
  49. package/dist/sqlite/localSqliteQuerierPool.js +8 -11
  50. package/dist/sqlite/nodeSqliteQuerierPool.d.ts +4 -2
  51. package/dist/sqlite/nodeSqliteQuerierPool.js +7 -4
  52. package/dist/sqlite/sqliteDialect.d.ts +9 -1
  53. package/dist/sqlite/sqliteDialect.js +42 -3
  54. package/dist/sqlite/sqliteQuerierPool.d.ts +6 -5
  55. package/dist/sqlite/sqliteQuerierPool.js +10 -7
  56. package/dist/turso/tursoLocalDialect.d.ts +1 -1
  57. package/dist/turso/tursoLocalDialect.js +1 -1
  58. package/dist/turso/tursoLocalQuerierPool.d.ts +4 -5
  59. package/dist/turso/tursoLocalQuerierPool.js +3 -10
  60. package/dist/type/dialect.d.ts +5 -0
  61. package/dist/type/entity.d.ts +14 -3
  62. package/dist/type/migration.d.ts +7 -9
  63. package/dist/type/queryAggregate.d.ts +4 -8
  64. package/dist/type/vector.d.ts +17 -0
  65. package/dist/type/vector.js +6 -0
  66. package/dist/util/ddlExpression.util.d.ts +2 -0
  67. package/dist/util/ddlExpression.util.js +5 -0
  68. package/dist/util/dialect.util.d.ts +11 -1
  69. package/dist/util/dialect.util.js +21 -1
  70. package/package.json +5 -3
  71. package/skills/uql-orm/SKILL.md +142 -0
@@ -8,12 +8,14 @@ export class MariaDialect extends MysqlLikeSqlDialect {
8
8
  // MariaDB 10.5+ has `INSERT ... RETURNING`, so ids come back exact per row - the upsert's too.
9
9
  insertIdSource = 'returning';
10
10
  /**
11
- * Unlike MySQL: `VECTOR(n)` takes its dimension, every column of a vector index has to be NOT NULL,
12
- * `CREATE INDEX` takes `IF NOT EXISTS`, and a lock cannot be narrowed to one table of a join.
11
+ * Unlike MySQL: `VECTOR(n)` takes its dimension and binds as packed float32 bytes, every column of a
12
+ * vector index has to be NOT NULL, `CREATE INDEX` takes `IF NOT EXISTS`, and a lock cannot be narrowed
13
+ * to one table of a join.
13
14
  */
14
15
  features = {
15
16
  ...MYSQL_FEATURES,
16
17
  vectorSupportsLength: true,
18
+ vectorBytes: true,
17
19
  vectorIndexRequiresNotNull: true,
18
20
  indexIfNotExists: true,
19
21
  rowLockOf: false,
@@ -69,17 +71,6 @@ export class MariaDialect extends MysqlLikeSqlDialect {
69
71
  ['cosine', { fn: 'VEC_DISTANCE_COSINE', index: 'cosine' }],
70
72
  ['l2', { fn: 'VEC_DISTANCE_EUCLIDEAN', index: 'euclidean' }],
71
73
  ]);
72
- /**
73
- * A `VECTOR` column holds a packed little-endian float32 blob, and MariaDB refuses text where one
74
- * belongs: inserting `'[1,2,3]'` fails with `Incorrect vector value`, and passing it to
75
- * `VEC_DISTANCE_COSINE` with `Illegal parameter data type varchar`. `VEC_FromText` is the
76
- * conversion, needed on both paths.
77
- */
78
- appendVectorValue(ctx, value) {
79
- ctx.append('VEC_FromText(');
80
- super.appendVectorValue(ctx, value);
81
- ctx.append(')');
82
- }
83
74
  /**
84
75
  * `mhnsw_ef_search` too, where a vector search is tuned. A setting scoped to one statement needs
85
76
  * neither a transaction nor a restore, and cannot leak to the next query on this pooled connection,
@@ -1,19 +1,25 @@
1
1
  import { AbstractPoolQuerier } from '../querier/abstractPoolQuerier.js';
2
2
  import { decodeBigInts } from '../util/wideNumber.js';
3
+ /** This driver binds only a `Buffer` as bytes: any other `Uint8Array` goes as the JSON of its indices. */
4
+ function toBindValues(values) {
5
+ return values?.map((value) => value instanceof Uint8Array && !Buffer.isBuffer(value)
6
+ ? Buffer.from(value.buffer, value.byteOffset, value.byteLength)
7
+ : value);
8
+ }
3
9
  export class MariadbQuerier extends AbstractPoolQuerier {
4
10
  async internalAll(query, values) {
5
- const rows = await this.getConn().query(query, values);
11
+ const rows = await this.getConn().query(query, toBindValues(values));
6
12
  return Array.from(rows, decodeBigInts);
7
13
  }
8
14
  async internalRun(query, values) {
9
- const res = await this.getConn().query(query, values);
15
+ const res = await this.getConn().query(query, toBindValues(values));
10
16
  // An OK packet reports `affectedRows`; a `RETURNING` statement answers rows instead, and counts by them.
11
17
  const changes = res.affectedRows ?? res.length;
12
18
  const rows = res.length ? Array.from(res, decodeBigInts) : [];
13
19
  return this.buildUpdateResult({ rows, changes, upsertStatus: res.affectedRows });
14
20
  }
15
21
  async *internalStream(query, values) {
16
- const stream = this.getConn().queryStream(query, values);
22
+ const stream = this.getConn().queryStream(query, toBindValues(values));
17
23
  try {
18
24
  for await (const row of stream) {
19
25
  yield decodeBigInts(row);
@@ -6,6 +6,7 @@ export { MsSqlIndexDdl } from './mssqlIndexDdl.js';
6
6
  export { MsSqlTableDdl } from './mssqlTableDdl.js';
7
7
  export { MariaIndexDdl, MySqlIndexDdl, MysqlLikeIndexDdl } from './mysqlIndexDdl.js';
8
8
  export { CockroachIndexDdl, PgIndexDdl } from './pgIndexDdl.js';
9
+ export { SqliteIndexDdl } from './sqliteIndexDdl.js';
9
10
  export { TableDdl } from './tableDdl.js';
10
11
  export declare function indexDdlFor(dialect: AbstractSqlDialect): IndexDdl;
11
12
  /**
@@ -3,16 +3,18 @@ import { MsSqlIndexDdl } from './mssqlIndexDdl.js';
3
3
  import { MsSqlTableDdl } from './mssqlTableDdl.js';
4
4
  import { MariaIndexDdl, MySqlIndexDdl } from './mysqlIndexDdl.js';
5
5
  import { CockroachIndexDdl, PgIndexDdl } from './pgIndexDdl.js';
6
+ import { SqliteIndexDdl } from './sqliteIndexDdl.js';
6
7
  import { TableDdl } from './tableDdl.js';
7
8
  export { IndexDdl } from './indexDdl.js';
8
9
  export { MsSqlIndexDdl } from './mssqlIndexDdl.js';
9
10
  export { MsSqlTableDdl } from './mssqlTableDdl.js';
10
11
  export { MariaIndexDdl, MySqlIndexDdl, MysqlLikeIndexDdl } from './mysqlIndexDdl.js';
11
12
  export { CockroachIndexDdl, PgIndexDdl } from './pgIndexDdl.js';
13
+ export { SqliteIndexDdl } from './sqliteIndexDdl.js';
12
14
  export { TableDdl } from './tableDdl.js';
13
15
  /**
14
16
  * Each engine's index DDL, by the `dialectName` a subclass inherits: by name, so this entry carries no
15
- * dialect, and exhaustive, so a new engine has to name its own. SQLite's is the portable form.
17
+ * dialect, and exhaustive, so a new engine has to name its own.
16
18
  */
17
19
  const INDEX_DDL = {
18
20
  postgres: PgIndexDdl,
@@ -20,7 +22,7 @@ const INDEX_DDL = {
20
22
  mysql: MySqlIndexDdl,
21
23
  mariadb: MariaIndexDdl,
22
24
  mssql: MsSqlIndexDdl,
23
- sqlite: IndexDdl,
25
+ sqlite: SqliteIndexDdl,
24
26
  };
25
27
  export function indexDdlFor(dialect) {
26
28
  return new INDEX_DDL[dialect.dialectName](dialect);
@@ -37,6 +37,8 @@ export declare class IndexDdl<D extends AbstractSqlDialect = AbstractSqlDialect>
37
37
  protected readonly indexTypeKeywords: ReadonlyMap<IndexType, string>;
38
38
  /** The keyword an index type replaces `INDEX` with, or `INDEX` for the types that do not. */
39
39
  protected indexKeyword(index: IndexSchema): string;
40
+ /** What the index is over, between the parentheses: a fulltext one's columns as a search matches them, else its entries. */
41
+ protected indexTarget(index: IndexSchema): string;
40
42
  /** One index entry: what is indexed, its operator class if any, then its stored order. */
41
43
  protected indexColumn(entry: IndexColumnSchema, index: IndexSchema): string;
42
44
  /**
@@ -1,7 +1,7 @@
1
1
  import { jsonTypeMode } from '../../dialect/jsonSql.js';
2
2
  import { INDEX_TYPES } from '../../schema/types.js';
3
3
  import { INDEX_FEATURE_LABELS, } from '../../type/index.js';
4
- import { getKeys } from '../../util/index.js';
4
+ import { fulltextConfig, getKeys } from '../../util/index.js';
5
5
  /**
6
6
  * What in an index asks for each feature. A `Record` over the feature union rather than a list, so a
7
7
  * feature added to {@link INDEX_FEATURE_LABELS} cannot reach a dialect without the test that decides
@@ -45,7 +45,7 @@ export class IndexDdl {
45
45
  assertIndexFeatures(index, this.indexFeatures, this.dialect.dialectName);
46
46
  const unique = index.unique ? 'UNIQUE ' : '';
47
47
  const ifNotExists = (opts.ifNotExists ?? this.dialect.features.indexIfNotExists) ? 'IF NOT EXISTS ' : '';
48
- const columns = index.entries.map((entry) => this.indexColumn(entry, index)).join(', ');
48
+ const columns = this.indexTarget(index);
49
49
  return (`CREATE ${unique}${this.indexKeyword(index)} ${ifNotExists}${this.dialect.escapeId(index.name)} ` +
50
50
  `ON ${this.dialect.escapeId(tableName)}${this.indexAccessMethod(index)} (${columns})` +
51
51
  `${this.indexInclude(index)}${this.indexTuning(index)}${this.indexPredicate(index)};`);
@@ -78,6 +78,14 @@ export class IndexDdl {
78
78
  indexKeyword(index) {
79
79
  return (index.type && this.indexTypeKeywords.get(index.type)) || 'INDEX';
80
80
  }
81
+ /** What the index is over, between the parentheses: a fulltext one's columns as a search matches them, else its entries. */
82
+ indexTarget(index) {
83
+ if (index.type === 'fulltext' && !index.entries.some((entry) => entry.expression)) {
84
+ const columns = index.entries.map((entry) => this.dialect.escapeId(entry.column));
85
+ return this.dialect.textSearchTarget(columns, fulltextConfig(index));
86
+ }
87
+ return index.entries.map((entry) => this.indexColumn(entry, index)).join(', ');
88
+ }
81
89
  /** One index entry: what is indexed, its operator class if any, then its stored order. */
82
90
  indexColumn(entry, index) {
83
91
  return `${this.indexColumnTarget(entry)}${this.indexColumnOpsClass(entry, index)}${this.indexColumnOrder(entry)}`;
@@ -2,9 +2,8 @@ import type { IndexColumnSchema, IndexSchema } from '../../type/index.js';
2
2
  import { IndexDdl } from './indexDdl.js';
3
3
  /** `CREATE INDEX ... USING hnsw ("embedding" vector_cosine_ops) WITH (m = ...)`, pgvector's form. */
4
4
  export declare class PgIndexDdl extends IndexDdl {
5
- /** Postgres 18's `pg_am`, with pgvector's two. */
5
+ /** Postgres 18's `pg_am`, with pgvector's two, and `fulltext`, which builds a `gin` one. */
6
6
  protected readonly indexTypes: Set<"brin" | "btree" | "fulltext" | "gin" | "gist" | "hash" | "hnsw" | "ivfflat" | "vector" | "vectorSearch">;
7
- protected readonly indexTypeHints: ReadonlyMap<"brin" | "btree" | "fulltext" | "gin" | "gist" | "hash" | "hnsw" | "ivfflat" | "vector" | "vectorSearch", string>;
8
7
  protected readonly indexFeatures: Set<"expression" | "include" | "jsonArray" | "jsonPath" | "nullsOrder" | "opsClass" | "partial" | "prefixLength">;
9
8
  /** pgvector's own index types; CockroachDB's native one widens this. */
10
9
  protected isVectorIndex(index: IndexSchema): boolean;
@@ -27,6 +26,6 @@ export declare class CockroachIndexDdl extends PgIndexDdl {
27
26
  protected isVectorIndex(index: IndexSchema): boolean;
28
27
  protected indexKeyword(index: IndexSchema): string;
29
28
  protected indexAccessMethod(index: IndexSchema): string;
30
- /** None of pgvector's knobs: `WITH (m = 16)` answers "invalid storage parameter", `hnsw` included. */
31
- protected indexTuning(): string;
29
+ /** Its build-time candidate list alone: pgvector's `WITH (m = 16)` answers "invalid storage parameter", `hnsw` included. */
30
+ protected indexTuning(index: IndexSchema): string;
32
31
  }
@@ -1,12 +1,8 @@
1
1
  import { unsupportedVectorMetric } from '../../type/vector.js';
2
2
  import { IndexDdl } from './indexDdl.js';
3
- /** `$text` computes its `TO_TSVECTOR` per row, which no index over the raw columns serves. */
4
- const PG_INDEX_TYPE_HINTS = new Map([
5
- ['fulltext', '. $text needs none there; name the columns it searches with $fields.'],
6
- ]);
7
3
  /** `CREATE INDEX ... USING hnsw ("embedding" vector_cosine_ops) WITH (m = ...)`, pgvector's form. */
8
4
  export class PgIndexDdl extends IndexDdl {
9
- /** Postgres 18's `pg_am`, with pgvector's two. */
5
+ /** Postgres 18's `pg_am`, with pgvector's two, and `fulltext`, which builds a `gin` one. */
10
6
  indexTypes = new Set([
11
7
  'btree',
12
8
  'hash',
@@ -15,8 +11,8 @@ export class PgIndexDdl extends IndexDdl {
15
11
  'brin',
16
12
  'hnsw',
17
13
  'ivfflat',
14
+ 'fulltext',
18
15
  ]);
19
- indexTypeHints = PG_INDEX_TYPE_HINTS;
20
16
  indexFeatures = new Set([
21
17
  'expression',
22
18
  'partial',
@@ -30,6 +26,9 @@ export class PgIndexDdl extends IndexDdl {
30
26
  return index.type === 'hnsw' || index.type === 'ivfflat';
31
27
  }
32
28
  indexAccessMethod(index) {
29
+ if (index.type === 'fulltext') {
30
+ return ' USING gin';
31
+ }
33
32
  return index.type ? ` USING ${index.type}` : '';
34
33
  }
35
34
  /**
@@ -75,9 +74,8 @@ export class PgIndexDdl extends IndexDdl {
75
74
  export class CockroachIndexDdl extends PgIndexDdl {
76
75
  indexFeatures = new Set(['expression', 'partial', 'include', 'jsonPath']);
77
76
  /** v26.3 answers `hash` and `brin` "unimplemented", `ivfflat` "unrecognized"; `hnsw` builds its vector index. */
78
- indexTypes = new Set(['btree', 'gin', 'gist', 'hnsw', 'vector']);
77
+ indexTypes = new Set(['btree', 'gin', 'gist', 'hnsw', 'vector', 'fulltext']);
79
78
  indexTypeHints = new Map([
80
- ...PG_INDEX_TYPE_HINTS,
81
79
  ['ivfflat', "; declare type: 'vector' instead"],
82
80
  ]);
83
81
  isNativeVectorIndex(index) {
@@ -92,8 +90,10 @@ export class CockroachIndexDdl extends PgIndexDdl {
92
90
  indexAccessMethod(index) {
93
91
  return this.isNativeVectorIndex(index) ? '' : super.indexAccessMethod(index);
94
92
  }
95
- /** None of pgvector's knobs: `WITH (m = 16)` answers "invalid storage parameter", `hnsw` included. */
96
- indexTuning() {
97
- return '';
93
+ /** Its build-time candidate list alone: pgvector's `WITH (m = 16)` answers "invalid storage parameter", `hnsw` included. */
94
+ indexTuning(index) {
95
+ return this.isVectorIndex(index) && index.efConstruction !== undefined
96
+ ? ` WITH (build_beam_size = ${index.efConstruction})`
97
+ : '';
98
98
  }
99
99
  }
@@ -0,0 +1,11 @@
1
+ import type { IndexColumnSchema, IndexSchema } from '../../type/index.js';
2
+ import { IndexDdl } from './indexDdl.js';
3
+ /**
4
+ * SQLite's `CREATE INDEX`, which names no index type. A vector index is libSQL's DiskANN where the dialect
5
+ * can index its metric, and a plain index on an engine with none, so an entity written for Postgres migrates.
6
+ */
7
+ export declare class SqliteIndexDdl extends IndexDdl {
8
+ protected indexColumn(entry: IndexColumnSchema, index: IndexSchema): string;
9
+ /** The metric libSQL's index names, or `undefined` where the index is no vector index or the engine has none. */
10
+ private vectorIndexMetric;
11
+ }
@@ -0,0 +1,38 @@
1
+ import { indexDistance, isVectorIndexType, unsupportedVectorMetric } from '../../type/vector.js';
2
+ import { IndexDdl } from './indexDdl.js';
3
+ /**
4
+ * SQLite's `CREATE INDEX`, which names no index type. A vector index is libSQL's DiskANN where the dialect
5
+ * can index its metric, and a plain index on an engine with none, so an entity written for Postgres migrates.
6
+ */
7
+ export class SqliteIndexDdl extends IndexDdl {
8
+ indexColumn(entry, index) {
9
+ const metric = this.vectorIndexMetric(index);
10
+ if (metric === undefined) {
11
+ return super.indexColumn(entry, index);
12
+ }
13
+ const options = [
14
+ `metric=${metric}`,
15
+ ...(index.m === undefined ? [] : [`max_neighbors=${index.m}`]),
16
+ ...(index.efConstruction === undefined ? [] : [`insert_l=${index.efConstruction}`]),
17
+ ];
18
+ const args = [this.dialect.escapeId(entry.column), ...options.map((option) => this.dialect.escape(option))];
19
+ return `libsql_vector_idx(${args.join(', ')})`;
20
+ }
21
+ /** The metric libSQL's index names, or `undefined` where the index is no vector index or the engine has none. */
22
+ vectorIndexMetric(index) {
23
+ const { vectorMetrics, dialectName } = this.dialect;
24
+ if (!isVectorIndexType(index.type) || !this.dialect.hasVectorIndex()) {
25
+ return undefined;
26
+ }
27
+ const distance = indexDistance(index);
28
+ const metric = vectorMetrics.get(distance)?.index;
29
+ if (!metric) {
30
+ throw unsupportedVectorMetric(dialectName, distance, index.name);
31
+ }
32
+ // Its tables are named after the index, unquoted: any other name fails with "unable to initialize diskann".
33
+ if (!/^\w+$/.test(index.name) || index.entries.length !== 1) {
34
+ throw new TypeError(`libSQL names a vector index only by letters, digits and underscores, over one column (index "${index.name}")`);
35
+ }
36
+ return metric;
37
+ }
38
+ }
@@ -6,6 +6,24 @@ export type MongoIndexOptions = {
6
6
  readonly name: string;
7
7
  readonly partialFilterExpression?: Readonly<Record<string, unknown>>;
8
8
  };
9
+ /** A field of an Atlas vector search index: the vector itself, or one its `filter` pre-filters on. */
10
+ export type MongoVectorSearchField = {
11
+ readonly type: 'vector';
12
+ readonly path: string;
13
+ readonly numDimensions: number;
14
+ readonly similarity: string;
15
+ } | {
16
+ readonly type: 'filter';
17
+ readonly path: string;
18
+ };
19
+ /** An Atlas search index as `createSearchIndex` takes one. */
20
+ export type MongoSearchIndex = {
21
+ readonly name: string;
22
+ readonly type: 'vectorSearch';
23
+ readonly definition: {
24
+ readonly fields: readonly MongoVectorSearchField[];
25
+ };
26
+ };
9
27
  /** The commands {@link MongoSchemaGenerator} emits as JSON, one per statement. */
10
28
  export type MongoCommand = {
11
29
  readonly action: 'createCollection';
@@ -27,6 +45,14 @@ export type MongoCommand = {
27
45
  readonly action: 'dropIndex';
28
46
  readonly collection: string;
29
47
  readonly name: string;
48
+ } | {
49
+ readonly action: 'createSearchIndex';
50
+ readonly collection: string;
51
+ readonly index: MongoSearchIndex;
52
+ } | {
53
+ readonly action: 'dropSearchIndex';
54
+ readonly collection: string;
55
+ readonly name: string;
30
56
  };
31
57
  export declare function serializeMongoCommand(command: MongoCommand): string;
32
58
  /**
@@ -41,6 +67,8 @@ export type MongoCommandTarget = {
41
67
  drop(): Promise<unknown>;
42
68
  createIndex(key: MongoIndexKey, options: MongoIndexOptions): Promise<unknown>;
43
69
  dropIndex(name: string): Promise<unknown>;
70
+ createSearchIndex(index: MongoSearchIndex): Promise<unknown>;
71
+ dropSearchIndex(name: string): Promise<unknown>;
44
72
  };
45
73
  };
46
74
  /** Execute one emitted command. */
@@ -25,6 +25,10 @@ export function runMongoCommand(db, statement) {
25
25
  return db.collection(command.collection).createIndex(command.key, command.options);
26
26
  case 'dropIndex':
27
27
  return db.collection(command.collection).dropIndex(command.name);
28
+ case 'createSearchIndex':
29
+ return db.collection(command.collection).createSearchIndex(command.index);
30
+ case 'dropSearchIndex':
31
+ return db.collection(command.collection).dropSearchIndex(command.name);
28
32
  default:
29
33
  // Unreachable for a command this module produced; a hand-written statement lands here rather
30
34
  // than being silently skipped.
@@ -46,6 +50,10 @@ export function mongoCommandSource(statement, db) {
46
50
  return `${db}.collection(${literal(command.collection)}).createIndex(${literal(command.key)}, ${literal(command.options)})`;
47
51
  case 'dropIndex':
48
52
  return `${db}.collection(${literal(command.collection)}).dropIndex(${literal(command.name)})`;
53
+ case 'createSearchIndex':
54
+ return `${db}.collection(${literal(command.collection)}).createSearchIndex(${literal(command.index)})`;
55
+ case 'dropSearchIndex':
56
+ return `${db}.collection(${literal(command.collection)}).dropSearchIndex(${literal(command.name)})`;
49
57
  default:
50
58
  throw unsupportedMongoCommand(statement);
51
59
  }
@@ -35,6 +35,8 @@ export declare class MongoSchemaGenerator extends MongoDialect implements Schema
35
35
  generateAlterTableDown(diff: SchemaDiff): string[];
36
36
  /** An index as MongoDB's key spec (`-1` descending, `'text'` full-text), refusing the SQL-only options. */
37
37
  generateCreateIndex(tableName: string, index: IndexSchema): string;
38
+ /** An Atlas vector search index: its vector field first, then each field a `$vectorSearch` pre-filters on. */
39
+ private generateCreateSearchIndex;
38
40
  generateDropIndex(tableName: string, indexName: string): string;
39
41
  /** A collection and its indexes, which is all a document store has: a column, a constraint or SQL throws. */
40
42
  generateOperation(operation: AnyMigrationOperation): string[];
@@ -1,16 +1,22 @@
1
1
  import { getMeta } from '../../entity/index.js';
2
2
  import { MongoDialect } from '../../mongo/mongoDialect.js';
3
3
  import { QueryRaw, } from '../../type/index.js';
4
- import { declaredIndexes, indexNameParts, renderIndexColumn } from '../../util/ddlExpression.util.js';
5
- import { derivedIndexName } from '../../util/sql.util.js';
4
+ import { indexDistance, unsupportedVectorMetric } from '../../type/vector.js';
5
+ import { declaredIndexes, declaredIndexName, renderIndexColumn } from '../../util/ddlExpression.util.js';
6
6
  import { assertIndexFeatures, assertIndexType } from '../ddl/indexDdl.js';
7
7
  import { assertIndexPredicate, refusedIndexPredicate } from '../indexPredicate.js';
8
8
  import { renderIndexDefinition } from './definitionToNode.js';
9
9
  import { serializeMongoCommand } from './mongoCommand.js';
10
- /** The index types a key spec can say: a plain key, or `'text'`. */
11
- const MONGO_INDEX_TYPES = new Set(['btree', 'fulltext']);
10
+ /** The index types a key spec can say, a plain key or `'text'`, and Atlas's vector search index. */
11
+ const MONGO_INDEX_TYPES = new Set(['btree', 'fulltext', 'vectorSearch']);
12
12
  /** A key spec's one feature beyond its keys: a partial filter. */
13
13
  const MONGO_INDEX_FEATURES = new Set(['partial']);
14
+ /** Atlas's name for each metric a vector search index scores by. */
15
+ const ATLAS_SIMILARITY = {
16
+ cosine: 'cosine',
17
+ l2: 'euclidean',
18
+ inner: 'dotProduct',
19
+ };
14
20
  export class MongoSchemaGenerator extends MongoDialect {
15
21
  defaultForeignKeyAction;
16
22
  constructor(namingStrategy, defaultForeignKeyAction) {
@@ -51,7 +57,11 @@ export class MongoSchemaGenerator extends MongoDialect {
51
57
  const entries = index.columns
52
58
  .map((entry) => renderIndexColumn(entry, () => this.compileDdl()))
53
59
  .map((entry) => ({ ...entry, column: this.columnOf(meta, entry.column) }));
54
- const name = index.name ?? derivedIndexName(collectionName, indexNameParts(entries));
60
+ const [first] = index.columns;
61
+ const vector = index.type === 'vectorSearch' && typeof first?.column === 'string' ? meta.fields[first.column] : undefined;
62
+ const name = vector
63
+ ? this.vectorSearchIndexName(index.name, entries[0].column)
64
+ : declaredIndexName(index.name, collectionName, entries);
55
65
  return {
56
66
  name,
57
67
  entries,
@@ -59,6 +69,8 @@ export class MongoSchemaGenerator extends MongoDialect {
59
69
  type: index.type,
60
70
  include: index.include,
61
71
  where: index.where && this.compileIndexPredicate(index.where, meta.entity, name),
72
+ distance: index.distance ?? vector?.distance,
73
+ dimensions: vector?.dimensions,
62
74
  };
63
75
  }
64
76
  /**
@@ -94,11 +106,16 @@ export class MongoSchemaGenerator extends MongoDialect {
94
106
  return (diff.indexesToAdd ?? []).map((index) => this.generateCreateIndex(diff.tableName, index));
95
107
  }
96
108
  generateAlterTableDown(diff) {
97
- return (diff.indexesToAdd ?? []).map((index) => this.generateDropIndex(diff.tableName, index.name));
109
+ return (diff.indexesToAdd ?? []).map((index) => index.type === 'vectorSearch'
110
+ ? serializeMongoCommand({ action: 'dropSearchIndex', collection: diff.tableName, name: index.name })
111
+ : this.generateDropIndex(diff.tableName, index.name));
98
112
  }
99
113
  /** An index as MongoDB's key spec (`-1` descending, `'text'` full-text), refusing the SQL-only options. */
100
114
  generateCreateIndex(tableName, index) {
101
115
  assertIndexType(index, MONGO_INDEX_TYPES, this.dialectName);
116
+ if (index.type === 'vectorSearch') {
117
+ return this.generateCreateSearchIndex(tableName, index);
118
+ }
102
119
  assertIndexFeatures(index, MONGO_INDEX_FEATURES, this.dialectName);
103
120
  const key = {};
104
121
  for (const entry of index.entries) {
@@ -116,6 +133,33 @@ export class MongoSchemaGenerator extends MongoDialect {
116
133
  },
117
134
  });
118
135
  }
136
+ /** An Atlas vector search index: its vector field first, then each field a `$vectorSearch` pre-filters on. */
137
+ generateCreateSearchIndex(tableName, index) {
138
+ assertIndexFeatures(index, new Set(), this.dialectName);
139
+ const [vector, ...filters] = index.entries;
140
+ if (!vector || index.dimensions === undefined) {
141
+ throw new TypeError(`an Atlas vector search index states its field's dimensions (index "${index.name}")`);
142
+ }
143
+ const distance = indexDistance(index);
144
+ const similarity = ATLAS_SIMILARITY[distance];
145
+ if (!similarity) {
146
+ throw unsupportedVectorMetric(this.dialectName, distance, index.name);
147
+ }
148
+ return serializeMongoCommand({
149
+ action: 'createSearchIndex',
150
+ collection: tableName,
151
+ index: {
152
+ name: index.name,
153
+ type: 'vectorSearch',
154
+ definition: {
155
+ fields: [
156
+ { type: 'vector', path: vector.column, numDimensions: index.dimensions, similarity },
157
+ ...filters.map((entry) => ({ type: 'filter', path: entry.column })),
158
+ ],
159
+ },
160
+ },
161
+ });
162
+ }
119
163
  generateDropIndex(tableName, indexName) {
120
164
  return serializeMongoCommand({ action: 'dropIndex', collection: tableName, name: indexName });
121
165
  }
@@ -1,5 +1,7 @@
1
1
  import { createTableNode, SchemaAST } from '../../schema/schemaAST.js';
2
2
  import { isMongoQuerier, } from '../../type/index.js';
3
+ /** What a server without Atlas Search answers a search index command with. */
4
+ const SEARCH_NOT_ENABLED = 31082;
3
5
  /**
4
6
  * MongoDB schema introspector.
5
7
  * MongoDB doesn't have a fixed schema, so this primarily focuses on collections and indexes.
@@ -29,15 +31,27 @@ export class MongoSchemaIntrospector {
29
31
  }
30
32
  // Annotated rather than inferred: the driver's `indexes()` is overloaded and resolves to `any` on
31
33
  // some versions, which silently made every field below unchecked.
32
- const indexes = await db.collection(tableName).indexes();
34
+ const collection = db.collection(tableName);
35
+ const indexes = await collection.indexes();
36
+ const searchIndexes = await listSearchIndexes(collection);
33
37
  return {
34
38
  name: tableName,
35
39
  columns: [],
36
- indexes: indexes.map((idx) => ({
37
- name: idx.name ?? Object.keys(idx.key).join('_'),
38
- entries: Object.keys(idx.key).map((column) => ({ column })),
39
- unique: !!idx.unique,
40
- })),
40
+ indexes: [
41
+ ...indexes.map((idx) => ({
42
+ name: idx.name ?? Object.keys(idx.key).join('_'),
43
+ entries: Object.keys(idx.key).map((column) => ({ column })),
44
+ unique: !!idx.unique,
45
+ })),
46
+ ...searchIndexes
47
+ .filter((idx) => idx.type === 'vectorSearch')
48
+ .map((idx) => ({
49
+ name: idx.name,
50
+ entries: idx.latestDefinition.fields.map(({ path }) => ({ column: path })),
51
+ unique: false,
52
+ type: 'vectorSearch',
53
+ })),
54
+ ],
41
55
  };
42
56
  });
43
57
  }
@@ -61,6 +75,18 @@ export class MongoSchemaIntrospector {
61
75
  });
62
76
  }
63
77
  }
78
+ /** A collection's Atlas search indexes, none where the server has no Atlas Search. */
79
+ async function listSearchIndexes(collection) {
80
+ try {
81
+ return await collection.aggregate([{ $listSearchIndexes: {} }]).toArray();
82
+ }
83
+ catch (error) {
84
+ if (error instanceof Error && 'code' in error && error.code === SEARCH_NOT_ENABLED) {
85
+ return [];
86
+ }
87
+ throw error;
88
+ }
89
+ }
64
90
  async function hasCollection(db, name) {
65
91
  const collections = await db.listCollections({ name, type: 'collection' }, { nameOnly: true }).toArray();
66
92
  return collections.length > 0;
@@ -83,7 +109,7 @@ function buildTable({ name, indexes = [] }) {
83
109
  });
84
110
  }
85
111
  }
86
- table.indexes.push({ name: index.name, table, entries: index.entries, unique: index.unique });
112
+ table.indexes.push({ name: index.name, table, entries: index.entries, unique: index.unique, type: index.type });
87
113
  }
88
114
  return table;
89
115
  }
@@ -199,6 +199,7 @@ export class PostgresSchemaIntrospector extends AbstractSqlSchemaIntrospector {
199
199
  type: INDEX_TYPES.find((type) => type === rows[0].method),
200
200
  where: rows[0].predicate ?? undefined,
201
201
  include: include.length > 0 ? include : undefined,
202
+ ...fulltextIndex(rows),
202
203
  };
203
204
  });
204
205
  }
@@ -267,6 +268,24 @@ const FOREIGN_KEY_ACTION_CODES = {
267
268
  };
268
269
  const NUMBER_DEFAULT = /^\(?(-?\d+(?:\.\d+)?)\)?$/;
269
270
  const QUOTED_NUMBER_DEFAULT = /^'(-?\d+(?:\.\d+)?)'::(?:smallint|integer|bigint|numeric|real|double precision)$/;
271
+ /**
272
+ * A `fulltext` index read back as declared, from the document its `GIN` index (`inverted` on CockroachDB)
273
+ * is over as UQL builds it, `to_tsvector('english'::regconfig, COALESCE(title, ''::text) || ...)`: its
274
+ * columns and config, so it compares with the entity. Any other expression stays the expression it is.
275
+ */
276
+ function fulltextIndex(rows) {
277
+ const [row] = rows;
278
+ const document = rows.length === 1 && TEXT_INDEX_METHODS.has(row.method) && row.is_expression;
279
+ const config = document ? /^\(?to_tsvector\('((?:[^']|'')*)'::/i.exec(row.entry) : null;
280
+ const columns = config
281
+ ? [...row.entry.matchAll(/COALESCE\(("(?:[^"]|"")+"|[^,()\s]+),/gi)].map(([, column]) => column.startsWith('"') ? column.slice(1, -1).replaceAll('""', '"') : column)
282
+ : [];
283
+ if (!config || !columns.length) {
284
+ return undefined;
285
+ }
286
+ return { type: 'fulltext', entries: columns.map((column) => ({ column })), config: config[1].replaceAll("''", "'") };
287
+ }
288
+ const TEXT_INDEX_METHODS = new Set(['gin', 'inverted']);
270
289
  /**
271
290
  * Postgres states every entry in full: a plain column still reports `order: 'asc'`, and only a
272
291
  * non-default operator class is named. The diff defaults the entity side to match, so an option
@@ -1,9 +1,13 @@
1
+ import type { IndexFacet } from '../../schema/indexDifferences.js';
1
2
  import type { ColumnSchema, ForeignKeySchema, IndexSchema } from '../../type/index.js';
2
3
  import { AbstractSqlSchemaIntrospector, type TableRowReader } from './abstractSqlSchemaIntrospector.js';
3
4
  /**
4
5
  * SQLite schema introspector
5
6
  */
6
7
  export declare class SqliteSchemaIntrospector extends AbstractSqlSchemaIntrospector {
8
+ /** Whether an index is libSQL's vector index, where the engine has one; elsewhere a declared one is built plain. */
9
+ readonly indexFacets: ReadonlySet<IndexFacet>;
10
+ /** Not SQLite's own tables, nor the ones libSQL keeps a vector index in: its metadata and `<index>_shadow`. */
7
11
  protected getTableNamesQuery(): string;
8
12
  protected tableExistsQuery(): string;
9
13
  protected parseTableExistsResult([row]: SqliteCountRow[]): boolean;
@@ -29,6 +33,8 @@ export declare class SqliteSchemaIntrospector extends AbstractSqlSchemaIntrospec
29
33
  protected mapForeignKeysResult(_read: TableRowReader, tableName: string, results: SqliteForeignKeyRow[]): Promise<ForeignKeySchema[]>;
30
34
  protected mapPrimaryKeyResult(results: SqliteColumnRow[]): string[] | undefined;
31
35
  private getUniqueColumns;
36
+ /** libSQL's `libsql_vector_idx(col, 'metric=...')`, read back from the statement that created it. */
37
+ private getVectorIndex;
32
38
  private getIndexColumns;
33
39
  protected normalizeType(type: string): string;
34
40
  protected extractLength(type: string): number | undefined;