sails-vectorai 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -0
- package/index.js +780 -0
- package/package.json +60 -0
- package/test/run-standard-tests.js +84 -0
package/README.md
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# sails-vectorai
|
|
2
|
+
|
|
3
|
+
Experimental Sails/Waterline adapter for proving the VectorAI datastore contract. It is currently developed inside PairLore and published as `sails-vectorai`.
|
|
4
|
+
|
|
5
|
+
It was generated with:
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
npx sails generate adapter vectorai
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Then it was narrowed into a local-first adapter inspired by `sails-sqlite`: register a datastore, keep a datastore manager, expose Waterline stage-3 methods, and keep the implementation small until the real backend contract settles.
|
|
12
|
+
|
|
13
|
+
## Current DX
|
|
14
|
+
|
|
15
|
+
Configure a datastore with `adapter: 'sails-vectorai'`, point models at that datastore, and use normal Waterline calls for local development:
|
|
16
|
+
|
|
17
|
+
```js
|
|
18
|
+
await CulturalReferenceEmbedding.createEach([
|
|
19
|
+
{
|
|
20
|
+
text: 'AI memory tool launch post',
|
|
21
|
+
embedding: [1, 0, 0],
|
|
22
|
+
topic: 'ai'
|
|
23
|
+
}
|
|
24
|
+
])
|
|
25
|
+
|
|
26
|
+
var matches = await CulturalReferenceEmbedding.find({
|
|
27
|
+
where: { topic: 'ai' },
|
|
28
|
+
limit: 5
|
|
29
|
+
}).meta({
|
|
30
|
+
vectorSearch: {
|
|
31
|
+
column: 'embedding',
|
|
32
|
+
nearestTo: [0.98, 0.02, 0],
|
|
33
|
+
metric: 'cosine',
|
|
34
|
+
scoreField: 'similarity',
|
|
35
|
+
distanceField: 'distance'
|
|
36
|
+
}
|
|
37
|
+
})
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
`meta.vectorSearch` is the adapter-specific escape hatch. It keeps the model API boring while leaving room for Actian VectorAI DB or another vector backend behind the same Waterline boundary.
|
|
41
|
+
|
|
42
|
+
## Implemented Methods
|
|
43
|
+
|
|
44
|
+
| Method | Status | Category |
|
|
45
|
+
| :---------------- | :---------- | :-------- |
|
|
46
|
+
| registerDatastore | Implemented | LIFECYCLE |
|
|
47
|
+
| teardown | Implemented | LIFECYCLE |
|
|
48
|
+
| create | Implemented | DML |
|
|
49
|
+
| createEach | Implemented | DML |
|
|
50
|
+
| update | Implemented | DML |
|
|
51
|
+
| destroy | Implemented | DML |
|
|
52
|
+
| find | Implemented | DQL |
|
|
53
|
+
| count | Implemented | DQL |
|
|
54
|
+
| sum | Implemented | DQL |
|
|
55
|
+
| avg | Implemented | DQL |
|
|
56
|
+
| define | Implemented | DDL |
|
|
57
|
+
| drop | Implemented | DDL |
|
|
58
|
+
|
|
59
|
+
## Not Yet
|
|
60
|
+
|
|
61
|
+
The adapter does not implement joins, associations, persistence, connection pooling, native queries, or Actian-specific SQL yet. Those should be added only when PairLore's product loop proves the need.
|
|
62
|
+
|
|
63
|
+
## License
|
|
64
|
+
|
|
65
|
+
This local adapter is available under the MIT license.
|
package/index.js
ADDED
|
@@ -0,0 +1,780 @@
|
|
|
1
|
+
var _ = require('@sailshq/lodash')
|
|
2
|
+
|
|
3
|
+
var registeredDatastores = {}
|
|
4
|
+
|
|
5
|
+
function makeUsageError(message) {
|
|
6
|
+
var err = new Error(message)
|
|
7
|
+
err.name = 'AdapterError'
|
|
8
|
+
return err
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function makeUniqueError(message, keys) {
|
|
12
|
+
var err = new Error(message)
|
|
13
|
+
err.code = 'E_UNIQUE'
|
|
14
|
+
err.footprint = {
|
|
15
|
+
identity: 'notUnique',
|
|
16
|
+
keys: keys || []
|
|
17
|
+
}
|
|
18
|
+
return err
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function clone(value) {
|
|
22
|
+
return _.cloneDeep(value)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function normalizeModel(modelKey, modelInfo) {
|
|
26
|
+
modelInfo = modelInfo || {}
|
|
27
|
+
|
|
28
|
+
var definition = modelInfo.definition || modelInfo.attributes || {}
|
|
29
|
+
var identity = modelInfo.identity || modelKey
|
|
30
|
+
var tableName = modelInfo.tableName || identity
|
|
31
|
+
|
|
32
|
+
return {
|
|
33
|
+
identity: identity,
|
|
34
|
+
primaryKey: modelInfo.primaryKey || 'id',
|
|
35
|
+
definition: definition,
|
|
36
|
+
attributes: definition,
|
|
37
|
+
tableName: tableName
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function registerModel(manager, modelKey, modelInfo) {
|
|
42
|
+
var model = normalizeModel(modelKey, modelInfo)
|
|
43
|
+
|
|
44
|
+
manager.modelsByIdentity[model.identity] = model
|
|
45
|
+
manager.modelsByTable[model.tableName] = model
|
|
46
|
+
ensureTable(manager, model.tableName)
|
|
47
|
+
|
|
48
|
+
return model
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function ensureTable(manager, tableName) {
|
|
52
|
+
if (!manager.tables[tableName]) {
|
|
53
|
+
manager.tables[tableName] = []
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (!manager.sequences[tableName]) {
|
|
57
|
+
manager.sequences[tableName] = 0
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return manager.tables[tableName]
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function getDatastore(datastoreName) {
|
|
64
|
+
var dsEntry = registeredDatastores[datastoreName]
|
|
65
|
+
|
|
66
|
+
if (!dsEntry) {
|
|
67
|
+
throw makeUsageError(
|
|
68
|
+
'Consistency violation: Cannot use datastore `' +
|
|
69
|
+
datastoreName +
|
|
70
|
+
'` because it is not registered with this adapter.'
|
|
71
|
+
)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return dsEntry
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function getModel(manager, tableName) {
|
|
78
|
+
return manager.modelsByTable[tableName] || normalizeModel(tableName, {})
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function getPkColumn(model) {
|
|
82
|
+
var primaryKey = model.primaryKey || 'id'
|
|
83
|
+
var attrs = model.attributes || model.definition || {}
|
|
84
|
+
var pkAttr = attrs[primaryKey] || {}
|
|
85
|
+
|
|
86
|
+
return pkAttr.columnName || primaryKey
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function hasOwn(obj, key) {
|
|
90
|
+
return Object.prototype.hasOwnProperty.call(obj, key)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function getTableForQuery(dsEntry, s3q) {
|
|
94
|
+
if (!s3q || !s3q.using) {
|
|
95
|
+
throw makeUsageError(
|
|
96
|
+
'Consistency violation: Received a stage-3 query without a `using` table name.'
|
|
97
|
+
)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return ensureTable(dsEntry.manager, s3q.using)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function insertRecord(manager, tableName, rawRecord) {
|
|
104
|
+
var table = ensureTable(manager, tableName)
|
|
105
|
+
var model = getModel(manager, tableName)
|
|
106
|
+
var pkColumn = getPkColumn(model)
|
|
107
|
+
var record = clone(rawRecord || {})
|
|
108
|
+
|
|
109
|
+
if (record[pkColumn] === undefined || record[pkColumn] === null) {
|
|
110
|
+
manager.sequences[tableName] += 1
|
|
111
|
+
record[pkColumn] = manager.sequences[tableName]
|
|
112
|
+
} else if (
|
|
113
|
+
_.isNumber(record[pkColumn]) &&
|
|
114
|
+
record[pkColumn] > manager.sequences[tableName]
|
|
115
|
+
) {
|
|
116
|
+
manager.sequences[tableName] = record[pkColumn]
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
var hasDuplicatePk = _.some(table, function (existingRecord) {
|
|
120
|
+
return _.isEqual(existingRecord[pkColumn], record[pkColumn])
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
if (hasDuplicatePk) {
|
|
124
|
+
throw makeUniqueError(
|
|
125
|
+
'Cannot insert duplicate primary key value for `' + pkColumn + '`.',
|
|
126
|
+
[pkColumn]
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
table.push(record)
|
|
131
|
+
|
|
132
|
+
return record
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function matchesCriteria(record, where) {
|
|
136
|
+
if (!where || _.isEmpty(where)) {
|
|
137
|
+
return true
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return _.every(where, function (expected, key) {
|
|
141
|
+
if (key === 'or') {
|
|
142
|
+
return _.some(expected || [], function (subcriteria) {
|
|
143
|
+
return matchesCriteria(record, subcriteria)
|
|
144
|
+
})
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if (key === 'and') {
|
|
148
|
+
return _.every(expected || [], function (subcriteria) {
|
|
149
|
+
return matchesCriteria(record, subcriteria)
|
|
150
|
+
})
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return matchesValue(record[key], expected)
|
|
154
|
+
})
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function matchesValue(actual, expected) {
|
|
158
|
+
if (_.isArray(expected)) {
|
|
159
|
+
return _.some(expected, function (candidate) {
|
|
160
|
+
return _.isEqual(actual, candidate)
|
|
161
|
+
})
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (_.isPlainObject(expected)) {
|
|
165
|
+
var operatorKeys = [
|
|
166
|
+
'<',
|
|
167
|
+
'<=',
|
|
168
|
+
'>',
|
|
169
|
+
'>=',
|
|
170
|
+
'!=',
|
|
171
|
+
'in',
|
|
172
|
+
'nin',
|
|
173
|
+
'contains',
|
|
174
|
+
'startsWith',
|
|
175
|
+
'endsWith'
|
|
176
|
+
]
|
|
177
|
+
var isOperatorCriteria = _.some(_.keys(expected), function (key) {
|
|
178
|
+
return _.includes(operatorKeys, key)
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
if (!isOperatorCriteria) {
|
|
182
|
+
return _.isEqual(actual, expected)
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
return _.every(expected, function (operand, operator) {
|
|
186
|
+
return matchesOperator(actual, operator, operand)
|
|
187
|
+
})
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return _.isEqual(actual, expected)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function matchesOperator(actual, operator, operand) {
|
|
194
|
+
switch (operator) {
|
|
195
|
+
case '<':
|
|
196
|
+
return actual < operand
|
|
197
|
+
case '<=':
|
|
198
|
+
return actual <= operand
|
|
199
|
+
case '>':
|
|
200
|
+
return actual > operand
|
|
201
|
+
case '>=':
|
|
202
|
+
return actual >= operand
|
|
203
|
+
case '!=':
|
|
204
|
+
if (_.isArray(operand)) {
|
|
205
|
+
return !_.some(operand, function (candidate) {
|
|
206
|
+
return _.isEqual(actual, candidate)
|
|
207
|
+
})
|
|
208
|
+
}
|
|
209
|
+
return !_.isEqual(actual, operand)
|
|
210
|
+
case 'in':
|
|
211
|
+
return _.some(operand || [], function (candidate) {
|
|
212
|
+
return _.isEqual(actual, candidate)
|
|
213
|
+
})
|
|
214
|
+
case 'nin':
|
|
215
|
+
return !_.some(operand || [], function (candidate) {
|
|
216
|
+
return _.isEqual(actual, candidate)
|
|
217
|
+
})
|
|
218
|
+
case 'contains':
|
|
219
|
+
return _.includes(String(actual || ''), String(operand))
|
|
220
|
+
case 'startsWith':
|
|
221
|
+
return String(actual || '').indexOf(String(operand)) === 0
|
|
222
|
+
case 'endsWith':
|
|
223
|
+
return (
|
|
224
|
+
String(actual || '').slice(-String(operand).length) === String(operand)
|
|
225
|
+
)
|
|
226
|
+
default:
|
|
227
|
+
return false
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function parseSort(sort) {
|
|
232
|
+
if (!sort) {
|
|
233
|
+
return []
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (_.isString(sort)) {
|
|
237
|
+
return _.compact(sort.split(',')).map(function (sortClause) {
|
|
238
|
+
var parts = sortClause.trim().split(/\s+/)
|
|
239
|
+
return {
|
|
240
|
+
field: parts[0],
|
|
241
|
+
direction: String(parts[1] || 'ASC').toUpperCase()
|
|
242
|
+
}
|
|
243
|
+
})
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (_.isPlainObject(sort)) {
|
|
247
|
+
return _.map(sort, function (direction, field) {
|
|
248
|
+
return {
|
|
249
|
+
field: field,
|
|
250
|
+
direction: String(direction || 'ASC').toUpperCase()
|
|
251
|
+
}
|
|
252
|
+
})
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
if (_.isArray(sort)) {
|
|
256
|
+
return sort.reduce(function (memo, sortDirective) {
|
|
257
|
+
return memo.concat(parseSort(sortDirective))
|
|
258
|
+
}, [])
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
return []
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function compareBySort(a, b, sortDirectives) {
|
|
265
|
+
for (var i = 0; i < sortDirectives.length; i += 1) {
|
|
266
|
+
var directive = sortDirectives[i]
|
|
267
|
+
var direction = directive.direction === 'DESC' ? -1 : 1
|
|
268
|
+
|
|
269
|
+
if (a[directive.field] < b[directive.field]) {
|
|
270
|
+
return -1 * direction
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
if (a[directive.field] > b[directive.field]) {
|
|
274
|
+
return 1 * direction
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
return 0
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function applyVectorSearch(records, dsEntry, meta) {
|
|
282
|
+
var vectorSearch = meta && meta.vectorSearch
|
|
283
|
+
|
|
284
|
+
if (!vectorSearch) {
|
|
285
|
+
return records
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
var nearestTo =
|
|
289
|
+
vectorSearch.nearestTo || vectorSearch.queryVector || vectorSearch.vector
|
|
290
|
+
var column =
|
|
291
|
+
vectorSearch.column ||
|
|
292
|
+
vectorSearch.vectorColumn ||
|
|
293
|
+
vectorSearch.field ||
|
|
294
|
+
dsEntry.config.vectorColumn ||
|
|
295
|
+
'embedding'
|
|
296
|
+
var metric = String(vectorSearch.metric || dsEntry.config.metric || 'cosine')
|
|
297
|
+
.toLowerCase()
|
|
298
|
+
.trim()
|
|
299
|
+
var scoreField =
|
|
300
|
+
vectorSearch.scoreField === false
|
|
301
|
+
? false
|
|
302
|
+
: vectorSearch.scoreField || 'similarity'
|
|
303
|
+
var distanceField =
|
|
304
|
+
vectorSearch.distanceField === false ? false : vectorSearch.distanceField
|
|
305
|
+
|
|
306
|
+
if (!_.isArray(nearestTo)) {
|
|
307
|
+
throw makeUsageError(
|
|
308
|
+
'Invalid `meta.vectorSearch`: expected `nearestTo` to be an array of numbers.'
|
|
309
|
+
)
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
return _.compact(
|
|
313
|
+
records.map(function (record) {
|
|
314
|
+
var recordVector = record[column]
|
|
315
|
+
|
|
316
|
+
if (!_.isArray(recordVector)) {
|
|
317
|
+
return undefined
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
var result = clone(record)
|
|
321
|
+
var match = calculateVectorMatch(recordVector, nearestTo, metric)
|
|
322
|
+
|
|
323
|
+
result.__vectorSearchScore = match.score
|
|
324
|
+
|
|
325
|
+
if (scoreField) {
|
|
326
|
+
result[scoreField] = match.similarity
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (distanceField) {
|
|
330
|
+
result[distanceField] = match.distance
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
return result
|
|
334
|
+
})
|
|
335
|
+
)
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function calculateVectorMatch(recordVector, queryVector, metric) {
|
|
339
|
+
if (recordVector.length !== queryVector.length) {
|
|
340
|
+
throw makeUsageError(
|
|
341
|
+
'Invalid `meta.vectorSearch`: vector dimensions must match.'
|
|
342
|
+
)
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
var distance = euclideanDistance(recordVector, queryVector)
|
|
346
|
+
var similarity
|
|
347
|
+
var score
|
|
348
|
+
|
|
349
|
+
switch (metric) {
|
|
350
|
+
case 'dot':
|
|
351
|
+
case 'inner-product':
|
|
352
|
+
similarity = dotProduct(recordVector, queryVector)
|
|
353
|
+
score = similarity
|
|
354
|
+
break
|
|
355
|
+
case 'euclidean':
|
|
356
|
+
case 'l2':
|
|
357
|
+
similarity = 1 / (1 + distance)
|
|
358
|
+
score = similarity
|
|
359
|
+
break
|
|
360
|
+
case 'cosine':
|
|
361
|
+
similarity = cosineSimilarity(recordVector, queryVector)
|
|
362
|
+
score = similarity
|
|
363
|
+
break
|
|
364
|
+
default:
|
|
365
|
+
throw makeUsageError(
|
|
366
|
+
'Invalid `meta.vectorSearch.metric`: expected cosine, dot, or euclidean.'
|
|
367
|
+
)
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
return {
|
|
371
|
+
distance: distance,
|
|
372
|
+
similarity: similarity,
|
|
373
|
+
score: score
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
function dotProduct(a, b) {
|
|
378
|
+
return _.sum(
|
|
379
|
+
a.map(function (value, index) {
|
|
380
|
+
return value * b[index]
|
|
381
|
+
})
|
|
382
|
+
)
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
function euclideanDistance(a, b) {
|
|
386
|
+
return Math.sqrt(
|
|
387
|
+
_.sum(
|
|
388
|
+
a.map(function (value, index) {
|
|
389
|
+
return Math.pow(value - b[index], 2)
|
|
390
|
+
})
|
|
391
|
+
)
|
|
392
|
+
)
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function cosineSimilarity(a, b) {
|
|
396
|
+
var denominator = Math.sqrt(dotProduct(a, a)) * Math.sqrt(dotProduct(b, b))
|
|
397
|
+
|
|
398
|
+
if (denominator === 0) {
|
|
399
|
+
return 0
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
return dotProduct(a, b) / denominator
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function runFind(dsEntry, s3q) {
|
|
406
|
+
var table = getTableForQuery(dsEntry, s3q)
|
|
407
|
+
var criteria = s3q.criteria || {}
|
|
408
|
+
var sortDirectives = parseSort(criteria.sort)
|
|
409
|
+
|
|
410
|
+
var records = table.filter(function (record) {
|
|
411
|
+
return matchesCriteria(record, criteria.where)
|
|
412
|
+
})
|
|
413
|
+
|
|
414
|
+
records = applyVectorSearch(records, dsEntry, s3q.meta)
|
|
415
|
+
|
|
416
|
+
records.sort(function (a, b) {
|
|
417
|
+
if (
|
|
418
|
+
a.__vectorSearchScore !== undefined ||
|
|
419
|
+
b.__vectorSearchScore !== undefined
|
|
420
|
+
) {
|
|
421
|
+
var scoreComparison =
|
|
422
|
+
(b.__vectorSearchScore || 0) - (a.__vectorSearchScore || 0)
|
|
423
|
+
|
|
424
|
+
if (scoreComparison !== 0) {
|
|
425
|
+
return scoreComparison
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
return compareBySort(a, b, sortDirectives)
|
|
430
|
+
})
|
|
431
|
+
|
|
432
|
+
if (criteria.skip) {
|
|
433
|
+
records = records.slice(criteria.skip)
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
if (_.isNumber(criteria.limit)) {
|
|
437
|
+
records = records.slice(0, criteria.limit)
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
return records.map(function (record) {
|
|
441
|
+
return projectRecord(record, s3q, getModel(dsEntry.manager, s3q.using))
|
|
442
|
+
})
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function projectRecord(record, s3q, model) {
|
|
446
|
+
var criteria = s3q.criteria || {}
|
|
447
|
+
var selectedFields = criteria.select
|
|
448
|
+
var projectedRecord
|
|
449
|
+
|
|
450
|
+
if (_.isArray(selectedFields) && selectedFields.length > 0) {
|
|
451
|
+
projectedRecord = {}
|
|
452
|
+
|
|
453
|
+
selectedFields.forEach(function (fieldName) {
|
|
454
|
+
if (hasOwn(record, fieldName)) {
|
|
455
|
+
projectedRecord[fieldName] = clone(record[fieldName])
|
|
456
|
+
}
|
|
457
|
+
})
|
|
458
|
+
|
|
459
|
+
var pkColumn = getPkColumn(model)
|
|
460
|
+
if (hasOwn(record, pkColumn)) {
|
|
461
|
+
projectedRecord[pkColumn] = clone(record[pkColumn])
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
addVectorScoreFields(record, projectedRecord, s3q.meta)
|
|
465
|
+
} else {
|
|
466
|
+
projectedRecord = clone(record)
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if (_.isArray(criteria.omit)) {
|
|
470
|
+
criteria.omit.forEach(function (fieldName) {
|
|
471
|
+
delete projectedRecord[fieldName]
|
|
472
|
+
})
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
delete projectedRecord.__vectorSearchScore
|
|
476
|
+
|
|
477
|
+
return projectedRecord
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
function addVectorScoreFields(source, target, meta) {
|
|
481
|
+
var vectorSearch = meta && meta.vectorSearch
|
|
482
|
+
|
|
483
|
+
if (!vectorSearch) {
|
|
484
|
+
return
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
var scoreField =
|
|
488
|
+
vectorSearch.scoreField === false
|
|
489
|
+
? false
|
|
490
|
+
: vectorSearch.scoreField || 'similarity'
|
|
491
|
+
var distanceField =
|
|
492
|
+
vectorSearch.distanceField === false ? false : vectorSearch.distanceField
|
|
493
|
+
|
|
494
|
+
if (scoreField && hasOwn(source, scoreField)) {
|
|
495
|
+
target[scoreField] = clone(source[scoreField])
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
if (distanceField && hasOwn(source, distanceField)) {
|
|
499
|
+
target[distanceField] = clone(source[distanceField])
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
function sumRecords(records, fieldName) {
|
|
504
|
+
return records.reduce(function (total, record) {
|
|
505
|
+
return total + (Number(record[fieldName]) || 0)
|
|
506
|
+
}, 0)
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
module.exports = {
|
|
510
|
+
identity: 'sails-vectorai',
|
|
511
|
+
|
|
512
|
+
adapterApiVersion: 1,
|
|
513
|
+
|
|
514
|
+
defaults: {
|
|
515
|
+
url: 'memory://pairlore-vectorai',
|
|
516
|
+
metric: 'cosine',
|
|
517
|
+
vectorColumn: 'embedding'
|
|
518
|
+
},
|
|
519
|
+
|
|
520
|
+
datastores: registeredDatastores,
|
|
521
|
+
|
|
522
|
+
registerDatastore: function registerDatastore(dsConfig, models, done) {
|
|
523
|
+
var datastoreName = dsConfig.identity
|
|
524
|
+
|
|
525
|
+
if (!datastoreName) {
|
|
526
|
+
return done(
|
|
527
|
+
makeUsageError(
|
|
528
|
+
'Consistency violation: A datastore must contain an `identity` property.'
|
|
529
|
+
)
|
|
530
|
+
)
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
if (registeredDatastores[datastoreName]) {
|
|
534
|
+
return done(
|
|
535
|
+
makeUsageError(
|
|
536
|
+
'Consistency violation: Cannot register datastore `' +
|
|
537
|
+
datastoreName +
|
|
538
|
+
'` because it is already registered with this adapter.'
|
|
539
|
+
)
|
|
540
|
+
)
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
var manager = {
|
|
544
|
+
config: dsConfig,
|
|
545
|
+
tables: {},
|
|
546
|
+
sequences: {},
|
|
547
|
+
modelsByIdentity: {},
|
|
548
|
+
modelsByTable: {}
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
_.each(models || {}, function (modelInfo, modelKey) {
|
|
552
|
+
registerModel(manager, modelKey, modelInfo)
|
|
553
|
+
})
|
|
554
|
+
|
|
555
|
+
registeredDatastores[datastoreName] = {
|
|
556
|
+
config: _.defaults({}, dsConfig, module.exports.defaults),
|
|
557
|
+
manager: manager,
|
|
558
|
+
driver: undefined
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
return done()
|
|
562
|
+
},
|
|
563
|
+
|
|
564
|
+
teardown: function teardown(datastoreName, done) {
|
|
565
|
+
if (!datastoreName) {
|
|
566
|
+
return done(
|
|
567
|
+
makeUsageError(
|
|
568
|
+
'Consistency violation: Cannot tear down a datastore without a datastore name.'
|
|
569
|
+
)
|
|
570
|
+
)
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
if (!registeredDatastores[datastoreName]) {
|
|
574
|
+
return done(
|
|
575
|
+
makeUsageError(
|
|
576
|
+
'Consistency violation: Cannot tear down datastore `' +
|
|
577
|
+
datastoreName +
|
|
578
|
+
'` because it is not registered with this adapter.'
|
|
579
|
+
)
|
|
580
|
+
)
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
delete registeredDatastores[datastoreName]
|
|
584
|
+
return done()
|
|
585
|
+
},
|
|
586
|
+
|
|
587
|
+
create: function create(datastoreName, s3q, done) {
|
|
588
|
+
try {
|
|
589
|
+
var dsEntry = getDatastore(datastoreName)
|
|
590
|
+
var record = insertRecord(dsEntry.manager, s3q.using, s3q.newRecord)
|
|
591
|
+
|
|
592
|
+
return done(null, s3q.meta && s3q.meta.fetch ? clone(record) : undefined)
|
|
593
|
+
} catch (err) {
|
|
594
|
+
return done(err)
|
|
595
|
+
}
|
|
596
|
+
},
|
|
597
|
+
|
|
598
|
+
createEach: function createEach(datastoreName, s3q, done) {
|
|
599
|
+
try {
|
|
600
|
+
var dsEntry = getDatastore(datastoreName)
|
|
601
|
+
var records = (s3q.newRecords || []).map(function (newRecord) {
|
|
602
|
+
return insertRecord(dsEntry.manager, s3q.using, newRecord)
|
|
603
|
+
})
|
|
604
|
+
|
|
605
|
+
return done(
|
|
606
|
+
null,
|
|
607
|
+
s3q.meta && s3q.meta.fetch ? records.map(clone) : undefined
|
|
608
|
+
)
|
|
609
|
+
} catch (err) {
|
|
610
|
+
return done(err)
|
|
611
|
+
}
|
|
612
|
+
},
|
|
613
|
+
|
|
614
|
+
update: function update(datastoreName, s3q, done) {
|
|
615
|
+
try {
|
|
616
|
+
var dsEntry = getDatastore(datastoreName)
|
|
617
|
+
var table = getTableForQuery(dsEntry, s3q)
|
|
618
|
+
var criteria = s3q.criteria || {}
|
|
619
|
+
var valuesToSet = s3q.valuesToSet || {}
|
|
620
|
+
var model = getModel(dsEntry.manager, s3q.using)
|
|
621
|
+
var pkColumn = getPkColumn(model)
|
|
622
|
+
var matchingRecords = table.filter(function (record) {
|
|
623
|
+
return matchesCriteria(record, criteria.where)
|
|
624
|
+
})
|
|
625
|
+
|
|
626
|
+
if (hasOwn(valuesToSet, pkColumn)) {
|
|
627
|
+
if (matchingRecords.length > 1) {
|
|
628
|
+
throw makeUniqueError(
|
|
629
|
+
'Cannot update multiple records to the same primary key value for `' +
|
|
630
|
+
pkColumn +
|
|
631
|
+
'`.',
|
|
632
|
+
[pkColumn]
|
|
633
|
+
)
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
var duplicatePk = _.some(table, function (record) {
|
|
637
|
+
return (
|
|
638
|
+
!_.includes(matchingRecords, record) &&
|
|
639
|
+
_.isEqual(record[pkColumn], valuesToSet[pkColumn])
|
|
640
|
+
)
|
|
641
|
+
})
|
|
642
|
+
|
|
643
|
+
if (duplicatePk) {
|
|
644
|
+
throw makeUniqueError(
|
|
645
|
+
'Cannot update duplicate primary key value for `' + pkColumn + '`.',
|
|
646
|
+
[pkColumn]
|
|
647
|
+
)
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
matchingRecords.forEach(function (record) {
|
|
652
|
+
_.assign(record, clone(valuesToSet))
|
|
653
|
+
})
|
|
654
|
+
|
|
655
|
+
return done(
|
|
656
|
+
null,
|
|
657
|
+
s3q.meta && s3q.meta.fetch ? matchingRecords.map(clone) : undefined
|
|
658
|
+
)
|
|
659
|
+
} catch (err) {
|
|
660
|
+
return done(err)
|
|
661
|
+
}
|
|
662
|
+
},
|
|
663
|
+
|
|
664
|
+
destroy: function destroy(datastoreName, s3q, done) {
|
|
665
|
+
try {
|
|
666
|
+
var dsEntry = getDatastore(datastoreName)
|
|
667
|
+
var table = getTableForQuery(dsEntry, s3q)
|
|
668
|
+
var criteria = s3q.criteria || {}
|
|
669
|
+
var destroyedRecords = []
|
|
670
|
+
|
|
671
|
+
for (var i = table.length - 1; i >= 0; i -= 1) {
|
|
672
|
+
if (matchesCriteria(table[i], criteria.where)) {
|
|
673
|
+
destroyedRecords.unshift(table[i])
|
|
674
|
+
table.splice(i, 1)
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
return done(
|
|
679
|
+
null,
|
|
680
|
+
s3q.meta && s3q.meta.fetch ? destroyedRecords.map(clone) : undefined
|
|
681
|
+
)
|
|
682
|
+
} catch (err) {
|
|
683
|
+
return done(err)
|
|
684
|
+
}
|
|
685
|
+
},
|
|
686
|
+
|
|
687
|
+
find: function find(datastoreName, s3q, done) {
|
|
688
|
+
try {
|
|
689
|
+
var dsEntry = getDatastore(datastoreName)
|
|
690
|
+
return done(null, runFind(dsEntry, s3q))
|
|
691
|
+
} catch (err) {
|
|
692
|
+
return done(err)
|
|
693
|
+
}
|
|
694
|
+
},
|
|
695
|
+
|
|
696
|
+
count: function count(datastoreName, s3q, done) {
|
|
697
|
+
try {
|
|
698
|
+
var dsEntry = getDatastore(datastoreName)
|
|
699
|
+
var table = getTableForQuery(dsEntry, s3q)
|
|
700
|
+
var criteria = s3q.criteria || {}
|
|
701
|
+
var total = table.filter(function (record) {
|
|
702
|
+
return matchesCriteria(record, criteria.where)
|
|
703
|
+
}).length
|
|
704
|
+
|
|
705
|
+
return done(null, total)
|
|
706
|
+
} catch (err) {
|
|
707
|
+
return done(err)
|
|
708
|
+
}
|
|
709
|
+
},
|
|
710
|
+
|
|
711
|
+
sum: function sum(datastoreName, s3q, done) {
|
|
712
|
+
try {
|
|
713
|
+
var dsEntry = getDatastore(datastoreName)
|
|
714
|
+
var records = runFind(dsEntry, _.assign({}, s3q, { meta: undefined }))
|
|
715
|
+
|
|
716
|
+
return done(null, sumRecords(records, s3q.numericAttrName))
|
|
717
|
+
} catch (err) {
|
|
718
|
+
return done(err)
|
|
719
|
+
}
|
|
720
|
+
},
|
|
721
|
+
|
|
722
|
+
avg: function avg(datastoreName, s3q, done) {
|
|
723
|
+
try {
|
|
724
|
+
var dsEntry = getDatastore(datastoreName)
|
|
725
|
+
var records = runFind(dsEntry, _.assign({}, s3q, { meta: undefined }))
|
|
726
|
+
|
|
727
|
+
if (records.length === 0) {
|
|
728
|
+
return done(null, 0)
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
return done(
|
|
732
|
+
null,
|
|
733
|
+
sumRecords(records, s3q.numericAttrName) / records.length
|
|
734
|
+
)
|
|
735
|
+
} catch (err) {
|
|
736
|
+
return done(err)
|
|
737
|
+
}
|
|
738
|
+
},
|
|
739
|
+
|
|
740
|
+
define: function define(datastoreName, tableName, definition, done) {
|
|
741
|
+
try {
|
|
742
|
+
var dsEntry = getDatastore(datastoreName)
|
|
743
|
+
ensureTable(dsEntry.manager, tableName)
|
|
744
|
+
registerModel(dsEntry.manager, tableName, {
|
|
745
|
+
identity: tableName,
|
|
746
|
+
primaryKey: 'id',
|
|
747
|
+
tableName: tableName,
|
|
748
|
+
definition: definition
|
|
749
|
+
})
|
|
750
|
+
|
|
751
|
+
return done()
|
|
752
|
+
} catch (err) {
|
|
753
|
+
return done(err)
|
|
754
|
+
}
|
|
755
|
+
},
|
|
756
|
+
|
|
757
|
+
drop: function drop(datastoreName, tableName, relations, done) {
|
|
758
|
+
if (_.isFunction(relations)) {
|
|
759
|
+
done = relations
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
try {
|
|
763
|
+
var dsEntry = getDatastore(datastoreName)
|
|
764
|
+
|
|
765
|
+
delete dsEntry.manager.tables[tableName]
|
|
766
|
+
delete dsEntry.manager.sequences[tableName]
|
|
767
|
+
delete dsEntry.manager.modelsByTable[tableName]
|
|
768
|
+
|
|
769
|
+
_.each(dsEntry.manager.modelsByIdentity, function (model, identity) {
|
|
770
|
+
if (model.tableName === tableName) {
|
|
771
|
+
delete dsEntry.manager.modelsByIdentity[identity]
|
|
772
|
+
}
|
|
773
|
+
})
|
|
774
|
+
|
|
775
|
+
return done()
|
|
776
|
+
} catch (err) {
|
|
777
|
+
return done(err)
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "sails-vectorai",
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"description": "Experimental Sails/Waterline adapter for VectorAI.",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"scripts": {
|
|
7
|
+
"test": "node test/run-standard-tests",
|
|
8
|
+
"pretest": "npm run lint",
|
|
9
|
+
"lint": "node ./node_modules/eslint/bin/eslint . --max-warnings=0"
|
|
10
|
+
},
|
|
11
|
+
"keywords": [
|
|
12
|
+
"actian",
|
|
13
|
+
"vector",
|
|
14
|
+
"vectorai",
|
|
15
|
+
"vector-database",
|
|
16
|
+
"semantic-search",
|
|
17
|
+
"adapter",
|
|
18
|
+
"sails",
|
|
19
|
+
"sails.js",
|
|
20
|
+
"waterline",
|
|
21
|
+
"orm",
|
|
22
|
+
"database-adapter"
|
|
23
|
+
],
|
|
24
|
+
"author": "Sailscasts HQ",
|
|
25
|
+
"license": "MIT",
|
|
26
|
+
"dependencies": {
|
|
27
|
+
"@sailshq/lodash": "^3.10.2"
|
|
28
|
+
},
|
|
29
|
+
"devDependencies": {
|
|
30
|
+
"eslint": "3.19.0",
|
|
31
|
+
"waterline-adapter-tests": "^1.0.0-6"
|
|
32
|
+
},
|
|
33
|
+
"waterlineAdapter": {
|
|
34
|
+
"type": "vectorai",
|
|
35
|
+
"waterlineVersion": "~0.13.0",
|
|
36
|
+
"interfaces": [
|
|
37
|
+
"semantic",
|
|
38
|
+
"queryable",
|
|
39
|
+
"migratable"
|
|
40
|
+
],
|
|
41
|
+
"features": []
|
|
42
|
+
},
|
|
43
|
+
"files": [
|
|
44
|
+
"index.js",
|
|
45
|
+
"README.md",
|
|
46
|
+
"test"
|
|
47
|
+
],
|
|
48
|
+
"repository": {
|
|
49
|
+
"type": "git",
|
|
50
|
+
"url": "git+https://github.com/sailscastshq/pairlore.git",
|
|
51
|
+
"directory": "api/adapters/vectorai"
|
|
52
|
+
},
|
|
53
|
+
"bugs": {
|
|
54
|
+
"url": "https://github.com/sailscastshq/pairlore/issues"
|
|
55
|
+
},
|
|
56
|
+
"homepage": "https://github.com/sailscastshq/pairlore/tree/main/api/adapters/vectorai#readme",
|
|
57
|
+
"engines": {
|
|
58
|
+
"node": ">=18.0.0"
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Module dependencies
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
var TestRunner = require('waterline-adapter-tests')
|
|
8
|
+
var packageMD = require('../package.json')
|
|
9
|
+
var Adapter = require('../')
|
|
10
|
+
|
|
11
|
+
// Log an intro message.
|
|
12
|
+
console.log('Testing `' + packageMD.name + '`, a Sails/Waterline adapter.')
|
|
13
|
+
console.log(
|
|
14
|
+
'Running `waterline-adapter-tests` against ' +
|
|
15
|
+
packageMD.waterlineAdapter.interfaces.length +
|
|
16
|
+
' interface(s) and ' +
|
|
17
|
+
packageMD.waterlineAdapter.features.length +
|
|
18
|
+
' feature(s)...'
|
|
19
|
+
)
|
|
20
|
+
console.log(
|
|
21
|
+
'| Interfaces: ' +
|
|
22
|
+
(packageMD.waterlineAdapter.interfaces.join(', ') || 'n/a') +
|
|
23
|
+
''
|
|
24
|
+
)
|
|
25
|
+
console.log(
|
|
26
|
+
'| Extra features: ' +
|
|
27
|
+
(packageMD.waterlineAdapter.features.join(', ') || 'n/a') +
|
|
28
|
+
''
|
|
29
|
+
)
|
|
30
|
+
console.log()
|
|
31
|
+
console.log('> More info about building Waterline adapters:')
|
|
32
|
+
console.log(
|
|
33
|
+
'> https://sailsjs.com/docs/concepts/extending-sails/adapters/custom-adapters'
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
// Ensure a `url` was specified.
|
|
37
|
+
// (https://sailsjs.com/config/datastores#?the-connection-url)
|
|
38
|
+
if (!process.env.WATERLINE_ADAPTER_TESTS_URL) {
|
|
39
|
+
console.error()
|
|
40
|
+
console.error('-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-')
|
|
41
|
+
console.error('Cannot run tests: No database connection `url` specified.')
|
|
42
|
+
console.error()
|
|
43
|
+
console.error('Tip: You can use an environment variable to configure this.')
|
|
44
|
+
console.error('For example:')
|
|
45
|
+
console.error('```')
|
|
46
|
+
console.error(
|
|
47
|
+
' WATERLINE_ADAPTER_TESTS_URL=root@localhost:3306/testdb npm test'
|
|
48
|
+
)
|
|
49
|
+
console.error('```')
|
|
50
|
+
console.error('-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-')
|
|
51
|
+
process.exit(1)
|
|
52
|
+
} //-•
|
|
53
|
+
|
|
54
|
+
// Use the `waterline-adapter-tests` module to
|
|
55
|
+
// run mocha tests against the specified interfaces
|
|
56
|
+
// of the currently-implemented Waterline adapter API.
|
|
57
|
+
new TestRunner({
|
|
58
|
+
// Load the adapter module.
|
|
59
|
+
adapter: Adapter,
|
|
60
|
+
|
|
61
|
+
// Adapter config to use for tests.
|
|
62
|
+
config: {
|
|
63
|
+
url: process.env.WATERLINE_ADAPTER_TESTS_URL
|
|
64
|
+
},
|
|
65
|
+
|
|
66
|
+
// The set of adapter interface layers & specific features to test against.
|
|
67
|
+
interfaces: packageMD.waterlineAdapter.interfaces,
|
|
68
|
+
features: packageMD.waterlineAdapter.features
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
|
72
|
+
//
|
|
73
|
+
// Most databases implement 'semantic' and 'queryable' interface layers.
|
|
74
|
+
//
|
|
75
|
+
// For full interface reference, see:
|
|
76
|
+
// https://sailsjs.com/docs/concepts/extending-sails/adapters/custom-adapters
|
|
77
|
+
//
|
|
78
|
+
// Some features are polyfilled if omitted; allowing optimizations at the adapter
|
|
79
|
+
// level for databases that support the feature. For example, if you don't implement
|
|
80
|
+
// a `join` method, it will be polyfilled for you by Waterline core (using the same
|
|
81
|
+
// "polypopulate" that it uses for cross-datastore joins.) For more on that, talk
|
|
82
|
+
// to an adapter maintainer @ https://sailsjs.com/support.
|
|
83
|
+
//
|
|
84
|
+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|