tsense 0.0.17 → 0.2.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -104
- package/dist/filters/filter-state.d.ts +22 -0
- package/dist/filters/filter-state.js +145 -0
- package/dist/filters/index.d.ts +40 -0
- package/dist/filters/index.js +128 -0
- package/dist/filters/url.d.ts +3 -0
- package/dist/filters/url.js +93 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -0
- package/dist/migrator.d.ts +1 -1
- package/dist/migrator.js +70 -88
- package/dist/rank.d.ts +7 -0
- package/dist/rank.js +12 -0
- package/dist/react/filter-builder.d.ts +60 -0
- package/dist/react/filter-builder.js +102 -0
- package/dist/react/index.d.ts +4 -0
- package/dist/react/index.js +2 -0
- package/dist/react/use-filter-builder.d.ts +23 -0
- package/dist/react/use-filter-builder.js +22 -0
- package/dist/tsense.d.ts +5 -4
- package/dist/tsense.js +388 -374
- package/dist/types.d.ts +37 -10
- package/package.json +69 -51
package/dist/tsense.js
CHANGED
|
@@ -1,13 +1,3 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
var _a;
|
|
11
1
|
import redaxios from "redaxios";
|
|
12
2
|
import { TSenseMigrator } from "./migrator.js";
|
|
13
3
|
import { defaultTransformers } from "./transformers/defaults.js";
|
|
@@ -18,7 +8,24 @@ function chunkArray(arr, size) {
|
|
|
18
8
|
}
|
|
19
9
|
return chunks;
|
|
20
10
|
}
|
|
21
|
-
const redaxiosInstance =
|
|
11
|
+
const redaxiosInstance = redaxios.default ?? redaxios;
|
|
12
|
+
function escapeFilterValue(value) {
|
|
13
|
+
if (typeof value === "string") {
|
|
14
|
+
return `\`${value.replaceAll("`", "")}\``;
|
|
15
|
+
}
|
|
16
|
+
if (Array.isArray(value)) {
|
|
17
|
+
return value.map(escapeFilterValue);
|
|
18
|
+
}
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
const filterOperators = {
|
|
22
|
+
not: (k, v) => `${k}:!=${v}`,
|
|
23
|
+
gt: (k, v) => `${k}:>${v}`,
|
|
24
|
+
gte: (k, v) => `${k}:>=${v}`,
|
|
25
|
+
lt: (k, v) => `${k}:<${v}`,
|
|
26
|
+
lte: (k, v) => `${k}:<=${v}`,
|
|
27
|
+
notIn: (k, v) => `${k}:!=[${v.join(",")}]`,
|
|
28
|
+
};
|
|
22
29
|
const arkToTsense = {
|
|
23
30
|
string: "string",
|
|
24
31
|
number: "float",
|
|
@@ -30,21 +37,20 @@ const arkToTsense = {
|
|
|
30
37
|
"boolean[]": "bool[]",
|
|
31
38
|
};
|
|
32
39
|
export class TSense {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
40
|
+
options;
|
|
41
|
+
fields = [];
|
|
42
|
+
axios;
|
|
43
|
+
synced = false;
|
|
44
|
+
fieldTransformers = new Map();
|
|
45
|
+
dataSyncConfig;
|
|
46
|
+
infer = undefined;
|
|
36
47
|
constructor(options) {
|
|
37
|
-
var _a;
|
|
38
48
|
this.options = options;
|
|
39
|
-
this.fields = [];
|
|
40
|
-
this.synced = false;
|
|
41
|
-
this.fieldTransformers = new Map();
|
|
42
|
-
this.infer = undefined;
|
|
43
49
|
this.axios = redaxiosInstance.create({
|
|
44
50
|
baseURL: `${options.connection.protocol}://${options.connection.host}:${options.connection.port}`,
|
|
45
51
|
headers: { "X-TYPESENSE-API-KEY": options.connection.apiKey },
|
|
46
52
|
});
|
|
47
|
-
this.
|
|
53
|
+
this.fields = this.extractFields(options.transformers ?? defaultTransformers);
|
|
48
54
|
this.dataSyncConfig = options.dataSync;
|
|
49
55
|
}
|
|
50
56
|
getBaseType(expression, domain) {
|
|
@@ -65,7 +71,7 @@ export class TSense {
|
|
|
65
71
|
return "string";
|
|
66
72
|
}
|
|
67
73
|
serializeDoc(doc) {
|
|
68
|
-
const result =
|
|
74
|
+
const result = { ...doc };
|
|
69
75
|
for (const [field, transformer] of this.fieldTransformers) {
|
|
70
76
|
if (result[field] != null) {
|
|
71
77
|
result[field] = transformer.serialize(result[field]);
|
|
@@ -88,124 +94,147 @@ export class TSense {
|
|
|
88
94
|
if (Array.isArray(value)) {
|
|
89
95
|
return value.map((v) => transformer.serialize(v));
|
|
90
96
|
}
|
|
91
|
-
if (typeof value === "object" &&
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
97
|
+
if (typeof value === "object" &&
|
|
98
|
+
value !== null &&
|
|
99
|
+
Object.getPrototypeOf(value) === Object.prototype) {
|
|
100
|
+
const result = {};
|
|
101
|
+
for (const [opKey, opValue] of Object.entries(value)) {
|
|
102
|
+
if (opValue == null) {
|
|
103
|
+
result[opKey] = opValue;
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (Array.isArray(opValue)) {
|
|
107
|
+
result[opKey] = opValue.map((item) => transformer.serialize(item));
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
result[opKey] = transformer.serialize(opValue);
|
|
111
|
+
}
|
|
96
112
|
}
|
|
97
|
-
const result = Object.assign({}, v);
|
|
98
|
-
if ("min" in v && v.min != null)
|
|
99
|
-
result.min = transformer.serialize(v.min);
|
|
100
|
-
if ("max" in v && v.max != null)
|
|
101
|
-
result.max = transformer.serialize(v.max);
|
|
102
|
-
if ("not" in v && v.not != null)
|
|
103
|
-
result.not = transformer.serialize(v.not);
|
|
104
113
|
return result;
|
|
105
114
|
}
|
|
106
115
|
return transformer.serialize(value);
|
|
107
116
|
}
|
|
108
117
|
extractFields(transformers) {
|
|
109
|
-
var _a, _b, _c, _d;
|
|
110
118
|
const internal = this.options.schema;
|
|
119
|
+
const fields = [];
|
|
111
120
|
for (const prop of internal.structure.props) {
|
|
112
|
-
const
|
|
113
|
-
const
|
|
121
|
+
const branches = prop.value.branches ?? [];
|
|
122
|
+
const enumValues = [];
|
|
123
|
+
for (const branch of branches) {
|
|
124
|
+
if (typeof branch.unit === "string") {
|
|
125
|
+
enumValues.push(branch.unit);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const innerType = branches[0] ?? prop.value;
|
|
129
|
+
const meta = (innerType.meta ?? prop.value.meta);
|
|
114
130
|
const expression = String(prop.value.expression);
|
|
115
131
|
const domain = prop.value.domain;
|
|
116
132
|
const baseType = this.getBaseType(expression, domain);
|
|
117
133
|
const transformer = transformers.find((t) => t.match(expression, domain) || t.match(baseType, domain));
|
|
118
134
|
if (transformer) {
|
|
119
135
|
this.fieldTransformers.set(prop.key, transformer);
|
|
120
|
-
|
|
136
|
+
fields.push({
|
|
121
137
|
name: prop.key,
|
|
122
138
|
type: transformer.storageType,
|
|
139
|
+
sourceExpression: expression,
|
|
123
140
|
optional: prop.kind === "optional",
|
|
124
|
-
facet: meta
|
|
125
|
-
sort: meta
|
|
126
|
-
index: meta
|
|
141
|
+
facet: meta?.facet,
|
|
142
|
+
sort: meta?.sort,
|
|
143
|
+
index: meta?.index,
|
|
144
|
+
enumValues,
|
|
127
145
|
});
|
|
128
146
|
continue;
|
|
129
147
|
}
|
|
130
|
-
const type =
|
|
131
|
-
|
|
148
|
+
const type = meta?.type ?? this.inferType(baseType);
|
|
149
|
+
fields.push({
|
|
132
150
|
name: prop.key,
|
|
133
151
|
type,
|
|
152
|
+
sourceExpression: expression,
|
|
134
153
|
optional: prop.kind === "optional",
|
|
135
|
-
facet: meta
|
|
136
|
-
sort: meta
|
|
137
|
-
index: meta
|
|
154
|
+
facet: meta?.facet,
|
|
155
|
+
sort: meta?.sort,
|
|
156
|
+
index: meta?.index,
|
|
157
|
+
enumValues,
|
|
138
158
|
});
|
|
139
159
|
}
|
|
160
|
+
return fields;
|
|
140
161
|
}
|
|
141
|
-
ensureSynced(force) {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
this.synced = true;
|
|
147
|
-
});
|
|
162
|
+
async ensureSynced(force) {
|
|
163
|
+
if (!force && (this.synced || !this.options.autoSyncSchema))
|
|
164
|
+
return;
|
|
165
|
+
await new TSenseMigrator(this.options.name, this.fields, this.options.defaultSortingField, this.axios).sync();
|
|
166
|
+
this.synced = true;
|
|
148
167
|
}
|
|
149
|
-
syncSchema() {
|
|
150
|
-
|
|
151
|
-
yield this.ensureSynced(true);
|
|
152
|
-
});
|
|
168
|
+
async syncSchema() {
|
|
169
|
+
await this.ensureSynced(true);
|
|
153
170
|
}
|
|
154
171
|
buildObjectFilter(key, value) {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
if (max != null) {
|
|
169
|
-
return `${key}:<=${max}`;
|
|
172
|
+
if (value.gte != null && value.lte != null) {
|
|
173
|
+
const escaped = escapeFilterValue(value.gte);
|
|
174
|
+
const escapedLte = escapeFilterValue(value.lte);
|
|
175
|
+
const parts = [`${key}:[${escaped}..${escapedLte}]`];
|
|
176
|
+
for (const [op, opValue] of Object.entries(value)) {
|
|
177
|
+
if (op === "gte" || op === "lte" || opValue == null)
|
|
178
|
+
continue;
|
|
179
|
+
const builder = filterOperators[op];
|
|
180
|
+
if (builder) {
|
|
181
|
+
parts.push(builder(key, escapeFilterValue(opValue)));
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return parts;
|
|
170
185
|
}
|
|
171
|
-
|
|
172
|
-
|
|
186
|
+
const parts = [];
|
|
187
|
+
for (const [op, opValue] of Object.entries(value)) {
|
|
188
|
+
if (opValue == null)
|
|
189
|
+
continue;
|
|
190
|
+
const builder = filterOperators[op];
|
|
191
|
+
if (builder) {
|
|
192
|
+
parts.push(builder(key, escapeFilterValue(opValue)));
|
|
193
|
+
}
|
|
173
194
|
}
|
|
195
|
+
return parts;
|
|
174
196
|
}
|
|
175
197
|
buildFilter(filter) {
|
|
176
198
|
const result = [];
|
|
177
|
-
for (const entry of Object.entries(filter
|
|
199
|
+
for (const entry of Object.entries(filter ?? {})) {
|
|
178
200
|
const [key, rawValue] = entry;
|
|
179
201
|
if (rawValue == null)
|
|
180
202
|
continue;
|
|
181
203
|
if (key === "OR") {
|
|
182
|
-
const
|
|
204
|
+
const orParts = [];
|
|
183
205
|
for (const condition of rawValue) {
|
|
184
206
|
const inner = this.buildFilter(condition);
|
|
185
|
-
|
|
207
|
+
orParts.push(`(${inner.join("&&")})`);
|
|
186
208
|
}
|
|
187
|
-
result.push(`(${
|
|
209
|
+
result.push(`(${orParts.join("||")})`);
|
|
188
210
|
continue;
|
|
189
211
|
}
|
|
190
212
|
const value = this.serializeFilterValue(key, rawValue);
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
break;
|
|
213
|
+
const escaped = escapeFilterValue(value);
|
|
214
|
+
if (typeof escaped === "string" ||
|
|
215
|
+
typeof escaped === "number" ||
|
|
216
|
+
typeof escaped === "boolean") {
|
|
217
|
+
result.push(`${key}:=${escaped}`);
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
if (Array.isArray(escaped)) {
|
|
221
|
+
result.push(`${key}:[${escaped.join(",")}]`);
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
if (typeof value === "object" && value !== null) {
|
|
225
|
+
result.push(...this.buildObjectFilter(key, value));
|
|
205
226
|
}
|
|
206
227
|
}
|
|
207
228
|
return result;
|
|
208
229
|
}
|
|
230
|
+
validateFields(fields) {
|
|
231
|
+
const valid = new Set(this.fields.map((f) => f.name));
|
|
232
|
+
for (const field of fields) {
|
|
233
|
+
if (field !== "score" && !valid.has(field)) {
|
|
234
|
+
throw new Error(`INVALID_FIELD: ${field}`);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
209
238
|
buildSort(options) {
|
|
210
239
|
if (!options.sortBy)
|
|
211
240
|
return;
|
|
@@ -219,322 +248,307 @@ export class TSense {
|
|
|
219
248
|
}
|
|
220
249
|
return result.join(",");
|
|
221
250
|
}
|
|
222
|
-
create() {
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
},
|
|
234
|
-
});
|
|
235
|
-
return this;
|
|
251
|
+
async create() {
|
|
252
|
+
const enableNested = this.fields.some((f) => f.type === "object" || f.type === "object[]");
|
|
253
|
+
await this.axios({
|
|
254
|
+
method: "POST",
|
|
255
|
+
url: "/collections",
|
|
256
|
+
data: {
|
|
257
|
+
name: this.options.name,
|
|
258
|
+
fields: this.fields,
|
|
259
|
+
default_sorting_field: this.options.defaultSortingField,
|
|
260
|
+
enable_nested_fields: enableNested,
|
|
261
|
+
},
|
|
236
262
|
});
|
|
263
|
+
return this;
|
|
237
264
|
}
|
|
238
|
-
drop() {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
url: `/collections/${this.options.name}`,
|
|
243
|
-
});
|
|
265
|
+
async drop() {
|
|
266
|
+
await this.axios({
|
|
267
|
+
method: "DELETE",
|
|
268
|
+
url: `/collections/${this.options.name}`,
|
|
244
269
|
});
|
|
245
270
|
}
|
|
246
|
-
get(id) {
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
throw e;
|
|
256
|
-
});
|
|
257
|
-
return data ? this.deserializeDoc(data) : null;
|
|
271
|
+
async get(id) {
|
|
272
|
+
await this.ensureSynced();
|
|
273
|
+
const { data } = await this.axios({
|
|
274
|
+
method: "GET",
|
|
275
|
+
url: `/collections/${this.options.name}/documents/${id}`,
|
|
276
|
+
}).catch((e) => {
|
|
277
|
+
if (e.status === 404)
|
|
278
|
+
return { data: null };
|
|
279
|
+
throw e;
|
|
258
280
|
});
|
|
281
|
+
return data ? this.deserializeDoc(data) : null;
|
|
259
282
|
}
|
|
260
|
-
delete(id) {
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
throw e;
|
|
270
|
-
});
|
|
271
|
-
return data != null;
|
|
283
|
+
async delete(id) {
|
|
284
|
+
await this.ensureSynced();
|
|
285
|
+
const { data } = await this.axios({
|
|
286
|
+
method: "DELETE",
|
|
287
|
+
url: `/collections/${this.options.name}/documents/${id}`,
|
|
288
|
+
}).catch((e) => {
|
|
289
|
+
if (e.status === 404)
|
|
290
|
+
return { data: null };
|
|
291
|
+
throw e;
|
|
272
292
|
});
|
|
293
|
+
return data != null;
|
|
273
294
|
}
|
|
274
|
-
deleteMany(filter) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
params: { filter_by: filterBy },
|
|
285
|
-
});
|
|
286
|
-
return { deleted: data.num_deleted };
|
|
295
|
+
async deleteMany(filter) {
|
|
296
|
+
await this.ensureSynced();
|
|
297
|
+
const filterBy = this.buildFilter(filter).join("&&");
|
|
298
|
+
if (!filterBy) {
|
|
299
|
+
throw new Error("FILTER_REQUIRED");
|
|
300
|
+
}
|
|
301
|
+
const { data } = await this.axios({
|
|
302
|
+
method: "DELETE",
|
|
303
|
+
url: `/collections/${this.options.name}/documents`,
|
|
304
|
+
params: { filter_by: filterBy },
|
|
287
305
|
});
|
|
306
|
+
return { deleted: data.num_deleted };
|
|
288
307
|
}
|
|
289
|
-
update(id, data) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
data: serialized,
|
|
297
|
-
});
|
|
298
|
-
return this.deserializeDoc(updated);
|
|
308
|
+
async update(id, data) {
|
|
309
|
+
await this.ensureSynced();
|
|
310
|
+
const serialized = this.serializeDoc(data);
|
|
311
|
+
const { data: updated } = await this.axios({
|
|
312
|
+
method: "PATCH",
|
|
313
|
+
url: `/collections/${this.options.name}/documents/${id}`,
|
|
314
|
+
data: serialized,
|
|
299
315
|
});
|
|
316
|
+
return this.deserializeDoc(updated);
|
|
300
317
|
}
|
|
301
|
-
updateMany(filter, data) {
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
data: serialized,
|
|
314
|
-
});
|
|
315
|
-
return { updated: result.num_updated };
|
|
318
|
+
async updateMany(filter, data) {
|
|
319
|
+
await this.ensureSynced();
|
|
320
|
+
const filterBy = this.buildFilter(filter).join("&&");
|
|
321
|
+
if (!filterBy) {
|
|
322
|
+
throw new Error("FILTER_REQUIRED");
|
|
323
|
+
}
|
|
324
|
+
const serialized = this.serializeDoc(data);
|
|
325
|
+
const { data: result } = await this.axios({
|
|
326
|
+
method: "PATCH",
|
|
327
|
+
url: `/collections/${this.options.name}/documents`,
|
|
328
|
+
params: { filter_by: filterBy },
|
|
329
|
+
data: serialized,
|
|
316
330
|
});
|
|
331
|
+
return { updated: result.num_updated };
|
|
317
332
|
}
|
|
318
|
-
search(options) {
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
333
|
+
async search(options) {
|
|
334
|
+
await this.ensureSynced();
|
|
335
|
+
const queryByFields = options.queryBy ?? [
|
|
336
|
+
this.options.defaultSearchField,
|
|
337
|
+
];
|
|
338
|
+
this.validateFields(queryByFields);
|
|
339
|
+
if (options.sortBy) {
|
|
340
|
+
this.validateFields(options.sortBy
|
|
341
|
+
.map((s) => s.split(":")[0])
|
|
342
|
+
.filter((f) => f !== "undefined"));
|
|
343
|
+
}
|
|
344
|
+
if (options.facetBy) {
|
|
345
|
+
this.validateFields(options.facetBy);
|
|
346
|
+
}
|
|
347
|
+
const queryBy = queryByFields.join(",");
|
|
348
|
+
const params = {
|
|
349
|
+
q: options.query ?? "*",
|
|
350
|
+
query_by: queryBy,
|
|
351
|
+
};
|
|
352
|
+
const sortBy = this.buildSort(options);
|
|
353
|
+
if (sortBy)
|
|
354
|
+
params.sort_by = sortBy;
|
|
355
|
+
const filterBy = this.buildFilter(options.filter).join("&&");
|
|
356
|
+
if (filterBy)
|
|
357
|
+
params.filter_by = filterBy;
|
|
358
|
+
if (options.page != null)
|
|
359
|
+
params.page = options.page;
|
|
360
|
+
if (options.limit != null)
|
|
361
|
+
params.per_page = options.limit;
|
|
362
|
+
const facetBy = options.facetBy?.join(",");
|
|
363
|
+
if (facetBy)
|
|
364
|
+
params.facet_by = facetBy;
|
|
365
|
+
if ("pick" in options && options.pick) {
|
|
366
|
+
params.include_fields = options.pick.join(",");
|
|
367
|
+
}
|
|
368
|
+
if ("omit" in options && options.omit) {
|
|
369
|
+
params.exclude_fields = options.omit.join(",");
|
|
370
|
+
}
|
|
371
|
+
const highlight = options.highlight;
|
|
372
|
+
const highlightOpts = typeof highlight === "object" ? highlight : undefined;
|
|
373
|
+
if (highlightOpts) {
|
|
374
|
+
if (highlightOpts.fields) {
|
|
375
|
+
params.highlight_fields = highlightOpts.fields.join(",");
|
|
341
376
|
}
|
|
342
|
-
if (
|
|
343
|
-
params.
|
|
377
|
+
if (highlightOpts.startTag) {
|
|
378
|
+
params.highlight_start_tag = highlightOpts.startTag;
|
|
344
379
|
}
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
let highlightOpts;
|
|
348
|
-
if (typeof highlight === "object") {
|
|
349
|
-
highlightOpts = highlight;
|
|
350
|
-
if (highlightOpts.fields) {
|
|
351
|
-
params.highlight_fields = highlightOpts.fields.join(",");
|
|
352
|
-
}
|
|
353
|
-
if (highlightOpts.startTag) {
|
|
354
|
-
params.highlight_start_tag = highlightOpts.startTag;
|
|
355
|
-
}
|
|
356
|
-
if (highlightOpts.endTag) {
|
|
357
|
-
params.highlight_end_tag = highlightOpts.endTag;
|
|
358
|
-
}
|
|
380
|
+
if (highlightOpts.endTag) {
|
|
381
|
+
params.highlight_end_tag = highlightOpts.endTag;
|
|
359
382
|
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
383
|
+
}
|
|
384
|
+
const { data: res } = await this.axios({
|
|
385
|
+
method: "GET",
|
|
386
|
+
url: `/collections/${this.options.name}/documents/search`,
|
|
387
|
+
params,
|
|
388
|
+
});
|
|
389
|
+
const data = [];
|
|
390
|
+
const scores = [];
|
|
391
|
+
for (const hit of res.hits ?? []) {
|
|
392
|
+
if (highlight) {
|
|
393
|
+
const fieldsToHighlight = highlightOpts?.fields;
|
|
394
|
+
for (const [key, value] of Object.entries(hit.highlight ?? {})) {
|
|
395
|
+
if (!value?.snippet)
|
|
396
|
+
continue;
|
|
397
|
+
if (fieldsToHighlight && !fieldsToHighlight.includes(key))
|
|
398
|
+
continue;
|
|
399
|
+
hit.document[key] = value.snippet;
|
|
377
400
|
}
|
|
378
|
-
const doc = this.deserializeDoc(hit.document);
|
|
379
|
-
data.push(doc);
|
|
380
|
-
scores.push((_f = hit.text_match) !== null && _f !== void 0 ? _f : 0);
|
|
381
401
|
}
|
|
382
|
-
const
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
402
|
+
const doc = this.deserializeDoc(hit.document);
|
|
403
|
+
data.push(doc);
|
|
404
|
+
scores.push(hit.text_match ?? 0);
|
|
405
|
+
}
|
|
406
|
+
const facets = {};
|
|
407
|
+
for (const facetCount of res.facet_counts ?? []) {
|
|
408
|
+
const fieldName = facetCount.field_name;
|
|
409
|
+
facets[fieldName] = {};
|
|
410
|
+
for (const item of facetCount.counts) {
|
|
411
|
+
facets[fieldName][item.value] = item.count;
|
|
389
412
|
}
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
}
|
|
413
|
+
}
|
|
414
|
+
return {
|
|
415
|
+
count: res.found,
|
|
416
|
+
data,
|
|
417
|
+
facets,
|
|
418
|
+
scores,
|
|
419
|
+
};
|
|
397
420
|
}
|
|
398
|
-
searchList(options) {
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
per_page: limit,
|
|
409
|
-
page,
|
|
410
|
-
sort_by: `${field}:${options.sort.direction}`,
|
|
411
|
-
};
|
|
412
|
-
const filterParts = this.buildFilter(options.filter);
|
|
413
|
-
const filterBy = filterParts.join("&&");
|
|
414
|
-
if (filterBy)
|
|
415
|
-
params.filter_by = filterBy;
|
|
416
|
-
const { data: res } = yield this.axios({
|
|
417
|
-
method: "GET",
|
|
418
|
-
url: `/collections/${this.options.name}/documents/search`,
|
|
419
|
-
params,
|
|
420
|
-
});
|
|
421
|
-
const hits = (_d = res.hits) !== null && _d !== void 0 ? _d : [];
|
|
422
|
-
const data = [];
|
|
423
|
-
for (const hit of hits) {
|
|
424
|
-
const doc = this.deserializeDoc(hit.document);
|
|
425
|
-
data.push(doc);
|
|
426
|
-
}
|
|
427
|
-
const hasMore = page * limit < res.found;
|
|
428
|
-
const nextCursor = hasMore ? String(page + 1) : null;
|
|
429
|
-
return { data, nextCursor, total: res.found };
|
|
421
|
+
async searchList(options) {
|
|
422
|
+
const page = options.cursor ? Number(options.cursor) : 1;
|
|
423
|
+
const limit = Math.min(options.limit ?? 20, 100);
|
|
424
|
+
const result = await this.search({
|
|
425
|
+
query: options.query,
|
|
426
|
+
queryBy: options.queryBy,
|
|
427
|
+
filter: options.filter,
|
|
428
|
+
sortBy: [options.sortBy],
|
|
429
|
+
page,
|
|
430
|
+
limit,
|
|
430
431
|
});
|
|
432
|
+
const hasMore = page * limit < result.count;
|
|
433
|
+
return {
|
|
434
|
+
data: result.data,
|
|
435
|
+
nextCursor: hasMore ? String(page + 1) : null,
|
|
436
|
+
total: result.count,
|
|
437
|
+
};
|
|
431
438
|
}
|
|
432
|
-
count(filter) {
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
const { data } = yield this.axios({
|
|
438
|
-
method: "GET",
|
|
439
|
-
url: `/collections/${this.options.name}`,
|
|
440
|
-
});
|
|
441
|
-
return data.num_documents;
|
|
442
|
-
}
|
|
443
|
-
const { data } = yield this.axios({
|
|
439
|
+
async count(filter) {
|
|
440
|
+
await this.ensureSynced();
|
|
441
|
+
const filterBy = this.buildFilter(filter).join("&&");
|
|
442
|
+
if (!filterBy) {
|
|
443
|
+
const { data } = await this.axios({
|
|
444
444
|
method: "GET",
|
|
445
|
-
url: `/collections/${this.options.name}
|
|
446
|
-
params: {
|
|
447
|
-
q: "*",
|
|
448
|
-
query_by: this.options.defaultSearchField,
|
|
449
|
-
per_page: 0,
|
|
450
|
-
filter_by: filterBy,
|
|
451
|
-
},
|
|
445
|
+
url: `/collections/${this.options.name}`,
|
|
452
446
|
});
|
|
453
|
-
return data.
|
|
447
|
+
return data.num_documents;
|
|
448
|
+
}
|
|
449
|
+
const params = {
|
|
450
|
+
q: "*",
|
|
451
|
+
query_by: this.options.defaultSearchField,
|
|
452
|
+
per_page: 0,
|
|
453
|
+
filter_by: filterBy,
|
|
454
|
+
};
|
|
455
|
+
const { data } = await this.axios({
|
|
456
|
+
method: "GET",
|
|
457
|
+
url: `/collections/${this.options.name}/documents/search`,
|
|
458
|
+
params,
|
|
454
459
|
});
|
|
460
|
+
return data.found;
|
|
455
461
|
}
|
|
456
|
-
upsert(docs) {
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
this.options.schema.assert(item);
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
const payload = items.map((item) => JSON.stringify(this.serializeDoc(item))).join("\n");
|
|
468
|
-
const params = { action: "upsert" };
|
|
469
|
-
if (this.options.batchSize) {
|
|
470
|
-
params.batch_size = this.options.batchSize;
|
|
462
|
+
async upsert(docs) {
|
|
463
|
+
await this.ensureSynced();
|
|
464
|
+
const items = Array.isArray(docs) ? docs : [docs];
|
|
465
|
+
if (!items.length)
|
|
466
|
+
return [];
|
|
467
|
+
if (this.options.validateOnUpsert) {
|
|
468
|
+
for (const item of items) {
|
|
469
|
+
this.options.schema.assert(item);
|
|
471
470
|
}
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
471
|
+
}
|
|
472
|
+
const payload = items
|
|
473
|
+
.map((item) => JSON.stringify(this.serializeDoc(item)))
|
|
474
|
+
.join("\n");
|
|
475
|
+
const params = { action: "upsert" };
|
|
476
|
+
if (this.options.batchSize) {
|
|
477
|
+
params.batch_size = this.options.batchSize;
|
|
478
|
+
}
|
|
479
|
+
const { data } = await this.axios({
|
|
480
|
+
method: "POST",
|
|
481
|
+
url: `/collections/${this.options.name}/documents/import`,
|
|
482
|
+
headers: { "Content-Type": "text/plain" },
|
|
483
|
+
params,
|
|
484
|
+
data: payload,
|
|
483
485
|
});
|
|
486
|
+
if (typeof data === "string") {
|
|
487
|
+
return data.split("\n").map((v) => JSON.parse(v));
|
|
488
|
+
}
|
|
489
|
+
return [data];
|
|
484
490
|
}
|
|
485
|
-
syncData(options) {
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
else
|
|
502
|
-
failed++;
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
let deleted = 0;
|
|
506
|
-
if (options === null || options === void 0 ? void 0 : options.purge) {
|
|
507
|
-
deleted = yield this.purgeOrphans(ids, chunkSize);
|
|
491
|
+
async syncData(options) {
|
|
492
|
+
if (!this.dataSyncConfig) {
|
|
493
|
+
throw new Error("DATA_SYNC_NOT_CONFIGURED");
|
|
494
|
+
}
|
|
495
|
+
const chunkSize = options?.chunkSize ?? this.dataSyncConfig.chunkSize ?? 500;
|
|
496
|
+
const ids = options?.ids ?? (await this.dataSyncConfig.getAllIds());
|
|
497
|
+
let upserted = 0;
|
|
498
|
+
let failed = 0;
|
|
499
|
+
for (const chunk of chunkArray(ids, chunkSize)) {
|
|
500
|
+
const items = await this.dataSyncConfig.getItems(chunk);
|
|
501
|
+
const results = await this.upsert(items);
|
|
502
|
+
for (const r of results) {
|
|
503
|
+
if (r.success)
|
|
504
|
+
upserted++;
|
|
505
|
+
else
|
|
506
|
+
failed++;
|
|
508
507
|
}
|
|
509
|
-
|
|
510
|
-
|
|
508
|
+
}
|
|
509
|
+
let deleted = 0;
|
|
510
|
+
if (options?.purge) {
|
|
511
|
+
deleted = await this.purgeOrphans(ids, chunkSize);
|
|
512
|
+
}
|
|
513
|
+
return { upserted, deleted, failed };
|
|
511
514
|
}
|
|
512
|
-
purgeOrphans(validIds, chunkSize) {
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
return deleted;
|
|
525
|
-
});
|
|
515
|
+
async purgeOrphans(validIds, chunkSize) {
|
|
516
|
+
const validSet = new Set(validIds);
|
|
517
|
+
const remoteIds = await this.exportIds();
|
|
518
|
+
const orphans = remoteIds.filter((id) => !validSet.has(id));
|
|
519
|
+
if (!orphans.length)
|
|
520
|
+
return 0;
|
|
521
|
+
let deleted = 0;
|
|
522
|
+
for (const chunk of chunkArray(orphans, chunkSize)) {
|
|
523
|
+
const result = await this.deleteMany({ id: chunk });
|
|
524
|
+
deleted += result.deleted;
|
|
525
|
+
}
|
|
526
|
+
return deleted;
|
|
526
527
|
}
|
|
527
|
-
exportIds() {
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
params: { include_fields: "id" },
|
|
533
|
-
});
|
|
534
|
-
return data
|
|
535
|
-
.split("\n")
|
|
536
|
-
.filter((line) => line.length)
|
|
537
|
-
.map((line) => JSON.parse(line).id);
|
|
528
|
+
async exportIds() {
|
|
529
|
+
const { data } = await this.axios({
|
|
530
|
+
method: "GET",
|
|
531
|
+
url: `/collections/${this.options.name}/documents/export`,
|
|
532
|
+
params: { include_fields: "id" },
|
|
538
533
|
});
|
|
534
|
+
return data
|
|
535
|
+
.split("\n")
|
|
536
|
+
.filter((line) => line.length)
|
|
537
|
+
.map((line) => JSON.parse(line).id);
|
|
538
|
+
}
|
|
539
|
+
scoped(baseFilter) {
|
|
540
|
+
return {
|
|
541
|
+
search: (options) => this.search({
|
|
542
|
+
...options,
|
|
543
|
+
filter: { ...options.filter, ...baseFilter },
|
|
544
|
+
}),
|
|
545
|
+
searchList: (options) => this.searchList({
|
|
546
|
+
...options,
|
|
547
|
+
filter: { ...options.filter, ...baseFilter },
|
|
548
|
+
}),
|
|
549
|
+
count: (filter) => this.count({ ...filter, ...baseFilter }),
|
|
550
|
+
deleteMany: (filter) => this.deleteMany({ ...filter, ...baseFilter }),
|
|
551
|
+
updateMany: (filter, data) => this.updateMany({ ...filter, ...baseFilter }, data),
|
|
552
|
+
};
|
|
539
553
|
}
|
|
540
554
|
}
|