para-cli 1.23.3 → 1.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +464 -335
- package/package.json +11 -2
- package/para-cli.js +57 -34
package/index.js
CHANGED
|
@@ -25,86 +25,90 @@
|
|
|
25
25
|
/* eslint indent: ["error", "tab"] */
|
|
26
26
|
/* eslint object-curly-spacing: ["error", "always"] */
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
import {
|
|
31
|
-
|
|
32
|
-
import
|
|
28
|
+
/* global console, process, Buffer */
|
|
29
|
+
|
|
30
|
+
import { readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
31
|
+
import { basename, relative, resolve } from 'node:path';
|
|
32
|
+
import { URL } from 'node:url';
|
|
33
|
+
import { TextEncoder } from 'node:util';
|
|
34
|
+
import input from '@inquirer/input';
|
|
35
|
+
import password from '@inquirer/password';
|
|
36
|
+
import chalk from 'chalk';
|
|
37
|
+
import { globbySync } from 'globby';
|
|
33
38
|
import { Parser } from 'htmlparser2';
|
|
34
|
-
import { createInterface } from 'readline';
|
|
35
|
-
import { Writable } from 'stream';
|
|
36
39
|
import jsonwebtoken from 'jsonwebtoken';
|
|
37
40
|
import { lookup } from 'mime-types';
|
|
38
|
-
import {
|
|
39
|
-
import
|
|
41
|
+
import { Pager, ParaClient, ParaObject } from 'para-client-js';
|
|
42
|
+
import striptags from 'striptags';
|
|
40
43
|
import apiClient from 'superagent';
|
|
41
|
-
import { URL } from 'url';
|
|
42
|
-
import { ParaClient, ParaObject, Pager } from 'para-client-js';
|
|
43
44
|
|
|
45
|
+
const encoder = new TextEncoder('utf-8');
|
|
44
46
|
const { cyan, red, yellow, green } = chalk;
|
|
45
47
|
const { sign } = jsonwebtoken;
|
|
46
|
-
|
|
48
|
+
const MAX_FILE_SIZE = 350 * 1024;
|
|
47
49
|
var defaultConfig = { accessKey: '', secretKey: '', endpoint: 'https://paraio.com' };
|
|
48
50
|
|
|
49
51
|
const _defaultConfig = defaultConfig;
|
|
52
|
+
|
|
50
53
|
export { _defaultConfig as defaultConfig };
|
|
51
54
|
|
|
52
55
|
export async function setup(config) {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
input
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
rl.close();
|
|
89
|
-
});
|
|
90
|
-
} else {
|
|
91
|
-
rl.close();
|
|
92
|
-
}
|
|
93
|
-
});
|
|
56
|
+
try {
|
|
57
|
+
const accessKey = await input({
|
|
58
|
+
message: 'Para Access Key:',
|
|
59
|
+
default: config.get('accessKey') || 'app:para'
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
const secretKey = await password({
|
|
63
|
+
message: 'Para Secret Key:',
|
|
64
|
+
mask: '*'
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
const endpoint = await input({
|
|
68
|
+
message: 'Para Endpoint:',
|
|
69
|
+
default: defaultConfig.endpoint
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
var access = (accessKey || config.get('accessKey') || 'app:para').trim();
|
|
73
|
+
var secret = (secretKey || config.get('secretKey')).trim();
|
|
74
|
+
var endpointValue = (endpoint || defaultConfig.endpoint).trim();
|
|
75
|
+
|
|
76
|
+
newJWT(access, secret, endpointValue, config);
|
|
77
|
+
ping(config);
|
|
78
|
+
|
|
79
|
+
if (access === 'app:para') {
|
|
80
|
+
listApps(config, {}, access, async () => {
|
|
81
|
+
// if none, ask to create one
|
|
82
|
+
const shouldCreate = await input({
|
|
83
|
+
message: 'Would you like to create a new Para app? [Y/n]',
|
|
84
|
+
default: 'Y'
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
const Yn = shouldCreate.trim();
|
|
88
|
+
if ('' === Yn || 'y' === Yn.toLowerCase()) {
|
|
89
|
+
const appname = await input({
|
|
90
|
+
message: 'App name:'
|
|
94
91
|
});
|
|
92
|
+
newApp(['', appname], config, {});
|
|
95
93
|
}
|
|
96
|
-
rl.close();
|
|
97
94
|
});
|
|
98
|
-
}
|
|
99
|
-
})
|
|
95
|
+
}
|
|
96
|
+
} catch (error) {
|
|
97
|
+
if (error.name === 'ExitPromptError') {
|
|
98
|
+
console.log('\nSetup cancelled.');
|
|
99
|
+
} else {
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
100
103
|
}
|
|
101
104
|
|
|
102
|
-
export function createAll(
|
|
105
|
+
export function createAll(input, config, flags = {}) {
|
|
103
106
|
if (!input[1]) {
|
|
104
107
|
fail('No files specified.');
|
|
105
108
|
return;
|
|
106
109
|
}
|
|
107
110
|
|
|
111
|
+
const pc = getClient(config, flags);
|
|
108
112
|
var files = globbySync(input[1], { realpath: true });
|
|
109
113
|
var totalSize = 0;
|
|
110
114
|
var totalObjects = 0;
|
|
@@ -120,7 +124,7 @@ export function createAll(pc, input, flags) {
|
|
|
120
124
|
var fileBody = '';
|
|
121
125
|
var id;
|
|
122
126
|
|
|
123
|
-
if (!stats
|
|
127
|
+
if (!stats?.isFile()) {
|
|
124
128
|
console.error(red('✖'), yellow(file), 'is not a file.');
|
|
125
129
|
continue;
|
|
126
130
|
}
|
|
@@ -141,13 +145,12 @@ export function createAll(pc, input, flags) {
|
|
|
141
145
|
json.text = json.text.replace(/[^0-9\p{L}]+/giu, ' ').replace(/[\s]+/gi, ' ');
|
|
142
146
|
}
|
|
143
147
|
|
|
144
|
-
id =
|
|
148
|
+
id = i === 0 && flags.id ? flags.id : json.url || filePath;
|
|
145
149
|
console.log(green('✔'), 'Creating', yellow(id));
|
|
146
150
|
var textEncoded = encoder.encode(json.text);
|
|
147
151
|
//batchSize += textEncoded.length;
|
|
148
152
|
if (textEncoded.length > MAX_FILE_SIZE) {
|
|
149
|
-
console.log(red('!'), yellow('File is larger than',
|
|
150
|
-
MAX_FILE_SIZE / 1024, 'KB - splitting into chunks...'));
|
|
153
|
+
console.log(red('!'), yellow('File is larger than', MAX_FILE_SIZE / 1024, 'KB - splitting into chunks...'));
|
|
151
154
|
sendFileChunk(1, textEncoded, json, id, flags, 0, MAX_FILE_SIZE, pc);
|
|
152
155
|
} else {
|
|
153
156
|
if (batchSize > MAX_FILE_SIZE) {
|
|
@@ -162,7 +165,7 @@ export function createAll(pc, input, flags) {
|
|
|
162
165
|
}
|
|
163
166
|
} else if (fileType === 'application/json') {
|
|
164
167
|
totalObjects++;
|
|
165
|
-
id =
|
|
168
|
+
id = i === 0 && flags.id ? flags.id : filePath;
|
|
166
169
|
totalSize += stats.size;
|
|
167
170
|
batchSize += stats.size;
|
|
168
171
|
if (batchSize > MAX_FILE_SIZE) {
|
|
@@ -175,47 +178,53 @@ export function createAll(pc, input, flags) {
|
|
|
175
178
|
addObjectsToBatch(batches[batchId], JSON.parse(readFile(file)), id, flags);
|
|
176
179
|
console.log(green('✔'), 'Creating', yellow(id));
|
|
177
180
|
} else {
|
|
178
|
-
console.error(red('✖'), 'Skipping', yellow(file),
|
|
181
|
+
console.error(red('✖'), 'Skipping', yellow(file), "- isn't JSON, HTML nor text.");
|
|
179
182
|
}
|
|
180
183
|
}
|
|
181
184
|
|
|
182
185
|
for (var k = 0; k < batches.length; k++) {
|
|
183
186
|
var objectsList = batches[k];
|
|
184
187
|
if (objectsList.length > 0) {
|
|
185
|
-
pc.createAll(objectsList)
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
188
|
+
pc.createAll(objectsList)
|
|
189
|
+
.then((data) => {
|
|
190
|
+
console.log(green('✔'), 'Created', data.length, 'objects.');
|
|
191
|
+
})
|
|
192
|
+
.catch((err) => {
|
|
193
|
+
fail('Failed to create documents:', err);
|
|
194
|
+
});
|
|
190
195
|
}
|
|
191
196
|
}
|
|
192
197
|
|
|
193
198
|
console.log(green('✔'), 'Created', totalObjects, 'objects with a total size of', Math.round(totalSize / 1024), 'KB.');
|
|
194
199
|
}
|
|
195
200
|
|
|
196
|
-
export function readAll(
|
|
201
|
+
export function readAll(config, flags = {}) {
|
|
202
|
+
const pc = getClient(config, flags);
|
|
197
203
|
if (flags.id) {
|
|
198
204
|
var readIds = flags.id;
|
|
199
|
-
if (!(readIds
|
|
205
|
+
if (!Array.isArray(readIds)) {
|
|
200
206
|
readIds = [readIds];
|
|
201
207
|
}
|
|
202
208
|
|
|
203
|
-
pc.readAll(readIds)
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
209
|
+
pc.readAll(readIds)
|
|
210
|
+
.then((data) => {
|
|
211
|
+
console.log(JSON.stringify(data, null, 2));
|
|
212
|
+
})
|
|
213
|
+
.catch((err) => {
|
|
214
|
+
fail('Failed to read object:', err);
|
|
215
|
+
});
|
|
208
216
|
} else {
|
|
209
217
|
fail('Must specify object id(s).');
|
|
210
218
|
}
|
|
211
219
|
}
|
|
212
220
|
|
|
213
|
-
export function updateAll(
|
|
221
|
+
export function updateAll(input, config, flags = {}) {
|
|
214
222
|
if (!input[1]) {
|
|
215
223
|
fail('No files specified.');
|
|
216
224
|
return;
|
|
217
225
|
}
|
|
218
226
|
|
|
227
|
+
const pc = getClient(config, flags);
|
|
219
228
|
var files = globbySync(input[1], { realpath: true });
|
|
220
229
|
var updateList = [];
|
|
221
230
|
|
|
@@ -230,52 +239,60 @@ export function updateAll(pc, input, flags) {
|
|
|
230
239
|
continue;
|
|
231
240
|
}
|
|
232
241
|
|
|
233
|
-
if (!stats
|
|
242
|
+
if (!stats?.isFile()) {
|
|
234
243
|
console.error(red('✖'), yellow(file), 'is not a file.');
|
|
235
244
|
continue;
|
|
236
245
|
}
|
|
237
246
|
|
|
238
247
|
var fileJSON = JSON.parse(readFile(file));
|
|
239
|
-
var id =
|
|
248
|
+
var id = fileJSON.id || defaultId;
|
|
240
249
|
addObjectsToBatch(updateList, fileJSON, id, flags);
|
|
241
250
|
console.log(green('✔'), 'Updating', yellow(id));
|
|
242
251
|
}
|
|
243
252
|
|
|
244
|
-
pc.updateAll(updateList)
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
253
|
+
pc.updateAll(updateList)
|
|
254
|
+
.then(() => {
|
|
255
|
+
console.log(green('✔'), 'Updated', updateList.length, 'files.');
|
|
256
|
+
})
|
|
257
|
+
.catch((err) => {
|
|
258
|
+
fail('Failed to read object:', err);
|
|
259
|
+
});
|
|
249
260
|
}
|
|
250
261
|
|
|
251
|
-
export function deleteAll(
|
|
262
|
+
export function deleteAll(input, config, flags = {}) {
|
|
263
|
+
const pc = getClient(config, flags);
|
|
252
264
|
if (flags.id || input[1]) {
|
|
253
265
|
var deleteIds = globbySync(input[1] || ' ', { realpath: true });
|
|
254
266
|
if (deleteIds.length === 0) {
|
|
255
|
-
deleteIds = flags.id
|
|
267
|
+
deleteIds = Array.isArray(flags.id) ? flags.id : [String(flags.id)];
|
|
256
268
|
}
|
|
257
269
|
|
|
258
270
|
for (var i = 0; i < deleteIds.length; i++) {
|
|
259
271
|
deleteIds[i] = basename(String(deleteIds[i]));
|
|
260
272
|
}
|
|
261
273
|
|
|
262
|
-
pc.deleteAll(deleteIds)
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
274
|
+
pc.deleteAll(deleteIds)
|
|
275
|
+
.then(() => {
|
|
276
|
+
console.log(green('✔'), 'Deleted objects "', deleteIds, '" from Para.');
|
|
277
|
+
})
|
|
278
|
+
.catch((err) => {
|
|
279
|
+
fail('Failed to delete objects:', err);
|
|
280
|
+
});
|
|
267
281
|
} else {
|
|
268
282
|
fail('No files specified.');
|
|
269
283
|
}
|
|
270
284
|
}
|
|
271
285
|
|
|
272
|
-
export function newKeys(
|
|
273
|
-
pc
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
286
|
+
export function newKeys(config, flags = {}) {
|
|
287
|
+
const pc = getClient(config, flags);
|
|
288
|
+
pc.newKeys()
|
|
289
|
+
.then((keys) => {
|
|
290
|
+
config.set('secretKey', keys.secretKey);
|
|
291
|
+
console.log(green('✔'), 'New JWT generated and saved in', yellow(config.path));
|
|
292
|
+
})
|
|
293
|
+
.catch((err) => {
|
|
294
|
+
fail('Failed to generate new secret key:', err);
|
|
295
|
+
});
|
|
279
296
|
}
|
|
280
297
|
|
|
281
298
|
export function newJWT(accessKey, secretKey, endpoint, config, flags) {
|
|
@@ -284,15 +301,15 @@ export function newJWT(accessKey, secretKey, endpoint, config, flags) {
|
|
|
284
301
|
return;
|
|
285
302
|
}
|
|
286
303
|
|
|
287
|
-
var now = Math.round(
|
|
304
|
+
var now = Math.round(Date.now() / 1000);
|
|
288
305
|
var sClaim = JSON.stringify({
|
|
289
|
-
exp: now +
|
|
306
|
+
exp: now + 7 * 24 * 60 * 60,
|
|
290
307
|
iat: now,
|
|
291
308
|
nbf: now - 5, // allow for 5 seconds time difference in clocks
|
|
292
309
|
appid: accessKey
|
|
293
310
|
});
|
|
294
311
|
var selectedApp = config.get('selectedApp');
|
|
295
|
-
if (selectedApp
|
|
312
|
+
if (selectedApp?.secretKey) {
|
|
296
313
|
selectedApp.accessKey = accessKey;
|
|
297
314
|
selectedApp.secretKey = secretKey;
|
|
298
315
|
config.set('selectedApp', selectedApp);
|
|
@@ -302,141 +319,176 @@ export function newJWT(accessKey, secretKey, endpoint, config, flags) {
|
|
|
302
319
|
}
|
|
303
320
|
config.set('endpoint', endpoint || config.get('endpoint'));
|
|
304
321
|
config.set('jwt', sign(sClaim, secretKey, { algorithm: 'HS256' }));
|
|
305
|
-
if (flags
|
|
322
|
+
if (flags?.print) {
|
|
306
323
|
console.log(yellow(config.get('jwt')));
|
|
307
324
|
} else {
|
|
308
325
|
console.log(green('✔'), 'New JWT generated and saved in', yellow(config.path));
|
|
309
326
|
}
|
|
310
327
|
}
|
|
311
328
|
|
|
312
|
-
export function newApp(
|
|
329
|
+
export function newApp(input, config, flags = {}) {
|
|
313
330
|
if (!input[1]) {
|
|
314
331
|
fail('App name not specified.');
|
|
315
332
|
return;
|
|
316
333
|
}
|
|
317
334
|
|
|
335
|
+
const pc = getClient(config, flags);
|
|
318
336
|
var appid = input[1];
|
|
319
|
-
var req = pc.invokeGet(
|
|
320
|
-
pc.getEntity(req)
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
337
|
+
var req = pc.invokeGet(`_setup/${appid}`, { name: flags.name || appid, shared: flags.shared || false });
|
|
338
|
+
pc.getEntity(req)
|
|
339
|
+
.then((resp) => {
|
|
340
|
+
if (resp?.secretKey) {
|
|
341
|
+
console.log(green('✔'), 'App created:');
|
|
342
|
+
console.log(JSON.stringify(resp, null, 2));
|
|
343
|
+
} else {
|
|
344
|
+
console.log(green('✔'), yellow(`App "${appid}" already exists.`));
|
|
345
|
+
}
|
|
346
|
+
})
|
|
347
|
+
.catch((err) => {
|
|
348
|
+
fail('Failed to create app:', err);
|
|
349
|
+
});
|
|
330
350
|
}
|
|
331
351
|
|
|
332
|
-
export function deleteApp(
|
|
333
|
-
if (!
|
|
352
|
+
export async function deleteApp(inputArgs, config, flags = {}) {
|
|
353
|
+
if (!inputArgs[1]) {
|
|
334
354
|
fail('App id not specified.');
|
|
335
355
|
return;
|
|
336
356
|
}
|
|
337
|
-
|
|
357
|
+
const pc = getClient(config, flags);
|
|
358
|
+
var appid = inputArgs[1];
|
|
338
359
|
if (appid.indexOf('app:') < 0) {
|
|
339
|
-
appid =
|
|
360
|
+
appid = `app:${appid}`;
|
|
340
361
|
}
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
if (confirm ===
|
|
348
|
-
pc.invokeDelete(
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
362
|
+
|
|
363
|
+
try {
|
|
364
|
+
const confirm = await input({
|
|
365
|
+
message: `${red.bold(`Are you sure you want to delete ${appid}? ALL DATA FOR THAT APP WILL BE LOST! `)}yes/No`
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
if (confirm === 'yes') {
|
|
369
|
+
pc.invokeDelete(`apps/${appid}`, {})
|
|
370
|
+
.then((resp) => {
|
|
371
|
+
if (resp?.ok) {
|
|
372
|
+
console.log(green('✔'), `App ${red.bold(appid)} was deleted!`);
|
|
373
|
+
} else {
|
|
374
|
+
console.log(green('✔'), yellow(`App "${appid}" could not be deleted.`));
|
|
375
|
+
}
|
|
376
|
+
})
|
|
377
|
+
.catch((err) => {
|
|
378
|
+
fail('Failed to delete app:', err);
|
|
379
|
+
});
|
|
357
380
|
}
|
|
358
|
-
|
|
359
|
-
|
|
381
|
+
} catch (error) {
|
|
382
|
+
if (error.name === 'ExitPromptError') {
|
|
383
|
+
console.log('\nDelete cancelled.');
|
|
384
|
+
} else {
|
|
385
|
+
throw error;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
360
388
|
}
|
|
361
389
|
|
|
362
|
-
export function ping(
|
|
363
|
-
pc
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
390
|
+
export function ping(config, flags = {}) {
|
|
391
|
+
const pc = getClient(config, flags);
|
|
392
|
+
pc.me()
|
|
393
|
+
.then((mee) => {
|
|
394
|
+
pc.getServerVersion()
|
|
395
|
+
.then((ver) => {
|
|
396
|
+
console.log(
|
|
397
|
+
green('✔'),
|
|
398
|
+
`Connected to Para server ${cyan.bold(`v${ver}`)}`,
|
|
399
|
+
`on ${cyan(pc.endpoint)}. Authenticated as:`,
|
|
400
|
+
cyan(`${mee.type} ${mee.name} (${mee.id})`)
|
|
401
|
+
);
|
|
402
|
+
})
|
|
403
|
+
.catch(() => {
|
|
404
|
+
fail('Connection failed. Run "para-cli setup" or check the configuration file', yellow(config.path));
|
|
405
|
+
process.exit(1);
|
|
406
|
+
});
|
|
407
|
+
})
|
|
408
|
+
.catch(() => {
|
|
369
409
|
fail('Connection failed. Run "para-cli setup" or check the configuration file', yellow(config.path));
|
|
370
410
|
process.exit(1);
|
|
371
411
|
});
|
|
372
|
-
}).catch(function () {
|
|
373
|
-
fail('Connection failed. Run "para-cli setup" or check the configuration file', yellow(config.path));
|
|
374
|
-
process.exit(1);
|
|
375
|
-
});
|
|
376
412
|
}
|
|
377
413
|
|
|
378
|
-
export function me(
|
|
379
|
-
pc
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
414
|
+
export function me(config, flags = {}) {
|
|
415
|
+
const pc = getClient(config, flags);
|
|
416
|
+
pc.me()
|
|
417
|
+
.then((mee) => {
|
|
418
|
+
console.log(JSON.stringify(mee, null, 2));
|
|
419
|
+
})
|
|
420
|
+
.catch(() => {
|
|
421
|
+
fail('Connection failed. Server might be down. Check the configuration file', yellow(config.path));
|
|
422
|
+
});
|
|
384
423
|
}
|
|
385
424
|
|
|
386
|
-
export function types(
|
|
387
|
-
const
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
425
|
+
export function types(config, flags = {}) {
|
|
426
|
+
const pc = getClient(config, flags);
|
|
427
|
+
const _types = pc
|
|
428
|
+
.getEntity(pc.invokeGet('_types'))
|
|
429
|
+
.then((data) => {
|
|
430
|
+
console.log(JSON.stringify(data, null, 2));
|
|
431
|
+
})
|
|
432
|
+
.catch(() => {
|
|
433
|
+
fail('Connection failed. Server might be down. Check the configuration file', yellow(config.path));
|
|
434
|
+
});
|
|
392
435
|
}
|
|
393
436
|
|
|
394
|
-
export function exportData(
|
|
395
|
-
pc
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
437
|
+
export function exportData(config, flags = {}) {
|
|
438
|
+
const pc = getClient(config, flags);
|
|
439
|
+
pc.invokeGet('/_export')
|
|
440
|
+
.then((data) => {
|
|
441
|
+
try {
|
|
442
|
+
var filename = data.headers['content-disposition'] || 'export.zip';
|
|
443
|
+
var filesize = Math.round(((data.headers['content-length'] || 0) / 1000000) * 100) / 100;
|
|
444
|
+
filename = filename.substring(filename.lastIndexOf('=') + 1);
|
|
445
|
+
writeFileSync(filename, data.body);
|
|
446
|
+
console.log(green('✔'), yellow(`Exported ${filesize}MB of data to file ${filename}`));
|
|
447
|
+
} catch (e) {
|
|
448
|
+
console.error(e);
|
|
449
|
+
}
|
|
450
|
+
})
|
|
451
|
+
.catch(() => {
|
|
452
|
+
fail('Connection failed. Server might be down. Check the configuration file', yellow(config.path));
|
|
453
|
+
});
|
|
408
454
|
}
|
|
409
455
|
|
|
410
|
-
export function importData(
|
|
456
|
+
export function importData(input, config, flags = {}) {
|
|
411
457
|
if (!input[1]) {
|
|
412
458
|
fail('No file to import.');
|
|
413
459
|
return;
|
|
414
460
|
}
|
|
461
|
+
const pc = getClient(config, flags);
|
|
415
462
|
if (!config.get('jwt')) {
|
|
416
463
|
newJWT(config.get('accessKey'), config.get('secretKey'), config.get('endpoint'), config);
|
|
417
464
|
}
|
|
418
465
|
var headers = {
|
|
419
466
|
'User-Agent': 'Para CLI tool',
|
|
420
467
|
'Content-Type': 'application/zip',
|
|
421
|
-
|
|
468
|
+
Authorization: `Bearer ${config.get('jwt')}`
|
|
422
469
|
};
|
|
423
470
|
try {
|
|
424
|
-
apiClient
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
471
|
+
apiClient
|
|
472
|
+
.put(`${pc.endpoint}/v1/_import`)
|
|
473
|
+
.set(headers)
|
|
474
|
+
.send(readFileSync(resolve(input[1])))
|
|
475
|
+
.then((res) => {
|
|
476
|
+
console.log(green('✔'), `${yellow(`Imported ${res.body.count} object into app "${res.body.appid}`)}"`);
|
|
477
|
+
})
|
|
478
|
+
.catch((e) => {
|
|
479
|
+
fail(`Import request failed. ${e}`);
|
|
480
|
+
});
|
|
429
481
|
} catch (e) {
|
|
430
|
-
fail(
|
|
482
|
+
fail(`Import request failed: ${e}`);
|
|
431
483
|
}
|
|
432
484
|
}
|
|
433
485
|
|
|
434
486
|
function promiseWhile(results, fn) {
|
|
435
|
-
return new Promise(
|
|
487
|
+
return new Promise((resolve, _reject) => {
|
|
436
488
|
function loop() {
|
|
437
|
-
return Promise.resolve(fn()).then(
|
|
489
|
+
return Promise.resolve(fn()).then((result) => {
|
|
438
490
|
if (result && result.length > 0) {
|
|
439
|
-
result.forEach(
|
|
491
|
+
result.forEach((res) => {
|
|
440
492
|
results.push(res);
|
|
441
493
|
});
|
|
442
494
|
return loop();
|
|
@@ -448,7 +500,8 @@ function promiseWhile(results, fn) {
|
|
|
448
500
|
});
|
|
449
501
|
}
|
|
450
502
|
|
|
451
|
-
export function search(
|
|
503
|
+
export function search(input, config, flags = {}) {
|
|
504
|
+
const pc = getClient(config, flags);
|
|
452
505
|
var p = new Pager(flags.page, flags.sort, flags.desc, flags.limit);
|
|
453
506
|
if (flags.lastKey) {
|
|
454
507
|
p.lastKey = flags.lastKey;
|
|
@@ -459,97 +512,118 @@ export function search(pc, input, flags) {
|
|
|
459
512
|
p.sortby = '_docid';
|
|
460
513
|
p.page = 1;
|
|
461
514
|
|
|
462
|
-
promiseWhile(results,
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
515
|
+
promiseWhile(results, () => pc.findQuery(getType(flags.type), String(input[1]) || '', p))
|
|
516
|
+
.then(() => {
|
|
517
|
+
console.log(JSON.stringify(results, null, 2));
|
|
518
|
+
})
|
|
519
|
+
.catch((err) => {
|
|
520
|
+
fail('Search failed.', err);
|
|
521
|
+
});
|
|
469
522
|
} else {
|
|
470
|
-
pc.findQuery(getType(flags.type), String(input[1]) || '', p)
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
523
|
+
pc.findQuery(getType(flags.type), String(input[1]) || '', p)
|
|
524
|
+
.then((resp) => {
|
|
525
|
+
console.log(JSON.stringify(resp, null, 2));
|
|
526
|
+
})
|
|
527
|
+
.catch((err) => {
|
|
528
|
+
fail('Search failed.', err);
|
|
529
|
+
});
|
|
475
530
|
}
|
|
476
531
|
}
|
|
477
532
|
|
|
478
|
-
export function appSettings(
|
|
479
|
-
pc
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
533
|
+
export function appSettings(config, flags = {}) {
|
|
534
|
+
const pc = getClient(config, flags);
|
|
535
|
+
pc.appSettings()
|
|
536
|
+
.then((settings) => {
|
|
537
|
+
console.log(JSON.stringify(settings, null, 2));
|
|
538
|
+
})
|
|
539
|
+
.catch(() => {
|
|
540
|
+
fail('Connection failed. Check the configuration file', yellow(config.path));
|
|
541
|
+
});
|
|
484
542
|
}
|
|
485
543
|
|
|
486
|
-
export function rebuildIndex(
|
|
487
|
-
pc
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
544
|
+
export function rebuildIndex(config, flags = {}) {
|
|
545
|
+
const pc = getClient(config, flags);
|
|
546
|
+
pc.rebuildIndex(flags.destinationIndex)
|
|
547
|
+
.then((response) => {
|
|
548
|
+
console.log(JSON.stringify(response, null, 2));
|
|
549
|
+
})
|
|
550
|
+
.catch((err) => {
|
|
551
|
+
fail('Reindex failed.', err);
|
|
552
|
+
});
|
|
492
553
|
}
|
|
493
554
|
|
|
494
555
|
export function listApps(config, flags, parentAccessKey, failureCallback) {
|
|
556
|
+
var pc = getClient(config, flags);
|
|
495
557
|
var selectedEndpoint = getSelectedEndpoint(config, flags);
|
|
496
|
-
var accessKey = selectedEndpoint.accessKey;
|
|
497
|
-
var secretKey = selectedEndpoint.secretKey;
|
|
498
|
-
var endpoint = selectedEndpoint.endpoint;
|
|
499
|
-
var pc = new ParaClient(accessKey, secretKey, parseEndpoint(endpoint));
|
|
500
558
|
var p = new Pager();
|
|
501
559
|
var results = [];
|
|
502
560
|
p.sortby = '_docid';
|
|
503
561
|
p.page = 1;
|
|
504
|
-
promiseWhile(results,
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
562
|
+
promiseWhile(results, () => pc.findQuery('app', '*', p))
|
|
563
|
+
.then(() => {
|
|
564
|
+
var apps = results.map((app) => app.appIdentifier.trim());
|
|
565
|
+
if (apps.length) {
|
|
566
|
+
console.log(
|
|
567
|
+
'Found',
|
|
568
|
+
p.count,
|
|
569
|
+
`apps on ${cyan(selectedEndpoint.endpoint)}:\n`,
|
|
570
|
+
yellow('[') + green(apps.join(yellow('] ['))) + yellow(']')
|
|
571
|
+
);
|
|
572
|
+
console.log(
|
|
573
|
+
'\nTyping',
|
|
574
|
+
cyan('para-cli select'),
|
|
575
|
+
green(apps[0]),
|
|
576
|
+
'will switch to that app. \nCurrent app:',
|
|
577
|
+
green(parentAccessKey)
|
|
578
|
+
);
|
|
579
|
+
process.exit(0);
|
|
580
|
+
} else {
|
|
581
|
+
failureCallback();
|
|
582
|
+
}
|
|
583
|
+
})
|
|
584
|
+
.catch((_err) => {
|
|
514
585
|
failureCallback();
|
|
515
|
-
}
|
|
516
|
-
}).catch(function (err) {
|
|
517
|
-
failureCallback();
|
|
518
|
-
});
|
|
586
|
+
});
|
|
519
587
|
}
|
|
520
588
|
|
|
521
589
|
export function selectApp(input, config, flags) {
|
|
522
590
|
var selectedEndpoint = getSelectedEndpoint(config, flags);
|
|
523
591
|
var accessKey = selectedEndpoint.accessKey;
|
|
524
592
|
var secretKey = selectedEndpoint.secretKey;
|
|
525
|
-
var endpoint = selectedEndpoint.endpoint;
|
|
526
593
|
if (accessKey === 'app:para' && secretKey) {
|
|
527
|
-
var selectedApp =
|
|
594
|
+
var selectedApp = `app:${(input[1] || 'para').trim()}`;
|
|
528
595
|
if (selectedApp === 'app:para') {
|
|
529
|
-
config.
|
|
596
|
+
config.set('selectedApp', selectedEndpoint);
|
|
530
597
|
console.log(green('✔'), 'Selected', green(selectedApp), 'as the current app.');
|
|
531
598
|
return;
|
|
532
599
|
}
|
|
533
|
-
var now = Math.round(
|
|
534
|
-
var jwt = sign(
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
600
|
+
var now = Math.round(Date.now() / 1000);
|
|
601
|
+
var jwt = sign(
|
|
602
|
+
JSON.stringify({
|
|
603
|
+
iat: now,
|
|
604
|
+
exp: now + 10,
|
|
605
|
+
nbf: now - 5, // allow for 5 seconds time difference in clocks
|
|
606
|
+
appid: accessKey,
|
|
607
|
+
getCredentials: selectedApp
|
|
608
|
+
}),
|
|
609
|
+
secretKey,
|
|
610
|
+
{ algorithm: 'HS256' }
|
|
611
|
+
);
|
|
612
|
+
var paraClient = getClient(config, flags);
|
|
542
613
|
paraClient.setAccessToken(jwt);
|
|
543
|
-
paraClient
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
614
|
+
paraClient
|
|
615
|
+
.me(jwt)
|
|
616
|
+
.then((data) => {
|
|
617
|
+
if (data?.credentials) {
|
|
618
|
+
config.set('selectedApp', data.credentials);
|
|
619
|
+
console.log(green('✔'), 'Selected', green(selectedApp), 'as the current app.');
|
|
620
|
+
} else {
|
|
621
|
+
fail(`That did not work -${red(input[1])} try updating Para to the latest version.`);
|
|
622
|
+
}
|
|
623
|
+
})
|
|
624
|
+
.catch((_err) => {
|
|
625
|
+
fail(`App ${red(input[1])} not found!`);
|
|
626
|
+
});
|
|
553
627
|
} else {
|
|
554
628
|
fail('This command only works when Para CLI is configured to use the keys for the root app.');
|
|
555
629
|
}
|
|
@@ -560,7 +634,7 @@ export function listEndpoints(config, flags, failureCallback) {
|
|
|
560
634
|
var secretKey = flags.secretKey || process.env.PARA_SECRET_KEY || config.get('secretKey');
|
|
561
635
|
var endpoint = flags.endpoint || process.env.PARA_ENDPOINT || config.get('endpoint');
|
|
562
636
|
var endpoints = config.get('endpoints') || [];
|
|
563
|
-
var list = [{endpoint: endpoint, accessKey: accessKey, secretKey: secretKey}].concat(endpoints);
|
|
637
|
+
var list = [{ endpoint: endpoint, accessKey: accessKey, secretKey: secretKey }].concat(endpoints);
|
|
564
638
|
if (list.length === 0) {
|
|
565
639
|
failureCallback();
|
|
566
640
|
return [];
|
|
@@ -569,55 +643,65 @@ export function listEndpoints(config, flags, failureCallback) {
|
|
|
569
643
|
var ep = list[i];
|
|
570
644
|
var selected = (config.get('selectedEndpoint') || 0) === i;
|
|
571
645
|
var rootAppConfigured = ep.accessKey === 'app:para' && ep.secretKey.length > 10;
|
|
572
|
-
console.log(
|
|
573
|
-
|
|
646
|
+
console.log(
|
|
647
|
+
yellow(selected ? ' ➤' : ' ', `${i + 1}. `) + cyan(ep.endpoint),
|
|
648
|
+
rootAppConfigured ? green('✔ root app configured') : red('root app not configured')
|
|
649
|
+
);
|
|
574
650
|
}
|
|
575
651
|
return list;
|
|
576
652
|
}
|
|
577
653
|
|
|
578
|
-
export function addEndpoint(config) {
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
rl.question(cyan.bold('Para Endpoint: '), function (endpoint) {
|
|
654
|
+
export async function addEndpoint(config) {
|
|
655
|
+
try {
|
|
656
|
+
const endpoint = await input({
|
|
657
|
+
message: 'Para Endpoint:'
|
|
658
|
+
});
|
|
659
|
+
|
|
585
660
|
if (!isValidUrl(endpoint)) {
|
|
586
661
|
fail('Endpoint must be a valid URL.');
|
|
587
|
-
rl.close();
|
|
588
662
|
return;
|
|
589
663
|
}
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
for (var i = 0; i < endpoints.length; i++) {
|
|
595
|
-
var ep = endpoints[i];
|
|
596
|
-
if (ep.endpoint === endpoint) {
|
|
597
|
-
ep.secretKey = secretKey;
|
|
598
|
-
existing = true;
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
if (!existing) {
|
|
602
|
-
endpoints.push({accessKey: 'app:para', secretKey: secretKey, endpoint: endpoint});
|
|
603
|
-
}
|
|
604
|
-
config.set('endpoints', endpoints);
|
|
605
|
-
ping(pc, config);
|
|
606
|
-
rl.close();
|
|
664
|
+
|
|
665
|
+
const secretKey = await password({
|
|
666
|
+
message: 'Para Secret Key (for root app app:para):',
|
|
667
|
+
mask: '*'
|
|
607
668
|
});
|
|
608
|
-
|
|
669
|
+
|
|
670
|
+
var endpoints = config.get('endpoints') || [];
|
|
671
|
+
var existing = false;
|
|
672
|
+
for (var i = 0; i < endpoints.length; i++) {
|
|
673
|
+
var ep = endpoints[i];
|
|
674
|
+
if (ep.endpoint === endpoint) {
|
|
675
|
+
ep.secretKey = secretKey;
|
|
676
|
+
existing = true;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
if (!existing) {
|
|
680
|
+
endpoints.push({ accessKey: 'app:para', secretKey: secretKey, endpoint: endpoint });
|
|
681
|
+
}
|
|
682
|
+
config.set('endpoints', endpoints);
|
|
683
|
+
ping(config);
|
|
684
|
+
} catch (error) {
|
|
685
|
+
if (error.name === 'ExitPromptError') {
|
|
686
|
+
console.log('\nAdd endpoint cancelled.');
|
|
687
|
+
} else {
|
|
688
|
+
throw error;
|
|
689
|
+
}
|
|
690
|
+
}
|
|
609
691
|
}
|
|
610
692
|
|
|
611
|
-
export function removeEndpoint(config, flags) {
|
|
612
|
-
var list = listEndpoints(config, flags,
|
|
613
|
-
|
|
614
|
-
input: process.stdin,
|
|
615
|
-
output: process.stdout
|
|
693
|
+
export async function removeEndpoint(config, flags) {
|
|
694
|
+
var list = listEndpoints(config, flags, () => {
|
|
695
|
+
console.log('No endpoints found.');
|
|
616
696
|
});
|
|
617
697
|
|
|
618
|
-
|
|
698
|
+
try {
|
|
699
|
+
const index = await input({
|
|
700
|
+
message: 'Type the number of the Para endpoint to remove:'
|
|
701
|
+
});
|
|
702
|
+
|
|
619
703
|
var selectedEndpoint = 0;
|
|
620
|
-
if (!isNaN(index) && index <= list.length && index >= 1) {
|
|
704
|
+
if (!Number.isNaN(index) && index <= list.length && index >= 1) {
|
|
621
705
|
selectedEndpoint = index - 1;
|
|
622
706
|
}
|
|
623
707
|
var url = list[selectedEndpoint].endpoint;
|
|
@@ -634,38 +718,51 @@ export function removeEndpoint(config, flags) {
|
|
|
634
718
|
list.shift();
|
|
635
719
|
config.set('endpoints', list);
|
|
636
720
|
}
|
|
637
|
-
console.log(
|
|
638
|
-
|
|
639
|
-
|
|
721
|
+
console.log(`Removed endpoint: ${cyan(url)}`);
|
|
722
|
+
} catch (error) {
|
|
723
|
+
if (error.name === 'ExitPromptError') {
|
|
724
|
+
console.log('\nRemove endpoint cancelled.');
|
|
725
|
+
} else {
|
|
726
|
+
throw error;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
640
729
|
}
|
|
641
730
|
|
|
642
|
-
export function selectEndpoint(config, flags) {
|
|
643
|
-
var list = listEndpoints(config, flags,
|
|
644
|
-
|
|
645
|
-
input: process.stdin,
|
|
646
|
-
output: process.stdout
|
|
731
|
+
export async function selectEndpoint(config, flags) {
|
|
732
|
+
var list = listEndpoints(config, flags, () => {
|
|
733
|
+
console.log('No endpoints found.');
|
|
647
734
|
});
|
|
648
|
-
|
|
735
|
+
|
|
736
|
+
try {
|
|
737
|
+
const index = await input({
|
|
738
|
+
message: 'Type the number of the Para endpoint to select:'
|
|
739
|
+
});
|
|
740
|
+
|
|
649
741
|
var selectedEndpoint = 0;
|
|
650
|
-
if (!isNaN(index) && index <= list.length && index >= 1) {
|
|
742
|
+
if (!Number.isNaN(index) && index <= list.length && index >= 1) {
|
|
651
743
|
selectedEndpoint = index - 1;
|
|
652
744
|
}
|
|
653
745
|
config.delete('selectedApp');
|
|
654
746
|
config.set('selectedEndpoint', selectedEndpoint);
|
|
655
|
-
console.log(
|
|
656
|
-
|
|
657
|
-
|
|
747
|
+
console.log(`Selected endpoint: ${cyan(list[selectedEndpoint].endpoint)}`);
|
|
748
|
+
} catch (error) {
|
|
749
|
+
if (error.name === 'ExitPromptError') {
|
|
750
|
+
console.log('\nSelect endpoint cancelled.');
|
|
751
|
+
} else {
|
|
752
|
+
throw error;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
658
755
|
}
|
|
659
756
|
|
|
660
757
|
export function parseEndpoint(endpoint) {
|
|
661
758
|
try {
|
|
662
759
|
var url = new URL(endpoint);
|
|
663
760
|
if (url.pathname !== '/') {
|
|
664
|
-
var x = { endpoint: url.protocol
|
|
761
|
+
var x = { endpoint: `${url.protocol}//${url.host}`, apiPath: `${url.pathname.replace(/\/*$/, '')}/v1/` };
|
|
665
762
|
return x;
|
|
666
763
|
}
|
|
667
764
|
} catch (e) {
|
|
668
|
-
fail(
|
|
765
|
+
fail(`Invalid Para endpoint: ${endpoint}`, e);
|
|
669
766
|
}
|
|
670
767
|
return { endpoint: endpoint };
|
|
671
768
|
}
|
|
@@ -674,15 +771,26 @@ function getSelectedEndpoint(config, flags) {
|
|
|
674
771
|
var accessKey = flags.accessKey || process.env.PARA_ACCESS_KEY || config.get('accessKey');
|
|
675
772
|
var secretKey = flags.secretKey || process.env.PARA_SECRET_KEY || config.get('secretKey');
|
|
676
773
|
var endpoint = flags.endpoint || process.env.PARA_ENDPOINT || config.get('endpoint');
|
|
677
|
-
var endpoints = [{endpoint: endpoint, accessKey: accessKey, secretKey: secretKey}].concat(
|
|
774
|
+
var endpoints = [{ endpoint: endpoint, accessKey: accessKey, secretKey: secretKey }].concat(
|
|
775
|
+
config.get('endpoints') || []
|
|
776
|
+
);
|
|
678
777
|
try {
|
|
679
778
|
return endpoints[config.get('selectedEndpoint') || 0];
|
|
680
|
-
} catch (
|
|
779
|
+
} catch (_e) {
|
|
681
780
|
config.delete('selectedEndpoint');
|
|
682
781
|
return endpoints[0];
|
|
683
782
|
}
|
|
684
783
|
}
|
|
685
784
|
|
|
785
|
+
function getClient(config, flags = {}) {
|
|
786
|
+
var selectedEndpoint = getSelectedEndpoint(config, flags);
|
|
787
|
+
return new ParaClient(
|
|
788
|
+
selectedEndpoint.accessKey,
|
|
789
|
+
selectedEndpoint.secretKey,
|
|
790
|
+
parseEndpoint(selectedEndpoint.endpoint)
|
|
791
|
+
);
|
|
792
|
+
}
|
|
793
|
+
|
|
686
794
|
function sendFileChunk(chunkId, textEncoded, json, id, flags, start, end, pc, decoder) {
|
|
687
795
|
if (start > 0 && textEncoded[start] !== 32) {
|
|
688
796
|
for (var i = 0; i < 100 && start - i >= 0; i++) {
|
|
@@ -712,23 +820,41 @@ function sendFileChunk(chunkId, textEncoded, json, id, flags, start, end, pc, de
|
|
|
712
820
|
|
|
713
821
|
var chunk = textEncoded.slice(start, end);
|
|
714
822
|
var text = decoder.decode(chunk);
|
|
715
|
-
var obj = getParaObject(Object.assign({}, json, { text: text }), id
|
|
823
|
+
var obj = getParaObject(Object.assign({}, json, { text: text }), `${id}_chunk${chunkId}`, flags);
|
|
716
824
|
if (text && text.trim().length > 0) {
|
|
717
825
|
obj.chunkid = chunkId;
|
|
718
|
-
pc.create(obj)
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
826
|
+
pc.create(obj)
|
|
827
|
+
.then(() => {
|
|
828
|
+
console.log(
|
|
829
|
+
green('✔'),
|
|
830
|
+
'Created object chunk',
|
|
831
|
+
yellow(chunkId),
|
|
832
|
+
'with size',
|
|
833
|
+
Math.round(chunk.length / 1024),
|
|
834
|
+
'KB.'
|
|
835
|
+
);
|
|
836
|
+
if (end < textEncoded.length) {
|
|
837
|
+
sendFileChunk(
|
|
838
|
+
++chunkId,
|
|
839
|
+
textEncoded,
|
|
840
|
+
json,
|
|
841
|
+
id,
|
|
842
|
+
flags,
|
|
843
|
+
start + MAX_FILE_SIZE,
|
|
844
|
+
end + MAX_FILE_SIZE,
|
|
845
|
+
pc,
|
|
846
|
+
decoder
|
|
847
|
+
);
|
|
848
|
+
}
|
|
849
|
+
})
|
|
850
|
+
.catch((err) => {
|
|
851
|
+
fail('Failed to create chunk:', err);
|
|
852
|
+
});
|
|
727
853
|
}
|
|
728
854
|
}
|
|
729
855
|
|
|
730
856
|
function addObjectsToBatch(list, json, id, flags) {
|
|
731
|
-
var objects = (json
|
|
857
|
+
var objects = Array.isArray(json) ? json : [json];
|
|
732
858
|
for (var i = 0; i < objects.length; i++) {
|
|
733
859
|
list.push(getParaObject(objects[i], id, flags));
|
|
734
860
|
}
|
|
@@ -738,7 +864,7 @@ function addObjectsToBatch(list, json, id, flags) {
|
|
|
738
864
|
|
|
739
865
|
function getParaObject(json, id, flags) {
|
|
740
866
|
var pobj = new ParaObject();
|
|
741
|
-
if (flags
|
|
867
|
+
if (flags?.type) {
|
|
742
868
|
pobj.setType(getType(flags.type));
|
|
743
869
|
}
|
|
744
870
|
|
|
@@ -768,29 +894,32 @@ function parseHTML(file) {
|
|
|
768
894
|
var text = '';
|
|
769
895
|
var inScript = false;
|
|
770
896
|
var inAnchor = false;
|
|
771
|
-
var parser = new Parser(
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
897
|
+
var parser = new Parser(
|
|
898
|
+
{
|
|
899
|
+
onopentag: (tag, attribs) => {
|
|
900
|
+
if (tag === 'meta' && attribs.property === 'og:title') {
|
|
901
|
+
title = attribs.content;
|
|
902
|
+
}
|
|
776
903
|
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
904
|
+
if (tag === 'meta' && attribs.property === 'og:url') {
|
|
905
|
+
url = attribs.content;
|
|
906
|
+
}
|
|
780
907
|
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
908
|
+
inScript = tag === 'script';
|
|
909
|
+
inAnchor = tag === 'a' && attribs.href && !attribs.href.match(/^http/i);
|
|
910
|
+
},
|
|
911
|
+
ontext: (txt) => {
|
|
912
|
+
if (!inScript && !inAnchor) {
|
|
913
|
+
text += ` ${txt}`;
|
|
914
|
+
}
|
|
915
|
+
},
|
|
916
|
+
onclosetag: () => {
|
|
917
|
+
inScript = false;
|
|
918
|
+
inAnchor = false;
|
|
787
919
|
}
|
|
788
920
|
},
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
inAnchor = false;
|
|
792
|
-
}
|
|
793
|
-
}, { decodeEntities: true });
|
|
921
|
+
{ decodeEntities: true }
|
|
922
|
+
);
|
|
794
923
|
parser.write(file);
|
|
795
924
|
parser.end();
|
|
796
925
|
return {
|
|
@@ -804,7 +933,7 @@ function isValidUrl(url) {
|
|
|
804
933
|
try {
|
|
805
934
|
new URL(url);
|
|
806
935
|
return true;
|
|
807
|
-
} catch (
|
|
936
|
+
} catch (_err) {
|
|
808
937
|
return false;
|
|
809
938
|
}
|
|
810
939
|
}
|
|
@@ -814,8 +943,8 @@ function readFile(filePath) {
|
|
|
814
943
|
}
|
|
815
944
|
|
|
816
945
|
function fail(msg, err) {
|
|
817
|
-
var errMsg = err
|
|
818
|
-
var code = err
|
|
946
|
+
var errMsg = err?.response?.body?.message ? err.response.body.message : err || '';
|
|
947
|
+
var code = err?.response?.status ? `(${err.response.status} ${err.response.res.statusMessage})` : '';
|
|
819
948
|
console.error(red('✖'), msg || 'Forgive me, I have failed you!', red(errMsg), red(code));
|
|
820
949
|
process.exitCode = 1;
|
|
821
950
|
}
|