scanoss 0.2.21 → 0.2.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/.cspell.json +3 -1
  2. package/build/main/bin/cli-bin.js +11 -4
  3. package/build/main/commands/fingerprint.d.ts +1 -0
  4. package/build/main/commands/fingerprint.js +49 -0
  5. package/build/main/commands/scan.js +4 -5
  6. package/build/main/index.d.ts +1 -0
  7. package/build/main/index.js +2 -1
  8. package/build/main/lib/dependencies/DependencyScanner.d.ts +1 -0
  9. package/build/main/lib/dependencies/DependencyScanner.js +18 -4
  10. package/build/main/lib/dependencies/LocalDependency/DependencyTypes.d.ts +1 -1
  11. package/build/main/lib/dependencies/LocalDependency/parsers/npmParser.js +3 -3
  12. package/build/main/lib/dependencies/LocalDependency/parsers/pyParser.js +7 -3
  13. package/build/main/lib/grpc/GrpcDependencyService.js +1 -2
  14. package/build/main/lib/scanner/Dispatcher/DispatchableItem.d.ts +4 -4
  15. package/build/main/lib/scanner/Dispatcher/DispatchableItem.js +5 -5
  16. package/build/main/lib/scanner/Scanner.d.ts +17 -19
  17. package/build/main/lib/scanner/Scanner.js +44 -33
  18. package/build/main/lib/scanner/ScannerCfg.js +3 -3
  19. package/build/main/lib/scanner/ScannerTypes.d.ts +1 -0
  20. package/build/main/lib/scanner/ScannerTypes.js +1 -1
  21. package/build/main/lib/scanner/WfpProvider/FingerprintPacket.d.ts +12 -0
  22. package/build/main/lib/scanner/WfpProvider/FingerprintPacket.js +38 -0
  23. package/build/main/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator.d.ts +20 -0
  24. package/build/main/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator.js +334 -0
  25. package/build/main/lib/scanner/WfpProvider/WfpProvider.d.ts +32 -0
  26. package/build/main/lib/scanner/WfpProvider/WfpProvider.js +63 -0
  27. package/build/main/lib/scanner/WfpProvider/WfpSplitter/WfpSplitter.d.ts +20 -0
  28. package/build/main/lib/scanner/WfpProvider/WfpSplitter/WfpSplitter.js +94 -0
  29. package/build/main/lib/scanner/Winnower/WinnowerExtractor.js +1 -1
  30. package/build/module/bin/cli-bin.js +11 -4
  31. package/build/module/commands/fingerprint.d.ts +1 -0
  32. package/build/module/commands/fingerprint.js +42 -0
  33. package/build/module/commands/scan.js +4 -5
  34. package/build/module/index.d.ts +1 -0
  35. package/build/module/index.js +2 -1
  36. package/build/module/lib/dependencies/DependencyScanner.d.ts +1 -0
  37. package/build/module/lib/dependencies/DependencyScanner.js +18 -4
  38. package/build/module/lib/dependencies/LocalDependency/DependencyTypes.d.ts +1 -1
  39. package/build/module/lib/dependencies/LocalDependency/parsers/npmParser.js +3 -3
  40. package/build/module/lib/dependencies/LocalDependency/parsers/pyParser.js +7 -3
  41. package/build/module/lib/grpc/GrpcDependencyService.js +1 -2
  42. package/build/module/lib/scanner/Dispatcher/DispatchableItem.d.ts +4 -4
  43. package/build/module/lib/scanner/Dispatcher/DispatchableItem.js +6 -6
  44. package/build/module/lib/scanner/Scanner.d.ts +17 -19
  45. package/build/module/lib/scanner/Scanner.js +50 -39
  46. package/build/module/lib/scanner/ScannerCfg.js +3 -3
  47. package/build/module/lib/scanner/ScannerTypes.d.ts +1 -0
  48. package/build/module/lib/scanner/ScannerTypes.js +1 -1
  49. package/build/module/lib/scanner/WfpProvider/FingerprintPacket.d.ts +12 -0
  50. package/build/module/lib/scanner/WfpProvider/FingerprintPacket.js +37 -0
  51. package/build/module/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator.d.ts +20 -0
  52. package/build/module/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator.js +330 -0
  53. package/build/module/lib/scanner/WfpProvider/WfpProvider.d.ts +32 -0
  54. package/build/module/lib/scanner/WfpProvider/WfpProvider.js +65 -0
  55. package/build/module/lib/scanner/WfpProvider/WfpSplitter/WfpSplitter.d.ts +20 -0
  56. package/build/module/lib/scanner/WfpProvider/WfpSplitter/WfpSplitter.js +93 -0
  57. package/build/module/lib/scanner/Winnower/WinnowerExtractor.js +1 -1
  58. package/build/tsconfig.module.tsbuildinfo +1 -1
  59. package/build/tsconfig.tsbuildinfo +1 -1
  60. package/package-lock.json +18588 -0
  61. package/package.json +1 -1
  62. package/src/bin/cli-bin.ts +11 -4
  63. package/src/commands/fingerprint.ts +55 -0
  64. package/src/commands/scan.ts +3 -3
  65. package/src/index.js +1 -0
  66. package/src/index.ts +1 -0
  67. package/src/lib/dependencies/DependencyScanner.ts +21 -3
  68. package/src/lib/dependencies/LocalDependency/DependencyTypes.ts +1 -1
  69. package/src/lib/dependencies/LocalDependency/parsers/npmParser.ts +2 -2
  70. package/src/lib/dependencies/LocalDependency/parsers/pyParser.ts +5 -2
  71. package/src/lib/grpc/GrpcDependencyService.ts +0 -1
  72. package/src/lib/scanner/Dispatcher/DispatchableItem.ts +7 -8
  73. package/src/lib/scanner/Scanner.ts +64 -53
  74. package/src/lib/scanner/ScannerCfg.ts +2 -2
  75. package/src/lib/scanner/ScannerTypes.ts +1 -0
  76. package/src/lib/scanner/{Winnower/WinnowerResponse.ts → WfpProvider/FingerprintPacket.ts} +11 -5
  77. package/src/lib/scanner/{Winnower/Winnower.ts → WfpProvider/WfpCalculator/WfpCalculator.ts} +32 -119
  78. package/src/lib/scanner/WfpProvider/WfpProvider.ts +91 -0
  79. package/src/lib/scanner/WfpProvider/WfpSplitter/WfpSplitter.ts +120 -0
  80. package/src/lib/scanner/Winnower/WinnowerExtractor.ts +0 -1
  81. package/yarn.lock +5289 -5202
@@ -0,0 +1,330 @@
1
+ import fs from 'fs';
2
+ import { Worker } from 'worker_threads';
3
+ import { ScannableItem } from '../../Scannable/ScannableItem';
4
+ import { ScannerCfg } from '../../ScannerCfg';
5
+ import { FingerprintPacket } from '../FingerprintPacket';
6
+ import { WfpProvider } from '../WfpProvider';
7
+ const stringWorker = `
8
+ const { parentPort } = require('worker_threads');
9
+
10
+ parentPort.on('message', async (scannableItem) => {
11
+
12
+ let fingerprint;
13
+ if ( scannableItem.winnowingMode === "FULL_WINNOWING") {
14
+ fingerprint = wfp_for_content(
15
+ scannableItem.content,
16
+ scannableItem.contentSource,
17
+ scannableItem.maxSizeWfp
18
+ );
19
+ } else {
20
+ fingerprint = wfp_only_md5(
21
+ scannableItem.content,
22
+ scannableItem.contentSource
23
+ );
24
+ }
25
+
26
+ scannableItem.fingerprint = fingerprint;
27
+
28
+ parentPort.postMessage(scannableItem);
29
+ });
30
+
31
+
32
+ const crypto = require('crypto');
33
+
34
+ const isWin = process.platform === 'win32';
35
+ const pathSeparator = isWin ? String.fromCharCode(92) : '/';
36
+
37
+ // Winnowing configuration. DO NOT CHANGE.
38
+ const GRAM = 30;
39
+ const WINDOW = 64;
40
+
41
+ // ASCII characters
42
+ const ASCII_0 = 48;
43
+ const ASCII_9 = 57;
44
+ const ASCII_A = 65;
45
+ const ASCII_Z = 90;
46
+ const ASCII_a = 97;
47
+ const ASCII_z = 122;
48
+ const ASCII_LF = 10;
49
+
50
+ function normalize(byte) {
51
+ if (byte < ASCII_0 || byte > ASCII_z) {
52
+ return 0;
53
+ }
54
+ if (byte <= ASCII_9 || byte >= ASCII_a) {
55
+ return byte;
56
+ }
57
+ if (byte >= ASCII_A && byte <= ASCII_Z) {
58
+ return byte + 32;
59
+ }
60
+ return 0;
61
+ }
62
+
63
+ function min_hex_array(array) {
64
+ let min = 'ffffffff';
65
+ for (let i = 0; i < array.length; i++) {
66
+ if (array[i] < min) {
67
+ min = array[i];
68
+ }
69
+ }
70
+ return min;
71
+ }
72
+
73
+ function wfp_for_content(content, contentSource, maxSize) {
74
+ let wfp = wfp_only_md5(content, contentSource);
75
+ wfp += calc_wfp(content, maxSize);
76
+ return wfp;
77
+ }
78
+
79
+ function wfp_only_md5(contents, contentSource) {
80
+ const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
81
+ let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
82
+ return wfp;
83
+ }
84
+
85
+ function calc_wfp(contents, maxSize) {
86
+ let gram = '';
87
+ const window = [];
88
+ let normalized = 0;
89
+ let line = 1;
90
+ let min_hash = 'ffffffff';
91
+ let last_hash = 'ffffffff';
92
+ let last_line = 0;
93
+ let output = '';
94
+ let gram_crc32 = 0;
95
+ let wfp = '';
96
+
97
+ for (let i = 0; i < contents.length; i++) {
98
+ if(wfp.length > maxSize)
99
+ return wfp;
100
+
101
+ const byte = contents[i];
102
+ if (byte == ASCII_LF) {
103
+ line += 1;
104
+ normalized = 0;
105
+ } else {
106
+ normalized = normalize(byte);
107
+ }
108
+ // Is this an useful byte?
109
+ if (normalized) {
110
+ gram += String.fromCharCode(normalized);
111
+
112
+ if (gram.length >= GRAM) {
113
+ gram_crc32 = crc32c_hex(gram);
114
+ window.push(gram_crc32);
115
+
116
+ if (window.length >= WINDOW) {
117
+ min_hash = min_hex_array(window);
118
+ if (min_hash !== last_hash) {
119
+ // Hashing the hash will result in a better balanced output data set
120
+ // as it will counter the winnowing effect which selects the "minimum"
121
+ // hash in each window
122
+ const min_hash_bytes_le = parseHexString(
123
+ toLittleEndianCRCHex(min_hash)
124
+ );
125
+ const crc_hex = crc32c_for_bytes_hex(min_hash_bytes_le);
126
+
127
+ if (last_line != line) {
128
+ if (output.length > 0) {
129
+ wfp += String(output) + String.fromCharCode(10);
130
+ }
131
+ output = String(line) + '=' + String(crc_hex);
132
+ } else {
133
+ output += ',' + String(crc_hex);
134
+ }
135
+ last_line = line;
136
+ last_hash = min_hash;
137
+ }
138
+ window.shift();
139
+ }
140
+ gram = gram.slice(1);
141
+ }
142
+ }
143
+ }
144
+ if (output.length > 0) {
145
+ wfp += String(output) + String.fromCharCode(10);
146
+ }
147
+
148
+ return wfp;
149
+ }
150
+
151
+ function parseHexString(str) {
152
+ const result = [];
153
+ while (str.length >= 2) {
154
+ result.push(parseInt(str.substring(0, 2), 16));
155
+ str = str.substring(2, str.length);
156
+ }
157
+
158
+ return result;
159
+ }
160
+
161
+ /**
162
+ *
163
+ * @param {string} hex
164
+ */
165
+ function toLittleEndianCRCHex(hex) {
166
+ return (
167
+ hex.charAt(6) +
168
+ hex.charAt(7) +
169
+ hex.charAt(4) +
170
+ hex.charAt(5) +
171
+ hex.charAt(2) +
172
+ hex.charAt(3) +
173
+ hex.charAt(0) +
174
+ hex.charAt(1)
175
+ );
176
+ }
177
+
178
+ let CRC_TABLE = [];
179
+
180
+ function makeCRCTable() {
181
+ let c;
182
+ const crcTable = [];
183
+ for (let n = 0; n < 256; n++) {
184
+ c = n;
185
+ for (let k = 0; k < 8; k++) {
186
+ c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1;
187
+ }
188
+ crcTable[n] = c;
189
+ }
190
+ return crcTable;
191
+ }
192
+
193
+ function crc32c(str) {
194
+ if (CRC_TABLE.length == 0) {
195
+ CRC_TABLE = makeCRCTable();
196
+ }
197
+ let crc = 0 ^ -1;
198
+
199
+ for (let i = 0; i < str.length; i++) {
200
+ crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xff];
201
+ }
202
+
203
+ return (crc ^ -1) >>> 0;
204
+ }
205
+
206
+ function crc32c_for_bytes(bytes) {
207
+ if (CRC_TABLE.length == 0) {
208
+ CRC_TABLE = makeCRCTable();
209
+ }
210
+ let crc = 0 ^ -1;
211
+
212
+ for (let i = 0; i < bytes.length; i++) {
213
+ crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ bytes[i]) & 0xff];
214
+ }
215
+
216
+ return (crc ^ -1) >>> 0;
217
+ }
218
+
219
+ function crc32c_for_bytes_hex(bytes) {
220
+ return crc32c_for_bytes(bytes).toString(16).padStart(8, '0');
221
+ }
222
+
223
+ function crc32c_hex(str) {
224
+ return crc32c(str).toString(16).padStart(8, '0');
225
+ }
226
+
227
+ `;
228
+ export class WfpCalculator extends WfpProvider {
229
+ fileList;
230
+ fileListIndex;
231
+ continue;
232
+ constructor(scannerCfg = new ScannerCfg()) {
233
+ super();
234
+ this.scannerCfg = scannerCfg;
235
+ }
236
+ init() {
237
+ super.init();
238
+ this.continue = true;
239
+ this.fileList = [];
240
+ this.fileListIndex = 0;
241
+ }
242
+ prepareWorker() {
243
+ this.worker = new Worker(stringWorker, { eval: true });
244
+ this.worker.on('message', async (scannableItem) => {
245
+ this.fingerprintPacker(scannableItem.fingerprint);
246
+ await this.nextStepMachine();
247
+ });
248
+ }
249
+ recoveryIndex() {
250
+ // Files: contains all files winnowed but not packed yet
251
+ const files = new FingerprintPacket(this.wfp, this.folderRoot).getFilesFingerprinted();
252
+ if (files.length) {
253
+ const lastFileWinnowed = files[files.length - 1];
254
+ let i = 0;
255
+ while (i <= files.length && lastFileWinnowed !== this.fileList[this.fileListIndex - i]) {
256
+ i += 1;
257
+ }
258
+ // If file already winnowed cannot be found in fileList emit an error.
259
+ if (i > files.length) {
260
+ this.sendError('Cannot recovery index on winnower');
261
+ return -1;
262
+ }
263
+ this.fileListIndex -= i;
264
+ if (this.fileList[this.fileListIndex] === lastFileWinnowed)
265
+ this.fileListIndex += 1;
266
+ }
267
+ return 0;
268
+ }
269
+ forceStopWorker() {
270
+ this.worker.removeAllListeners();
271
+ this.worker.terminate();
272
+ }
273
+ processPackedWfp(content) {
274
+ const fingerprint = new FingerprintPacket(content, this.folderRoot);
275
+ this.sendFingerprint(fingerprint);
276
+ }
277
+ async getNextScannableItem() {
278
+ if (this.fileListIndex >= this.fileList.length)
279
+ return null;
280
+ const path = this.fileList[this.fileListIndex];
281
+ const contentSource = path.replace(`${this.folderRoot}`, '');
282
+ const content = await fs.promises.readFile(path);
283
+ this.fileListIndex += 1;
284
+ const scannable = new ScannableItem(content, contentSource, this.winnowingMode, this.scannerCfg.WFP_FILE_MAX_SIZE);
285
+ return scannable;
286
+ }
287
+ async nextStepMachine() {
288
+ if (!this.continue)
289
+ return;
290
+ const scannableItem = await this.getNextScannableItem();
291
+ if (scannableItem)
292
+ this.worker.postMessage(scannableItem);
293
+ else {
294
+ this.finishWinnowing();
295
+ this.forceStopWorker();
296
+ this.sendLog('[ SCANNER ]: WFP Calculator finished...');
297
+ }
298
+ }
299
+ start(params) {
300
+ if (!params.fileList)
301
+ this.sendError('File list is required');
302
+ this.sendLog('[ SCANNER ]: WFP Calculator starting...');
303
+ this.init();
304
+ this.prepareWorker();
305
+ if (params.winnowingMode)
306
+ this.setWinnowingMode(params.winnowingMode);
307
+ this.pendingFiles = true;
308
+ this.folderRoot = params.folderRoot;
309
+ this.fileList = params.fileList;
310
+ this.nextStepMachine();
311
+ }
312
+ pause() {
313
+ this.sendLog('[ SCANNER ]: WFP Calculator paused...');
314
+ this.continue = false;
315
+ }
316
+ resume() {
317
+ this.sendLog('[ SCANNER ]: WFP Calculator resumed...');
318
+ this.continue = true;
319
+ this.recoveryIndex();
320
+ this.nextStepMachine();
321
+ }
322
+ stop() {
323
+ this.continue = false;
324
+ this.pendingFiles = false;
325
+ this.forceStopWorker();
326
+ this.prepareWorker();
327
+ this.init();
328
+ }
329
+ }
330
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwQ2FsY3VsYXRvci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uLy4uL3NyYy9saWIvc2Nhbm5lci9XZnBQcm92aWRlci9XZnBDYWxjdWxhdG9yL1dmcENhbGN1bGF0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLE1BQU0sSUFBSSxDQUFDO0FBQ3BCLE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUV4QyxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFDOUQsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRzlDLE9BQU8sRUFBRSxpQkFBaUIsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQ3pELE9BQU8sRUFBcUIsV0FBVyxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFHaEUsTUFBTSxZQUFZLEdBQUc7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Q0E0TnBCLENBQUM7QUFFRixNQUFNLE9BQU8sYUFBYyxTQUFRLFdBQVc7SUFDcEMsUUFBUSxDQUFNO0lBRWQsYUFBYSxDQUFTO0lBRXRCLFFBQVEsQ0FBVTtJQUUxQixZQUFZLFVBQVUsR0FBRyxJQUFJLFVBQVUsRUFBRTtRQUN2QyxLQUFLLEVBQUUsQ0FBQztRQUNSLElBQUksQ0FBQyxVQUFVLEdBQUcsVUFBVSxDQUFDO0lBQy9CLENBQUM7SUFFRCxJQUFJO1FBQ0YsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ2IsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLENBQUM7UUFDckIsSUFBSSxDQUFDLFFBQVEsR0FBRyxFQUFFLENBQUM7UUFDbkIsSUFBSSxDQUFDLGFBQWEsR0FBRyxDQUFDLENBQUM7SUFDekIsQ0FBQztJQUVELGFBQWE7UUFDWCxJQUFJLENBQUMsTUFBTSxHQUFHLElBQUksTUFBTSxDQUFDLFlBQVksRUFBRSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDO1FBQ3ZELElBQUksQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLFNBQVMsRUFBRSxLQUFLLEVBQUUsYUFBYSxFQUFFLEVBQUU7WUFDaEQsSUFBSSxDQUFDLGlCQUFpQixDQUFDLGFBQWEsQ0FBQyxXQUFXLENBQUMsQ0FBQztZQUNsRCxNQUFNLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztRQUMvQixDQUFDLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRCxhQUFhO1FBQ1gsd0RBQXdEO1FBQ3hELE1BQU0sS0FBSyxHQUFHLElBQUksaUJBQWlCLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUMscUJBQXFCLEVBQUUsQ0FBQztRQUN2RixJQUFJLEtBQUssQ0FBQyxNQUFNLEVBQUU7WUFDaEIsTUFBTSxnQkFBZ0IsR0FBRyxLQUFLLENBQUMsS0FBSyxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUMsQ0FBQztZQUNqRCxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7WUFDVixPQUFPLENBQUMsSUFBSSxLQUFLLENBQUMsTUFBTSxJQUFJLGdCQUFnQixLQUFLLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsR0FBRyxDQUFDLENBQUMsRUFBRTtnQkFDdEYsQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNSO1lBQ0Qsc0VBQXNFO1lBQ3RFLElBQUksQ0FBQyxHQUFHLEtBQUssQ0FBQyxNQUFNLEVBQUU7Z0JBQ3BCLElBQUksQ0FBQyxTQUFTLENBQUMsbUNBQW1DLENBQUMsQ0FBQztnQkFDcEQsT0FBTyxDQUFDLENBQUMsQ0FBQzthQUNYO1lBQ0QsSUFBSSxDQUFDLGFBQWEsSUFBSSxDQUFDLENBQUM7WUFDeEIsSUFBSSxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsS0FBSyxnQkFBZ0I7Z0JBQUUsSUFBSSxDQUFDLGFBQWEsSUFBSSxDQUFDLENBQUM7U0FDckY7UUFDRCxPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7SUFFRCxlQUFlO1FBQ2IsSUFBSSxDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsRUFBRSxDQUFDO1FBQ2pDLElBQUksQ0FBQyxNQUFNLENBQUMsU0FBUyxFQUFFLENBQUM7SUFDMUIsQ0FBQztJQUVTLGdCQUFnQixDQUFDLE9BQU87UUFDaEMsTUFBTSxXQUFXLEdBQUcsSUFBSSxpQkFBaUIsQ0FBQyxPQUFPLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3BFLElBQUksQ0FBQyxlQUFlLENBQUMsV0FBVyxDQUFDLENBQUM7SUFDcEMsQ0FBQztJQUVELEtBQUssQ0FBQyxvQkFBb0I7UUFDeEIsSUFBSSxJQUFJLENBQUMsYUFBYSxJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTTtZQUFFLE9BQU8sSUFBSSxDQUFDO1FBQzVELE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBQy9DLE1BQU0sYUFBYSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxJQUFJLENBQUMsVUFBVSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7UUFDN0QsTUFBTSxPQUFPLEdBQUcsTUFBTSxFQUFFLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNqRCxJQUFJLENBQUMsYUFBYSxJQUFJLENBQUMsQ0FBQztRQUN4QixNQUFNLFNBQVMsR0FBRyxJQUFJLGFBQWEsQ0FBQyxPQUFPLEVBQUUsYUFBYSxFQUFFLElBQUksQ0FBQyxhQUFhLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxpQkFBaUIsQ0FBQyxDQUFDO1FBQ25ILE9BQU8sU0FBUyxDQUFDO0lBQ25CLENBQUM7SUFFRCxLQUFLLENBQUMsZUFBZTtRQUNuQixJQUFJLENBQUMsSUFBSSxDQUFDLFFBQVE7WUFBRSxPQUFPO1FBQzNCLE1BQU0sYUFBYSxHQUFHLE1BQU0sSUFBSSxDQUFDLG9CQUFvQixFQUFFLENBQUM7UUFDeEQsSUFBSSxhQUFhO1lBQUUsSUFBSSxDQUFDLE1BQU0sQ0FBQyxXQUFXLENBQUMsYUFBYSxDQUFDLENBQUM7YUFDckQ7WUFDSCxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7WUFDdkIsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1lBQ3ZCLElBQUksQ0FBQyxPQUFPLENBQUMseUNBQXlDLENBQUMsQ0FBQztTQUV6RDtJQUNILENBQUM7SUFHTSxLQUFLLENBQUMsTUFBeUI7UUFFcEMsSUFBRyxDQUFDLE1BQU0sQ0FBQyxRQUFRO1lBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDO1FBQzdELElBQUksQ0FBQyxPQUFPLENBQUMseUNBQXlDLENBQUMsQ0FBQztRQUV4RCxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDWixJQUFJLENBQUMsYUFBYSxFQUFFLENBQUM7UUFFckIsSUFBRyxNQUFNLENBQUMsYUFBYTtZQUFFLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxNQUFNLENBQUMsYUFBYSxDQUFDLENBQUM7UUFDckUsSUFBSSxDQUFDLFlBQVksR0FBRyxJQUFJLENBQUM7UUFDekIsSUFBSSxDQUFDLFVBQVUsR0FBRyxNQUFNLENBQUMsVUFBVSxDQUFDO1FBQ3BDLElBQUksQ0FBQyxRQUFRLEdBQUcsTUFBTSxDQUFDLFFBQVEsQ0FBQztRQUNoQyxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7SUFDekIsQ0FBQztJQUtNLEtBQUs7UUFDVixJQUFJLENBQUMsT0FBTyxDQUFDLHVDQUF1QyxDQUFDLENBQUE7UUFDckQsSUFBSSxDQUFDLFFBQVEsR0FBRyxLQUFLLENBQUM7SUFDeEIsQ0FBQztJQUVNLE1BQU07UUFDWCxJQUFJLENBQUMsT0FBTyxDQUFDLHdDQUF3QyxDQUFDLENBQUE7UUFDdEQsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLENBQUM7UUFDckIsSUFBSSxDQUFDLGFBQWEsRUFBRSxDQUFDO1FBQ3JCLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztJQUN6QixDQUFDO0lBRU0sSUFBSTtRQUNULElBQUksQ0FBQyxRQUFRLEdBQUcsS0FBSyxDQUFDO1FBQ3RCLElBQUksQ0FBQyxZQUFZLEdBQUcsS0FBSyxDQUFDO1FBQzFCLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztRQUN2QixJQUFJLENBQUMsYUFBYSxFQUFFLENBQUM7UUFDckIsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ2QsQ0FBQztDQUVGIn0=
@@ -0,0 +1,32 @@
1
+ /// <reference types="node" />
2
+ import { Worker } from 'worker_threads';
3
+ import { EventEmitter } from "stream";
4
+ import { ScannerCfg } from "../ScannerCfg";
5
+ import { WinnowingMode } from "../ScannerTypes";
6
+ import { FingerprintPacket } from "./FingerprintPacket";
7
+ export interface IWfpProviderInput {
8
+ wfpPath?: string;
9
+ folderRoot?: string;
10
+ fileList?: Array<string>;
11
+ winnowingMode?: WinnowingMode;
12
+ }
13
+ export declare abstract class WfpProvider extends EventEmitter {
14
+ protected wfp: string;
15
+ protected scannerCfg: ScannerCfg;
16
+ protected folderRoot: string;
17
+ protected worker: Worker;
18
+ protected pendingFiles: boolean;
19
+ protected winnowingMode: WinnowingMode;
20
+ protected init(): void;
21
+ protected fingerprintPacker(fingerprint: string): boolean;
22
+ protected finishWinnowing(): void;
23
+ protected sendFingerprint(fingerprintPacket: FingerprintPacket): void;
24
+ protected sendLog(logMsg: string): void;
25
+ protected sendError(errorMsg: string): void;
26
+ protected setWinnowingMode(mode: WinnowingMode): void;
27
+ abstract start(params: IWfpProviderInput): void;
28
+ abstract stop(): void;
29
+ abstract pause(): void;
30
+ abstract resume(): void;
31
+ hasPendingFiles(): boolean;
32
+ }
@@ -0,0 +1,65 @@
1
+ import { EventEmitter } from "stream";
2
+ import { ScannerEvents, WinnowingMode } from "../ScannerTypes";
3
+ import { FingerprintPacket } from "./FingerprintPacket";
4
+ export class WfpProvider extends EventEmitter {
5
+ wfp;
6
+ scannerCfg;
7
+ folderRoot;
8
+ worker;
9
+ pendingFiles;
10
+ winnowingMode;
11
+ init() {
12
+ this.wfp = '';
13
+ this.folderRoot = '';
14
+ this.pendingFiles = false;
15
+ this.winnowingMode = WinnowingMode.FULL_WINNOWING;
16
+ }
17
+ // returns true if the function emitted a new fingerprint packet
18
+ fingerprintPacker(fingerprint) {
19
+ // When the fingerprint of one file is bigger than 64Kb, truncate to the last 64Kb line.
20
+ if (fingerprint.length > this.scannerCfg.WFP_FILE_MAX_SIZE) {
21
+ let truncateStringOnIndex = this.scannerCfg.WFP_FILE_MAX_SIZE;
22
+ let keepRemovingCharacters = true;
23
+ while (keepRemovingCharacters) {
24
+ if (fingerprint[truncateStringOnIndex] === '\n')
25
+ keepRemovingCharacters = false;
26
+ truncateStringOnIndex -= 1;
27
+ }
28
+ truncateStringOnIndex += 1;
29
+ // eslint-disable-next-line no-param-reassign
30
+ fingerprint = fingerprint.substring(0, truncateStringOnIndex);
31
+ // eslint-disable-next-line no-param-reassign
32
+ fingerprint += '\n';
33
+ }
34
+ if (this.wfp.length + fingerprint.length >= this.scannerCfg.WFP_FILE_MAX_SIZE) {
35
+ this.sendFingerprint(new FingerprintPacket(this.wfp, this.folderRoot));
36
+ this.wfp = '';
37
+ }
38
+ this.wfp += fingerprint;
39
+ if (this.wfp !== fingerprint)
40
+ return false;
41
+ return true;
42
+ }
43
+ finishWinnowing() {
44
+ if (this.wfp.length !== 0)
45
+ this.sendFingerprint(new FingerprintPacket(this.wfp, this.folderRoot));
46
+ this.pendingFiles = false;
47
+ this.emit(ScannerEvents.WINNOWING_FINISHED);
48
+ }
49
+ sendFingerprint(fingerprintPacket) {
50
+ this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPacket);
51
+ }
52
+ sendLog(logMsg) {
53
+ this.emit(ScannerEvents.WINNOWER_LOG, logMsg);
54
+ }
55
+ sendError(errorMsg) {
56
+ this.emit(ScannerEvents.ERROR, new Error(errorMsg));
57
+ }
58
+ setWinnowingMode(mode) {
59
+ this.winnowingMode = mode;
60
+ }
61
+ hasPendingFiles() {
62
+ return this.pendingFiles;
63
+ }
64
+ }
65
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwUHJvdmlkZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi9zcmMvbGliL3NjYW5uZXIvV2ZwUHJvdmlkZXIvV2ZwUHJvdmlkZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLFFBQVEsQ0FBQztBQUV0QyxPQUFPLEVBQUUsYUFBYSxFQUFnQixhQUFhLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUM3RSxPQUFPLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQVV4RCxNQUFNLE9BQWdCLFdBQVksU0FBUSxZQUFZO0lBRTFDLEdBQUcsQ0FBUztJQUNaLFVBQVUsQ0FBYTtJQUN2QixVQUFVLENBQVM7SUFDbkIsTUFBTSxDQUFTO0lBQ2YsWUFBWSxDQUFVO0lBQ3RCLGFBQWEsQ0FBZ0I7SUFFN0IsSUFBSTtRQUNaLElBQUksQ0FBQyxHQUFHLEdBQUcsRUFBRSxDQUFDO1FBQ2QsSUFBSSxDQUFDLFVBQVUsR0FBRyxFQUFFLENBQUM7UUFDckIsSUFBSSxDQUFDLFlBQVksR0FBRyxLQUFLLENBQUM7UUFDMUIsSUFBSSxDQUFDLGFBQWEsR0FBRyxhQUFhLENBQUMsY0FBYyxDQUFDO0lBQ3BELENBQUM7SUFFRCxnRUFBZ0U7SUFDdEQsaUJBQWlCLENBQUMsV0FBbUI7UUFDN0Msd0ZBQXdGO1FBQ3hGLElBQUksV0FBVyxDQUFDLE1BQU0sR0FBRyxJQUFJLENBQUMsVUFBVSxDQUFDLGlCQUFpQixFQUFFO1lBQzFELElBQUkscUJBQXFCLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxpQkFBaUIsQ0FBQztZQUM5RCxJQUFJLHNCQUFzQixHQUFHLElBQUksQ0FBQztZQUNsQyxPQUFPLHNCQUFzQixFQUFFO2dCQUM3QixJQUFJLFdBQVcsQ0FBQyxxQkFBcUIsQ0FBQyxLQUFLLElBQUk7b0JBQUUsc0JBQXNCLEdBQUcsS0FBSyxDQUFDO2dCQUNoRixxQkFBcUIsSUFBSSxDQUFDLENBQUM7YUFDNUI7WUFDRCxxQkFBcUIsSUFBSSxDQUFDLENBQUM7WUFDM0IsNkNBQTZDO1lBQzdDLFdBQVcsR0FBRyxXQUFXLENBQUMsU0FBUyxDQUFDLENBQUMsRUFBRSxxQkFBcUIsQ0FBQyxDQUFDO1lBQzlELDZDQUE2QztZQUM3QyxXQUFXLElBQUksSUFBSSxDQUFDO1NBQ3JCO1FBRUQsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sR0FBRyxXQUFXLENBQUMsTUFBTSxJQUFJLElBQUksQ0FBQyxVQUFVLENBQUMsaUJBQWlCLEVBQUU7WUFDN0UsSUFBSSxDQUFDLGVBQWUsQ0FBQyxJQUFJLGlCQUFpQixDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUM7WUFDdkUsSUFBSSxDQUFDLEdBQUcsR0FBRyxFQUFFLENBQUM7U0FDZjtRQUNELElBQUksQ0FBQyxHQUFHLElBQUksV0FBVyxDQUFDO1FBRXhCLElBQUcsSUFBSSxDQUFDLEdBQUcsS0FBSyxXQUFXO1lBQUUsT0FBTyxLQUFLLENBQUM7UUFDMUMsT0FBTyxJQUFJLENBQUM7SUFDZCxDQUFDO0lBRVMsZUFBZTtRQUN2QixJQUFJLElBQUksQ0FBQyxHQUFHLENBQUMsTUFBTSxLQUFLLENBQUM7WUFBRSxJQUFJLENBQUMsZUFBZSxDQUFDLElBQUksaUJBQWlCLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUMsQ0FBQztRQUNsRyxJQUFJLENBQUMsWUFBWSxHQUFHLEtBQUssQ0FBQztRQUMxQixJQUFJLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDO0lBQzlDLENBQUM7SUFHUyxlQUFlLENBQUMsaUJBQW9DO1FBQzVELElBQUksQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLHFCQUFxQixFQUFFLGlCQUFpQixDQUFDLENBQUM7SUFDcEUsQ0FBQztJQUVTLE9BQU8sQ0FBQyxNQUFjO1FBQzlCLElBQUksQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLFlBQVksRUFBRSxNQUFNLENBQUMsQ0FBQztJQUNoRCxDQUFDO0lBRVMsU0FBUyxDQUFDLFFBQWdCO1FBQ2xDLElBQUksQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLEtBQUssRUFBRSxJQUFJLEtBQUssQ0FBQyxRQUFRLENBQUMsQ0FBQyxDQUFDO0lBQ3RELENBQUM7SUFFUyxnQkFBZ0IsQ0FBQyxJQUFtQjtRQUM1QyxJQUFJLENBQUMsYUFBYSxHQUFHLElBQUksQ0FBQztJQUM1QixDQUFDO0lBUU0sZUFBZTtRQUNwQixPQUFPLElBQUksQ0FBQyxZQUFZLENBQUM7SUFDM0IsQ0FBQztDQUVGIn0=
@@ -0,0 +1,20 @@
1
+ import { IWfpProviderInput, WfpProvider } from "../WfpProvider";
2
+ import { ScannerCfg } from "../../ScannerCfg";
3
+ export declare class WfpSplitter extends WfpProvider {
4
+ private continue;
5
+ private wfpStream;
6
+ private chunkDataRead;
7
+ private fingerprints;
8
+ private fingerprintIndex;
9
+ private timer;
10
+ constructor(scannerCfg?: ScannerCfg);
11
+ start(params: IWfpProviderInput): void;
12
+ stop(): void;
13
+ pause(): void;
14
+ resume(): void;
15
+ private sendFingerprints;
16
+ private stopSendFingerprints;
17
+ private streamBufferFlush;
18
+ private setStreamListeners;
19
+ private splitFingerprints;
20
+ }
@@ -0,0 +1,93 @@
1
+ import { WfpProvider } from "../WfpProvider";
2
+ import fs from "fs";
3
+ import { ScannerCfg } from "../../ScannerCfg";
4
+ export class WfpSplitter extends WfpProvider {
5
+ continue;
6
+ wfpStream;
7
+ chunkDataRead;
8
+ fingerprints;
9
+ fingerprintIndex;
10
+ timer;
11
+ constructor(scannerCfg = new ScannerCfg()) {
12
+ super();
13
+ this.scannerCfg = scannerCfg;
14
+ }
15
+ start(params) {
16
+ this.sendLog('[ SCANNER ]: WFP Splitter starting...');
17
+ this.init();
18
+ this.pendingFiles = true;
19
+ this.chunkDataRead = '';
20
+ this.fingerprints = [];
21
+ this.continue = true;
22
+ this.fingerprintIndex = 0;
23
+ const wfpPath = params.wfpPath;
24
+ if (!wfpPath)
25
+ this.sendError('WFP path is not defined');
26
+ this.wfpStream = fs.createReadStream(wfpPath, { encoding: 'utf8' });
27
+ this.setStreamListeners();
28
+ }
29
+ stop() {
30
+ this.continue = false;
31
+ }
32
+ pause() {
33
+ this.sendLog('[ SCANNER ]: WFP Splitter paused...');
34
+ this.continue = false;
35
+ }
36
+ resume() {
37
+ this.sendLog('[ SCANNER ]: WFP Splitter resumed...');
38
+ this.continue = true;
39
+ this.sendFingerprints();
40
+ this.streamBufferFlush();
41
+ }
42
+ sendFingerprints() {
43
+ if (this.timer === undefined) {
44
+ this.timer = setInterval(() => {
45
+ if (this.fingerprintIndex < this.fingerprints.length && this.continue) {
46
+ this.fingerprintPacker(this.fingerprints[this.fingerprintIndex]);
47
+ this.fingerprintIndex++;
48
+ }
49
+ else {
50
+ this.stopSendFingerprints();
51
+ }
52
+ });
53
+ }
54
+ }
55
+ stopSendFingerprints() {
56
+ clearInterval(this.timer);
57
+ this.timer = undefined;
58
+ if (this.fingerprintIndex === this.fingerprints.length) {
59
+ this.finishWinnowing();
60
+ this.sendLog('[ SCANNER ]: WFP Splitter finished...');
61
+ }
62
+ }
63
+ streamBufferFlush() {
64
+ // Use a loop to make sure we read all currently available data
65
+ while (this.continue && null !== (this.chunkDataRead = this.wfpStream.read(1 * 1024 * 1024))) { // Read chunks of 1MB
66
+ // Extract the first portion of the chunk until the first file=
67
+ // Then append the chunk to the last fingerprint.
68
+ if (!this.chunkDataRead.startsWith('file=')) {
69
+ const chunkOfData = this.chunkDataRead.substring(0, this.chunkDataRead.indexOf('file='));
70
+ this.fingerprints[this.fingerprints.length - 1] += chunkOfData;
71
+ }
72
+ this.fingerprints = [...this.fingerprints, ...this.splitFingerprints(this.chunkDataRead)];
73
+ this.sendFingerprints();
74
+ }
75
+ }
76
+ setStreamListeners() {
77
+ this.wfpStream.on('readable', () => {
78
+ this.streamBufferFlush();
79
+ });
80
+ this.wfpStream.on('end', () => { });
81
+ }
82
+ splitFingerprints(chunkOfData) {
83
+ const reg = /file=/g;
84
+ const blocksIndex = [...((chunkOfData).matchAll(reg))].map(x => x.index);
85
+ const fingerprints = [];
86
+ for (let i = 0; i < blocksIndex.length; i++) {
87
+ const fingerprint = chunkOfData.substring(blocksIndex[i], blocksIndex[i + 1]);
88
+ fingerprints.push(fingerprint);
89
+ }
90
+ return fingerprints;
91
+ }
92
+ }
93
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwU3BsaXR0ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi8uLi9zcmMvbGliL3NjYW5uZXIvV2ZwUHJvdmlkZXIvV2ZwU3BsaXR0ZXIvV2ZwU3BsaXR0ZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFxQixXQUFXLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUNoRSxPQUFPLEVBQUUsTUFBTSxJQUFJLENBQUM7QUFDcEIsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRzlDLE1BQU0sT0FBTyxXQUFZLFNBQVEsV0FBVztJQUVsQyxRQUFRLENBQVU7SUFFbEIsU0FBUyxDQUFXO0lBRXBCLGFBQWEsQ0FBUztJQUV0QixZQUFZLENBQWdCO0lBRTVCLGdCQUFnQixDQUFTO0lBRXpCLEtBQUssQ0FBZ0M7SUFFN0MsWUFBWSxVQUFVLEdBQUcsSUFBSSxVQUFVLEVBQUU7UUFDdkMsS0FBSyxFQUFFLENBQUM7UUFDUixJQUFJLENBQUMsVUFBVSxHQUFHLFVBQVUsQ0FBQztJQUMvQixDQUFDO0lBRU0sS0FBSyxDQUFDLE1BQXlCO1FBQ3BDLElBQUksQ0FBQyxPQUFPLENBQUMsdUNBQXVDLENBQUMsQ0FBQztRQUV0RCxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDWixJQUFJLENBQUMsWUFBWSxHQUFHLElBQUksQ0FBQztRQUN6QixJQUFJLENBQUMsYUFBYSxHQUFHLEVBQUUsQ0FBQztRQUN4QixJQUFJLENBQUMsWUFBWSxHQUFHLEVBQUUsQ0FBQztRQUN2QixJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQztRQUNyQixJQUFJLENBQUMsZ0JBQWdCLEdBQUcsQ0FBQyxDQUFDO1FBQzFCLE1BQU0sT0FBTyxHQUFHLE1BQU0sQ0FBQyxPQUFPLENBQUM7UUFDL0IsSUFBSSxDQUFDLE9BQU87WUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLHlCQUF5QixDQUFDLENBQUM7UUFFeEQsSUFBSSxDQUFDLFNBQVMsR0FBRyxFQUFFLENBQUMsZ0JBQWdCLENBQUMsT0FBTyxFQUFFLEVBQUUsUUFBUSxFQUFFLE1BQU0sRUFBRSxDQUFDLENBQUM7UUFDcEUsSUFBSSxDQUFDLGtCQUFrQixFQUFFLENBQUM7SUFDNUIsQ0FBQztJQUVNLElBQUk7UUFDVCxJQUFJLENBQUMsUUFBUSxHQUFHLEtBQUssQ0FBQztJQUN4QixDQUFDO0lBRU0sS0FBSztRQUNWLElBQUksQ0FBQyxPQUFPLENBQUMscUNBQXFDLENBQUMsQ0FBQTtRQUNuRCxJQUFJLENBQUMsUUFBUSxHQUFHLEtBQUssQ0FBQztJQUN4QixDQUFDO0lBRU0sTUFBTTtRQUNYLElBQUksQ0FBQyxPQUFPLENBQUMsc0NBQXNDLENBQUMsQ0FBQTtRQUNwRCxJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQztRQUNyQixJQUFJLENBQUMsZ0JBQWdCLEVBQUUsQ0FBQztRQUN4QixJQUFJLENBQUMsaUJBQWlCLEVBQUUsQ0FBQztJQUMzQixDQUFDO0lBR08sZ0JBQWdCO1FBQ3RCLElBQUcsSUFBSSxDQUFDLEtBQUssS0FBSyxTQUFTLEVBQUU7WUFDM0IsSUFBSSxDQUFDLEtBQUssR0FBRyxXQUFXLENBQUMsR0FBRyxFQUFFO2dCQUM1QixJQUFHLElBQUksQ0FBQyxnQkFBZ0IsR0FBRyxJQUFJLENBQUMsWUFBWSxDQUFDLE1BQU0sSUFBSSxJQUFJLENBQUMsUUFBUSxFQUFFO29CQUNwRSxJQUFJLENBQUMsaUJBQWlCLENBQUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxJQUFJLENBQUMsZ0JBQWdCLENBQUMsQ0FBQyxDQUFDO29CQUNqRSxJQUFJLENBQUMsZ0JBQWdCLEVBQUUsQ0FBQztpQkFDekI7cUJBQU07b0JBQ0wsSUFBSSxDQUFDLG9CQUFvQixFQUFFLENBQUM7aUJBQzdCO1lBQ0gsQ0FBQyxDQUFDLENBQUM7U0FDSjtJQUdILENBQUM7SUFFTyxvQkFBb0I7UUFDMUIsYUFBYSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUMxQixJQUFJLENBQUMsS0FBSyxHQUFHLFNBQVMsQ0FBQztRQUN2QixJQUFHLElBQUksQ0FBQyxnQkFBZ0IsS0FBSyxJQUFJLENBQUMsWUFBWSxDQUFDLE1BQU0sRUFBQztZQUNwRCxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7WUFDdkIsSUFBSSxDQUFDLE9BQU8sQ0FBQyx1Q0FBdUMsQ0FBQyxDQUFDO1NBQ3ZEO0lBQ0gsQ0FBQztJQUVPLGlCQUFpQjtRQUN2QiwrREFBK0Q7UUFDL0QsT0FBTyxJQUFJLENBQUMsUUFBUSxJQUFJLElBQUksS0FBSyxDQUFDLElBQUksQ0FBQyxhQUFhLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxHQUFHLElBQUksR0FBRyxJQUFJLENBQUMsQ0FBQyxFQUFFLEVBQUcscUJBQXFCO1lBR3BILCtEQUErRDtZQUMvRCxpREFBaUQ7WUFDakQsSUFBSSxDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsVUFBVSxDQUFDLE9BQU8sQ0FBQyxFQUFFO2dCQUMzQyxNQUFNLFdBQVcsR0FBRyxJQUFJLENBQUMsYUFBYSxDQUFDLFNBQVMsQ0FBQyxDQUFDLEVBQUUsSUFBSSxDQUFDLGFBQWEsQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQztnQkFDekYsSUFBSSxDQUFDLFlBQVksQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLE1BQU0sR0FBQyxDQUFDLENBQUMsSUFBSSxXQUFXLENBQUM7YUFDOUQ7WUFDRCxJQUFJLENBQUMsWUFBWSxHQUFHLENBQUMsR0FBRyxJQUFJLENBQUMsWUFBWSxFQUFFLEdBQUcsSUFBSSxDQUFDLGlCQUFpQixDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsQ0FBQyxDQUFDO1lBQzFGLElBQUksQ0FBQyxnQkFBZ0IsRUFBRSxDQUFDO1NBQ3pCO0lBQ0gsQ0FBQztJQUVPLGtCQUFrQjtRQUN4QixJQUFJLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQyxVQUFVLEVBQUUsR0FBRyxFQUFFO1lBQ2pDLElBQUksQ0FBQyxpQkFBaUIsRUFBRSxDQUFDO1FBQzNCLENBQUMsQ0FBQyxDQUFDO1FBRUgsSUFBSSxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUMsS0FBSyxFQUFFLEdBQUcsRUFBRSxHQUFFLENBQUMsQ0FBQyxDQUFDO0lBQ3JDLENBQUM7SUFFTyxpQkFBaUIsQ0FBQyxXQUFrQjtRQUMxQyxNQUFNLEdBQUcsR0FBRyxRQUFRLENBQUM7UUFDckIsTUFBTSxXQUFXLEdBQUcsQ0FBQyxHQUFHLENBQUMsQ0FBQyxXQUFXLENBQUMsQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUN6RSxNQUFNLFlBQVksR0FBRyxFQUFFLENBQUM7UUFDeEIsS0FBSyxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxHQUFHLFdBQVcsQ0FBQyxNQUFNLEVBQUcsQ0FBQyxFQUFFLEVBQUU7WUFDNUMsTUFBTSxXQUFXLEdBQUcsV0FBVyxDQUFDLFNBQVMsQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLEVBQUUsV0FBVyxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDO1lBQzlFLFlBQVksQ0FBQyxJQUFJLENBQUMsV0FBVyxDQUFDLENBQUM7U0FDaEM7UUFDRCxPQUFPLFlBQVksQ0FBQztJQUN0QixDQUFDO0NBR0YifQ==
@@ -27,4 +27,4 @@ export class WinnowerExtractor {
27
27
  return res;
28
28
  }
29
29
  }
30
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2lubm93ZXJFeHRyYWN0b3IuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi9zcmMvbGliL3NjYW5uZXIvV2lubm93ZXIvV2lubm93ZXJFeHRyYWN0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLE1BQU0sSUFBSSxDQUFDO0FBRXBCLE1BQU0sT0FBTyxpQkFBaUI7SUFFcEIsU0FBUyxDQUFTO0lBRWxCLGlCQUFpQixDQUFTO0lBRTFCLFdBQVcsQ0FBZ0I7SUFFM0IsWUFBWSxDQUFTO0lBRTdCO1FBQ0UsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ2QsQ0FBQztJQUVPLElBQUk7UUFDVixJQUFJLENBQUMsU0FBUyxHQUFHLEVBQUUsQ0FBQztRQUNwQixJQUFJLENBQUMsWUFBWSxHQUFHLENBQUMsQ0FBQztJQUN4QixDQUFDO0lBRU0sUUFBUSxDQUFDLElBQVk7UUFDMUIsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ1osSUFBSSxDQUFDLGlCQUFpQixHQUFHLElBQUksQ0FBQztRQUM5QixJQUFJLENBQUMsU0FBUyxHQUFHLEVBQUUsQ0FBQyxZQUFZLENBQUMsSUFBSSxFQUFFLE9BQU8sQ0FBQyxDQUFDO1FBQ2hELE1BQU0sR0FBRyxHQUFHLFFBQVEsQ0FBQztRQUNyQixJQUFJLENBQUMsV0FBVyxHQUFHLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBRTdFLENBQUM7SUFBQSxDQUFDO0lBRUssZUFBZTtRQUNwQixJQUFJLEdBQUcsR0FBRyxFQUFFLENBQUM7UUFDYixJQUFJLElBQUksQ0FBQyxZQUFZLEdBQUcsSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEdBQUMsQ0FBQztZQUMvQyxHQUFHLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLEVBQUUsSUFBSSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsWUFBWSxHQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFDN0csSUFBSSxDQUFDLFlBQVksSUFBRSxDQUFDLENBQUM7UUFDckIsT0FBTyxHQUFHLENBQUM7SUFDYixDQUFDO0NBQ0YifQ==
30
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2lubm93ZXJFeHRyYWN0b3IuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi9zcmMvbGliL3NjYW5uZXIvV2lubm93ZXIvV2lubm93ZXJFeHRyYWN0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLE1BQU0sSUFBSSxDQUFDO0FBRXBCLE1BQU0sT0FBTyxpQkFBaUI7SUFFcEIsU0FBUyxDQUFTO0lBRWxCLGlCQUFpQixDQUFTO0lBRTFCLFdBQVcsQ0FBZ0I7SUFFM0IsWUFBWSxDQUFTO0lBRTdCO1FBQ0UsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ2QsQ0FBQztJQUVPLElBQUk7UUFDVixJQUFJLENBQUMsU0FBUyxHQUFHLEVBQUUsQ0FBQztRQUNwQixJQUFJLENBQUMsWUFBWSxHQUFHLENBQUMsQ0FBQztJQUN4QixDQUFDO0lBRU0sUUFBUSxDQUFDLElBQVk7UUFDMUIsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ1osSUFBSSxDQUFDLGlCQUFpQixHQUFHLElBQUksQ0FBQztRQUM5QixJQUFJLENBQUMsU0FBUyxHQUFHLEVBQUUsQ0FBQyxZQUFZLENBQUMsSUFBSSxFQUFFLE9BQU8sQ0FBQyxDQUFDO1FBQ2hELE1BQU0sR0FBRyxHQUFHLFFBQVEsQ0FBQztRQUNyQixJQUFJLENBQUMsV0FBVyxHQUFHLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQzdFLENBQUM7SUFBQSxDQUFDO0lBRUssZUFBZTtRQUNwQixJQUFJLEdBQUcsR0FBRyxFQUFFLENBQUM7UUFDYixJQUFJLElBQUksQ0FBQyxZQUFZLEdBQUcsSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEdBQUMsQ0FBQztZQUMvQyxHQUFHLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLEVBQUUsSUFBSSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsWUFBWSxHQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFDN0csSUFBSSxDQUFDLFlBQVksSUFBRSxDQUFDLENBQUM7UUFDckIsT0FBTyxHQUFHLENBQUM7SUFDYixDQUFDO0NBQ0YifQ==