querysub 0.719.0 → 0.721.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/3-path-functions/pathFunctionLoader.ts +42 -15
- package/src/deployManager/components/ProcessesView.tsx +4 -1
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +7 -19
- package/src/diagnostics/logs/errorNotifications2/ErrorWarning.tsx +4 -5
- package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +27 -47
- package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +168 -0
package/package.json
CHANGED
|
@@ -12,7 +12,7 @@ import { MaybePromise } from "socket-function/src/types";
|
|
|
12
12
|
import yargs from "yargs";
|
|
13
13
|
import { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
|
|
14
14
|
import { MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
|
|
15
|
-
import { isNode, isNodeTrue, nextId, timeInSecond } from "socket-function/src/misc";
|
|
15
|
+
import { isNode, isNodeTrue, nextId, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
16
16
|
import { getPathStr2, getPathStr3 } from "../path";
|
|
17
17
|
import { consistentHash } from "../misc/hash";
|
|
18
18
|
import { setExternalHotReloading } from "socket-function/hot/HotReloadController";
|
|
@@ -22,10 +22,15 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
|
|
|
22
22
|
import { getControllerNodeId, getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
|
|
23
23
|
import { sha256 } from "js-sha256";
|
|
24
24
|
import os from "os";
|
|
25
|
-
import { formatTime } from "socket-function/src/formatting/format";
|
|
25
|
+
import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
|
|
26
26
|
import path from "path";
|
|
27
27
|
import pathMod from "path";
|
|
28
28
|
|
|
29
|
+
const MODULE_LOAD_ERROR_RETRY_DELAY = timeInMinute * 5;
|
|
30
|
+
const YARN_INSTALL_INITIAL_RETRY_DELAY = timeInSecond * 10;
|
|
31
|
+
const YARN_INSTALL_MAX_RETRY_DELAY = timeInMinute;
|
|
32
|
+
const YARN_INSTALL_MAX_ATTEMPTS = 10;
|
|
33
|
+
|
|
29
34
|
export type LoadFunctionSpec = {
|
|
30
35
|
gitURL: string;
|
|
31
36
|
gitRef: string;
|
|
@@ -87,6 +92,7 @@ export function getModuleFromSpec(spec: FunctionSpec): MaybePromise<NodeJS.Modul
|
|
|
87
92
|
let moduleCache = new Map<string, {
|
|
88
93
|
result: MaybePromise<NodeJS.Module>;
|
|
89
94
|
error: Error | undefined;
|
|
95
|
+
errorTime: number;
|
|
90
96
|
}>();
|
|
91
97
|
function getSpecKey(spec: LoadFunctionSpec) {
|
|
92
98
|
// Only include these specific keys AND order them, so the hash hits no matter how this function is called
|
|
@@ -104,18 +110,23 @@ export function getModuleFromConfig(spec: LoadFunctionSpec): MaybePromise<NodeJS
|
|
|
104
110
|
let value = moduleCache.get(key);
|
|
105
111
|
if (!value) {
|
|
106
112
|
let promise = getModuleFromSpecBase(spec);
|
|
107
|
-
value = { result: promise, error: undefined };
|
|
113
|
+
value = { result: promise, error: undefined, errorTime: 0 };
|
|
108
114
|
moduleCache.set(key, value);
|
|
109
115
|
promise.then(
|
|
110
116
|
result => {
|
|
111
|
-
moduleCache.set(key, { result, error: undefined });
|
|
117
|
+
moduleCache.set(key, { result, error: undefined, errorTime: 0 });
|
|
112
118
|
},
|
|
113
119
|
error => {
|
|
114
|
-
moduleCache.set(key, { result:
|
|
120
|
+
moduleCache.set(key, { result: promise, error, errorTime: Date.now() });
|
|
115
121
|
}
|
|
116
122
|
);
|
|
117
123
|
}
|
|
118
|
-
if (value.error)
|
|
124
|
+
if (value.error) {
|
|
125
|
+
if (Date.now() - value.errorTime < MODULE_LOAD_ERROR_RETRY_DELAY) throw value.error;
|
|
126
|
+
console.log(red(`Retrying module load for ${key}, the previous attempt failed at ${formatDateTime(value.errorTime)}, more than ${formatTime(MODULE_LOAD_ERROR_RETRY_DELAY)} ago, with:\n${value.error.stack}`));
|
|
127
|
+
moduleCache.delete(key);
|
|
128
|
+
return getModuleFromConfig(spec);
|
|
129
|
+
}
|
|
119
130
|
return value.result;
|
|
120
131
|
}
|
|
121
132
|
|
|
@@ -466,17 +477,33 @@ const hotreloadIfChanged = batchFunction({ delay: 100, name: "hotreloadIfChanged
|
|
|
466
477
|
// A corrupt entry in the shared global yarn cache (e.g. "Extracting tar content ... the file appears to be
|
|
467
478
|
// corrupt") fails every install identically, because re-cloning the repo folder never touches the global
|
|
468
479
|
// cache. Detect that, purge the cache with `yarn cache clean`, and retry once so the loader heals itself.
|
|
480
|
+
// Retrying with delays could make this function very slow. However, the alternative is just all the function calls failing, which is the worst case scenario.
|
|
469
481
|
async function yarnInstallWithHealing(repoPath: string): Promise<void> {
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
482
|
+
let cleanedCache = false;
|
|
483
|
+
let failures = 0;
|
|
484
|
+
let retryDelay = YARN_INSTALL_INITIAL_RETRY_DELAY;
|
|
485
|
+
while (true) {
|
|
486
|
+
try {
|
|
487
|
+
await executeCommand("yarn", ["install", "--mutex", "network"], { cwd: repoPath });
|
|
488
|
+
if (failures) {
|
|
489
|
+
console.log(blue(`yarn install in ${repoPath} succeeded after ${failures} failures`));
|
|
490
|
+
}
|
|
491
|
+
return;
|
|
492
|
+
} catch (e) {
|
|
493
|
+
failures++;
|
|
494
|
+
let errorText = String((e as Error)?.stack || e);
|
|
495
|
+
if (failures >= YARN_INSTALL_MAX_ATTEMPTS) throw e;
|
|
496
|
+
if (!cleanedCache && /appears to be corrupt|Extracting tar content/.test(errorText)) {
|
|
497
|
+
cleanedCache = true;
|
|
498
|
+
console.log(red(`yarn install failed with a corrupt yarn cache, cleaning cache and retrying:\n${errorText}`));
|
|
499
|
+
await executeCommand("yarn", ["cache", "clean"]);
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
console.log(red(`yarn install in ${repoPath} failed (failure ${failures}/${YARN_INSTALL_MAX_ATTEMPTS}), retrying in ${formatTime(retryDelay)}:\n${errorText}`));
|
|
503
|
+
await delay(retryDelay);
|
|
504
|
+
retryDelay = Math.min(retryDelay * 2, YARN_INSTALL_MAX_RETRY_DELAY);
|
|
505
|
+
}
|
|
478
506
|
}
|
|
479
|
-
await executeCommand("yarn", ["install", "--mutex", "network"], { cwd: repoPath });
|
|
480
507
|
}
|
|
481
508
|
|
|
482
509
|
async function which(command: string): Promise<string> {
|
|
@@ -8,6 +8,7 @@ import { sort, timeInSecond, nextId } from "socket-function/src/misc";
|
|
|
8
8
|
import { MachineController, watchProcessOutput, stopWatchingProcessOutput } from "../machineController";
|
|
9
9
|
import type { ProcessRecord } from "../processLogs";
|
|
10
10
|
import { Button } from "../../library-components/Button";
|
|
11
|
+
import { getMachineInfoLive, getMachineNameLive } from "../machineSchema";
|
|
11
12
|
import { parseAnsiColors, rgbToHsl } from "../../diagnostics/logs/ansiFormat";
|
|
12
13
|
|
|
13
14
|
module.hotreload = true;
|
|
@@ -168,9 +169,11 @@ class MachineProcesses extends qreact.Component<{ machineId: string; applyNodeId
|
|
|
168
169
|
let historical = shown.filter(x => x.deadTime !== undefined);
|
|
169
170
|
sort(running, x => -x.startTime);
|
|
170
171
|
sort(historical, x => -(x.deadTime || 0));
|
|
172
|
+
let machineName = getMachineNameLive(this.props.machineId);
|
|
171
173
|
return <div className={css.vbox(8).fillWidth}>
|
|
172
174
|
<div className={css.hbox(10).wrap.alignItems("center")}>
|
|
173
|
-
<div className={css.boldStyle}>{
|
|
175
|
+
<div className={css.boldStyle + (!machineName && css.colorhsl(0, 0, 45))}>{machineName || "(unnamed machine)"}</div>
|
|
176
|
+
<div>{this.props.machineId} ({getMachineInfoLive(this.props.machineId)?.info["getExternalIP"]})</div>
|
|
174
177
|
<div className={css.colorhsl(0, 0, 45)}>{running.length} running</div>
|
|
175
178
|
{historical.length > 0 && <Button flavor="tiny" onClick={() => this.state.showHistorical = !this.state.showHistorical}>
|
|
176
179
|
{this.state.showHistorical && `Hide ${historical.length} historical` || `Show ${historical.length} historical`}
|
|
@@ -10,8 +10,7 @@ import { isPublic } from "../../../config";
|
|
|
10
10
|
import { MachineThreadInfo } from "../../MachineThreadInfo";
|
|
11
11
|
import { ErrorWarning } from "./ErrorWarning";
|
|
12
12
|
import { getErrorNotificationsManager } from "./errorWatcher";
|
|
13
|
-
import {
|
|
14
|
-
import { cacheLimited } from "socket-function/src/caching";
|
|
13
|
+
import { createPatternSetMatcher, getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
|
|
15
14
|
import { managementPageURL, showingManagementURL } from "../../managementPages";
|
|
16
15
|
import { searchTextURL, readProductionLogsURL } from "../IndexedLogs/LogViewerParams";
|
|
17
16
|
import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
|
|
@@ -57,13 +56,6 @@ function TicketButton(props: {
|
|
|
57
56
|
}
|
|
58
57
|
qreact.inline(TicketButton);
|
|
59
58
|
|
|
60
|
-
export let getMatcher = cacheLimited(100, (pattern: string) =>
|
|
61
|
-
createMatchesPattern(Buffer.from(normalizeMatchWhitespace(pattern)), false)
|
|
62
|
-
);
|
|
63
|
-
|
|
64
|
-
export function getMatchBuffer(datum: LogDatum): Buffer {
|
|
65
|
-
return Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum)));
|
|
66
|
-
}
|
|
67
59
|
|
|
68
60
|
export class LogDatumRenderer extends qreact.Component<{
|
|
69
61
|
datum: LogDatum;
|
|
@@ -81,15 +73,11 @@ export class LogDatumRenderer extends qreact.Component<{
|
|
|
81
73
|
let datum = this.props.datum;
|
|
82
74
|
let mainMessage = datum.param0 && String(datum.param0) || "(no message)";
|
|
83
75
|
|
|
84
|
-
let
|
|
76
|
+
let suppressionEntries = this.manager.state.suppressionEntries;
|
|
77
|
+
let matchingSuppressions: typeof suppressionEntries = [];
|
|
85
78
|
if (!this.manager.state.isLoading && !this.props.hideSuppression) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
let errorBuffer = getMatchBuffer(datum);
|
|
89
|
-
if (matcher(errorBuffer)) {
|
|
90
|
-
matchingSuppressions.push(suppression);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
79
|
+
let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(datum));
|
|
80
|
+
matchingSuppressions = matchedIndexes.map(index => suppressionEntries[index]);
|
|
93
81
|
}
|
|
94
82
|
|
|
95
83
|
let timedOutMatches = matchingSuppressions.filter(s => datum.time > s.timeout);
|
|
@@ -464,8 +452,8 @@ class SuppressionItem extends qreact.Component<{
|
|
|
464
452
|
let example = this.props.matchData?.examples[0];
|
|
465
453
|
if (example) return example;
|
|
466
454
|
// No example recorded for this suppression — find any live error matching the pattern, so the ticket stores a real, full log entry instead of a synthetic stub.
|
|
467
|
-
let matcher =
|
|
468
|
-
let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)));
|
|
455
|
+
let matcher = createPatternSetMatcher([suppression.pattern]);
|
|
456
|
+
let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)).length > 0);
|
|
469
457
|
return fallback || {
|
|
470
458
|
time: Date.now(),
|
|
471
459
|
__LOG_TYPE: "error",
|
|
@@ -4,7 +4,8 @@ import { css } from "typesafecss";
|
|
|
4
4
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
5
5
|
import { ATag } from "../../../library-components/ATag";
|
|
6
6
|
import { managementPageURL, showingManagementURL } from "../../managementPages";
|
|
7
|
-
import {
|
|
7
|
+
import { LogDatumRenderer } from "./ErrorNotificationPage";
|
|
8
|
+
import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
|
|
8
9
|
import { getErrorNotificationsManager } from "./errorWatcher";
|
|
9
10
|
import { LogDatum } from "../diskLogger";
|
|
10
11
|
import { TicketsController } from "../errorTickets/tickets";
|
|
@@ -57,10 +58,8 @@ export class ErrorWarning extends qreact.Component {
|
|
|
57
58
|
if (!ticket.suppressionPattern) continue;
|
|
58
59
|
openTicketPatterns.push(ticket.suppressionPattern);
|
|
59
60
|
}
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
return openTicketPatterns.some(pattern => getMatcher(pattern)(errorBuffer));
|
|
63
|
-
};
|
|
61
|
+
const openTicketMatcher = getPatternSetMatcher(openTicketPatterns);
|
|
62
|
+
const isCoveredByOpenTicket = (datum: LogDatum) => openTicketMatcher(getMatchBuffer(datum)).length > 0;
|
|
64
63
|
|
|
65
64
|
let errorWarning: preact.ComponentChild = undefined;
|
|
66
65
|
let firstError = !this.manager.state.isLoading && this.manager.state.unmatchedErrors.find(x => !isCoveredByOpenTicket(x)) || undefined;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { lazy } from "socket-function/src/caching";
|
|
2
2
|
import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
|
|
3
3
|
import { Querysub } from "../../../4-querysub/Querysub";
|
|
4
4
|
import { getDomain, isPublic } from "../../../config";
|
|
5
5
|
import { MachineInfo } from "../../../deployManager/machineSchema";
|
|
6
|
-
import {
|
|
6
|
+
import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
|
|
7
7
|
import { LogDatum, getErrorLogs } from "../diskLogger";
|
|
8
8
|
import { watchAllValues } from "./logWatcher";
|
|
9
9
|
import { batchFunction, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
@@ -55,8 +55,6 @@ export async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
|
|
|
55
55
|
return await suppression.values();
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
let getMatcher = cacheLimited(10000, (text: string) => createMatchesPattern(Buffer.from(normalizeMatchWhitespace(text)), false));
|
|
59
|
-
|
|
60
58
|
// The watch loop checks every incoming error against the suppression list. Reading the suppression archive once per error produced thousands of storage listings per minute, so the hot path is served from a cached copy: the first pass tolerates 15 minute staleness, and only errors that pass unsuppressed force a fresher (30 second) re-read.
|
|
61
59
|
const SUPPRESSION_CACHE_TTL = timeInMinute * 15;
|
|
62
60
|
const SUPPRESSION_FRESH_CACHE_TTL = 1000 * 30;
|
|
@@ -87,16 +85,9 @@ function invalidateSuppressionEntriesCaches() {
|
|
|
87
85
|
suppressionFreshCacheTime = 0;
|
|
88
86
|
}
|
|
89
87
|
|
|
90
|
-
// Side-effect-free version of applySuppression's "suppressed" verdict, for deciding which entry set the watch loop should apply against.
|
|
91
88
|
function isSuppressedByAny(error: LogDatum, suppressionEntries: SuppressionEntry[]): boolean {
|
|
92
|
-
let
|
|
93
|
-
|
|
94
|
-
let isMatch = getMatcher(suppressionEntry.pattern);
|
|
95
|
-
if (isMatch(errorBuffer) && error.time <= suppressionEntry.timeout) {
|
|
96
|
-
return true;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
return false;
|
|
89
|
+
let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(error));
|
|
90
|
+
return matchedIndexes.some(index => error.time <= suppressionEntries[index].timeout);
|
|
100
91
|
}
|
|
101
92
|
|
|
102
93
|
const MAX_UNMATCHED = 10000;
|
|
@@ -122,13 +113,22 @@ export function getChunkEndTime(chunk: number) {
|
|
|
122
113
|
return chunk + chunkUnit;
|
|
123
114
|
}
|
|
124
115
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
let
|
|
129
|
-
let
|
|
130
|
-
|
|
116
|
+
function applySuppressions(error: LogDatum, suppressionEntries: SuppressionEntry[]): { suppressed: boolean; matchedIds: string[] } {
|
|
117
|
+
let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(error));
|
|
118
|
+
let suppressed = false;
|
|
119
|
+
let matchedIds: string[] = [];
|
|
120
|
+
for (let index of matchedIndexes) {
|
|
121
|
+
let suppressionEntry = suppressionEntries[index];
|
|
122
|
+
recordSuppressionMatch(error, suppressionEntry);
|
|
123
|
+
matchedIds.push(suppressionEntry.id);
|
|
124
|
+
if (error.time <= suppressionEntry.timeout) {
|
|
125
|
+
suppressed = true;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return { suppressed, matchedIds };
|
|
129
|
+
}
|
|
131
130
|
|
|
131
|
+
function recordSuppressionMatch(error: LogDatum, suppressionEntry: SuppressionEntry) {
|
|
132
132
|
let history = suppressionMatches.get(suppressionEntry.id);
|
|
133
133
|
if (!history) {
|
|
134
134
|
history = {
|
|
@@ -154,12 +154,6 @@ function applySuppression(error: LogDatum, suppressionEntry: SuppressionEntry):
|
|
|
154
154
|
history.history.set(historyChunk, historyEntry);
|
|
155
155
|
}
|
|
156
156
|
historyEntry.count++;
|
|
157
|
-
|
|
158
|
-
if (error.time > suppressionEntry.timeout) {
|
|
159
|
-
return "matched-timeout-passed";
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
return "suppressed";
|
|
163
157
|
}
|
|
164
158
|
|
|
165
159
|
async function reapplySuppressions() {
|
|
@@ -168,15 +162,9 @@ async function reapplySuppressions() {
|
|
|
168
162
|
let changedSuppressionIds = new Set<string>();
|
|
169
163
|
|
|
170
164
|
for (let error of unmatchedErrors) {
|
|
171
|
-
let suppressed =
|
|
172
|
-
for (let
|
|
173
|
-
|
|
174
|
-
if (status === "suppressed" || status === "matched-timeout-passed") {
|
|
175
|
-
changedSuppressionIds.add(suppressionEntry.id);
|
|
176
|
-
}
|
|
177
|
-
if (status === "suppressed") {
|
|
178
|
-
suppressed = true;
|
|
179
|
-
}
|
|
165
|
+
let { suppressed, matchedIds } = applySuppressions(error, suppressionEntries);
|
|
166
|
+
for (let id of matchedIds) {
|
|
167
|
+
changedSuppressionIds.add(id);
|
|
180
168
|
}
|
|
181
169
|
if (!suppressed) {
|
|
182
170
|
newUnmatchedErrors.push(error);
|
|
@@ -355,19 +343,11 @@ export async function exposeErrorWatchService() {
|
|
|
355
343
|
if (!isSuppressedByAny(error, suppressionEntries)) {
|
|
356
344
|
suppressionEntries = await getSuppressionEntriesFresh();
|
|
357
345
|
}
|
|
358
|
-
let suppressed =
|
|
359
|
-
let
|
|
360
|
-
|
|
361
|
-
for (let suppressionEntry of suppressionEntries) {
|
|
362
|
-
let status = applySuppression(error, suppressionEntry);
|
|
363
|
-
if (status === "suppressed" || status === "matched-timeout-passed") {
|
|
364
|
-
matchedSuppressionId = suppressionEntry.id;
|
|
365
|
-
void ErrorNotificationService.triggerSuppressionUpdate(suppressionEntry.id);
|
|
366
|
-
}
|
|
367
|
-
if (status === "suppressed") {
|
|
368
|
-
suppressed = true;
|
|
369
|
-
}
|
|
346
|
+
let { suppressed, matchedIds } = applySuppressions(error, suppressionEntries);
|
|
347
|
+
for (let id of matchedIds) {
|
|
348
|
+
void ErrorNotificationService.triggerSuppressionUpdate(id);
|
|
370
349
|
}
|
|
350
|
+
let matchedSuppressionId: string | undefined = matchedIds[matchedIds.length - 1];
|
|
371
351
|
//console.log(`Watch loop received error, ${suppressed ? "suppressed" : "not suppressed"}, matched suppression id: ${matchedSuppressionId}: ${error.param0}`);
|
|
372
352
|
|
|
373
353
|
if (suppressed) continue;
|
|
@@ -392,7 +372,7 @@ export async function exposeErrorWatchService() {
|
|
|
392
372
|
|
|
393
373
|
await suppression.set(newEntry.id, newEntry);
|
|
394
374
|
invalidateSuppressionEntriesCaches();
|
|
395
|
-
|
|
375
|
+
applySuppressions(error, [newEntry]);
|
|
396
376
|
queueDiscordNotification(newEntry.id, 1, true);
|
|
397
377
|
|
|
398
378
|
void ErrorNotificationService.triggerSuppressionUpdate(newEntry.id);
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
|
|
2
|
+
import { LogDatum } from "../diskLogger";
|
|
3
|
+
|
|
4
|
+
const ALPHABET_SIZE = 256;
|
|
5
|
+
const MATCHER_CACHE_SIZE = 10;
|
|
6
|
+
const SEARCH_OR_CHAR = "|";
|
|
7
|
+
const SEARCH_AND_CHAR = "&";
|
|
8
|
+
const WILD_CARD_CHAR = "*";
|
|
9
|
+
|
|
10
|
+
type Structure = { type: "or" | "and"; parts: Structure[] } | { type: "wildcard"; parts: Buffer[] } | { type: "exact"; value: Buffer };
|
|
11
|
+
type IdStructure = { type: "or" | "and"; parts: IdStructure[] } | { type: "wildcard"; ids: number[]; parts: Buffer[] } | { type: "exact"; id: number };
|
|
12
|
+
|
|
13
|
+
export type PatternSetMatcher = (matchBuffer: Buffer) => number[];
|
|
14
|
+
|
|
15
|
+
let matchBuffers = new WeakMap<LogDatum, Buffer>();
|
|
16
|
+
export function getMatchBuffer(datum: LogDatum): Buffer {
|
|
17
|
+
let buffer = matchBuffers.get(datum);
|
|
18
|
+
if (buffer) return buffer;
|
|
19
|
+
buffer = lowercaseAscii(Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum))));
|
|
20
|
+
matchBuffers.set(datum, buffer);
|
|
21
|
+
return buffer;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
let matcherCache: { patterns: string[]; matcher: PatternSetMatcher }[] = [];
|
|
25
|
+
export function getPatternSetMatcher(patterns: string[]): PatternSetMatcher {
|
|
26
|
+
let cached = matcherCache.find(x => x.patterns.length === patterns.length && x.patterns.every((pattern, i) => pattern === patterns[i]));
|
|
27
|
+
if (cached) return cached.matcher;
|
|
28
|
+
let matcher = createPatternSetMatcher(patterns);
|
|
29
|
+
matcherCache.push({ patterns: patterns.slice(), matcher });
|
|
30
|
+
if (matcherCache.length > MATCHER_CACHE_SIZE) matcherCache.shift();
|
|
31
|
+
return matcher;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function createPatternSetMatcher(patterns: string[]): PatternSetMatcher {
|
|
35
|
+
let literalIds = new Map<string, number>();
|
|
36
|
+
let literals: Buffer[] = [];
|
|
37
|
+
function getLiteralId(value: Buffer): number {
|
|
38
|
+
let key = value.toString("latin1");
|
|
39
|
+
let id = literalIds.get(key);
|
|
40
|
+
if (id !== undefined) return id;
|
|
41
|
+
id = literals.length;
|
|
42
|
+
literalIds.set(key, id);
|
|
43
|
+
literals.push(value);
|
|
44
|
+
return id;
|
|
45
|
+
}
|
|
46
|
+
function toIds(structure: Structure): IdStructure {
|
|
47
|
+
if (structure.type === "exact") return { type: "exact", id: getLiteralId(structure.value) };
|
|
48
|
+
if (structure.type === "wildcard") return { type: "wildcard", ids: structure.parts.map(getLiteralId), parts: structure.parts };
|
|
49
|
+
return { type: structure.type, parts: structure.parts.map(toIds) };
|
|
50
|
+
}
|
|
51
|
+
let structures = patterns.map(pattern => toIds(compilePattern(normalizeMatchWhitespace(pattern))));
|
|
52
|
+
let automaton = buildAutomaton(literals);
|
|
53
|
+
let foundStamp = new Int32Array(literals.length);
|
|
54
|
+
let stamp = 0;
|
|
55
|
+
|
|
56
|
+
return function matchPatternSet(matchBuffer: Buffer): number[] {
|
|
57
|
+
stamp++;
|
|
58
|
+
for (let id of automaton.outputs[0]) {
|
|
59
|
+
foundStamp[id] = stamp;
|
|
60
|
+
}
|
|
61
|
+
let state = 0;
|
|
62
|
+
for (let i = 0; i < matchBuffer.length; i++) {
|
|
63
|
+
state = automaton.transitions[state * ALPHABET_SIZE + matchBuffer[i]];
|
|
64
|
+
let outputState = automaton.hasOwnOutput[state] && state || automaton.dictionaryLink[state];
|
|
65
|
+
while (outputState > 0) {
|
|
66
|
+
let outputs = automaton.outputs[outputState];
|
|
67
|
+
for (let j = 0; j < outputs.length; j++) {
|
|
68
|
+
foundStamp[outputs[j]] = stamp;
|
|
69
|
+
}
|
|
70
|
+
outputState = automaton.dictionaryLink[outputState];
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
let matched: number[] = [];
|
|
74
|
+
for (let patternIndex = 0; patternIndex < structures.length; patternIndex++) {
|
|
75
|
+
if (matchesWithIds(structures[patternIndex])) matched.push(patternIndex);
|
|
76
|
+
}
|
|
77
|
+
return matched;
|
|
78
|
+
|
|
79
|
+
function matchesWithIds(structure: IdStructure): boolean {
|
|
80
|
+
if (structure.type === "exact") return foundStamp[structure.id] === stamp;
|
|
81
|
+
if (structure.type === "or") return structure.parts.some(matchesWithIds);
|
|
82
|
+
if (structure.type === "and") return structure.parts.every(matchesWithIds);
|
|
83
|
+
if (!structure.ids.every(id => foundStamp[id] === stamp)) return false;
|
|
84
|
+
let position = 0;
|
|
85
|
+
for (let segment of structure.parts) {
|
|
86
|
+
let found = matchBuffer.indexOf(segment, position);
|
|
87
|
+
if (found === -1) return false;
|
|
88
|
+
position = found + segment.length;
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function lowercaseAscii(buffer: Buffer): Buffer {
|
|
96
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
97
|
+
let byte = buffer[i];
|
|
98
|
+
if (65 <= byte && byte <= 90) buffer[i] = byte + 32;
|
|
99
|
+
}
|
|
100
|
+
return buffer;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function compilePattern(pattern: string): Structure {
|
|
104
|
+
let lowered = lowercaseAscii(Buffer.from(pattern)).toString("utf8");
|
|
105
|
+
if (!lowered.includes(SEARCH_OR_CHAR) && !lowered.includes(SEARCH_AND_CHAR) && !lowered.includes(WILD_CARD_CHAR)) {
|
|
106
|
+
return { type: "exact", value: Buffer.from(lowered) };
|
|
107
|
+
}
|
|
108
|
+
return parseStructure(lowered);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function parseStructure(text: string): Structure {
|
|
112
|
+
let orParts = text.split(SEARCH_OR_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
113
|
+
if (orParts.length > 1) return { type: "or", parts: orParts.map(parseStructure) };
|
|
114
|
+
let andParts = text.split(SEARCH_AND_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
115
|
+
if (andParts.length > 1) return { type: "and", parts: andParts.map(parseStructure) };
|
|
116
|
+
let wildcardParts = text.split(WILD_CARD_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
117
|
+
if (wildcardParts.length > 1) return { type: "wildcard", parts: wildcardParts.map(x => Buffer.from(x)) };
|
|
118
|
+
return { type: "exact", value: Buffer.from(text) };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function buildAutomaton(literals: Buffer[]) {
|
|
122
|
+
let children: Map<number, number>[] = [new Map()];
|
|
123
|
+
let outputs: number[][] = [[]];
|
|
124
|
+
for (let literalIndex = 0; literalIndex < literals.length; literalIndex++) {
|
|
125
|
+
let state = 0;
|
|
126
|
+
for (let byte of literals[literalIndex]) {
|
|
127
|
+
let next = children[state].get(byte);
|
|
128
|
+
if (next === undefined) {
|
|
129
|
+
next = children.length;
|
|
130
|
+
children.push(new Map());
|
|
131
|
+
outputs.push([]);
|
|
132
|
+
children[state].set(byte, next);
|
|
133
|
+
}
|
|
134
|
+
state = next;
|
|
135
|
+
}
|
|
136
|
+
outputs[state].push(literalIndex);
|
|
137
|
+
}
|
|
138
|
+
let stateCount = children.length;
|
|
139
|
+
let transitions = new Int32Array(stateCount * ALPHABET_SIZE);
|
|
140
|
+
let failure = new Int32Array(stateCount);
|
|
141
|
+
let dictionaryLink = new Int32Array(stateCount);
|
|
142
|
+
let hasOwnOutput = new Uint8Array(stateCount);
|
|
143
|
+
for (let state = 0; state < stateCount; state++) {
|
|
144
|
+
if (outputs[state].length) hasOwnOutput[state] = 1;
|
|
145
|
+
}
|
|
146
|
+
let queue: number[] = [];
|
|
147
|
+
for (let byte = 0; byte < ALPHABET_SIZE; byte++) {
|
|
148
|
+
let child = children[0].get(byte);
|
|
149
|
+
transitions[byte] = child ?? 0;
|
|
150
|
+
if (child !== undefined) queue.push(child);
|
|
151
|
+
}
|
|
152
|
+
for (let queueIndex = 0; queueIndex < queue.length; queueIndex++) {
|
|
153
|
+
let state = queue[queueIndex];
|
|
154
|
+
let failState = failure[state];
|
|
155
|
+
dictionaryLink[state] = hasOwnOutput[failState] && failState || dictionaryLink[failState];
|
|
156
|
+
for (let byte = 0; byte < ALPHABET_SIZE; byte++) {
|
|
157
|
+
let child = children[state].get(byte);
|
|
158
|
+
if (child === undefined) {
|
|
159
|
+
transitions[state * ALPHABET_SIZE + byte] = transitions[failState * ALPHABET_SIZE + byte];
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
transitions[state * ALPHABET_SIZE + byte] = child;
|
|
163
|
+
failure[child] = transitions[failState * ALPHABET_SIZE + byte];
|
|
164
|
+
queue.push(child);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return { transitions, dictionaryLink, hasOwnOutput, outputs };
|
|
168
|
+
}
|