querysub 0.720.0 → 0.721.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "querysub",
3
- "version": "0.720.0",
3
+ "version": "0.721.0",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -10,8 +10,7 @@ import { isPublic } from "../../../config";
10
10
  import { MachineThreadInfo } from "../../MachineThreadInfo";
11
11
  import { ErrorWarning } from "./ErrorWarning";
12
12
  import { getErrorNotificationsManager } from "./errorWatcher";
13
- import { createMatchesPattern, normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
14
- import { cacheLimited } from "socket-function/src/caching";
13
+ import { createPatternSetMatcher, getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
15
14
  import { managementPageURL, showingManagementURL } from "../../managementPages";
16
15
  import { searchTextURL, readProductionLogsURL } from "../IndexedLogs/LogViewerParams";
17
16
  import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
@@ -57,13 +56,6 @@ function TicketButton(props: {
57
56
  }
58
57
  qreact.inline(TicketButton);
59
58
 
60
- export let getMatcher = cacheLimited(100, (pattern: string) =>
61
- createMatchesPattern(Buffer.from(normalizeMatchWhitespace(pattern)), false)
62
- );
63
-
64
- export function getMatchBuffer(datum: LogDatum): Buffer {
65
- return Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum)));
66
- }
67
59
 
68
60
  export class LogDatumRenderer extends qreact.Component<{
69
61
  datum: LogDatum;
@@ -81,15 +73,11 @@ export class LogDatumRenderer extends qreact.Component<{
81
73
  let datum = this.props.datum;
82
74
  let mainMessage = datum.param0 && String(datum.param0) || "(no message)";
83
75
 
84
- let matchingSuppressions = [];
76
+ let suppressionEntries = this.manager.state.suppressionEntries;
77
+ let matchingSuppressions: typeof suppressionEntries = [];
85
78
  if (!this.manager.state.isLoading && !this.props.hideSuppression) {
86
- for (let suppression of this.manager.state.suppressionEntries) {
87
- let matcher = getMatcher(suppression.pattern);
88
- let errorBuffer = getMatchBuffer(datum);
89
- if (matcher(errorBuffer)) {
90
- matchingSuppressions.push(suppression);
91
- }
92
- }
79
+ let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(datum));
80
+ matchingSuppressions = matchedIndexes.map(index => suppressionEntries[index]);
93
81
  }
94
82
 
95
83
  let timedOutMatches = matchingSuppressions.filter(s => datum.time > s.timeout);
@@ -464,8 +452,8 @@ class SuppressionItem extends qreact.Component<{
464
452
  let example = this.props.matchData?.examples[0];
465
453
  if (example) return example;
466
454
  // No example recorded for this suppression — find any live error matching the pattern, so the ticket stores a real, full log entry instead of a synthetic stub.
467
- let matcher = getMatcher(suppression.pattern);
468
- let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)));
455
+ let matcher = createPatternSetMatcher([suppression.pattern]);
456
+ let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)).length > 0);
469
457
  return fallback || {
470
458
  time: Date.now(),
471
459
  __LOG_TYPE: "error",
@@ -4,7 +4,8 @@ import { css } from "typesafecss";
4
4
  import { SocketFunction } from "socket-function/SocketFunction";
5
5
  import { ATag } from "../../../library-components/ATag";
6
6
  import { managementPageURL, showingManagementURL } from "../../managementPages";
7
- import { getMatcher, getMatchBuffer, LogDatumRenderer } from "./ErrorNotificationPage";
7
+ import { LogDatumRenderer } from "./ErrorNotificationPage";
8
+ import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
8
9
  import { getErrorNotificationsManager } from "./errorWatcher";
9
10
  import { LogDatum } from "../diskLogger";
10
11
  import { TicketsController } from "../errorTickets/tickets";
@@ -57,10 +58,8 @@ export class ErrorWarning extends qreact.Component {
57
58
  if (!ticket.suppressionPattern) continue;
58
59
  openTicketPatterns.push(ticket.suppressionPattern);
59
60
  }
60
- const isCoveredByOpenTicket = (datum: LogDatum) => {
61
- let errorBuffer = getMatchBuffer(datum);
62
- return openTicketPatterns.some(pattern => getMatcher(pattern)(errorBuffer));
63
- };
61
+ const openTicketMatcher = getPatternSetMatcher(openTicketPatterns);
62
+ const isCoveredByOpenTicket = (datum: LogDatum) => openTicketMatcher(getMatchBuffer(datum)).length > 0;
64
63
 
65
64
  let errorWarning: preact.ComponentChild = undefined;
66
65
  let firstError = !this.manager.state.isLoading && this.manager.state.unmatchedErrors.find(x => !isCoveredByOpenTicket(x)) || undefined;
@@ -1,9 +1,9 @@
1
- import { cacheLimited, lazy } from "socket-function/src/caching";
1
+ import { lazy } from "socket-function/src/caching";
2
2
  import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
3
3
  import { Querysub } from "../../../4-querysub/Querysub";
4
4
  import { getDomain, isPublic } from "../../../config";
5
5
  import { MachineInfo } from "../../../deployManager/machineSchema";
6
- import { createMatchesPattern, normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
6
+ import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
7
7
  import { LogDatum, getErrorLogs } from "../diskLogger";
8
8
  import { watchAllValues } from "./logWatcher";
9
9
  import { batchFunction, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
@@ -55,8 +55,6 @@ export async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
55
55
  return await suppression.values();
56
56
  }
57
57
 
58
- let getMatcher = cacheLimited(10000, (text: string) => createMatchesPattern(Buffer.from(normalizeMatchWhitespace(text)), false));
59
-
60
58
  // The watch loop checks every incoming error against the suppression list. Reading the suppression archive once per error produced thousands of storage listings per minute, so the hot path is served from a cached copy: the first pass tolerates 15 minute staleness, and only errors that pass unsuppressed force a fresher (30 second) re-read.
61
59
  const SUPPRESSION_CACHE_TTL = timeInMinute * 15;
62
60
  const SUPPRESSION_FRESH_CACHE_TTL = 1000 * 30;
@@ -87,16 +85,9 @@ function invalidateSuppressionEntriesCaches() {
87
85
  suppressionFreshCacheTime = 0;
88
86
  }
89
87
 
90
- // Side-effect-free version of applySuppression's "suppressed" verdict, for deciding which entry set the watch loop should apply against.
91
88
  function isSuppressedByAny(error: LogDatum, suppressionEntries: SuppressionEntry[]): boolean {
92
- let errorBuffer = Buffer.from(normalizeMatchWhitespace(JSON.stringify(error)));
93
- for (let suppressionEntry of suppressionEntries) {
94
- let isMatch = getMatcher(suppressionEntry.pattern);
95
- if (isMatch(errorBuffer) && error.time <= suppressionEntry.timeout) {
96
- return true;
97
- }
98
- }
99
- return false;
89
+ let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(error));
90
+ return matchedIndexes.some(index => error.time <= suppressionEntries[index].timeout);
100
91
  }
101
92
 
102
93
  const MAX_UNMATCHED = 10000;
@@ -122,13 +113,22 @@ export function getChunkEndTime(chunk: number) {
122
113
  return chunk + chunkUnit;
123
114
  }
124
115
 
125
- type SuppressionStatus = "suppressed" | "matched-timeout-passed" | "no-match";
126
-
127
- function applySuppression(error: LogDatum, suppressionEntry: SuppressionEntry): SuppressionStatus {
128
- let errorBuffer = Buffer.from(normalizeMatchWhitespace(JSON.stringify(error)));
129
- let isMatch = getMatcher(suppressionEntry.pattern);
130
- if (!isMatch(errorBuffer)) return "no-match";
116
+ function applySuppressions(error: LogDatum, suppressionEntries: SuppressionEntry[]): { suppressed: boolean; matchedIds: string[] } {
117
+ let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(error));
118
+ let suppressed = false;
119
+ let matchedIds: string[] = [];
120
+ for (let index of matchedIndexes) {
121
+ let suppressionEntry = suppressionEntries[index];
122
+ recordSuppressionMatch(error, suppressionEntry);
123
+ matchedIds.push(suppressionEntry.id);
124
+ if (error.time <= suppressionEntry.timeout) {
125
+ suppressed = true;
126
+ }
127
+ }
128
+ return { suppressed, matchedIds };
129
+ }
131
130
 
131
+ function recordSuppressionMatch(error: LogDatum, suppressionEntry: SuppressionEntry) {
132
132
  let history = suppressionMatches.get(suppressionEntry.id);
133
133
  if (!history) {
134
134
  history = {
@@ -154,12 +154,6 @@ function applySuppression(error: LogDatum, suppressionEntry: SuppressionEntry):
154
154
  history.history.set(historyChunk, historyEntry);
155
155
  }
156
156
  historyEntry.count++;
157
-
158
- if (error.time > suppressionEntry.timeout) {
159
- return "matched-timeout-passed";
160
- }
161
-
162
- return "suppressed";
163
157
  }
164
158
 
165
159
  async function reapplySuppressions() {
@@ -168,15 +162,9 @@ async function reapplySuppressions() {
168
162
  let changedSuppressionIds = new Set<string>();
169
163
 
170
164
  for (let error of unmatchedErrors) {
171
- let suppressed = false;
172
- for (let suppressionEntry of suppressionEntries) {
173
- let status = applySuppression(error, suppressionEntry);
174
- if (status === "suppressed" || status === "matched-timeout-passed") {
175
- changedSuppressionIds.add(suppressionEntry.id);
176
- }
177
- if (status === "suppressed") {
178
- suppressed = true;
179
- }
165
+ let { suppressed, matchedIds } = applySuppressions(error, suppressionEntries);
166
+ for (let id of matchedIds) {
167
+ changedSuppressionIds.add(id);
180
168
  }
181
169
  if (!suppressed) {
182
170
  newUnmatchedErrors.push(error);
@@ -355,19 +343,11 @@ export async function exposeErrorWatchService() {
355
343
  if (!isSuppressedByAny(error, suppressionEntries)) {
356
344
  suppressionEntries = await getSuppressionEntriesFresh();
357
345
  }
358
- let suppressed = false;
359
- let matchedSuppressionId: string | undefined = undefined;
360
-
361
- for (let suppressionEntry of suppressionEntries) {
362
- let status = applySuppression(error, suppressionEntry);
363
- if (status === "suppressed" || status === "matched-timeout-passed") {
364
- matchedSuppressionId = suppressionEntry.id;
365
- void ErrorNotificationService.triggerSuppressionUpdate(suppressionEntry.id);
366
- }
367
- if (status === "suppressed") {
368
- suppressed = true;
369
- }
346
+ let { suppressed, matchedIds } = applySuppressions(error, suppressionEntries);
347
+ for (let id of matchedIds) {
348
+ void ErrorNotificationService.triggerSuppressionUpdate(id);
370
349
  }
350
+ let matchedSuppressionId: string | undefined = matchedIds[matchedIds.length - 1];
371
351
  //console.log(`Watch loop received error, ${suppressed ? "suppressed" : "not suppressed"}, matched suppression id: ${matchedSuppressionId}: ${error.param0}`);
372
352
 
373
353
  if (suppressed) continue;
@@ -392,7 +372,7 @@ export async function exposeErrorWatchService() {
392
372
 
393
373
  await suppression.set(newEntry.id, newEntry);
394
374
  invalidateSuppressionEntriesCaches();
395
- applySuppression(error, newEntry);
375
+ applySuppressions(error, [newEntry]);
396
376
  queueDiscordNotification(newEntry.id, 1, true);
397
377
 
398
378
  void ErrorNotificationService.triggerSuppressionUpdate(newEntry.id);
@@ -0,0 +1,168 @@
1
+ import { normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
2
+ import { LogDatum } from "../diskLogger";
3
+
4
+ const ALPHABET_SIZE = 256;
5
+ const MATCHER_CACHE_SIZE = 10;
6
+ const SEARCH_OR_CHAR = "|";
7
+ const SEARCH_AND_CHAR = "&";
8
+ const WILD_CARD_CHAR = "*";
9
+
10
+ type Structure = { type: "or" | "and"; parts: Structure[] } | { type: "wildcard"; parts: Buffer[] } | { type: "exact"; value: Buffer };
11
+ type IdStructure = { type: "or" | "and"; parts: IdStructure[] } | { type: "wildcard"; ids: number[]; parts: Buffer[] } | { type: "exact"; id: number };
12
+
13
+ export type PatternSetMatcher = (matchBuffer: Buffer) => number[];
14
+
15
+ let matchBuffers = new WeakMap<LogDatum, Buffer>();
16
+ export function getMatchBuffer(datum: LogDatum): Buffer {
17
+ let buffer = matchBuffers.get(datum);
18
+ if (buffer) return buffer;
19
+ buffer = lowercaseAscii(Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum))));
20
+ matchBuffers.set(datum, buffer);
21
+ return buffer;
22
+ }
23
+
24
+ let matcherCache: { patterns: string[]; matcher: PatternSetMatcher }[] = [];
25
+ export function getPatternSetMatcher(patterns: string[]): PatternSetMatcher {
26
+ let cached = matcherCache.find(x => x.patterns.length === patterns.length && x.patterns.every((pattern, i) => pattern === patterns[i]));
27
+ if (cached) return cached.matcher;
28
+ let matcher = createPatternSetMatcher(patterns);
29
+ matcherCache.push({ patterns: patterns.slice(), matcher });
30
+ if (matcherCache.length > MATCHER_CACHE_SIZE) matcherCache.shift();
31
+ return matcher;
32
+ }
33
+
34
+ export function createPatternSetMatcher(patterns: string[]): PatternSetMatcher {
35
+ let literalIds = new Map<string, number>();
36
+ let literals: Buffer[] = [];
37
+ function getLiteralId(value: Buffer): number {
38
+ let key = value.toString("latin1");
39
+ let id = literalIds.get(key);
40
+ if (id !== undefined) return id;
41
+ id = literals.length;
42
+ literalIds.set(key, id);
43
+ literals.push(value);
44
+ return id;
45
+ }
46
+ function toIds(structure: Structure): IdStructure {
47
+ if (structure.type === "exact") return { type: "exact", id: getLiteralId(structure.value) };
48
+ if (structure.type === "wildcard") return { type: "wildcard", ids: structure.parts.map(getLiteralId), parts: structure.parts };
49
+ return { type: structure.type, parts: structure.parts.map(toIds) };
50
+ }
51
+ let structures = patterns.map(pattern => toIds(compilePattern(normalizeMatchWhitespace(pattern))));
52
+ let automaton = buildAutomaton(literals);
53
+ let foundStamp = new Int32Array(literals.length);
54
+ let stamp = 0;
55
+
56
+ return function matchPatternSet(matchBuffer: Buffer): number[] {
57
+ stamp++;
58
+ for (let id of automaton.outputs[0]) {
59
+ foundStamp[id] = stamp;
60
+ }
61
+ let state = 0;
62
+ for (let i = 0; i < matchBuffer.length; i++) {
63
+ state = automaton.transitions[state * ALPHABET_SIZE + matchBuffer[i]];
64
+ let outputState = automaton.hasOwnOutput[state] && state || automaton.dictionaryLink[state];
65
+ while (outputState > 0) {
66
+ let outputs = automaton.outputs[outputState];
67
+ for (let j = 0; j < outputs.length; j++) {
68
+ foundStamp[outputs[j]] = stamp;
69
+ }
70
+ outputState = automaton.dictionaryLink[outputState];
71
+ }
72
+ }
73
+ let matched: number[] = [];
74
+ for (let patternIndex = 0; patternIndex < structures.length; patternIndex++) {
75
+ if (matchesWithIds(structures[patternIndex])) matched.push(patternIndex);
76
+ }
77
+ return matched;
78
+
79
+ function matchesWithIds(structure: IdStructure): boolean {
80
+ if (structure.type === "exact") return foundStamp[structure.id] === stamp;
81
+ if (structure.type === "or") return structure.parts.some(matchesWithIds);
82
+ if (structure.type === "and") return structure.parts.every(matchesWithIds);
83
+ if (!structure.ids.every(id => foundStamp[id] === stamp)) return false;
84
+ let position = 0;
85
+ for (let segment of structure.parts) {
86
+ let found = matchBuffer.indexOf(segment, position);
87
+ if (found === -1) return false;
88
+ position = found + segment.length;
89
+ }
90
+ return true;
91
+ }
92
+ };
93
+ }
94
+
95
+ function lowercaseAscii(buffer: Buffer): Buffer {
96
+ for (let i = 0; i < buffer.length; i++) {
97
+ let byte = buffer[i];
98
+ if (65 <= byte && byte <= 90) buffer[i] = byte + 32;
99
+ }
100
+ return buffer;
101
+ }
102
+
103
+ function compilePattern(pattern: string): Structure {
104
+ let lowered = lowercaseAscii(Buffer.from(pattern)).toString("utf8");
105
+ if (!lowered.includes(SEARCH_OR_CHAR) && !lowered.includes(SEARCH_AND_CHAR) && !lowered.includes(WILD_CARD_CHAR)) {
106
+ return { type: "exact", value: Buffer.from(lowered) };
107
+ }
108
+ return parseStructure(lowered);
109
+ }
110
+
111
+ function parseStructure(text: string): Structure {
112
+ let orParts = text.split(SEARCH_OR_CHAR).map(x => x.trim()).filter(x => x.length > 0);
113
+ if (orParts.length > 1) return { type: "or", parts: orParts.map(parseStructure) };
114
+ let andParts = text.split(SEARCH_AND_CHAR).map(x => x.trim()).filter(x => x.length > 0);
115
+ if (andParts.length > 1) return { type: "and", parts: andParts.map(parseStructure) };
116
+ let wildcardParts = text.split(WILD_CARD_CHAR).map(x => x.trim()).filter(x => x.length > 0);
117
+ if (wildcardParts.length > 1) return { type: "wildcard", parts: wildcardParts.map(x => Buffer.from(x)) };
118
+ return { type: "exact", value: Buffer.from(text) };
119
+ }
120
+
121
+ function buildAutomaton(literals: Buffer[]) {
122
+ let children: Map<number, number>[] = [new Map()];
123
+ let outputs: number[][] = [[]];
124
+ for (let literalIndex = 0; literalIndex < literals.length; literalIndex++) {
125
+ let state = 0;
126
+ for (let byte of literals[literalIndex]) {
127
+ let next = children[state].get(byte);
128
+ if (next === undefined) {
129
+ next = children.length;
130
+ children.push(new Map());
131
+ outputs.push([]);
132
+ children[state].set(byte, next);
133
+ }
134
+ state = next;
135
+ }
136
+ outputs[state].push(literalIndex);
137
+ }
138
+ let stateCount = children.length;
139
+ let transitions = new Int32Array(stateCount * ALPHABET_SIZE);
140
+ let failure = new Int32Array(stateCount);
141
+ let dictionaryLink = new Int32Array(stateCount);
142
+ let hasOwnOutput = new Uint8Array(stateCount);
143
+ for (let state = 0; state < stateCount; state++) {
144
+ if (outputs[state].length) hasOwnOutput[state] = 1;
145
+ }
146
+ let queue: number[] = [];
147
+ for (let byte = 0; byte < ALPHABET_SIZE; byte++) {
148
+ let child = children[0].get(byte);
149
+ transitions[byte] = child ?? 0;
150
+ if (child !== undefined) queue.push(child);
151
+ }
152
+ for (let queueIndex = 0; queueIndex < queue.length; queueIndex++) {
153
+ let state = queue[queueIndex];
154
+ let failState = failure[state];
155
+ dictionaryLink[state] = hasOwnOutput[failState] && failState || dictionaryLink[failState];
156
+ for (let byte = 0; byte < ALPHABET_SIZE; byte++) {
157
+ let child = children[state].get(byte);
158
+ if (child === undefined) {
159
+ transitions[state * ALPHABET_SIZE + byte] = transitions[failState * ALPHABET_SIZE + byte];
160
+ continue;
161
+ }
162
+ transitions[state * ALPHABET_SIZE + byte] = child;
163
+ failure[child] = transitions[failState * ALPHABET_SIZE + byte];
164
+ queue.push(child);
165
+ }
166
+ }
167
+ return { transitions, dictionaryLink, hasOwnOutput, outputs };
168
+ }