querysub 0.719.0 → 0.721.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "querysub",
3
- "version": "0.719.0",
3
+ "version": "0.721.0",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -12,7 +12,7 @@ import { MaybePromise } from "socket-function/src/types";
12
12
  import yargs from "yargs";
13
13
  import { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
14
14
  import { MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
15
- import { isNode, isNodeTrue, nextId, timeInSecond } from "socket-function/src/misc";
15
+ import { isNode, isNodeTrue, nextId, timeInMinute, timeInSecond } from "socket-function/src/misc";
16
16
  import { getPathStr2, getPathStr3 } from "../path";
17
17
  import { consistentHash } from "../misc/hash";
18
18
  import { setExternalHotReloading } from "socket-function/hot/HotReloadController";
@@ -22,10 +22,15 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
22
22
  import { getControllerNodeId, getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
23
23
  import { sha256 } from "js-sha256";
24
24
  import os from "os";
25
- import { formatTime } from "socket-function/src/formatting/format";
25
+ import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
26
26
  import path from "path";
27
27
  import pathMod from "path";
28
28
 
29
+ const MODULE_LOAD_ERROR_RETRY_DELAY = timeInMinute * 5;
30
+ const YARN_INSTALL_INITIAL_RETRY_DELAY = timeInSecond * 10;
31
+ const YARN_INSTALL_MAX_RETRY_DELAY = timeInMinute;
32
+ const YARN_INSTALL_MAX_ATTEMPTS = 10;
33
+
29
34
  export type LoadFunctionSpec = {
30
35
  gitURL: string;
31
36
  gitRef: string;
@@ -87,6 +92,7 @@ export function getModuleFromSpec(spec: FunctionSpec): MaybePromise<NodeJS.Modul
87
92
  let moduleCache = new Map<string, {
88
93
  result: MaybePromise<NodeJS.Module>;
89
94
  error: Error | undefined;
95
+ errorTime: number;
90
96
  }>();
91
97
  function getSpecKey(spec: LoadFunctionSpec) {
92
98
  // Only include these specific keys AND order them, so the hash hits no matter how this function is called
@@ -104,18 +110,23 @@ export function getModuleFromConfig(spec: LoadFunctionSpec): MaybePromise<NodeJS
104
110
  let value = moduleCache.get(key);
105
111
  if (!value) {
106
112
  let promise = getModuleFromSpecBase(spec);
107
- value = { result: promise, error: undefined };
113
+ value = { result: promise, error: undefined, errorTime: 0 };
108
114
  moduleCache.set(key, value);
109
115
  promise.then(
110
116
  result => {
111
- moduleCache.set(key, { result, error: undefined });
117
+ moduleCache.set(key, { result, error: undefined, errorTime: 0 });
112
118
  },
113
119
  error => {
114
- moduleCache.set(key, { result: null as any, error });
120
+ moduleCache.set(key, { result: promise, error, errorTime: Date.now() });
115
121
  }
116
122
  );
117
123
  }
118
- if (value.error) throw value.error;
124
+ if (value.error) {
125
+ if (Date.now() - value.errorTime < MODULE_LOAD_ERROR_RETRY_DELAY) throw value.error;
126
+ console.log(red(`Retrying module load for ${key}, the previous attempt failed at ${formatDateTime(value.errorTime)}, more than ${formatTime(MODULE_LOAD_ERROR_RETRY_DELAY)} ago, with:\n${value.error.stack}`));
127
+ moduleCache.delete(key);
128
+ return getModuleFromConfig(spec);
129
+ }
119
130
  return value.result;
120
131
  }
121
132
 
@@ -466,17 +477,33 @@ const hotreloadIfChanged = batchFunction({ delay: 100, name: "hotreloadIfChanged
466
477
  // A corrupt entry in the shared global yarn cache (e.g. "Extracting tar content ... the file appears to be
467
478
  // corrupt") fails every install identically, because re-cloning the repo folder never touches the global
468
479
  // cache. Detect that, purge the cache with `yarn cache clean`, and retry once so the loader heals itself.
480
+ // Retrying with delays could make this function very slow. However, the alternative is just all the function calls failing, which is the worst case scenario.
469
481
  async function yarnInstallWithHealing(repoPath: string): Promise<void> {
470
- try {
471
- await executeCommand("yarn", ["install", "--mutex", "network"], { cwd: repoPath });
472
- return;
473
- } catch (e) {
474
- let errorText = String((e as Error)?.stack || e);
475
- if (!/appears to be corrupt|Extracting tar content/.test(errorText)) throw e;
476
- console.log(red(`yarn install failed with a corrupt yarn cache, cleaning cache and retrying:\n${errorText}`));
477
- await executeCommand("yarn", ["cache", "clean"]);
482
+ let cleanedCache = false;
483
+ let failures = 0;
484
+ let retryDelay = YARN_INSTALL_INITIAL_RETRY_DELAY;
485
+ while (true) {
486
+ try {
487
+ await executeCommand("yarn", ["install", "--mutex", "network"], { cwd: repoPath });
488
+ if (failures) {
489
+ console.log(blue(`yarn install in ${repoPath} succeeded after ${failures} failures`));
490
+ }
491
+ return;
492
+ } catch (e) {
493
+ failures++;
494
+ let errorText = String((e as Error)?.stack || e);
495
+ if (failures >= YARN_INSTALL_MAX_ATTEMPTS) throw e;
496
+ if (!cleanedCache && /appears to be corrupt|Extracting tar content/.test(errorText)) {
497
+ cleanedCache = true;
498
+ console.log(red(`yarn install failed with a corrupt yarn cache, cleaning cache and retrying:\n${errorText}`));
499
+ await executeCommand("yarn", ["cache", "clean"]);
500
+ continue;
501
+ }
502
+ console.log(red(`yarn install in ${repoPath} failed (failure ${failures}/${YARN_INSTALL_MAX_ATTEMPTS}), retrying in ${formatTime(retryDelay)}:\n${errorText}`));
503
+ await delay(retryDelay);
504
+ retryDelay = Math.min(retryDelay * 2, YARN_INSTALL_MAX_RETRY_DELAY);
505
+ }
478
506
  }
479
- await executeCommand("yarn", ["install", "--mutex", "network"], { cwd: repoPath });
480
507
  }
481
508
 
482
509
  async function which(command: string): Promise<string> {
@@ -8,6 +8,7 @@ import { sort, timeInSecond, nextId } from "socket-function/src/misc";
8
8
  import { MachineController, watchProcessOutput, stopWatchingProcessOutput } from "../machineController";
9
9
  import type { ProcessRecord } from "../processLogs";
10
10
  import { Button } from "../../library-components/Button";
11
+ import { getMachineInfoLive, getMachineNameLive } from "../machineSchema";
11
12
  import { parseAnsiColors, rgbToHsl } from "../../diagnostics/logs/ansiFormat";
12
13
 
13
14
  module.hotreload = true;
@@ -168,9 +169,11 @@ class MachineProcesses extends qreact.Component<{ machineId: string; applyNodeId
168
169
  let historical = shown.filter(x => x.deadTime !== undefined);
169
170
  sort(running, x => -x.startTime);
170
171
  sort(historical, x => -(x.deadTime || 0));
172
+ let machineName = getMachineNameLive(this.props.machineId);
171
173
  return <div className={css.vbox(8).fillWidth}>
172
174
  <div className={css.hbox(10).wrap.alignItems("center")}>
173
- <div className={css.boldStyle}>{this.props.machineId}</div>
175
+ <div className={css.boldStyle + (!machineName && css.colorhsl(0, 0, 45))}>{machineName || "(unnamed machine)"}</div>
176
+ <div>{this.props.machineId} ({getMachineInfoLive(this.props.machineId)?.info["getExternalIP"]})</div>
174
177
  <div className={css.colorhsl(0, 0, 45)}>{running.length} running</div>
175
178
  {historical.length > 0 && <Button flavor="tiny" onClick={() => this.state.showHistorical = !this.state.showHistorical}>
176
179
  {this.state.showHistorical && `Hide ${historical.length} historical` || `Show ${historical.length} historical`}
@@ -10,8 +10,7 @@ import { isPublic } from "../../../config";
10
10
  import { MachineThreadInfo } from "../../MachineThreadInfo";
11
11
  import { ErrorWarning } from "./ErrorWarning";
12
12
  import { getErrorNotificationsManager } from "./errorWatcher";
13
- import { createMatchesPattern, normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
14
- import { cacheLimited } from "socket-function/src/caching";
13
+ import { createPatternSetMatcher, getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
15
14
  import { managementPageURL, showingManagementURL } from "../../managementPages";
16
15
  import { searchTextURL, readProductionLogsURL } from "../IndexedLogs/LogViewerParams";
17
16
  import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
@@ -57,13 +56,6 @@ function TicketButton(props: {
57
56
  }
58
57
  qreact.inline(TicketButton);
59
58
 
60
- export let getMatcher = cacheLimited(100, (pattern: string) =>
61
- createMatchesPattern(Buffer.from(normalizeMatchWhitespace(pattern)), false)
62
- );
63
-
64
- export function getMatchBuffer(datum: LogDatum): Buffer {
65
- return Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum)));
66
- }
67
59
 
68
60
  export class LogDatumRenderer extends qreact.Component<{
69
61
  datum: LogDatum;
@@ -81,15 +73,11 @@ export class LogDatumRenderer extends qreact.Component<{
81
73
  let datum = this.props.datum;
82
74
  let mainMessage = datum.param0 && String(datum.param0) || "(no message)";
83
75
 
84
- let matchingSuppressions = [];
76
+ let suppressionEntries = this.manager.state.suppressionEntries;
77
+ let matchingSuppressions: typeof suppressionEntries = [];
85
78
  if (!this.manager.state.isLoading && !this.props.hideSuppression) {
86
- for (let suppression of this.manager.state.suppressionEntries) {
87
- let matcher = getMatcher(suppression.pattern);
88
- let errorBuffer = getMatchBuffer(datum);
89
- if (matcher(errorBuffer)) {
90
- matchingSuppressions.push(suppression);
91
- }
92
- }
79
+ let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(datum));
80
+ matchingSuppressions = matchedIndexes.map(index => suppressionEntries[index]);
93
81
  }
94
82
 
95
83
  let timedOutMatches = matchingSuppressions.filter(s => datum.time > s.timeout);
@@ -464,8 +452,8 @@ class SuppressionItem extends qreact.Component<{
464
452
  let example = this.props.matchData?.examples[0];
465
453
  if (example) return example;
466
454
  // No example recorded for this suppression — find any live error matching the pattern, so the ticket stores a real, full log entry instead of a synthetic stub.
467
- let matcher = getMatcher(suppression.pattern);
468
- let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)));
455
+ let matcher = createPatternSetMatcher([suppression.pattern]);
456
+ let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)).length > 0);
469
457
  return fallback || {
470
458
  time: Date.now(),
471
459
  __LOG_TYPE: "error",
@@ -4,7 +4,8 @@ import { css } from "typesafecss";
4
4
  import { SocketFunction } from "socket-function/SocketFunction";
5
5
  import { ATag } from "../../../library-components/ATag";
6
6
  import { managementPageURL, showingManagementURL } from "../../managementPages";
7
- import { getMatcher, getMatchBuffer, LogDatumRenderer } from "./ErrorNotificationPage";
7
+ import { LogDatumRenderer } from "./ErrorNotificationPage";
8
+ import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
8
9
  import { getErrorNotificationsManager } from "./errorWatcher";
9
10
  import { LogDatum } from "../diskLogger";
10
11
  import { TicketsController } from "../errorTickets/tickets";
@@ -57,10 +58,8 @@ export class ErrorWarning extends qreact.Component {
57
58
  if (!ticket.suppressionPattern) continue;
58
59
  openTicketPatterns.push(ticket.suppressionPattern);
59
60
  }
60
- const isCoveredByOpenTicket = (datum: LogDatum) => {
61
- let errorBuffer = getMatchBuffer(datum);
62
- return openTicketPatterns.some(pattern => getMatcher(pattern)(errorBuffer));
63
- };
61
+ const openTicketMatcher = getPatternSetMatcher(openTicketPatterns);
62
+ const isCoveredByOpenTicket = (datum: LogDatum) => openTicketMatcher(getMatchBuffer(datum)).length > 0;
64
63
 
65
64
  let errorWarning: preact.ComponentChild = undefined;
66
65
  let firstError = !this.manager.state.isLoading && this.manager.state.unmatchedErrors.find(x => !isCoveredByOpenTicket(x)) || undefined;
@@ -1,9 +1,9 @@
1
- import { cacheLimited, lazy } from "socket-function/src/caching";
1
+ import { lazy } from "socket-function/src/caching";
2
2
  import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
3
3
  import { Querysub } from "../../../4-querysub/Querysub";
4
4
  import { getDomain, isPublic } from "../../../config";
5
5
  import { MachineInfo } from "../../../deployManager/machineSchema";
6
- import { createMatchesPattern, normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
6
+ import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
7
7
  import { LogDatum, getErrorLogs } from "../diskLogger";
8
8
  import { watchAllValues } from "./logWatcher";
9
9
  import { batchFunction, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
@@ -55,8 +55,6 @@ export async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
55
55
  return await suppression.values();
56
56
  }
57
57
 
58
- let getMatcher = cacheLimited(10000, (text: string) => createMatchesPattern(Buffer.from(normalizeMatchWhitespace(text)), false));
59
-
60
58
  // The watch loop checks every incoming error against the suppression list. Reading the suppression archive once per error produced thousands of storage listings per minute, so the hot path is served from a cached copy: the first pass tolerates 15 minute staleness, and only errors that pass unsuppressed force a fresher (30 second) re-read.
61
59
  const SUPPRESSION_CACHE_TTL = timeInMinute * 15;
62
60
  const SUPPRESSION_FRESH_CACHE_TTL = 1000 * 30;
@@ -87,16 +85,9 @@ function invalidateSuppressionEntriesCaches() {
87
85
  suppressionFreshCacheTime = 0;
88
86
  }
89
87
 
90
- // Side-effect-free version of applySuppression's "suppressed" verdict, for deciding which entry set the watch loop should apply against.
91
88
  function isSuppressedByAny(error: LogDatum, suppressionEntries: SuppressionEntry[]): boolean {
92
- let errorBuffer = Buffer.from(normalizeMatchWhitespace(JSON.stringify(error)));
93
- for (let suppressionEntry of suppressionEntries) {
94
- let isMatch = getMatcher(suppressionEntry.pattern);
95
- if (isMatch(errorBuffer) && error.time <= suppressionEntry.timeout) {
96
- return true;
97
- }
98
- }
99
- return false;
89
+ let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(error));
90
+ return matchedIndexes.some(index => error.time <= suppressionEntries[index].timeout);
100
91
  }
101
92
 
102
93
  const MAX_UNMATCHED = 10000;
@@ -122,13 +113,22 @@ export function getChunkEndTime(chunk: number) {
122
113
  return chunk + chunkUnit;
123
114
  }
124
115
 
125
- type SuppressionStatus = "suppressed" | "matched-timeout-passed" | "no-match";
126
-
127
- function applySuppression(error: LogDatum, suppressionEntry: SuppressionEntry): SuppressionStatus {
128
- let errorBuffer = Buffer.from(normalizeMatchWhitespace(JSON.stringify(error)));
129
- let isMatch = getMatcher(suppressionEntry.pattern);
130
- if (!isMatch(errorBuffer)) return "no-match";
116
+ function applySuppressions(error: LogDatum, suppressionEntries: SuppressionEntry[]): { suppressed: boolean; matchedIds: string[] } {
117
+ let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(error));
118
+ let suppressed = false;
119
+ let matchedIds: string[] = [];
120
+ for (let index of matchedIndexes) {
121
+ let suppressionEntry = suppressionEntries[index];
122
+ recordSuppressionMatch(error, suppressionEntry);
123
+ matchedIds.push(suppressionEntry.id);
124
+ if (error.time <= suppressionEntry.timeout) {
125
+ suppressed = true;
126
+ }
127
+ }
128
+ return { suppressed, matchedIds };
129
+ }
131
130
 
131
+ function recordSuppressionMatch(error: LogDatum, suppressionEntry: SuppressionEntry) {
132
132
  let history = suppressionMatches.get(suppressionEntry.id);
133
133
  if (!history) {
134
134
  history = {
@@ -154,12 +154,6 @@ function applySuppression(error: LogDatum, suppressionEntry: SuppressionEntry):
154
154
  history.history.set(historyChunk, historyEntry);
155
155
  }
156
156
  historyEntry.count++;
157
-
158
- if (error.time > suppressionEntry.timeout) {
159
- return "matched-timeout-passed";
160
- }
161
-
162
- return "suppressed";
163
157
  }
164
158
 
165
159
  async function reapplySuppressions() {
@@ -168,15 +162,9 @@ async function reapplySuppressions() {
168
162
  let changedSuppressionIds = new Set<string>();
169
163
 
170
164
  for (let error of unmatchedErrors) {
171
- let suppressed = false;
172
- for (let suppressionEntry of suppressionEntries) {
173
- let status = applySuppression(error, suppressionEntry);
174
- if (status === "suppressed" || status === "matched-timeout-passed") {
175
- changedSuppressionIds.add(suppressionEntry.id);
176
- }
177
- if (status === "suppressed") {
178
- suppressed = true;
179
- }
165
+ let { suppressed, matchedIds } = applySuppressions(error, suppressionEntries);
166
+ for (let id of matchedIds) {
167
+ changedSuppressionIds.add(id);
180
168
  }
181
169
  if (!suppressed) {
182
170
  newUnmatchedErrors.push(error);
@@ -355,19 +343,11 @@ export async function exposeErrorWatchService() {
355
343
  if (!isSuppressedByAny(error, suppressionEntries)) {
356
344
  suppressionEntries = await getSuppressionEntriesFresh();
357
345
  }
358
- let suppressed = false;
359
- let matchedSuppressionId: string | undefined = undefined;
360
-
361
- for (let suppressionEntry of suppressionEntries) {
362
- let status = applySuppression(error, suppressionEntry);
363
- if (status === "suppressed" || status === "matched-timeout-passed") {
364
- matchedSuppressionId = suppressionEntry.id;
365
- void ErrorNotificationService.triggerSuppressionUpdate(suppressionEntry.id);
366
- }
367
- if (status === "suppressed") {
368
- suppressed = true;
369
- }
346
+ let { suppressed, matchedIds } = applySuppressions(error, suppressionEntries);
347
+ for (let id of matchedIds) {
348
+ void ErrorNotificationService.triggerSuppressionUpdate(id);
370
349
  }
350
+ let matchedSuppressionId: string | undefined = matchedIds[matchedIds.length - 1];
371
351
  //console.log(`Watch loop received error, ${suppressed ? "suppressed" : "not suppressed"}, matched suppression id: ${matchedSuppressionId}: ${error.param0}`);
372
352
 
373
353
  if (suppressed) continue;
@@ -392,7 +372,7 @@ export async function exposeErrorWatchService() {
392
372
 
393
373
  await suppression.set(newEntry.id, newEntry);
394
374
  invalidateSuppressionEntriesCaches();
395
- applySuppression(error, newEntry);
375
+ applySuppressions(error, [newEntry]);
396
376
  queueDiscordNotification(newEntry.id, 1, true);
397
377
 
398
378
  void ErrorNotificationService.triggerSuppressionUpdate(newEntry.id);
@@ -0,0 +1,168 @@
1
+ import { normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
2
+ import { LogDatum } from "../diskLogger";
3
+
4
+ const ALPHABET_SIZE = 256;
5
+ const MATCHER_CACHE_SIZE = 10;
6
+ const SEARCH_OR_CHAR = "|";
7
+ const SEARCH_AND_CHAR = "&";
8
+ const WILD_CARD_CHAR = "*";
9
+
10
+ type Structure = { type: "or" | "and"; parts: Structure[] } | { type: "wildcard"; parts: Buffer[] } | { type: "exact"; value: Buffer };
11
+ type IdStructure = { type: "or" | "and"; parts: IdStructure[] } | { type: "wildcard"; ids: number[]; parts: Buffer[] } | { type: "exact"; id: number };
12
+
13
+ export type PatternSetMatcher = (matchBuffer: Buffer) => number[];
14
+
15
+ let matchBuffers = new WeakMap<LogDatum, Buffer>();
16
+ export function getMatchBuffer(datum: LogDatum): Buffer {
17
+ let buffer = matchBuffers.get(datum);
18
+ if (buffer) return buffer;
19
+ buffer = lowercaseAscii(Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum))));
20
+ matchBuffers.set(datum, buffer);
21
+ return buffer;
22
+ }
23
+
24
+ let matcherCache: { patterns: string[]; matcher: PatternSetMatcher }[] = [];
25
+ export function getPatternSetMatcher(patterns: string[]): PatternSetMatcher {
26
+ let cached = matcherCache.find(x => x.patterns.length === patterns.length && x.patterns.every((pattern, i) => pattern === patterns[i]));
27
+ if (cached) return cached.matcher;
28
+ let matcher = createPatternSetMatcher(patterns);
29
+ matcherCache.push({ patterns: patterns.slice(), matcher });
30
+ if (matcherCache.length > MATCHER_CACHE_SIZE) matcherCache.shift();
31
+ return matcher;
32
+ }
33
+
34
+ export function createPatternSetMatcher(patterns: string[]): PatternSetMatcher {
35
+ let literalIds = new Map<string, number>();
36
+ let literals: Buffer[] = [];
37
+ function getLiteralId(value: Buffer): number {
38
+ let key = value.toString("latin1");
39
+ let id = literalIds.get(key);
40
+ if (id !== undefined) return id;
41
+ id = literals.length;
42
+ literalIds.set(key, id);
43
+ literals.push(value);
44
+ return id;
45
+ }
46
+ function toIds(structure: Structure): IdStructure {
47
+ if (structure.type === "exact") return { type: "exact", id: getLiteralId(structure.value) };
48
+ if (structure.type === "wildcard") return { type: "wildcard", ids: structure.parts.map(getLiteralId), parts: structure.parts };
49
+ return { type: structure.type, parts: structure.parts.map(toIds) };
50
+ }
51
+ let structures = patterns.map(pattern => toIds(compilePattern(normalizeMatchWhitespace(pattern))));
52
+ let automaton = buildAutomaton(literals);
53
+ let foundStamp = new Int32Array(literals.length);
54
+ let stamp = 0;
55
+
56
+ return function matchPatternSet(matchBuffer: Buffer): number[] {
57
+ stamp++;
58
+ for (let id of automaton.outputs[0]) {
59
+ foundStamp[id] = stamp;
60
+ }
61
+ let state = 0;
62
+ for (let i = 0; i < matchBuffer.length; i++) {
63
+ state = automaton.transitions[state * ALPHABET_SIZE + matchBuffer[i]];
64
+ let outputState = automaton.hasOwnOutput[state] && state || automaton.dictionaryLink[state];
65
+ while (outputState > 0) {
66
+ let outputs = automaton.outputs[outputState];
67
+ for (let j = 0; j < outputs.length; j++) {
68
+ foundStamp[outputs[j]] = stamp;
69
+ }
70
+ outputState = automaton.dictionaryLink[outputState];
71
+ }
72
+ }
73
+ let matched: number[] = [];
74
+ for (let patternIndex = 0; patternIndex < structures.length; patternIndex++) {
75
+ if (matchesWithIds(structures[patternIndex])) matched.push(patternIndex);
76
+ }
77
+ return matched;
78
+
79
+ function matchesWithIds(structure: IdStructure): boolean {
80
+ if (structure.type === "exact") return foundStamp[structure.id] === stamp;
81
+ if (structure.type === "or") return structure.parts.some(matchesWithIds);
82
+ if (structure.type === "and") return structure.parts.every(matchesWithIds);
83
+ if (!structure.ids.every(id => foundStamp[id] === stamp)) return false;
84
+ let position = 0;
85
+ for (let segment of structure.parts) {
86
+ let found = matchBuffer.indexOf(segment, position);
87
+ if (found === -1) return false;
88
+ position = found + segment.length;
89
+ }
90
+ return true;
91
+ }
92
+ };
93
+ }
94
+
95
+ function lowercaseAscii(buffer: Buffer): Buffer {
96
+ for (let i = 0; i < buffer.length; i++) {
97
+ let byte = buffer[i];
98
+ if (65 <= byte && byte <= 90) buffer[i] = byte + 32;
99
+ }
100
+ return buffer;
101
+ }
102
+
103
+ function compilePattern(pattern: string): Structure {
104
+ let lowered = lowercaseAscii(Buffer.from(pattern)).toString("utf8");
105
+ if (!lowered.includes(SEARCH_OR_CHAR) && !lowered.includes(SEARCH_AND_CHAR) && !lowered.includes(WILD_CARD_CHAR)) {
106
+ return { type: "exact", value: Buffer.from(lowered) };
107
+ }
108
+ return parseStructure(lowered);
109
+ }
110
+
111
+ function parseStructure(text: string): Structure {
112
+ let orParts = text.split(SEARCH_OR_CHAR).map(x => x.trim()).filter(x => x.length > 0);
113
+ if (orParts.length > 1) return { type: "or", parts: orParts.map(parseStructure) };
114
+ let andParts = text.split(SEARCH_AND_CHAR).map(x => x.trim()).filter(x => x.length > 0);
115
+ if (andParts.length > 1) return { type: "and", parts: andParts.map(parseStructure) };
116
+ let wildcardParts = text.split(WILD_CARD_CHAR).map(x => x.trim()).filter(x => x.length > 0);
117
+ if (wildcardParts.length > 1) return { type: "wildcard", parts: wildcardParts.map(x => Buffer.from(x)) };
118
+ return { type: "exact", value: Buffer.from(text) };
119
+ }
120
+
121
+ function buildAutomaton(literals: Buffer[]) {
122
+ let children: Map<number, number>[] = [new Map()];
123
+ let outputs: number[][] = [[]];
124
+ for (let literalIndex = 0; literalIndex < literals.length; literalIndex++) {
125
+ let state = 0;
126
+ for (let byte of literals[literalIndex]) {
127
+ let next = children[state].get(byte);
128
+ if (next === undefined) {
129
+ next = children.length;
130
+ children.push(new Map());
131
+ outputs.push([]);
132
+ children[state].set(byte, next);
133
+ }
134
+ state = next;
135
+ }
136
+ outputs[state].push(literalIndex);
137
+ }
138
+ let stateCount = children.length;
139
+ let transitions = new Int32Array(stateCount * ALPHABET_SIZE);
140
+ let failure = new Int32Array(stateCount);
141
+ let dictionaryLink = new Int32Array(stateCount);
142
+ let hasOwnOutput = new Uint8Array(stateCount);
143
+ for (let state = 0; state < stateCount; state++) {
144
+ if (outputs[state].length) hasOwnOutput[state] = 1;
145
+ }
146
+ let queue: number[] = [];
147
+ for (let byte = 0; byte < ALPHABET_SIZE; byte++) {
148
+ let child = children[0].get(byte);
149
+ transitions[byte] = child ?? 0;
150
+ if (child !== undefined) queue.push(child);
151
+ }
152
+ for (let queueIndex = 0; queueIndex < queue.length; queueIndex++) {
153
+ let state = queue[queueIndex];
154
+ let failState = failure[state];
155
+ dictionaryLink[state] = hasOwnOutput[failState] && failState || dictionaryLink[failState];
156
+ for (let byte = 0; byte < ALPHABET_SIZE; byte++) {
157
+ let child = children[state].get(byte);
158
+ if (child === undefined) {
159
+ transitions[state * ALPHABET_SIZE + byte] = transitions[failState * ALPHABET_SIZE + byte];
160
+ continue;
161
+ }
162
+ transitions[state * ALPHABET_SIZE + byte] = child;
163
+ failure[child] = transitions[failState * ALPHABET_SIZE + byte];
164
+ queue.push(child);
165
+ }
166
+ }
167
+ return { transitions, dictionaryLink, hasOwnOutput, outputs };
168
+ }