querysub 0.720.0 → 0.722.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/flushlogs.js +7 -0
- package/debugging.md +43 -0
- package/package.json +9 -3
- package/src/-f-node-discovery/NodeDiscovery.ts +1 -1
- package/src/0-path-value-core/AuthorityLookup.ts +0 -1
- package/src/0-path-value-core/PathRouter.ts +0 -1
- package/src/0-path-value-core/PathWatcher.ts +0 -3
- package/src/0-path-value-core/ValidStateComputer.ts +0 -3
- package/src/2-proxy/PathValueProxyWatcher.ts +0 -3
- package/src/2-proxy/TransactionDelayer.ts +0 -1
- package/src/4-deploy/edgeNodes.ts +0 -1
- package/src/4-querysub/querysubPrediction.ts +0 -1
- package/src/config.ts +0 -1
- package/src/diagnostics/StatsHeader.tsx +2 -1
- package/src/diagnostics/StatsOverrides.ts +35 -0
- package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +0 -1
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +36 -23
- package/src/diagnostics/logs/IndexedLogs/flushLogsEntry.ts +42 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +6 -0
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +7 -19
- package/src/diagnostics/logs/errorNotifications2/ErrorWarning.tsx +4 -5
- package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +27 -47
- package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +168 -0
- package/src/diagnostics/network/logFilesMain.ts +57 -0
- package/src/diagnostics/network/logSearchMain.ts +86 -0
- package/src/diagnostics/network/networkNodesMain.ts +225 -0
- package/src/diagnostics/network/networkProbe.ts +31 -0
- package/src/diagnostics/network/networkServicesMain.ts +170 -0
- package/src/diagnostics/network/portMappingsMain.ts +28 -0
- package/src/diagnostics/statsDefinitions.tsx +0 -1
- package/src/library-components/LazyComponent.tsx +0 -1
- package/src/library-components/URLParam.ts +0 -2
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { lazy } from "socket-function/src/caching";
|
|
2
2
|
import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
|
|
3
3
|
import { Querysub } from "../../../4-querysub/Querysub";
|
|
4
4
|
import { getDomain, isPublic } from "../../../config";
|
|
5
5
|
import { MachineInfo } from "../../../deployManager/machineSchema";
|
|
6
|
-
import {
|
|
6
|
+
import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
|
|
7
7
|
import { LogDatum, getErrorLogs } from "../diskLogger";
|
|
8
8
|
import { watchAllValues } from "./logWatcher";
|
|
9
9
|
import { batchFunction, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
@@ -55,8 +55,6 @@ export async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
|
|
|
55
55
|
return await suppression.values();
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
let getMatcher = cacheLimited(10000, (text: string) => createMatchesPattern(Buffer.from(normalizeMatchWhitespace(text)), false));
|
|
59
|
-
|
|
60
58
|
// The watch loop checks every incoming error against the suppression list. Reading the suppression archive once per error produced thousands of storage listings per minute, so the hot path is served from a cached copy: the first pass tolerates 15 minute staleness, and only errors that pass unsuppressed force a fresher (30 second) re-read.
|
|
61
59
|
const SUPPRESSION_CACHE_TTL = timeInMinute * 15;
|
|
62
60
|
const SUPPRESSION_FRESH_CACHE_TTL = 1000 * 30;
|
|
@@ -87,16 +85,9 @@ function invalidateSuppressionEntriesCaches() {
|
|
|
87
85
|
suppressionFreshCacheTime = 0;
|
|
88
86
|
}
|
|
89
87
|
|
|
90
|
-
// Side-effect-free version of applySuppression's "suppressed" verdict, for deciding which entry set the watch loop should apply against.
|
|
91
88
|
function isSuppressedByAny(error: LogDatum, suppressionEntries: SuppressionEntry[]): boolean {
|
|
92
|
-
let
|
|
93
|
-
|
|
94
|
-
let isMatch = getMatcher(suppressionEntry.pattern);
|
|
95
|
-
if (isMatch(errorBuffer) && error.time <= suppressionEntry.timeout) {
|
|
96
|
-
return true;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
return false;
|
|
89
|
+
let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(error));
|
|
90
|
+
return matchedIndexes.some(index => error.time <= suppressionEntries[index].timeout);
|
|
100
91
|
}
|
|
101
92
|
|
|
102
93
|
const MAX_UNMATCHED = 10000;
|
|
@@ -122,13 +113,22 @@ export function getChunkEndTime(chunk: number) {
|
|
|
122
113
|
return chunk + chunkUnit;
|
|
123
114
|
}
|
|
124
115
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
let
|
|
129
|
-
let
|
|
130
|
-
|
|
116
|
+
function applySuppressions(error: LogDatum, suppressionEntries: SuppressionEntry[]): { suppressed: boolean; matchedIds: string[] } {
|
|
117
|
+
let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(error));
|
|
118
|
+
let suppressed = false;
|
|
119
|
+
let matchedIds: string[] = [];
|
|
120
|
+
for (let index of matchedIndexes) {
|
|
121
|
+
let suppressionEntry = suppressionEntries[index];
|
|
122
|
+
recordSuppressionMatch(error, suppressionEntry);
|
|
123
|
+
matchedIds.push(suppressionEntry.id);
|
|
124
|
+
if (error.time <= suppressionEntry.timeout) {
|
|
125
|
+
suppressed = true;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return { suppressed, matchedIds };
|
|
129
|
+
}
|
|
131
130
|
|
|
131
|
+
function recordSuppressionMatch(error: LogDatum, suppressionEntry: SuppressionEntry) {
|
|
132
132
|
let history = suppressionMatches.get(suppressionEntry.id);
|
|
133
133
|
if (!history) {
|
|
134
134
|
history = {
|
|
@@ -154,12 +154,6 @@ function applySuppression(error: LogDatum, suppressionEntry: SuppressionEntry):
|
|
|
154
154
|
history.history.set(historyChunk, historyEntry);
|
|
155
155
|
}
|
|
156
156
|
historyEntry.count++;
|
|
157
|
-
|
|
158
|
-
if (error.time > suppressionEntry.timeout) {
|
|
159
|
-
return "matched-timeout-passed";
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
return "suppressed";
|
|
163
157
|
}
|
|
164
158
|
|
|
165
159
|
async function reapplySuppressions() {
|
|
@@ -168,15 +162,9 @@ async function reapplySuppressions() {
|
|
|
168
162
|
let changedSuppressionIds = new Set<string>();
|
|
169
163
|
|
|
170
164
|
for (let error of unmatchedErrors) {
|
|
171
|
-
let suppressed =
|
|
172
|
-
for (let
|
|
173
|
-
|
|
174
|
-
if (status === "suppressed" || status === "matched-timeout-passed") {
|
|
175
|
-
changedSuppressionIds.add(suppressionEntry.id);
|
|
176
|
-
}
|
|
177
|
-
if (status === "suppressed") {
|
|
178
|
-
suppressed = true;
|
|
179
|
-
}
|
|
165
|
+
let { suppressed, matchedIds } = applySuppressions(error, suppressionEntries);
|
|
166
|
+
for (let id of matchedIds) {
|
|
167
|
+
changedSuppressionIds.add(id);
|
|
180
168
|
}
|
|
181
169
|
if (!suppressed) {
|
|
182
170
|
newUnmatchedErrors.push(error);
|
|
@@ -355,19 +343,11 @@ export async function exposeErrorWatchService() {
|
|
|
355
343
|
if (!isSuppressedByAny(error, suppressionEntries)) {
|
|
356
344
|
suppressionEntries = await getSuppressionEntriesFresh();
|
|
357
345
|
}
|
|
358
|
-
let suppressed =
|
|
359
|
-
let
|
|
360
|
-
|
|
361
|
-
for (let suppressionEntry of suppressionEntries) {
|
|
362
|
-
let status = applySuppression(error, suppressionEntry);
|
|
363
|
-
if (status === "suppressed" || status === "matched-timeout-passed") {
|
|
364
|
-
matchedSuppressionId = suppressionEntry.id;
|
|
365
|
-
void ErrorNotificationService.triggerSuppressionUpdate(suppressionEntry.id);
|
|
366
|
-
}
|
|
367
|
-
if (status === "suppressed") {
|
|
368
|
-
suppressed = true;
|
|
369
|
-
}
|
|
346
|
+
let { suppressed, matchedIds } = applySuppressions(error, suppressionEntries);
|
|
347
|
+
for (let id of matchedIds) {
|
|
348
|
+
void ErrorNotificationService.triggerSuppressionUpdate(id);
|
|
370
349
|
}
|
|
350
|
+
let matchedSuppressionId: string | undefined = matchedIds[matchedIds.length - 1];
|
|
371
351
|
//console.log(`Watch loop received error, ${suppressed ? "suppressed" : "not suppressed"}, matched suppression id: ${matchedSuppressionId}: ${error.param0}`);
|
|
372
352
|
|
|
373
353
|
if (suppressed) continue;
|
|
@@ -392,7 +372,7 @@ export async function exposeErrorWatchService() {
|
|
|
392
372
|
|
|
393
373
|
await suppression.set(newEntry.id, newEntry);
|
|
394
374
|
invalidateSuppressionEntriesCaches();
|
|
395
|
-
|
|
375
|
+
applySuppressions(error, [newEntry]);
|
|
396
376
|
queueDiscordNotification(newEntry.id, 1, true);
|
|
397
377
|
|
|
398
378
|
void ErrorNotificationService.triggerSuppressionUpdate(newEntry.id);
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
|
|
2
|
+
import { LogDatum } from "../diskLogger";
|
|
3
|
+
|
|
4
|
+
const ALPHABET_SIZE = 256;
|
|
5
|
+
const MATCHER_CACHE_SIZE = 10;
|
|
6
|
+
const SEARCH_OR_CHAR = "|";
|
|
7
|
+
const SEARCH_AND_CHAR = "&";
|
|
8
|
+
const WILD_CARD_CHAR = "*";
|
|
9
|
+
|
|
10
|
+
type Structure = { type: "or" | "and"; parts: Structure[] } | { type: "wildcard"; parts: Buffer[] } | { type: "exact"; value: Buffer };
|
|
11
|
+
type IdStructure = { type: "or" | "and"; parts: IdStructure[] } | { type: "wildcard"; ids: number[]; parts: Buffer[] } | { type: "exact"; id: number };
|
|
12
|
+
|
|
13
|
+
export type PatternSetMatcher = (matchBuffer: Buffer) => number[];
|
|
14
|
+
|
|
15
|
+
let matchBuffers = new WeakMap<LogDatum, Buffer>();
|
|
16
|
+
export function getMatchBuffer(datum: LogDatum): Buffer {
|
|
17
|
+
let buffer = matchBuffers.get(datum);
|
|
18
|
+
if (buffer) return buffer;
|
|
19
|
+
buffer = lowercaseAscii(Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum))));
|
|
20
|
+
matchBuffers.set(datum, buffer);
|
|
21
|
+
return buffer;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
let matcherCache: { patterns: string[]; matcher: PatternSetMatcher }[] = [];
|
|
25
|
+
export function getPatternSetMatcher(patterns: string[]): PatternSetMatcher {
|
|
26
|
+
let cached = matcherCache.find(x => x.patterns.length === patterns.length && x.patterns.every((pattern, i) => pattern === patterns[i]));
|
|
27
|
+
if (cached) return cached.matcher;
|
|
28
|
+
let matcher = createPatternSetMatcher(patterns);
|
|
29
|
+
matcherCache.push({ patterns: patterns.slice(), matcher });
|
|
30
|
+
if (matcherCache.length > MATCHER_CACHE_SIZE) matcherCache.shift();
|
|
31
|
+
return matcher;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function createPatternSetMatcher(patterns: string[]): PatternSetMatcher {
|
|
35
|
+
let literalIds = new Map<string, number>();
|
|
36
|
+
let literals: Buffer[] = [];
|
|
37
|
+
function getLiteralId(value: Buffer): number {
|
|
38
|
+
let key = value.toString("latin1");
|
|
39
|
+
let id = literalIds.get(key);
|
|
40
|
+
if (id !== undefined) return id;
|
|
41
|
+
id = literals.length;
|
|
42
|
+
literalIds.set(key, id);
|
|
43
|
+
literals.push(value);
|
|
44
|
+
return id;
|
|
45
|
+
}
|
|
46
|
+
function toIds(structure: Structure): IdStructure {
|
|
47
|
+
if (structure.type === "exact") return { type: "exact", id: getLiteralId(structure.value) };
|
|
48
|
+
if (structure.type === "wildcard") return { type: "wildcard", ids: structure.parts.map(getLiteralId), parts: structure.parts };
|
|
49
|
+
return { type: structure.type, parts: structure.parts.map(toIds) };
|
|
50
|
+
}
|
|
51
|
+
let structures = patterns.map(pattern => toIds(compilePattern(normalizeMatchWhitespace(pattern))));
|
|
52
|
+
let automaton = buildAutomaton(literals);
|
|
53
|
+
let foundStamp = new Int32Array(literals.length);
|
|
54
|
+
let stamp = 0;
|
|
55
|
+
|
|
56
|
+
return function matchPatternSet(matchBuffer: Buffer): number[] {
|
|
57
|
+
stamp++;
|
|
58
|
+
for (let id of automaton.outputs[0]) {
|
|
59
|
+
foundStamp[id] = stamp;
|
|
60
|
+
}
|
|
61
|
+
let state = 0;
|
|
62
|
+
for (let i = 0; i < matchBuffer.length; i++) {
|
|
63
|
+
state = automaton.transitions[state * ALPHABET_SIZE + matchBuffer[i]];
|
|
64
|
+
let outputState = automaton.hasOwnOutput[state] && state || automaton.dictionaryLink[state];
|
|
65
|
+
while (outputState > 0) {
|
|
66
|
+
let outputs = automaton.outputs[outputState];
|
|
67
|
+
for (let j = 0; j < outputs.length; j++) {
|
|
68
|
+
foundStamp[outputs[j]] = stamp;
|
|
69
|
+
}
|
|
70
|
+
outputState = automaton.dictionaryLink[outputState];
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
let matched: number[] = [];
|
|
74
|
+
for (let patternIndex = 0; patternIndex < structures.length; patternIndex++) {
|
|
75
|
+
if (matchesWithIds(structures[patternIndex])) matched.push(patternIndex);
|
|
76
|
+
}
|
|
77
|
+
return matched;
|
|
78
|
+
|
|
79
|
+
function matchesWithIds(structure: IdStructure): boolean {
|
|
80
|
+
if (structure.type === "exact") return foundStamp[structure.id] === stamp;
|
|
81
|
+
if (structure.type === "or") return structure.parts.some(matchesWithIds);
|
|
82
|
+
if (structure.type === "and") return structure.parts.every(matchesWithIds);
|
|
83
|
+
if (!structure.ids.every(id => foundStamp[id] === stamp)) return false;
|
|
84
|
+
let position = 0;
|
|
85
|
+
for (let segment of structure.parts) {
|
|
86
|
+
let found = matchBuffer.indexOf(segment, position);
|
|
87
|
+
if (found === -1) return false;
|
|
88
|
+
position = found + segment.length;
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function lowercaseAscii(buffer: Buffer): Buffer {
|
|
96
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
97
|
+
let byte = buffer[i];
|
|
98
|
+
if (65 <= byte && byte <= 90) buffer[i] = byte + 32;
|
|
99
|
+
}
|
|
100
|
+
return buffer;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function compilePattern(pattern: string): Structure {
|
|
104
|
+
let lowered = lowercaseAscii(Buffer.from(pattern)).toString("utf8");
|
|
105
|
+
if (!lowered.includes(SEARCH_OR_CHAR) && !lowered.includes(SEARCH_AND_CHAR) && !lowered.includes(WILD_CARD_CHAR)) {
|
|
106
|
+
return { type: "exact", value: Buffer.from(lowered) };
|
|
107
|
+
}
|
|
108
|
+
return parseStructure(lowered);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function parseStructure(text: string): Structure {
|
|
112
|
+
let orParts = text.split(SEARCH_OR_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
113
|
+
if (orParts.length > 1) return { type: "or", parts: orParts.map(parseStructure) };
|
|
114
|
+
let andParts = text.split(SEARCH_AND_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
115
|
+
if (andParts.length > 1) return { type: "and", parts: andParts.map(parseStructure) };
|
|
116
|
+
let wildcardParts = text.split(WILD_CARD_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
117
|
+
if (wildcardParts.length > 1) return { type: "wildcard", parts: wildcardParts.map(x => Buffer.from(x)) };
|
|
118
|
+
return { type: "exact", value: Buffer.from(text) };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function buildAutomaton(literals: Buffer[]) {
|
|
122
|
+
let children: Map<number, number>[] = [new Map()];
|
|
123
|
+
let outputs: number[][] = [[]];
|
|
124
|
+
for (let literalIndex = 0; literalIndex < literals.length; literalIndex++) {
|
|
125
|
+
let state = 0;
|
|
126
|
+
for (let byte of literals[literalIndex]) {
|
|
127
|
+
let next = children[state].get(byte);
|
|
128
|
+
if (next === undefined) {
|
|
129
|
+
next = children.length;
|
|
130
|
+
children.push(new Map());
|
|
131
|
+
outputs.push([]);
|
|
132
|
+
children[state].set(byte, next);
|
|
133
|
+
}
|
|
134
|
+
state = next;
|
|
135
|
+
}
|
|
136
|
+
outputs[state].push(literalIndex);
|
|
137
|
+
}
|
|
138
|
+
let stateCount = children.length;
|
|
139
|
+
let transitions = new Int32Array(stateCount * ALPHABET_SIZE);
|
|
140
|
+
let failure = new Int32Array(stateCount);
|
|
141
|
+
let dictionaryLink = new Int32Array(stateCount);
|
|
142
|
+
let hasOwnOutput = new Uint8Array(stateCount);
|
|
143
|
+
for (let state = 0; state < stateCount; state++) {
|
|
144
|
+
if (outputs[state].length) hasOwnOutput[state] = 1;
|
|
145
|
+
}
|
|
146
|
+
let queue: number[] = [];
|
|
147
|
+
for (let byte = 0; byte < ALPHABET_SIZE; byte++) {
|
|
148
|
+
let child = children[0].get(byte);
|
|
149
|
+
transitions[byte] = child ?? 0;
|
|
150
|
+
if (child !== undefined) queue.push(child);
|
|
151
|
+
}
|
|
152
|
+
for (let queueIndex = 0; queueIndex < queue.length; queueIndex++) {
|
|
153
|
+
let state = queue[queueIndex];
|
|
154
|
+
let failState = failure[state];
|
|
155
|
+
dictionaryLink[state] = hasOwnOutput[failState] && failState || dictionaryLink[failState];
|
|
156
|
+
for (let byte = 0; byte < ALPHABET_SIZE; byte++) {
|
|
157
|
+
let child = children[state].get(byte);
|
|
158
|
+
if (child === undefined) {
|
|
159
|
+
transitions[state * ALPHABET_SIZE + byte] = transitions[failState * ALPHABET_SIZE + byte];
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
transitions[state * ALPHABET_SIZE + byte] = child;
|
|
163
|
+
failure[child] = transitions[failState * ALPHABET_SIZE + byte];
|
|
164
|
+
queue.push(child);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return { transitions, dictionaryLink, hasOwnOutput, outputs };
|
|
168
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
{
|
|
4
|
+
let argv = process.argv;
|
|
5
|
+
let cwdIdx = argv.indexOf("--cwd");
|
|
6
|
+
if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
|
|
7
|
+
process.chdir(argv[cwdIdx + 1]);
|
|
8
|
+
argv.splice(cwdIdx, 2);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
import "../../inject";
|
|
13
|
+
|
|
14
|
+
import yargs from "yargs";
|
|
15
|
+
import { formatTime } from "socket-function/src/formatting/format";
|
|
16
|
+
import { magenta } from "socket-function/src/formatting/logColors";
|
|
17
|
+
import { getArchives2 } from "../../-a-archives/archives2";
|
|
18
|
+
import { decodeLogFilePath } from "../logs/IndexedLogs/TimeFileTree";
|
|
19
|
+
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
20
|
+
|
|
21
|
+
let args = yargs(process.argv.slice(2))
|
|
22
|
+
.option("day", { type: "string", demandOption: true, describe: "UTC day folder, ex 2026/11/20" })
|
|
23
|
+
.option("logger", { type: "string", default: "logs/log" })
|
|
24
|
+
.strict()
|
|
25
|
+
.argv;
|
|
26
|
+
|
|
27
|
+
async function listByMachine(config: { label: string; list: () => Promise<string[]> }) {
|
|
28
|
+
let start = Date.now();
|
|
29
|
+
let files = await config.list();
|
|
30
|
+
let byMachine = new Map<string, number>();
|
|
31
|
+
for (let file of files) {
|
|
32
|
+
let decoded = decodeLogFilePath(file);
|
|
33
|
+
if (!decoded) continue;
|
|
34
|
+
let machineId = decoded.machineId || "(none)";
|
|
35
|
+
byMachine.set(machineId, (byMachine.get(machineId) || 0) + 1);
|
|
36
|
+
}
|
|
37
|
+
console.log(magenta(`${config.label}: ${files.length} files in ${formatTime(Date.now() - start)}`), [...byMachine].map(([id, count]) => `${id}=${count}`).join(", "));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function main() {
|
|
41
|
+
disablePathAuditer();
|
|
42
|
+
let archives = getArchives2(`final-indexed-logs/${args.logger}`);
|
|
43
|
+
let folder = args.day.endsWith("/") ? args.day : args.day + "/";
|
|
44
|
+
console.log(magenta(`Listing ${archives.getDebugName()}${folder}`));
|
|
45
|
+
await Promise.all([
|
|
46
|
+
listByMachine({ label: "find", list: () => archives.find(folder, { shallow: true, type: "files" }) }),
|
|
47
|
+
listByMachine({ label: "find with fallbacks", list: () => archives.find(folder, { shallow: true, type: "files", fallbacks: true }) }),
|
|
48
|
+
]);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
main().then(
|
|
52
|
+
() => process.exit(),
|
|
53
|
+
e => {
|
|
54
|
+
console.error((e as Error).stack || e);
|
|
55
|
+
process.exit(1);
|
|
56
|
+
},
|
|
57
|
+
);
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
{
|
|
4
|
+
let argv = process.argv;
|
|
5
|
+
let cwdIdx = argv.indexOf("--cwd");
|
|
6
|
+
if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
|
|
7
|
+
process.chdir(argv[cwdIdx + 1]);
|
|
8
|
+
argv.splice(cwdIdx, 2);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
import "../../inject";
|
|
13
|
+
|
|
14
|
+
import yargs from "yargs";
|
|
15
|
+
import { formatDateTime } from "socket-function/src/formatting/format";
|
|
16
|
+
import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
|
|
17
|
+
import { timeInMinute } from "socket-function/src/misc";
|
|
18
|
+
import { MCPIndexedLogs, normalizeTime } from "../logs/IndexedLogs/MCPIndexedLogs";
|
|
19
|
+
import { machineNames } from "../../deployManager/machineSchema";
|
|
20
|
+
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
21
|
+
|
|
22
|
+
const DEFAULT_LIMIT = 200;
|
|
23
|
+
const DEFAULT_COLUMNS = ["__entry", "__threadId", "param0", "param1", "param2", "param3"];
|
|
24
|
+
const DEFAULT_END_AGE = timeInMinute * 2;
|
|
25
|
+
|
|
26
|
+
let args = yargs(process.argv.slice(2))
|
|
27
|
+
.option("machine", { type: "string", demandOption: true, describe: "machine id, a substring of a machine name (ex \"5600x\"), or \"local\"" })
|
|
28
|
+
.option("query", { type: "string", default: "", describe: "| = OR, & = AND, * = ordered wildcard; case-insensitive substrings" })
|
|
29
|
+
.option("start", { type: "string", demandOption: true, describe: "epoch ms or any Date string (no timezone = local time)" })
|
|
30
|
+
.option("end", { type: "string", describe: `epoch ms or any Date string (default ${DEFAULT_END_AGE / timeInMinute} minutes ago)` })
|
|
31
|
+
.option("direction", { choices: ["fromStart", "fromEnd"] as const, default: "fromEnd" as const })
|
|
32
|
+
.option("columns", { type: "array", string: true, describe: `fields to print (default ${DEFAULT_COLUMNS.join(", ")})` })
|
|
33
|
+
.option("limit", { type: "number", default: DEFAULT_LIMIT })
|
|
34
|
+
.option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders instead of using the cached file listing" })
|
|
35
|
+
.strict()
|
|
36
|
+
.argv;
|
|
37
|
+
|
|
38
|
+
async function resolveMachineId(machine: string): Promise<string> {
|
|
39
|
+
if (machine === "local") return machine;
|
|
40
|
+
let names = await machineNames.values();
|
|
41
|
+
if (names.some(x => x.machineId === machine)) return machine;
|
|
42
|
+
let matches = names.filter(x => x.name.toLowerCase().includes(machine.toLowerCase()));
|
|
43
|
+
if (matches.length === 1) {
|
|
44
|
+
console.log(magenta(`Machine "${machine}" is ${matches[0].name} (${matches[0].machineId})`));
|
|
45
|
+
return matches[0].machineId;
|
|
46
|
+
}
|
|
47
|
+
if (matches.length > 1) {
|
|
48
|
+
throw new Error(`Machine "${machine}" matches several machines: ${matches.map(x => `${x.name} (${x.machineId})`).join(", ")}`);
|
|
49
|
+
}
|
|
50
|
+
console.warn(yellow(`No machine is named like "${machine}", treating it as a machine id`));
|
|
51
|
+
return machine;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async function main() {
|
|
55
|
+
disablePathAuditer();
|
|
56
|
+
let machine = await resolveMachineId(args.machine);
|
|
57
|
+
let columns = (args.columns || DEFAULT_COLUMNS).map(String);
|
|
58
|
+
let endTime = args.end && normalizeTime(args.end, "end") || Date.now() - DEFAULT_END_AGE;
|
|
59
|
+
let result = await new MCPIndexedLogs().search({
|
|
60
|
+
query: args.query,
|
|
61
|
+
machine,
|
|
62
|
+
startTime: normalizeTime(args.start, "start"),
|
|
63
|
+
endTime,
|
|
64
|
+
direction: args.direction,
|
|
65
|
+
columns,
|
|
66
|
+
limit: args.limit,
|
|
67
|
+
forceRefresh: args.refresh,
|
|
68
|
+
});
|
|
69
|
+
let rows = result.results.slice();
|
|
70
|
+
rows.sort((a, b) => Number(a.time) - Number(b.time));
|
|
71
|
+
for (let row of rows) {
|
|
72
|
+
let fields = columns.filter(x => row[x] !== undefined).map(x => x.startsWith("param") ? row[x] : `${x}=${row[x]}`);
|
|
73
|
+
console.log(blue(formatDateTime(Number(row.time))), ...fields);
|
|
74
|
+
}
|
|
75
|
+
console.log(magenta(`${rows.length} rows`), JSON.stringify(result.files));
|
|
76
|
+
if (result.note) console.log(yellow(result.note));
|
|
77
|
+
console.log(magenta(`Columns seen: ${result.allColumns.join(", ")}`));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
main().then(
|
|
81
|
+
() => process.exit(),
|
|
82
|
+
e => {
|
|
83
|
+
console.error((e as Error).stack || e);
|
|
84
|
+
process.exit(1);
|
|
85
|
+
},
|
|
86
|
+
);
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
{
|
|
4
|
+
let argv = process.argv;
|
|
5
|
+
let cwdIdx = argv.indexOf("--cwd");
|
|
6
|
+
if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
|
|
7
|
+
process.chdir(argv[cwdIdx + 1]);
|
|
8
|
+
argv.splice(cwdIdx, 2);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
import "../../inject";
|
|
13
|
+
|
|
14
|
+
import { decodeNodeId } from "sliftutils/misc/https/certs";
|
|
15
|
+
import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
|
|
16
|
+
import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
|
|
17
|
+
import { sort } from "socket-function/src/misc";
|
|
18
|
+
import { getArchives2Buffered } from "../../-a-archives/archives2";
|
|
19
|
+
import { DEAD_THRESHOLD, NodeDiscoveryController } from "../../-f-node-discovery/NodeDiscovery";
|
|
20
|
+
import { NodeCapabilitiesController, NodeMetadata } from "../../-g-core-values/NodeCapabilities";
|
|
21
|
+
import { getDomain } from "../../config";
|
|
22
|
+
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
23
|
+
import { probeAsync, ProbeResult, serviceFolderFromEntryPoint, shortEntryPoint } from "./networkProbe";
|
|
24
|
+
|
|
25
|
+
const NODE_CALL_TIMEOUT_MS = 10_000;
|
|
26
|
+
|
|
27
|
+
type NodeReport = {
|
|
28
|
+
nodeId: string;
|
|
29
|
+
machineId: string;
|
|
30
|
+
port: number;
|
|
31
|
+
heartbeatTime: number;
|
|
32
|
+
alive: ProbeResult<boolean>;
|
|
33
|
+
metadata?: ProbeResult<NodeMetadata>;
|
|
34
|
+
knownNodes?: ProbeResult<string[]>;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
async function main() {
|
|
38
|
+
disablePathAuditer();
|
|
39
|
+
let domain = getDomain();
|
|
40
|
+
let archive = getArchives2Buffered("nodes/");
|
|
41
|
+
|
|
42
|
+
console.log(magenta(`Reading the node list from ${archive.getDebugName()} (domain ${domain})`));
|
|
43
|
+
let listStart = Date.now();
|
|
44
|
+
let nodeIds = await archive.find("", { fallbacks: true });
|
|
45
|
+
console.log(magenta(`Node list has ${nodeIds.length} nodes, read in ${formatTime(Date.now() - listStart)}`));
|
|
46
|
+
|
|
47
|
+
let reports = await Promise.all(nodeIds.map(async nodeId => {
|
|
48
|
+
let parts = decodeNodeId(nodeId, domain);
|
|
49
|
+
let heartbeatPromise = archive.get(nodeId).then(x => Number(x?.toString()) || 0, () => 0);
|
|
50
|
+
let alive = await probeAsync({ timeoutMs: NODE_CALL_TIMEOUT_MS, call: () => NodeDiscoveryController.nodes[nodeId].isAlive() });
|
|
51
|
+
let report: NodeReport = {
|
|
52
|
+
nodeId,
|
|
53
|
+
machineId: parts?.machineId || "",
|
|
54
|
+
port: parts?.port || 0,
|
|
55
|
+
heartbeatTime: 0,
|
|
56
|
+
alive,
|
|
57
|
+
};
|
|
58
|
+
if (alive.value) {
|
|
59
|
+
let [metadata, knownNodes] = await Promise.all([
|
|
60
|
+
probeAsync({ timeoutMs: NODE_CALL_TIMEOUT_MS, call: () => NodeCapabilitiesController.nodes[nodeId].getMetadata() }),
|
|
61
|
+
probeAsync({ timeoutMs: NODE_CALL_TIMEOUT_MS, call: () => NodeDiscoveryController.nodes[nodeId].getAllNodeIds() }),
|
|
62
|
+
]);
|
|
63
|
+
report.metadata = metadata;
|
|
64
|
+
report.knownNodes = knownNodes;
|
|
65
|
+
}
|
|
66
|
+
report.heartbeatTime = await heartbeatPromise;
|
|
67
|
+
logNodeReport(report);
|
|
68
|
+
return report;
|
|
69
|
+
}));
|
|
70
|
+
|
|
71
|
+
logSummary({ reports, nodeIds });
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function heartbeatText(heartbeatTime: number): string {
|
|
75
|
+
if (!heartbeatTime) return red("no heartbeat file");
|
|
76
|
+
let age = Date.now() - heartbeatTime;
|
|
77
|
+
let text = `heartbeat ${formatTime(age)} ago`;
|
|
78
|
+
if (age > DEAD_THRESHOLD) return red(`${text} (stale, dead threshold ${formatTime(DEAD_THRESHOLD)})`);
|
|
79
|
+
return text;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function logNodeReport(report: NodeReport) {
|
|
83
|
+
let where = `${report.machineId}:${report.port}`;
|
|
84
|
+
if (!report.alive.value) {
|
|
85
|
+
console.log(red(`✗ ${report.nodeId}`), where, heartbeatText(report.heartbeatTime), red(`unreachable after ${formatTime(report.alive.ms)}: ${report.alive.error}`));
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
let parts = [green(`✓ ${report.nodeId}`), where, heartbeatText(report.heartbeatTime), `ping ${formatTime(report.alive.ms)}`];
|
|
89
|
+
let metadata = report.metadata?.value;
|
|
90
|
+
if (metadata) {
|
|
91
|
+
let service = serviceFolderFromEntryPoint(metadata.entryPoint);
|
|
92
|
+
if (service) parts.push(`service ${service}`);
|
|
93
|
+
parts.push(`entry ${shortEntryPoint(metadata.entryPoint)}`, `up ${formatTime(Date.now() - metadata.startupTime)}`, `pid ${metadata.processId}`, `git ${metadata.gitRef.slice(0, 8)}`);
|
|
94
|
+
if (metadata.scheduledShutdownTime) parts.push(yellow(`scheduled shutdown ${formatDateTime(metadata.scheduledShutdownTime)}`));
|
|
95
|
+
} else if (report.metadata) {
|
|
96
|
+
parts.push(yellow(`metadata failed: ${report.metadata.error}`));
|
|
97
|
+
}
|
|
98
|
+
let known = report.knownNodes?.value;
|
|
99
|
+
if (known) {
|
|
100
|
+
parts.push(`knows ${known.length} nodes`);
|
|
101
|
+
} else if (report.knownNodes) {
|
|
102
|
+
parts.push(yellow(`node list failed: ${report.knownNodes.error}`));
|
|
103
|
+
}
|
|
104
|
+
console.log(...parts);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function logSummary(config: { reports: NodeReport[]; nodeIds: string[] }) {
|
|
108
|
+
let { reports, nodeIds } = config;
|
|
109
|
+
let listed = new Set(nodeIds);
|
|
110
|
+
let reachable = reports.filter(x => x.alive.value);
|
|
111
|
+
let unreachable = reports.filter(x => !x.alive.value);
|
|
112
|
+
let isFresh = (x: NodeReport) => !!x.heartbeatTime && Date.now() - x.heartbeatTime <= DEAD_THRESHOLD;
|
|
113
|
+
|
|
114
|
+
console.log();
|
|
115
|
+
console.log(magenta(`===== Summary =====`));
|
|
116
|
+
console.log(`${nodeIds.length} listed, ${green(`${reachable.length} reachable`)}, ${unreachable.length && red(`${unreachable.length} unreachable`) || "0 unreachable"}`);
|
|
117
|
+
|
|
118
|
+
let struggling = unreachable.filter(isFresh);
|
|
119
|
+
if (struggling.length) {
|
|
120
|
+
console.log(red(`Unreachable but still writing heartbeats (alive, but not answering calls):`));
|
|
121
|
+
for (let report of struggling) {
|
|
122
|
+
console.log(red(` ${report.nodeId}`), heartbeatText(report.heartbeatTime), report.alive.error?.split("\n")[0]);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
let stale = reports.filter(x => !isFresh(x));
|
|
126
|
+
if (stale.length) {
|
|
127
|
+
console.log(yellow(`Stale or missing heartbeats (dead, waiting to be removed):`));
|
|
128
|
+
for (let report of stale) {
|
|
129
|
+
console.log(yellow(` ${report.nodeId}`), heartbeatText(report.heartbeatTime), report.alive.value && green("but it answered calls") || "");
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
let byPort = new Map<string, NodeReport[]>();
|
|
134
|
+
for (let report of reports) {
|
|
135
|
+
let key = `${report.machineId}:${report.port}`;
|
|
136
|
+
let list = byPort.get(key);
|
|
137
|
+
if (!list) {
|
|
138
|
+
list = [];
|
|
139
|
+
byPort.set(key, list);
|
|
140
|
+
}
|
|
141
|
+
list.push(report);
|
|
142
|
+
}
|
|
143
|
+
let duplicates = [...byPort].filter(([, list]) => list.length > 1);
|
|
144
|
+
if (duplicates.length) {
|
|
145
|
+
console.log(yellow(`Several threads listed on the same machine and port (all but one are dead threads):`));
|
|
146
|
+
for (let [key, list] of duplicates) {
|
|
147
|
+
console.log(yellow(` ${key}`), list.map(x => `${x.nodeId} (${x.alive.value && "reachable" || "unreachable"})`).join(", "));
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
let byMachine = new Map<string, NodeReport[]>();
|
|
152
|
+
for (let report of reports) {
|
|
153
|
+
let list = byMachine.get(report.machineId);
|
|
154
|
+
if (!list) {
|
|
155
|
+
list = [];
|
|
156
|
+
byMachine.set(report.machineId, list);
|
|
157
|
+
}
|
|
158
|
+
list.push(report);
|
|
159
|
+
}
|
|
160
|
+
console.log(blue(`By machine:`));
|
|
161
|
+
let machines = [...byMachine];
|
|
162
|
+
sort(machines, ([machineId]) => machineId);
|
|
163
|
+
for (let [machineId, list] of machines) {
|
|
164
|
+
let up = list.filter(x => x.alive.value).length;
|
|
165
|
+
let entries = list.map(x => {
|
|
166
|
+
let entryPoint = x.metadata?.value?.entryPoint;
|
|
167
|
+
let label = entryPoint && (serviceFolderFromEntryPoint(entryPoint) || shortEntryPoint(entryPoint)) || "?";
|
|
168
|
+
return `${x.port}=${label}${x.alive.value ? "" : "✗"}`;
|
|
169
|
+
});
|
|
170
|
+
sort(entries, x => x);
|
|
171
|
+
let color = up === list.length && green || up && yellow || red;
|
|
172
|
+
console.log(color(` ${machineId || "(undecodable node id)"} ${up}/${list.length} reachable`), entries.join(" "));
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
let gitRefs = new Map<string, number>();
|
|
176
|
+
for (let report of reachable) {
|
|
177
|
+
let gitRef = report.metadata?.value?.gitRef;
|
|
178
|
+
if (!gitRef) continue;
|
|
179
|
+
gitRefs.set(gitRef, (gitRefs.get(gitRef) || 0) + 1);
|
|
180
|
+
}
|
|
181
|
+
if (gitRefs.size > 1) {
|
|
182
|
+
console.log(blue(`Git refs running:`), [...gitRefs].map(([ref, count]) => `${ref.slice(0, 8)} x${count}`).join(", "));
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
let reporters = reachable.filter(x => x.knownNodes?.value);
|
|
186
|
+
let ghosts = new Map<string, string[]>();
|
|
187
|
+
let missingFromPeers = new Map<string, number>();
|
|
188
|
+
for (let report of reporters) {
|
|
189
|
+
let known = new Set(report.knownNodes?.value || []);
|
|
190
|
+
for (let nodeId of known) {
|
|
191
|
+
if (listed.has(nodeId)) continue;
|
|
192
|
+
let list = ghosts.get(nodeId);
|
|
193
|
+
if (!list) {
|
|
194
|
+
list = [];
|
|
195
|
+
ghosts.set(nodeId, list);
|
|
196
|
+
}
|
|
197
|
+
list.push(report.nodeId);
|
|
198
|
+
}
|
|
199
|
+
for (let nodeId of nodeIds) {
|
|
200
|
+
if (known.has(nodeId)) continue;
|
|
201
|
+
missingFromPeers.set(nodeId, (missingFromPeers.get(nodeId) || 0) + 1);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (ghosts.size) {
|
|
205
|
+
console.log(yellow(`Nodes peers still know about that are not in the node list (ghosts in their memory):`));
|
|
206
|
+
for (let [nodeId, by] of ghosts) {
|
|
207
|
+
console.log(yellow(` ${nodeId}`), `known by ${by.length}/${reporters.length} peers`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
let missing = [...missingFromPeers];
|
|
211
|
+
if (missing.length) {
|
|
212
|
+
console.log(yellow(`Listed nodes that some peers don't know about:`));
|
|
213
|
+
for (let [nodeId, count] of missing) {
|
|
214
|
+
console.log(yellow(` ${nodeId}`), `missing from ${count}/${reporters.length} peers`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
main().then(
|
|
220
|
+
() => process.exit(),
|
|
221
|
+
e => {
|
|
222
|
+
console.error((e as Error).stack || e);
|
|
223
|
+
process.exit(1);
|
|
224
|
+
},
|
|
225
|
+
);
|