querysub 0.720.0 → 0.722.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/flushlogs.js +7 -0
- package/debugging.md +43 -0
- package/package.json +9 -3
- package/src/-f-node-discovery/NodeDiscovery.ts +1 -1
- package/src/0-path-value-core/AuthorityLookup.ts +0 -1
- package/src/0-path-value-core/PathRouter.ts +0 -1
- package/src/0-path-value-core/PathWatcher.ts +0 -3
- package/src/0-path-value-core/ValidStateComputer.ts +0 -3
- package/src/2-proxy/PathValueProxyWatcher.ts +0 -3
- package/src/2-proxy/TransactionDelayer.ts +0 -1
- package/src/4-deploy/edgeNodes.ts +0 -1
- package/src/4-querysub/querysubPrediction.ts +0 -1
- package/src/config.ts +0 -1
- package/src/diagnostics/StatsHeader.tsx +2 -1
- package/src/diagnostics/StatsOverrides.ts +35 -0
- package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +0 -1
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +36 -23
- package/src/diagnostics/logs/IndexedLogs/flushLogsEntry.ts +42 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +6 -0
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +7 -19
- package/src/diagnostics/logs/errorNotifications2/ErrorWarning.tsx +4 -5
- package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +27 -47
- package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +168 -0
- package/src/diagnostics/network/logFilesMain.ts +57 -0
- package/src/diagnostics/network/logSearchMain.ts +86 -0
- package/src/diagnostics/network/networkNodesMain.ts +225 -0
- package/src/diagnostics/network/networkProbe.ts +31 -0
- package/src/diagnostics/network/networkServicesMain.ts +170 -0
- package/src/diagnostics/network/portMappingsMain.ts +28 -0
- package/src/diagnostics/statsDefinitions.tsx +0 -1
- package/src/library-components/LazyComponent.tsx +0 -1
- package/src/library-components/URLParam.ts +0 -2
package/bin/flushlogs.js
ADDED
package/debugging.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Debugging the Network
|
|
2
|
+
|
|
3
|
+
When the site won't load, the debug pages are unreachable, or nodes keep dying or being removed, work through these steps in order. Each step needs less of the network to be up than the one after it, so start at the top and only move on when the current step doesn't explain the problem.
|
|
4
|
+
|
|
5
|
+
Every script takes `--cwd <project root>` so it runs with that project's domain, keys, and certs (for the cyoa network: `--cwd D:/repos/qs-cyoa/`). Both scripts are read-only, except that, like every server-mode querysub process, they run the node-discovery audit, which deletes node list entries whose port has provably been taken by a newer thread.
|
|
6
|
+
|
|
7
|
+
## 1. What nodes exist
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
yarn network-nodes --cwd D:/repos/qs-cyoa/
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Reads the node list (the `nodes/` folder of the fast archive bucket, which falls back to Backblaze if the storage servers are down), then calls every node and logs each one as it answers: heartbeat age, ping, service folder, entry point, uptime, git ref, and how many nodes it knows about. It ends with a summary:
|
|
14
|
+
|
|
15
|
+
- **Unreachable but still writing heartbeats**: the process is alive but nobody can call it. Usually networking (a port not forwarded, a firewall, another process answering on that port).
|
|
16
|
+
- **Stale or missing heartbeats**: dead processes waiting to be removed. Entries that stay listed with no heartbeat file across several runs mean deletes aren't sticking in storage.
|
|
17
|
+
- **Several threads on the same machine and port**: all but one are dead threads.
|
|
18
|
+
- **By machine**: which ports each machine is running and what is on them.
|
|
19
|
+
- **Ghosts / missing**: disagreement between the node list and what the nodes themselves hold in memory.
|
|
20
|
+
|
|
21
|
+
The node list lives on the storage servers, so if this step is slow or fails, the storage servers are the first suspect.
|
|
22
|
+
|
|
23
|
+
## 2. Which services are healthy
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
yarn network-services --cwd D:/repos/qs-cyoa/
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Reads the service configs and machine heartbeats (also from the storage servers, falling back to Backblaze, so it can be slow when they are down), works out every instance that should be running, and checks each instance's recorded node against the node list and with a call. It reports machines with stale heartbeats, instances a machine never reported, instances with no node id yet, and instances whose node is unreachable or missing from the node list. A node id that the machine recorded but whose port is now held by another process usually means the machine's record is stale, not that the service is down; check the step 1 output for a reachable node with the same service folder.
|
|
30
|
+
|
|
31
|
+
`overview.md` describes the roles that should exist (path value servers, function runners, the client-facing querysub server, error notifications, logging, machine management, storage); every one of them should show up here as a healthy service.
|
|
32
|
+
|
|
33
|
+
## 3. Look at the machine directly
|
|
34
|
+
|
|
35
|
+
If the problem still isn't apparent, SSH into the machine (`ssh root@<machine>`). Every service instance runs in its own tmux session named `<service key>-<instance index>-dply` (for example `cyoa-0-dply`, `serverall-2-dply`), and the machine's own deploy process runs in `machine-alwaysup`.
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
tmux ls
|
|
39
|
+
tmux attach -t cyoa-0-dply
|
|
40
|
+
tmux capture-pane -t cyoa-0-dply -p -S -2000
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Each instance's folder is `~/machine-services/<session name>/`, holding the `parameters.json` it was started with and `serviceNodeId.txt`, the node id it registered as.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.722.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -28,6 +28,11 @@
|
|
|
28
28
|
"thin": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --nobreak --public",
|
|
29
29
|
"thin-watch": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --nobreak --public --watch",
|
|
30
30
|
"thin-dry": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --nobreak --public --dry-run",
|
|
31
|
+
"network-nodes": "yarn typenode ./src/diagnostics/network/networkNodesMain.ts",
|
|
32
|
+
"network-services": "yarn typenode ./src/diagnostics/network/networkServicesMain.ts",
|
|
33
|
+
"port-mappings": "yarn typenode ./src/diagnostics/network/portMappingsMain.ts",
|
|
34
|
+
"log-search": "yarn typenode ./src/diagnostics/network/logSearchMain.ts",
|
|
35
|
+
"log-files": "yarn typenode ./src/diagnostics/network/logFilesMain.ts",
|
|
31
36
|
"parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
|
|
32
37
|
"setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
|
|
33
38
|
"stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
|
|
@@ -50,6 +55,7 @@
|
|
|
50
55
|
"join": "./bin/join.js",
|
|
51
56
|
"join-public": "./bin/join-public.js",
|
|
52
57
|
"movelogs": "./bin/movelogs.js",
|
|
58
|
+
"flushlogs": "./bin/flushlogs.js",
|
|
53
59
|
"addsuperuser": "./bin/addsuperuser.js",
|
|
54
60
|
"error-watch": "./bin/error-watch.js",
|
|
55
61
|
"error-watch-public": "./bin/error-watch-public.js",
|
|
@@ -81,10 +87,10 @@
|
|
|
81
87
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
82
88
|
"pako": "^2.1.0",
|
|
83
89
|
"peggy": "^5.0.6",
|
|
84
|
-
"sliftutils": "^1.9.
|
|
90
|
+
"sliftutils": "^1.9.17",
|
|
85
91
|
"socket-function": "^1.3.1",
|
|
86
92
|
"terser": "^5.31.0",
|
|
87
|
-
"typenode": "^6.6.
|
|
93
|
+
"typenode": "^6.6.3",
|
|
88
94
|
"typesafecss": "^0.32.0",
|
|
89
95
|
"yaml": "^2.5.0",
|
|
90
96
|
"yargs": "^15.3.1"
|
|
@@ -28,7 +28,7 @@ let HEARTBEAT_INTERVAL = timeInMinute * 1;
|
|
|
28
28
|
// Interval which we check other heartbeats
|
|
29
29
|
let CHECK_INTERVAL = HEARTBEAT_INTERVAL;
|
|
30
30
|
// If the heartbeat is older than thing, it fails the dead check
|
|
31
|
-
|
|
31
|
+
export const DEAD_THRESHOLD = HEARTBEAT_INTERVAL * 2;
|
|
32
32
|
// If we find another node is dead this number of checks, we remove it from storage, and tell
|
|
33
33
|
// all other nodes to remove from their memory.
|
|
34
34
|
let DEAD_CHECK_COUNT = 4;
|
|
@@ -231,7 +231,6 @@ export class PathRouter {
|
|
|
231
231
|
if (prefix) {
|
|
232
232
|
let key = getPathIndex(path, prefix.childKeyIndex);
|
|
233
233
|
if (key === undefined) {
|
|
234
|
-
require("debugbreak")(2);
|
|
235
234
|
debugger;
|
|
236
235
|
throw new Error(`Impossible, hash index ${prefix.childKeyIndex} is out of range for path ${path}, but it matched the prefix ${prefix.originalPrefix}`);
|
|
237
236
|
}
|
|
@@ -52,8 +52,6 @@ function incrementWatcherSequence() {
|
|
|
52
52
|
|
|
53
53
|
if (!registerResource) {
|
|
54
54
|
debugger;
|
|
55
|
-
require("debugbreak")(2);
|
|
56
|
-
debugger;
|
|
57
55
|
}
|
|
58
56
|
|
|
59
57
|
// WATCH CASES
|
|
@@ -112,7 +110,6 @@ class PathWatcher {
|
|
|
112
110
|
}) {
|
|
113
111
|
this.ensureUnwatchingOnDisconnect(config.nodeId);
|
|
114
112
|
if (config.nodeId.startsWith(getInternalValueWatchPrefix()) && (!config.noInitialTrigger || config.initialTrigger)) {
|
|
115
|
-
require("debugbreak")(2);
|
|
116
113
|
debugger;
|
|
117
114
|
// No initial trigger because if it's a remote value, then we won't have it. If it's a local value, then we've already processed it in valid state computers, there's no reason to trigger it again.
|
|
118
115
|
throw new Error("Internal value watches (INTERNAL_VALUE_WATCH_PREFIX) cannot have initial triggers");
|
|
@@ -209,7 +209,6 @@ class ValidStateComputer {
|
|
|
209
209
|
while (dependenciesChanged.size > 0) {
|
|
210
210
|
loopCount++;
|
|
211
211
|
if (loopCount > 3000) {
|
|
212
|
-
require("debugbreak")(2);
|
|
213
212
|
debugger;
|
|
214
213
|
console.error("Too many loops in valid state loop. aborting.", {
|
|
215
214
|
loopCount,
|
|
@@ -350,7 +349,6 @@ class ValidStateComputer {
|
|
|
350
349
|
let lockTime = locks[0].endTime;
|
|
351
350
|
for (let lock of locks) {
|
|
352
351
|
if (compareTime(lock.endTime, lockTime) !== 0) {
|
|
353
|
-
require("debugbreak")(2);
|
|
354
352
|
debugger;
|
|
355
353
|
console.error(`Lock has inconsistent end times. ${debugTime(lock.endTime)} !== ${debugTime(lockTime)}. Path: ${lock.path}`);
|
|
356
354
|
valid = false;
|
|
@@ -359,7 +357,6 @@ class ValidStateComputer {
|
|
|
359
357
|
|
|
360
358
|
for (let value of valueGroup) {
|
|
361
359
|
if (!PathRouter.isSelfAuthority(value.path)) {
|
|
362
|
-
require("debugbreak")(2);
|
|
363
360
|
debugger;
|
|
364
361
|
console.error(`Somehow we're attempting to compute the valid state of a value we don't own. This should be impossible. ${debugPathValue(value)}`);
|
|
365
362
|
valid = false;
|
|
@@ -1739,7 +1739,6 @@ export class PathValueProxyWatcher {
|
|
|
1739
1739
|
for (let lock of locks) {
|
|
1740
1740
|
if (compareTime(lock.endTime, writeTime) > 0) {
|
|
1741
1741
|
// I'm curious how this could even happen?
|
|
1742
|
-
require("debugbreak")(2);
|
|
1743
1742
|
debugger;
|
|
1744
1743
|
console.error(`Write time is before the end time of a lock. This is not allowed. Shifting write time forward ${debugTime(writeTime)} < ${debugTime(lock.endTime)}`);
|
|
1745
1744
|
writeTime = clientWatcher.getFreeWriteTime({
|
|
@@ -2464,8 +2463,6 @@ export function noAtomicSchema<T>(code: () => T) {
|
|
|
2464
2463
|
|
|
2465
2464
|
export const proxyWatcher = new PathValueProxyWatcher();
|
|
2466
2465
|
if ((globalThis as any).proxyWatcher) {
|
|
2467
|
-
debugger;
|
|
2468
|
-
require("debugbreak")(2);
|
|
2469
2466
|
debugger;
|
|
2470
2467
|
console.error(`Loaded PathValueProxyWatcher twice. This will break this. In ${module.filename}`);
|
|
2471
2468
|
}
|
|
@@ -14,7 +14,6 @@ export function isMissingTransactionPart(pathValues: PathValue[], timeoutCheck?:
|
|
|
14
14
|
for (const pathValue of pathValues) {
|
|
15
15
|
if (!pathValue.valid) {
|
|
16
16
|
if (timeoutCheck) return undefined;
|
|
17
|
-
require("debugbreak")(2);
|
|
18
17
|
debugger;
|
|
19
18
|
throw new Error(`Cannot validate a transaction where part of it is invalid. We shouldn't have even been passed this value. Value: ${debugPathValue(pathValue)}`);
|
|
20
19
|
}
|
|
@@ -452,7 +452,6 @@ export function getCallWrites(config: {
|
|
|
452
452
|
if (config.useFinishReordering && !config.metadata?.noFinishInStartOrder && call.fromProxy) {
|
|
453
453
|
let triggerCaller = getCurrentCallCreationProxy();
|
|
454
454
|
if (!triggerCaller) {
|
|
455
|
-
require("debugbreak")(2);
|
|
456
455
|
debugger;
|
|
457
456
|
throw new Error(`getCallWrites did not happen synchronously when triggering a call. It's very important that we synchronously start the proxy watcher, so we can make sure that predictions happen in a consistent order. FIX THIS! Find the async weighting that was added and move it inside the watch function using cacheAsync to make it watched. For: ${debugName}`);
|
|
458
457
|
}
|
package/src/config.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { timeInHour } from "socket-function/src/misc";
|
|
|
5
5
|
import { isCurrentUserSuperUser } from "../user-implementation/userData";
|
|
6
6
|
import { Querysub } from "../4-querysub/Querysub";
|
|
7
7
|
import { StatWarning, StatWarningAction } from "./StatWarning";
|
|
8
|
-
import { getStatOverridesSync, setStatOverride, statOffenderKey } from "./StatsOverrides";
|
|
8
|
+
import { getStatOverridesSync, isStatStoragePersistentSync, setStatOverride, statOffenderKey } from "./StatsOverrides";
|
|
9
9
|
|
|
10
10
|
module.hotreload = true;
|
|
11
11
|
|
|
@@ -181,6 +181,7 @@ export class StatsHeader extends qreact.Component<{ getStats: () => StatDefiniti
|
|
|
181
181
|
|
|
182
182
|
render() {
|
|
183
183
|
if (!isCurrentUserSuperUser()) return undefined;
|
|
184
|
+
if (!isStatStoragePersistentSync()) return undefined;
|
|
184
185
|
|
|
185
186
|
let { categories, byCategory, infos } = this.buildInfos();
|
|
186
187
|
let activeWarnings = infos.filter(info =>
|
|
@@ -2,6 +2,8 @@ import { Querysub } from "../4-querysub/Querysub";
|
|
|
2
2
|
import { t } from "../2-proxy/schema2";
|
|
3
3
|
|
|
4
4
|
const LOCAL_STORAGE_KEY = "StatsHeader.overrides.v2";
|
|
5
|
+
const DEFAULT_HEAP_LIMIT_BYTES = 1024 * 1024 * 1024;
|
|
6
|
+
const INCOGNITO_QUOTA_HEAP_MULTIPLIER = 2;
|
|
5
7
|
|
|
6
8
|
export type StatOverride = { thresholdOverride?: number; ignoreUntilMs?: number };
|
|
7
9
|
type OverridesMap = { [key: string]: StatOverride };
|
|
@@ -10,8 +12,41 @@ let cachedOverrides: OverridesMap | undefined;
|
|
|
10
12
|
|
|
11
13
|
let overridesSchema = Querysub.createLocalSchema("statsOverrides", {
|
|
12
14
|
sequenceNumber: t.number,
|
|
15
|
+
storagePersists: t.boolean,
|
|
13
16
|
});
|
|
14
17
|
|
|
18
|
+
if (typeof window !== "undefined") {
|
|
19
|
+
void detectStoragePersists().then(persists => {
|
|
20
|
+
if (!persists) {
|
|
21
|
+
console.log(`StatsOverrides: private browsing detected, stat warnings are hidden because their overrides would not persist`);
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
Querysub.localCommit(() => {
|
|
25
|
+
overridesSchema().storagePersists = true;
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async function detectStoragePersists(): Promise<boolean> {
|
|
31
|
+
if (typeof navigator.storage?.getDirectory === "function") {
|
|
32
|
+
try {
|
|
33
|
+
await navigator.storage.getDirectory();
|
|
34
|
+
} catch {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (typeof navigator.storage?.estimate !== "function") return true;
|
|
39
|
+
let { quota } = await navigator.storage.estimate();
|
|
40
|
+
if (!quota) return true;
|
|
41
|
+
let memory = (performance as unknown as { memory?: { jsHeapSizeLimit: number } }).memory;
|
|
42
|
+
let heapLimit = memory && memory.jsHeapSizeLimit || DEFAULT_HEAP_LIMIT_BYTES;
|
|
43
|
+
return quota >= heapLimit * INCOGNITO_QUOTA_HEAP_MULTIPLIER;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function isStatStoragePersistentSync(): boolean {
|
|
47
|
+
return overridesSchema().storagePersists;
|
|
48
|
+
}
|
|
49
|
+
|
|
15
50
|
function loadFromStorage(): OverridesMap {
|
|
16
51
|
if (cachedOverrides) {
|
|
17
52
|
return cachedOverrides;
|
|
@@ -24,6 +24,7 @@ const MOVE_GRACE = timeInMinute;
|
|
|
24
24
|
const PATHS_CACHE_TTL = timeInMinute * 15;
|
|
25
25
|
// During the read/scan loops, emit a progress line at most this often.
|
|
26
26
|
const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
|
|
27
|
+
const FLUSH_CALL_TIMEOUT = timeInSecond * 30;
|
|
27
28
|
|
|
28
29
|
const LOGGER_NAMES = ["logs/log", "logs/info", "logs/warn", "logs/error"] as const;
|
|
29
30
|
type LoggerName = typeof LOGGER_NAMES[number];
|
|
@@ -195,6 +196,7 @@ export class MCPIndexedLogs {
|
|
|
195
196
|
|
|
196
197
|
let pathsStart = Date.now();
|
|
197
198
|
let totalPathsSeen = 0;
|
|
199
|
+
let pathsByMachine = new Map<string, number>();
|
|
198
200
|
await Promise.all(LOGGER_NAMES.map(async (loggerName) => {
|
|
199
201
|
let logger = this.getLoggerByName(loggers, loggerName);
|
|
200
202
|
let archives = logger.debugGetCachedLogs({ type: useType });
|
|
@@ -210,13 +212,17 @@ export class MCPIndexedLogs {
|
|
|
210
212
|
totalPathsSeen += paths.length;
|
|
211
213
|
|
|
212
214
|
for (let p of paths) {
|
|
213
|
-
if (p.machineId !== machineId) continue;
|
|
214
215
|
if (p.endTime < startTime || p.startTime > endTime) continue;
|
|
216
|
+
pathsByMachine.set(p.machineId || "(none)", (pathsByMachine.get(p.machineId || "(none)") || 0) + 1);
|
|
217
|
+
if (p.machineId !== machineId) continue;
|
|
215
218
|
allFiles.push({ path: p, loggerName, archives });
|
|
216
219
|
fileCounts[loggerName].total++;
|
|
217
220
|
}
|
|
218
221
|
}));
|
|
219
222
|
console.log(`[search] paths gathered in ${formatTime(Date.now() - pathsStart)} (totalSeen=${totalPathsSeen} matchedMachine=${allFiles.length})`);
|
|
223
|
+
if (!allFiles.length) {
|
|
224
|
+
console.warn(`[search] no ${useType} log files in range are from machine ${machineId}. Files in range by machine: ${[...pathsByMachine].map(([id, count]) => `${id}=${count}`).join(", ") || "(none at all)"}`);
|
|
225
|
+
}
|
|
220
226
|
|
|
221
227
|
if (config.direction === "fromStart") {
|
|
222
228
|
allFiles.sort((a, b) => a.path.startTime - b.path.startTime);
|
|
@@ -482,52 +488,59 @@ export class MCPIndexedLogs {
|
|
|
482
488
|
// (e.g. older versions still running). Records moved-through up to
|
|
483
489
|
// now - MOVE_GRACE so we skip this on subsequent calls covering the same
|
|
484
490
|
// window.
|
|
485
|
-
private async ensureMovedThrough(machineId: string, endTime: number, forceRefresh?: boolean): Promise<"cached" | "no-node" | "moved"> {
|
|
491
|
+
private async ensureMovedThrough(machineId: string, endTime: number, forceRefresh?: boolean): Promise<"cached" | "no-node" | "unreachable" | "moved"> {
|
|
486
492
|
let lastMoved = this.movedThroughByMachine.get(machineId) ?? 0;
|
|
487
493
|
if (!forceRefresh && lastMoved >= endTime) return "cached";
|
|
488
494
|
|
|
489
495
|
let nodeIds = await this.findRemoteNodesOnMachine(machineId);
|
|
490
496
|
if (nodeIds.length === 0) {
|
|
491
|
-
console.warn(`MCPIndexedLogs: no
|
|
497
|
+
console.warn(`MCPIndexedLogs: no nodes in the node list are on machine ${machineId}, so its pending logs could NOT be flushed. Searching only what it already moved to public storage.`);
|
|
492
498
|
return "no-node";
|
|
493
499
|
}
|
|
500
|
+
console.log(`MCPIndexedLogs: flushing machine ${machineId} through ${formatDateTime(endTime)} via one of ${nodeIds.length} node(s): ${nodeIds.join(", ")}`);
|
|
494
501
|
|
|
502
|
+
let unflushed: string[] = [];
|
|
495
503
|
for (let loggerName of LOGGER_NAMES) {
|
|
496
|
-
let
|
|
504
|
+
let flushedBy = "";
|
|
505
|
+
let failures: string[] = [];
|
|
497
506
|
for (let nodeId of nodeIds) {
|
|
507
|
+
let start = Date.now();
|
|
498
508
|
try {
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
await IndexedLogShimController.nodes[nodeId].forceMoveLogsToPublic({
|
|
502
|
-
indexedLogsName: loggerName,
|
|
503
|
-
});
|
|
504
|
-
answered = true;
|
|
505
|
-
break;
|
|
506
|
-
}
|
|
507
|
-
let hasPending = await timeoutToUndefinedSilent(
|
|
508
|
-
5000,
|
|
509
|
+
let hasPending = !!forceRefresh || await timeoutToError(
|
|
510
|
+
FLUSH_CALL_TIMEOUT,
|
|
509
511
|
IndexedLogShimController.nodes[nodeId].hasPendingInRange({
|
|
510
512
|
indexedLogsName: loggerName,
|
|
511
513
|
startTime: 0,
|
|
512
514
|
endTime,
|
|
513
515
|
}),
|
|
516
|
+
() => new Error(`hasPendingInRange timed out after ${formatTime(FLUSH_CALL_TIMEOUT)}`),
|
|
514
517
|
);
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
518
|
+
if (hasPending) {
|
|
519
|
+
await timeoutToError(
|
|
520
|
+
FLUSH_CALL_TIMEOUT,
|
|
521
|
+
IndexedLogShimController.nodes[nodeId].forceMoveLogsToPublic({ indexedLogsName: loggerName }),
|
|
522
|
+
() => new Error(`forceMoveLogsToPublic timed out after ${formatTime(FLUSH_CALL_TIMEOUT)}`),
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
flushedBy = nodeId;
|
|
526
|
+
console.log(`MCPIndexedLogs: ${loggerName} on ${nodeId}: ${hasPending && "had pending logs, moved them to public" || "nothing pending"} (${formatTime(Date.now() - start)})`);
|
|
521
527
|
break;
|
|
522
528
|
} catch (e) {
|
|
523
|
-
|
|
529
|
+
failures.push(`${nodeId} after ${formatTime(Date.now() - start)}: ${String((e as Error).stack ?? e).split("\n")[0]}`);
|
|
524
530
|
}
|
|
525
531
|
}
|
|
526
|
-
if (!
|
|
527
|
-
|
|
532
|
+
if (!flushedBy) {
|
|
533
|
+
unflushed.push(loggerName);
|
|
534
|
+
console.warn(`MCPIndexedLogs: could NOT flush ${loggerName} on machine ${machineId}, every node failed:\n ${failures.join("\n ")}`);
|
|
535
|
+
} else if (failures.length) {
|
|
536
|
+
console.warn(`MCPIndexedLogs: ${loggerName} needed ${failures.length} failed node(s) before ${flushedBy} answered:\n ${failures.join("\n ")}`);
|
|
528
537
|
}
|
|
529
538
|
}
|
|
530
539
|
|
|
540
|
+
if (unflushed.length) {
|
|
541
|
+
console.warn(`MCPIndexedLogs: machine ${machineId} could not be flushed for ${unflushed.join(", ")}. Results only include what it had already moved to public storage on its own.`);
|
|
542
|
+
return "unreachable";
|
|
543
|
+
}
|
|
531
544
|
this.movedThroughByMachine.set(machineId, Date.now() - MOVE_GRACE);
|
|
532
545
|
return "moved";
|
|
533
546
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import "../../../inject";
|
|
2
|
+
|
|
3
|
+
import { formatTime } from "socket-function/src/formatting/format";
|
|
4
|
+
import { green, magenta, red } from "socket-function/src/formatting/logColors";
|
|
5
|
+
import { isPublic } from "../../../config";
|
|
6
|
+
import { getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
|
|
7
|
+
import { getLoggers2Async, LogDatum } from "../diskLogger";
|
|
8
|
+
import { IndexedLogs } from "./IndexedLogs";
|
|
9
|
+
|
|
10
|
+
async function countPending(logs: IndexedLogs<LogDatum>): Promise<number> {
|
|
11
|
+
let paths = await logs.getPaths({ startTime: 0, endTime: Date.now(), only: "local" });
|
|
12
|
+
return paths.filter(x => x.logCount === undefined).length;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async function main() {
|
|
16
|
+
if (!isPublic()) {
|
|
17
|
+
throw new Error(`flushlogs must run with --public, or the logs would be "moved" into a folder on this machine's own disk instead of public storage`);
|
|
18
|
+
}
|
|
19
|
+
console.log(magenta(`Flushing every pending log file on machine ${getOwnMachineId()} to public storage`));
|
|
20
|
+
let loggers = await getLoggers2Async();
|
|
21
|
+
let stuck = 0;
|
|
22
|
+
for (let [name, logs] of Object.entries(loggers)) {
|
|
23
|
+
let before = await countPending(logs);
|
|
24
|
+
let start = Date.now();
|
|
25
|
+
await logs.moveLogsToPublic(true);
|
|
26
|
+
let after = await countPending(logs);
|
|
27
|
+
let color = after && red || green;
|
|
28
|
+
console.log(color(`${name}: ${before} pending files before, ${after} after (${formatTime(Date.now() - start)})`));
|
|
29
|
+
stuck += after;
|
|
30
|
+
}
|
|
31
|
+
if (stuck) {
|
|
32
|
+
throw new Error(`${stuck} log files are still pending after the flush. See the move logs above for why.`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
main().then(
|
|
37
|
+
() => process.exit(),
|
|
38
|
+
e => {
|
|
39
|
+
console.error((e as Error).stack || e);
|
|
40
|
+
process.exit(1);
|
|
41
|
+
},
|
|
42
|
+
);
|
|
@@ -77,6 +77,9 @@ export async function moveLogsToPublic(config: {
|
|
|
77
77
|
let validMovingFiles = await checkAndCleanMovingFiles();
|
|
78
78
|
// 2) If they're not past our timeout threshold at the top of the file, Then we just return (NOOP).
|
|
79
79
|
if (validMovingFiles.length > 0) {
|
|
80
|
+
if (forceAll) {
|
|
81
|
+
console.warn(`Forced log move for ${config.loggerName} skipped: another thread holds the move lock (${validMovingFiles.join(", ")}), which goes stale after ${formatTime(movingTimeout)}`);
|
|
82
|
+
}
|
|
80
83
|
return false;
|
|
81
84
|
}
|
|
82
85
|
|
|
@@ -94,6 +97,9 @@ export async function moveLogsToPublic(config: {
|
|
|
94
97
|
// 7) Also, out of all of the moving files which aren't beyond the timeout, we need to make sure ours is the oldest, but not older than half the timeout. If this isn't true, then we just return.
|
|
95
98
|
let oldestMovingFile = validMovingFiles[0];
|
|
96
99
|
if (oldestMovingFile !== ourMovingFileName) {
|
|
100
|
+
if (forceAll) {
|
|
101
|
+
console.warn(`Forced log move for ${config.loggerName} skipped: ${oldestMovingFile} took the move lock before us (${ourMovingFileName})`);
|
|
102
|
+
}
|
|
97
103
|
// Clean up our moving file since we didn't get the lock
|
|
98
104
|
await localLogs.del(ourMovingFileName);
|
|
99
105
|
return false;
|
|
@@ -10,8 +10,7 @@ import { isPublic } from "../../../config";
|
|
|
10
10
|
import { MachineThreadInfo } from "../../MachineThreadInfo";
|
|
11
11
|
import { ErrorWarning } from "./ErrorWarning";
|
|
12
12
|
import { getErrorNotificationsManager } from "./errorWatcher";
|
|
13
|
-
import {
|
|
14
|
-
import { cacheLimited } from "socket-function/src/caching";
|
|
13
|
+
import { createPatternSetMatcher, getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
|
|
15
14
|
import { managementPageURL, showingManagementURL } from "../../managementPages";
|
|
16
15
|
import { searchTextURL, readProductionLogsURL } from "../IndexedLogs/LogViewerParams";
|
|
17
16
|
import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
|
|
@@ -57,13 +56,6 @@ function TicketButton(props: {
|
|
|
57
56
|
}
|
|
58
57
|
qreact.inline(TicketButton);
|
|
59
58
|
|
|
60
|
-
export let getMatcher = cacheLimited(100, (pattern: string) =>
|
|
61
|
-
createMatchesPattern(Buffer.from(normalizeMatchWhitespace(pattern)), false)
|
|
62
|
-
);
|
|
63
|
-
|
|
64
|
-
export function getMatchBuffer(datum: LogDatum): Buffer {
|
|
65
|
-
return Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum)));
|
|
66
|
-
}
|
|
67
59
|
|
|
68
60
|
export class LogDatumRenderer extends qreact.Component<{
|
|
69
61
|
datum: LogDatum;
|
|
@@ -81,15 +73,11 @@ export class LogDatumRenderer extends qreact.Component<{
|
|
|
81
73
|
let datum = this.props.datum;
|
|
82
74
|
let mainMessage = datum.param0 && String(datum.param0) || "(no message)";
|
|
83
75
|
|
|
84
|
-
let
|
|
76
|
+
let suppressionEntries = this.manager.state.suppressionEntries;
|
|
77
|
+
let matchingSuppressions: typeof suppressionEntries = [];
|
|
85
78
|
if (!this.manager.state.isLoading && !this.props.hideSuppression) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
let errorBuffer = getMatchBuffer(datum);
|
|
89
|
-
if (matcher(errorBuffer)) {
|
|
90
|
-
matchingSuppressions.push(suppression);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
79
|
+
let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(datum));
|
|
80
|
+
matchingSuppressions = matchedIndexes.map(index => suppressionEntries[index]);
|
|
93
81
|
}
|
|
94
82
|
|
|
95
83
|
let timedOutMatches = matchingSuppressions.filter(s => datum.time > s.timeout);
|
|
@@ -464,8 +452,8 @@ class SuppressionItem extends qreact.Component<{
|
|
|
464
452
|
let example = this.props.matchData?.examples[0];
|
|
465
453
|
if (example) return example;
|
|
466
454
|
// No example recorded for this suppression — find any live error matching the pattern, so the ticket stores a real, full log entry instead of a synthetic stub.
|
|
467
|
-
let matcher =
|
|
468
|
-
let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)));
|
|
455
|
+
let matcher = createPatternSetMatcher([suppression.pattern]);
|
|
456
|
+
let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)).length > 0);
|
|
469
457
|
return fallback || {
|
|
470
458
|
time: Date.now(),
|
|
471
459
|
__LOG_TYPE: "error",
|
|
@@ -4,7 +4,8 @@ import { css } from "typesafecss";
|
|
|
4
4
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
5
5
|
import { ATag } from "../../../library-components/ATag";
|
|
6
6
|
import { managementPageURL, showingManagementURL } from "../../managementPages";
|
|
7
|
-
import {
|
|
7
|
+
import { LogDatumRenderer } from "./ErrorNotificationPage";
|
|
8
|
+
import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
|
|
8
9
|
import { getErrorNotificationsManager } from "./errorWatcher";
|
|
9
10
|
import { LogDatum } from "../diskLogger";
|
|
10
11
|
import { TicketsController } from "../errorTickets/tickets";
|
|
@@ -57,10 +58,8 @@ export class ErrorWarning extends qreact.Component {
|
|
|
57
58
|
if (!ticket.suppressionPattern) continue;
|
|
58
59
|
openTicketPatterns.push(ticket.suppressionPattern);
|
|
59
60
|
}
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
return openTicketPatterns.some(pattern => getMatcher(pattern)(errorBuffer));
|
|
63
|
-
};
|
|
61
|
+
const openTicketMatcher = getPatternSetMatcher(openTicketPatterns);
|
|
62
|
+
const isCoveredByOpenTicket = (datum: LogDatum) => openTicketMatcher(getMatchBuffer(datum)).length > 0;
|
|
64
63
|
|
|
65
64
|
let errorWarning: preact.ComponentChild = undefined;
|
|
66
65
|
let firstError = !this.manager.state.isLoading && this.manager.state.unmatchedErrors.find(x => !isCoveredByOpenTicket(x)) || undefined;
|