querysub 0.720.0 → 0.722.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/bin/flushlogs.js +7 -0
  2. package/debugging.md +43 -0
  3. package/package.json +9 -3
  4. package/src/-f-node-discovery/NodeDiscovery.ts +1 -1
  5. package/src/0-path-value-core/AuthorityLookup.ts +0 -1
  6. package/src/0-path-value-core/PathRouter.ts +0 -1
  7. package/src/0-path-value-core/PathWatcher.ts +0 -3
  8. package/src/0-path-value-core/ValidStateComputer.ts +0 -3
  9. package/src/2-proxy/PathValueProxyWatcher.ts +0 -3
  10. package/src/2-proxy/TransactionDelayer.ts +0 -1
  11. package/src/4-deploy/edgeNodes.ts +0 -1
  12. package/src/4-querysub/querysubPrediction.ts +0 -1
  13. package/src/config.ts +0 -1
  14. package/src/diagnostics/StatsHeader.tsx +2 -1
  15. package/src/diagnostics/StatsOverrides.ts +35 -0
  16. package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +0 -1
  17. package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +36 -23
  18. package/src/diagnostics/logs/IndexedLogs/flushLogsEntry.ts +42 -0
  19. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +6 -0
  20. package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +7 -19
  21. package/src/diagnostics/logs/errorNotifications2/ErrorWarning.tsx +4 -5
  22. package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +27 -47
  23. package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +168 -0
  24. package/src/diagnostics/network/logFilesMain.ts +57 -0
  25. package/src/diagnostics/network/logSearchMain.ts +86 -0
  26. package/src/diagnostics/network/networkNodesMain.ts +225 -0
  27. package/src/diagnostics/network/networkProbe.ts +31 -0
  28. package/src/diagnostics/network/networkServicesMain.ts +170 -0
  29. package/src/diagnostics/network/portMappingsMain.ts +28 -0
  30. package/src/diagnostics/statsDefinitions.tsx +0 -1
  31. package/src/library-components/LazyComponent.tsx +0 -1
  32. package/src/library-components/URLParam.ts +0 -2
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env node
2
+
3
+ process.argv.push("--nobreak");
4
+ process.argv.push("--public");
5
+
6
+ require("typenode");
7
+ require("../src/diagnostics/logs/IndexedLogs/flushLogsEntry");
package/debugging.md ADDED
@@ -0,0 +1,43 @@
1
+ # Debugging the Network
2
+
3
+ When the site won't load, the debug pages are unreachable, or nodes keep dying or being removed, work through these steps in order. Each step needs less of the network to be up than the one after it, so start at the top and only move on when the current step doesn't explain the problem.
4
+
5
+ Every script takes `--cwd <project root>` so it runs with that project's domain, keys, and certs (for the cyoa network: `--cwd D:/repos/qs-cyoa/`). Both scripts are read-only, except that, like every server-mode querysub process, they run the node-discovery audit, which deletes node list entries whose port has provably been taken by a newer thread.
6
+
7
+ ## 1. What nodes exist
8
+
9
+ ```
10
+ yarn network-nodes --cwd D:/repos/qs-cyoa/
11
+ ```
12
+
13
+ Reads the node list (the `nodes/` folder of the fast archive bucket, which falls back to Backblaze if the storage servers are down), then calls every node and logs each one as it answers: heartbeat age, ping, service folder, entry point, uptime, git ref, and how many nodes it knows about. It ends with a summary:
14
+
15
+ - **Unreachable but still writing heartbeats**: the process is alive but nobody can call it. Usually networking (a port not forwarded, a firewall, another process answering on that port).
16
+ - **Stale or missing heartbeats**: dead processes waiting to be removed. Entries that stay listed with no heartbeat file across several runs mean deletes aren't sticking in storage.
17
+ - **Several threads on the same machine and port**: all but one are dead threads.
18
+ - **By machine**: which ports each machine is running and what is on them.
19
+ - **Ghosts / missing**: disagreement between the node list and what the nodes themselves hold in memory.
20
+
21
+ The node list lives on the storage servers, so if this step is slow or fails, the storage servers are the first suspect.
22
+
23
+ ## 2. Which services are healthy
24
+
25
+ ```
26
+ yarn network-services --cwd D:/repos/qs-cyoa/
27
+ ```
28
+
29
+ Reads the service configs and machine heartbeats (also from the storage servers, falling back to Backblaze, so it can be slow when they are down), works out every instance that should be running, and checks each instance's recorded node against the node list and with a call. It reports machines with stale heartbeats, instances a machine never reported, instances with no node id yet, and instances whose node is unreachable or missing from the node list. A node id that the machine recorded but whose port is now held by another process usually means the machine's record is stale, not that the service is down; check the step 1 output for a reachable node with the same service folder.
30
+
31
+ `overview.md` describes the roles that should exist (path value servers, function runners, the client-facing querysub server, error notifications, logging, machine management, storage); every one of them should show up here as a healthy service.
32
+
33
+ ## 3. Look at the machine directly
34
+
35
+ If the problem still isn't apparent, SSH into the machine (`ssh root@<machine>`). Every service instance runs in its own tmux session named `<service key>-<instance index>-dply` (for example `cyoa-0-dply`, `serverall-2-dply`), and the machine's own deploy process runs in `machine-alwaysup`.
36
+
37
+ ```
38
+ tmux ls
39
+ tmux attach -t cyoa-0-dply
40
+ tmux capture-pane -t cyoa-0-dply -p -S -2000
41
+ ```
42
+
43
+ Each instance's folder is `~/machine-services/<session name>/`, holding the `parameters.json` it was started with and `serviceNodeId.txt`, the node id it registered as.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "querysub",
3
- "version": "0.720.0",
3
+ "version": "0.722.0",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -28,6 +28,11 @@
28
28
  "thin": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --nobreak --public",
29
29
  "thin-watch": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --nobreak --public --watch",
30
30
  "thin-dry": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --nobreak --public --dry-run",
31
+ "network-nodes": "yarn typenode ./src/diagnostics/network/networkNodesMain.ts",
32
+ "network-services": "yarn typenode ./src/diagnostics/network/networkServicesMain.ts",
33
+ "port-mappings": "yarn typenode ./src/diagnostics/network/portMappingsMain.ts",
34
+ "log-search": "yarn typenode ./src/diagnostics/network/logSearchMain.ts",
35
+ "log-files": "yarn typenode ./src/diagnostics/network/logFilesMain.ts",
31
36
  "parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
32
37
  "setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
33
38
  "stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
@@ -50,6 +55,7 @@
50
55
  "join": "./bin/join.js",
51
56
  "join-public": "./bin/join-public.js",
52
57
  "movelogs": "./bin/movelogs.js",
58
+ "flushlogs": "./bin/flushlogs.js",
53
59
  "addsuperuser": "./bin/addsuperuser.js",
54
60
  "error-watch": "./bin/error-watch.js",
55
61
  "error-watch-public": "./bin/error-watch-public.js",
@@ -81,10 +87,10 @@
81
87
  "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
82
88
  "pako": "^2.1.0",
83
89
  "peggy": "^5.0.6",
84
- "sliftutils": "^1.9.5",
90
+ "sliftutils": "^1.9.17",
85
91
  "socket-function": "^1.3.1",
86
92
  "terser": "^5.31.0",
87
- "typenode": "^6.6.1",
93
+ "typenode": "^6.6.3",
88
94
  "typesafecss": "^0.32.0",
89
95
  "yaml": "^2.5.0",
90
96
  "yargs": "^15.3.1"
@@ -28,7 +28,7 @@ let HEARTBEAT_INTERVAL = timeInMinute * 1;
28
28
  // Interval which we check other heartbeats
29
29
  let CHECK_INTERVAL = HEARTBEAT_INTERVAL;
30
30
  // If the heartbeat is older than thing, it fails the dead check
31
- let DEAD_THRESHOLD = HEARTBEAT_INTERVAL * 2;
31
+ export const DEAD_THRESHOLD = HEARTBEAT_INTERVAL * 2;
32
32
  // If we find another node is dead this number of checks, we remove it from storage, and tell
33
33
  // all other nodes to remove from their memory.
34
34
  let DEAD_CHECK_COUNT = 4;
@@ -58,7 +58,6 @@ class AuthorityLookup {
58
58
 
59
59
  public getTopologySync() {
60
60
  if (!this.didInitialSync) {
61
- require("debugbreak")(2);
62
61
  debugger;
63
62
  throw new Error("Cannot call getTopologySync without awaiting syncAllNow or startSyncing.");
64
63
  }
@@ -231,7 +231,6 @@ export class PathRouter {
231
231
  if (prefix) {
232
232
  let key = getPathIndex(path, prefix.childKeyIndex);
233
233
  if (key === undefined) {
234
- require("debugbreak")(2);
235
234
  debugger;
236
235
  throw new Error(`Impossible, hash index ${prefix.childKeyIndex} is out of range for path ${path}, but it matched the prefix ${prefix.originalPrefix}`);
237
236
  }
@@ -52,8 +52,6 @@ function incrementWatcherSequence() {
52
52
 
53
53
  if (!registerResource) {
54
54
  debugger;
55
- require("debugbreak")(2);
56
- debugger;
57
55
  }
58
56
 
59
57
  // WATCH CASES
@@ -112,7 +110,6 @@ class PathWatcher {
112
110
  }) {
113
111
  this.ensureUnwatchingOnDisconnect(config.nodeId);
114
112
  if (config.nodeId.startsWith(getInternalValueWatchPrefix()) && (!config.noInitialTrigger || config.initialTrigger)) {
115
- require("debugbreak")(2);
116
113
  debugger;
117
114
  // No initial trigger because if it's a remote value, then we won't have it. If it's a local value, then we've already processed it in valid state computers, there's no reason to trigger it again.
118
115
  throw new Error("Internal value watches (INTERNAL_VALUE_WATCH_PREFIX) cannot have initial triggers");
@@ -209,7 +209,6 @@ class ValidStateComputer {
209
209
  while (dependenciesChanged.size > 0) {
210
210
  loopCount++;
211
211
  if (loopCount > 3000) {
212
- require("debugbreak")(2);
213
212
  debugger;
214
213
  console.error("Too many loops in valid state loop. aborting.", {
215
214
  loopCount,
@@ -350,7 +349,6 @@ class ValidStateComputer {
350
349
  let lockTime = locks[0].endTime;
351
350
  for (let lock of locks) {
352
351
  if (compareTime(lock.endTime, lockTime) !== 0) {
353
- require("debugbreak")(2);
354
352
  debugger;
355
353
  console.error(`Lock has inconsistent end times. ${debugTime(lock.endTime)} !== ${debugTime(lockTime)}. Path: ${lock.path}`);
356
354
  valid = false;
@@ -359,7 +357,6 @@ class ValidStateComputer {
359
357
 
360
358
  for (let value of valueGroup) {
361
359
  if (!PathRouter.isSelfAuthority(value.path)) {
362
- require("debugbreak")(2);
363
360
  debugger;
364
361
  console.error(`Somehow we're attempting to compute the valid state of a value we don't own. This should be impossible. ${debugPathValue(value)}`);
365
362
  valid = false;
@@ -1739,7 +1739,6 @@ export class PathValueProxyWatcher {
1739
1739
  for (let lock of locks) {
1740
1740
  if (compareTime(lock.endTime, writeTime) > 0) {
1741
1741
  // I'm curious how this could even happen?
1742
- require("debugbreak")(2);
1743
1742
  debugger;
1744
1743
  console.error(`Write time is before the end time of a lock. This is not allowed. Shifting write time forward ${debugTime(writeTime)} < ${debugTime(lock.endTime)}`);
1745
1744
  writeTime = clientWatcher.getFreeWriteTime({
@@ -2464,8 +2463,6 @@ export function noAtomicSchema<T>(code: () => T) {
2464
2463
 
2465
2464
  export const proxyWatcher = new PathValueProxyWatcher();
2466
2465
  if ((globalThis as any).proxyWatcher) {
2467
- debugger;
2468
- require("debugbreak")(2);
2469
2466
  debugger;
2470
2467
  console.error(`Loaded PathValueProxyWatcher twice. This will break this. In ${module.filename}`);
2471
2468
  }
@@ -14,7 +14,6 @@ export function isMissingTransactionPart(pathValues: PathValue[], timeoutCheck?:
14
14
  for (const pathValue of pathValues) {
15
15
  if (!pathValue.valid) {
16
16
  if (timeoutCheck) return undefined;
17
- require("debugbreak")(2);
18
17
  debugger;
19
18
  throw new Error(`Cannot validate a transaction where part of it is invalid. We shouldn't have even been passed this value. Value: ${debugPathValue(pathValue)}`);
20
19
  }
@@ -298,7 +298,6 @@ async function updateEdgeNodesFile() {
298
298
  return atomic(deploySchema()[getDomain()].deploy.live.hash);
299
299
  });
300
300
  if (!liveHash) {
301
- require("debugbreak")(2);
302
301
  debugger;
303
302
  throw new Error(`No live hash set?`);
304
303
  }
@@ -452,7 +452,6 @@ export function getCallWrites(config: {
452
452
  if (config.useFinishReordering && !config.metadata?.noFinishInStartOrder && call.fromProxy) {
453
453
  let triggerCaller = getCurrentCallCreationProxy();
454
454
  if (!triggerCaller) {
455
- require("debugbreak")(2);
456
455
  debugger;
457
456
  throw new Error(`getCallWrites did not happen synchronously when triggering a call. It's very important that we synchronously start the proxy watcher, so we can make sure that predictions happen in a consistent order. FIX THIS! Find the async weighting that was added and move it inside the watch function using cacheAsync to make it watched. For: ${debugName}`);
458
457
  }
package/src/config.ts CHANGED
@@ -155,7 +155,6 @@ export function devDebugbreak() {
155
155
  return;
156
156
  }
157
157
  if (yargObj.nobreak) return;
158
- require("debugbreak")(2);
159
158
  debugger;
160
159
  }
161
160
  let debuggedEnabled = false;
@@ -5,7 +5,7 @@ import { timeInHour } from "socket-function/src/misc";
5
5
  import { isCurrentUserSuperUser } from "../user-implementation/userData";
6
6
  import { Querysub } from "../4-querysub/Querysub";
7
7
  import { StatWarning, StatWarningAction } from "./StatWarning";
8
- import { getStatOverridesSync, setStatOverride, statOffenderKey } from "./StatsOverrides";
8
+ import { getStatOverridesSync, isStatStoragePersistentSync, setStatOverride, statOffenderKey } from "./StatsOverrides";
9
9
 
10
10
  module.hotreload = true;
11
11
 
@@ -181,6 +181,7 @@ export class StatsHeader extends qreact.Component<{ getStats: () => StatDefiniti
181
181
 
182
182
  render() {
183
183
  if (!isCurrentUserSuperUser()) return undefined;
184
+ if (!isStatStoragePersistentSync()) return undefined;
184
185
 
185
186
  let { categories, byCategory, infos } = this.buildInfos();
186
187
  let activeWarnings = infos.filter(info =>
@@ -2,6 +2,8 @@ import { Querysub } from "../4-querysub/Querysub";
2
2
  import { t } from "../2-proxy/schema2";
3
3
 
4
4
  const LOCAL_STORAGE_KEY = "StatsHeader.overrides.v2";
5
+ const DEFAULT_HEAP_LIMIT_BYTES = 1024 * 1024 * 1024;
6
+ const INCOGNITO_QUOTA_HEAP_MULTIPLIER = 2;
5
7
 
6
8
  export type StatOverride = { thresholdOverride?: number; ignoreUntilMs?: number };
7
9
  type OverridesMap = { [key: string]: StatOverride };
@@ -10,8 +12,41 @@ let cachedOverrides: OverridesMap | undefined;
10
12
 
11
13
  let overridesSchema = Querysub.createLocalSchema("statsOverrides", {
12
14
  sequenceNumber: t.number,
15
+ storagePersists: t.boolean,
13
16
  });
14
17
 
18
+ if (typeof window !== "undefined") {
19
+ void detectStoragePersists().then(persists => {
20
+ if (!persists) {
21
+ console.log(`StatsOverrides: private browsing detected, stat warnings are hidden because their overrides would not persist`);
22
+ return;
23
+ }
24
+ Querysub.localCommit(() => {
25
+ overridesSchema().storagePersists = true;
26
+ });
27
+ });
28
+ }
29
+
30
+ async function detectStoragePersists(): Promise<boolean> {
31
+ if (typeof navigator.storage?.getDirectory === "function") {
32
+ try {
33
+ await navigator.storage.getDirectory();
34
+ } catch {
35
+ return false;
36
+ }
37
+ }
38
+ if (typeof navigator.storage?.estimate !== "function") return true;
39
+ let { quota } = await navigator.storage.estimate();
40
+ if (!quota) return true;
41
+ let memory = (performance as unknown as { memory?: { jsHeapSizeLimit: number } }).memory;
42
+ let heapLimit = memory && memory.jsHeapSizeLimit || DEFAULT_HEAP_LIMIT_BYTES;
43
+ return quota >= heapLimit * INCOGNITO_QUOTA_HEAP_MULTIPLIER;
44
+ }
45
+
46
+ export function isStatStoragePersistentSync(): boolean {
47
+ return overridesSchema().storagePersists;
48
+ }
49
+
15
50
  function loadFromStorage(): OverridesMap {
16
51
  if (cachedOverrides) {
17
52
  return cachedOverrides;
@@ -65,7 +65,6 @@ const getCppAddon = lazy(async () => {
65
65
  fs.writeFileSync(versionFilePath, currentHash);
66
66
  } catch (e) {
67
67
  if (!isPublic()) {
68
- require("debugbreak")(2);
69
68
  debugger;
70
69
  console.error(`Error building C++ addon: ${e}`);
71
70
  }
@@ -24,6 +24,7 @@ const MOVE_GRACE = timeInMinute;
24
24
  const PATHS_CACHE_TTL = timeInMinute * 15;
25
25
  // During the read/scan loops, emit a progress line at most this often.
26
26
  const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
27
+ const FLUSH_CALL_TIMEOUT = timeInSecond * 30;
27
28
 
28
29
  const LOGGER_NAMES = ["logs/log", "logs/info", "logs/warn", "logs/error"] as const;
29
30
  type LoggerName = typeof LOGGER_NAMES[number];
@@ -195,6 +196,7 @@ export class MCPIndexedLogs {
195
196
 
196
197
  let pathsStart = Date.now();
197
198
  let totalPathsSeen = 0;
199
+ let pathsByMachine = new Map<string, number>();
198
200
  await Promise.all(LOGGER_NAMES.map(async (loggerName) => {
199
201
  let logger = this.getLoggerByName(loggers, loggerName);
200
202
  let archives = logger.debugGetCachedLogs({ type: useType });
@@ -210,13 +212,17 @@ export class MCPIndexedLogs {
210
212
  totalPathsSeen += paths.length;
211
213
 
212
214
  for (let p of paths) {
213
- if (p.machineId !== machineId) continue;
214
215
  if (p.endTime < startTime || p.startTime > endTime) continue;
216
+ pathsByMachine.set(p.machineId || "(none)", (pathsByMachine.get(p.machineId || "(none)") || 0) + 1);
217
+ if (p.machineId !== machineId) continue;
215
218
  allFiles.push({ path: p, loggerName, archives });
216
219
  fileCounts[loggerName].total++;
217
220
  }
218
221
  }));
219
222
  console.log(`[search] paths gathered in ${formatTime(Date.now() - pathsStart)} (totalSeen=${totalPathsSeen} matchedMachine=${allFiles.length})`);
223
+ if (!allFiles.length) {
224
+ console.warn(`[search] no ${useType} log files in range are from machine ${machineId}. Files in range by machine: ${[...pathsByMachine].map(([id, count]) => `${id}=${count}`).join(", ") || "(none at all)"}`);
225
+ }
220
226
 
221
227
  if (config.direction === "fromStart") {
222
228
  allFiles.sort((a, b) => a.path.startTime - b.path.startTime);
@@ -482,52 +488,59 @@ export class MCPIndexedLogs {
482
488
  // (e.g. older versions still running). Records moved-through up to
483
489
  // now - MOVE_GRACE so we skip this on subsequent calls covering the same
484
490
  // window.
485
- private async ensureMovedThrough(machineId: string, endTime: number, forceRefresh?: boolean): Promise<"cached" | "no-node" | "moved"> {
491
+ private async ensureMovedThrough(machineId: string, endTime: number, forceRefresh?: boolean): Promise<"cached" | "no-node" | "unreachable" | "moved"> {
486
492
  let lastMoved = this.movedThroughByMachine.get(machineId) ?? 0;
487
493
  if (!forceRefresh && lastMoved >= endTime) return "cached";
488
494
 
489
495
  let nodeIds = await this.findRemoteNodesOnMachine(machineId);
490
496
  if (nodeIds.length === 0) {
491
- console.warn(`MCPIndexedLogs: no remote nodes available for machine ${machineId}; falling back to whatever's in storage now`);
497
+ console.warn(`MCPIndexedLogs: no nodes in the node list are on machine ${machineId}, so its pending logs could NOT be flushed. Searching only what it already moved to public storage.`);
492
498
  return "no-node";
493
499
  }
500
+ console.log(`MCPIndexedLogs: flushing machine ${machineId} through ${formatDateTime(endTime)} via one of ${nodeIds.length} node(s): ${nodeIds.join(", ")}`);
494
501
 
502
+ let unflushed: string[] = [];
495
503
  for (let loggerName of LOGGER_NAMES) {
496
- let answered = false;
504
+ let flushedBy = "";
505
+ let failures: string[] = [];
497
506
  for (let nodeId of nodeIds) {
507
+ let start = Date.now();
498
508
  try {
499
- if (forceRefresh) {
500
- console.log(`MCPIndexedLogs: forceRefresh — flushing ${loggerName} on ${nodeId} unconditionally`);
501
- await IndexedLogShimController.nodes[nodeId].forceMoveLogsToPublic({
502
- indexedLogsName: loggerName,
503
- });
504
- answered = true;
505
- break;
506
- }
507
- let hasPending = await timeoutToUndefinedSilent(
508
- 5000,
509
+ let hasPending = !!forceRefresh || await timeoutToError(
510
+ FLUSH_CALL_TIMEOUT,
509
511
  IndexedLogShimController.nodes[nodeId].hasPendingInRange({
510
512
  indexedLogsName: loggerName,
511
513
  startTime: 0,
512
514
  endTime,
513
515
  }),
516
+ () => new Error(`hasPendingInRange timed out after ${formatTime(FLUSH_CALL_TIMEOUT)}`),
514
517
  );
515
- answered = true;
516
- if (!hasPending) break;
517
- console.log(`MCPIndexedLogs: hasPendingInRange returned true for ${loggerName} on ${nodeId}, forcing move to public`);
518
- await IndexedLogShimController.nodes[nodeId].forceMoveLogsToPublic({
519
- indexedLogsName: loggerName,
520
- });
518
+ if (hasPending) {
519
+ await timeoutToError(
520
+ FLUSH_CALL_TIMEOUT,
521
+ IndexedLogShimController.nodes[nodeId].forceMoveLogsToPublic({ indexedLogsName: loggerName }),
522
+ () => new Error(`forceMoveLogsToPublic timed out after ${formatTime(FLUSH_CALL_TIMEOUT)}`),
523
+ );
524
+ }
525
+ flushedBy = nodeId;
526
+ console.log(`MCPIndexedLogs: ${loggerName} on ${nodeId}: ${hasPending && "had pending logs, moved them to public" || "nothing pending"} (${formatTime(Date.now() - start)})`);
521
527
  break;
522
528
  } catch (e) {
523
- console.warn(`MCPIndexedLogs: hasPendingRange failed, trying next node: ${(e as Error).stack ?? e}`, { loggerName, nodeId });
529
+ failures.push(`${nodeId} after ${formatTime(Date.now() - start)}: ${String((e as Error).stack ?? e).split("\n")[0]}`);
524
530
  }
525
531
  }
526
- if (!answered) {
527
- console.warn(`MCPIndexedLogs: no node on machine ${machineId} could service hasPendingInRange/forceMoveLogsToPublic for ${loggerName}`);
532
+ if (!flushedBy) {
533
+ unflushed.push(loggerName);
534
+ console.warn(`MCPIndexedLogs: could NOT flush ${loggerName} on machine ${machineId}, every node failed:\n ${failures.join("\n ")}`);
535
+ } else if (failures.length) {
536
+ console.warn(`MCPIndexedLogs: ${loggerName} needed ${failures.length} failed node(s) before ${flushedBy} answered:\n ${failures.join("\n ")}`);
528
537
  }
529
538
  }
530
539
 
540
+ if (unflushed.length) {
541
+ console.warn(`MCPIndexedLogs: machine ${machineId} could not be flushed for ${unflushed.join(", ")}. Results only include what it had already moved to public storage on its own.`);
542
+ return "unreachable";
543
+ }
531
544
  this.movedThroughByMachine.set(machineId, Date.now() - MOVE_GRACE);
532
545
  return "moved";
533
546
  }
@@ -0,0 +1,42 @@
1
+ import "../../../inject";
2
+
3
+ import { formatTime } from "socket-function/src/formatting/format";
4
+ import { green, magenta, red } from "socket-function/src/formatting/logColors";
5
+ import { isPublic } from "../../../config";
6
+ import { getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
7
+ import { getLoggers2Async, LogDatum } from "../diskLogger";
8
+ import { IndexedLogs } from "./IndexedLogs";
9
+
10
+ async function countPending(logs: IndexedLogs<LogDatum>): Promise<number> {
11
+ let paths = await logs.getPaths({ startTime: 0, endTime: Date.now(), only: "local" });
12
+ return paths.filter(x => x.logCount === undefined).length;
13
+ }
14
+
15
+ async function main() {
16
+ if (!isPublic()) {
17
+ throw new Error(`flushlogs must run with --public, or the logs would be "moved" into a folder on this machine's own disk instead of public storage`);
18
+ }
19
+ console.log(magenta(`Flushing every pending log file on machine ${getOwnMachineId()} to public storage`));
20
+ let loggers = await getLoggers2Async();
21
+ let stuck = 0;
22
+ for (let [name, logs] of Object.entries(loggers)) {
23
+ let before = await countPending(logs);
24
+ let start = Date.now();
25
+ await logs.moveLogsToPublic(true);
26
+ let after = await countPending(logs);
27
+ let color = after && red || green;
28
+ console.log(color(`${name}: ${before} pending files before, ${after} after (${formatTime(Date.now() - start)})`));
29
+ stuck += after;
30
+ }
31
+ if (stuck) {
32
+ throw new Error(`${stuck} log files are still pending after the flush. See the move logs above for why.`);
33
+ }
34
+ }
35
+
36
+ main().then(
37
+ () => process.exit(),
38
+ e => {
39
+ console.error((e as Error).stack || e);
40
+ process.exit(1);
41
+ },
42
+ );
@@ -77,6 +77,9 @@ export async function moveLogsToPublic(config: {
77
77
  let validMovingFiles = await checkAndCleanMovingFiles();
78
78
  // 2) If they're not past our timeout threshold at the top of the file, Then we just return (NOOP).
79
79
  if (validMovingFiles.length > 0) {
80
+ if (forceAll) {
81
+ console.warn(`Forced log move for ${config.loggerName} skipped: another thread holds the move lock (${validMovingFiles.join(", ")}), which goes stale after ${formatTime(movingTimeout)}`);
82
+ }
80
83
  return false;
81
84
  }
82
85
 
@@ -94,6 +97,9 @@ export async function moveLogsToPublic(config: {
94
97
  // 7) Also, out of all of the moving files which aren't beyond the timeout, we need to make sure ours is the oldest, but not older than half the timeout. If this isn't true, then we just return.
95
98
  let oldestMovingFile = validMovingFiles[0];
96
99
  if (oldestMovingFile !== ourMovingFileName) {
100
+ if (forceAll) {
101
+ console.warn(`Forced log move for ${config.loggerName} skipped: ${oldestMovingFile} took the move lock before us (${ourMovingFileName})`);
102
+ }
97
103
  // Clean up our moving file since we didn't get the lock
98
104
  await localLogs.del(ourMovingFileName);
99
105
  return false;
@@ -10,8 +10,7 @@ import { isPublic } from "../../../config";
10
10
  import { MachineThreadInfo } from "../../MachineThreadInfo";
11
11
  import { ErrorWarning } from "./ErrorWarning";
12
12
  import { getErrorNotificationsManager } from "./errorWatcher";
13
- import { createMatchesPattern, normalizeMatchWhitespace } from "../IndexedLogs/bufferSearchFindMatcher";
14
- import { cacheLimited } from "socket-function/src/caching";
13
+ import { createPatternSetMatcher, getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
15
14
  import { managementPageURL, showingManagementURL } from "../../managementPages";
16
15
  import { searchTextURL, readProductionLogsURL } from "../IndexedLogs/LogViewerParams";
17
16
  import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
@@ -57,13 +56,6 @@ function TicketButton(props: {
57
56
  }
58
57
  qreact.inline(TicketButton);
59
58
 
60
- export let getMatcher = cacheLimited(100, (pattern: string) =>
61
- createMatchesPattern(Buffer.from(normalizeMatchWhitespace(pattern)), false)
62
- );
63
-
64
- export function getMatchBuffer(datum: LogDatum): Buffer {
65
- return Buffer.from(normalizeMatchWhitespace(JSON.stringify(datum)));
66
- }
67
59
 
68
60
  export class LogDatumRenderer extends qreact.Component<{
69
61
  datum: LogDatum;
@@ -81,15 +73,11 @@ export class LogDatumRenderer extends qreact.Component<{
81
73
  let datum = this.props.datum;
82
74
  let mainMessage = datum.param0 && String(datum.param0) || "(no message)";
83
75
 
84
- let matchingSuppressions = [];
76
+ let suppressionEntries = this.manager.state.suppressionEntries;
77
+ let matchingSuppressions: typeof suppressionEntries = [];
85
78
  if (!this.manager.state.isLoading && !this.props.hideSuppression) {
86
- for (let suppression of this.manager.state.suppressionEntries) {
87
- let matcher = getMatcher(suppression.pattern);
88
- let errorBuffer = getMatchBuffer(datum);
89
- if (matcher(errorBuffer)) {
90
- matchingSuppressions.push(suppression);
91
- }
92
- }
79
+ let matchedIndexes = getPatternSetMatcher(suppressionEntries.map(x => x.pattern))(getMatchBuffer(datum));
80
+ matchingSuppressions = matchedIndexes.map(index => suppressionEntries[index]);
93
81
  }
94
82
 
95
83
  let timedOutMatches = matchingSuppressions.filter(s => datum.time > s.timeout);
@@ -464,8 +452,8 @@ class SuppressionItem extends qreact.Component<{
464
452
  let example = this.props.matchData?.examples[0];
465
453
  if (example) return example;
466
454
  // No example recorded for this suppression — find any live error matching the pattern, so the ticket stores a real, full log entry instead of a synthetic stub.
467
- let matcher = getMatcher(suppression.pattern);
468
- let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)));
455
+ let matcher = createPatternSetMatcher([suppression.pattern]);
456
+ let fallback = this.manager.state.unmatchedErrors.find(e => matcher(getMatchBuffer(e)).length > 0);
469
457
  return fallback || {
470
458
  time: Date.now(),
471
459
  __LOG_TYPE: "error",
@@ -4,7 +4,8 @@ import { css } from "typesafecss";
4
4
  import { SocketFunction } from "socket-function/SocketFunction";
5
5
  import { ATag } from "../../../library-components/ATag";
6
6
  import { managementPageURL, showingManagementURL } from "../../managementPages";
7
- import { getMatcher, getMatchBuffer, LogDatumRenderer } from "./ErrorNotificationPage";
7
+ import { LogDatumRenderer } from "./ErrorNotificationPage";
8
+ import { getMatchBuffer, getPatternSetMatcher } from "./suppressionPatternMatcher";
8
9
  import { getErrorNotificationsManager } from "./errorWatcher";
9
10
  import { LogDatum } from "../diskLogger";
10
11
  import { TicketsController } from "../errorTickets/tickets";
@@ -57,10 +58,8 @@ export class ErrorWarning extends qreact.Component {
57
58
  if (!ticket.suppressionPattern) continue;
58
59
  openTicketPatterns.push(ticket.suppressionPattern);
59
60
  }
60
- const isCoveredByOpenTicket = (datum: LogDatum) => {
61
- let errorBuffer = getMatchBuffer(datum);
62
- return openTicketPatterns.some(pattern => getMatcher(pattern)(errorBuffer));
63
- };
61
+ const openTicketMatcher = getPatternSetMatcher(openTicketPatterns);
62
+ const isCoveredByOpenTicket = (datum: LogDatum) => openTicketMatcher(getMatchBuffer(datum)).length > 0;
64
63
 
65
64
  let errorWarning: preact.ComponentChild = undefined;
66
65
  let firstError = !this.manager.state.isLoading && this.manager.state.unmatchedErrors.find(x => !isCoveredByOpenTicket(x)) || undefined;