querysub 0.725.0 → 0.727.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/backfill-log-categories.js +7 -0
  2. package/package.json +8 -2
  3. package/src/-a-archives/archiveCache.ts +7 -7
  4. package/src/-a-archives/archivesDisk.ts +20 -7
  5. package/src/0-path-value-core/AuthorityLookup.ts +113 -19
  6. package/src/deployManager/components/DeployPage.tsx +2 -2
  7. package/src/deployManager/components/ServiceDetailPage.tsx +17 -4
  8. package/src/deployManager/components/ServicesListPage.tsx +61 -8
  9. package/src/deployManager/components/deployButtons.tsx +166 -101
  10. package/src/deployManager/machineSchema.ts +13 -2
  11. package/src/deployManager/publishQuerysub.ts +121 -49
  12. package/src/deployManager/serviceCategories.ts +90 -1
  13. package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +2 -0
  14. package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +2 -1
  15. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +40 -1
  16. package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +6 -2
  17. package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +2 -6
  18. package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +34 -0
  19. package/src/diagnostics/logs/IndexedLogs/limitPublicLogsSize.ts +119 -0
  20. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +13 -5
  21. package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +2 -0
  22. package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +93 -21
  23. package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts +4 -2
  24. package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +1 -1
  25. package/src/diagnostics/logs/logCategories/LogBreakdownPage.tsx +421 -0
  26. package/src/diagnostics/logs/logCategories/logCategories.ts +273 -0
  27. package/src/diagnostics/managementPages.tsx +6 -0
  28. package/src/diagnostics/network/backfillLogCategoriesMain.ts +128 -0
  29. package/src/diagnostics/network/deleteOldLogsMain.ts +57 -0
  30. package/src/diagnostics/network/listLogFilesMain.ts +60 -0
  31. package/src/diagnostics/network/logFilesMain.ts +13 -15
  32. package/src/diagnostics/network/probeLogDeleteMain.ts +71 -0
  33. package/src/diagnostics/network/watchServiceMain.ts +106 -0
  34. package/src/fs.ts +29 -0
  35. package/src/misc/moveFolderContents.ts +67 -0
@@ -0,0 +1,106 @@
1
+ import yargs from "yargs";
2
+ import { runPromise } from "socket-function/src/runPromise";
3
+ import { delay } from "socket-function/src/batching";
4
+ import { formatNumber } from "socket-function/src/formatting/format";
5
+ import { red, yellow } from "socket-function/src/formatting/logColors";
6
+ import { timeInMinute, timeInSecond } from "socket-function/src/misc";
7
+
8
+ const DEFAULT_INTERVAL = timeInSecond * 10;
9
+ const LOW_MEMORY_FRACTION = 0.15;
10
+ const SIGNIFICANT_MEMORY_CHANGE_FRACTION = 0.02;
11
+ const HEARTBEAT_INTERVAL = timeInMinute * 5;
12
+ const PROGRESS_PRINT_INTERVAL = timeInMinute;
13
+ const PROGRESS_LINE = /^Moved \d+\/\d+ files.*\((logs\/\w+)\)$/;
14
+ const BATCH_LINE = /^(\(\d+\/\d+\) Wrote|Merging and moving)/;
15
+ const DEFAULT_MATCH = "Moving|Reading log file sizes|Read \\d+ log file sizes|Moved \\d+|Wrote \\d+ log files|skipped|error|Error|heap|memory";
16
+ const SAFE_NAME = /^[\w.@-]+$/;
17
+
18
+ let args = yargs(process.argv.slice(2))
19
+ .option("host", { type: "string", demandOption: true, describe: "machine, ex 10.0.0.188" })
20
+ .option("session", { type: "string", demandOption: true, describe: "tmux session of the service, ex movelogs-0-dply" })
21
+ .option("process", { type: "string", describe: "substring of the service's command line, used to find its memory (default: the session name's service key)" })
22
+ .option("match", { type: "string", default: DEFAULT_MATCH, describe: "regex for pane lines worth printing" })
23
+ .option("interval", { type: "number", default: DEFAULT_INTERVAL })
24
+ .strict()
25
+ .argv;
26
+
27
+ function assertSafe(value: string, label: string): string {
28
+ if (!SAFE_NAME.test(value)) {
29
+ throw new Error(`Expected ${label} to match ${SAFE_NAME}, was ${JSON.stringify(value)}`);
30
+ }
31
+ return value;
32
+ }
33
+
34
+ async function main() {
35
+ let host = assertSafe(args.host.includes("@") ? args.host : `root@${args.host}`, "host");
36
+ let session = assertSafe(args.session, "session");
37
+ let processMatch = args.process || session.replace(/-\d+-dply$/, "");
38
+ let pattern = new RegExp(args.match);
39
+ let ssh = (command: string) => runPromise(`ssh -o BatchMode=yes -o ConnectTimeout=20 ${host} "${command}"`, { quiet: true, nothrow: true, includeStderrInOutput: true });
40
+ let seenLines = new Set<string>();
41
+ let firstPoll = true;
42
+ let lastPids = "-";
43
+ let lastRss = 0;
44
+ let lastAvailable = 0;
45
+ let lastPrintTime = 0;
46
+ let latestProgress = new Map<string, string>();
47
+ let lastProgressPrint = 0;
48
+ while (true) {
49
+ let [ps, free, pane] = await Promise.all([
50
+ ssh("ps -eo pid,rss,etime,args"),
51
+ ssh("free -b"),
52
+ ssh(`tmux capture-pane -J -p -S -2000 -t ${session}`),
53
+ ]);
54
+ let processes = ps.split("\n").filter(line => line.includes(processMatch) && !line.includes("ps -eo"));
55
+ let memLine = free.split("\n").find(line => line.startsWith("Mem:"))?.split(/\s+/) || [];
56
+ let total = Number(memLine[1]) || 0;
57
+ let available = Number(memLine[6]) || 0;
58
+ let pids = processes.map(line => line.trim().split(/\s+/)[0]).join(",");
59
+ let rss = processes.reduce((acc, line) => acc + Number(line.trim().split(/\s+/)[1]) * 1024, 0);
60
+ let processText = processes.map(line => {
61
+ let [pid, rssKb, etime] = line.trim().split(/\s+/);
62
+ return `pid ${pid} rss ${formatNumber(Number(rssKb) * 1024)}B up ${etime}`;
63
+ }).join(", ") || red(`no process matching "${processMatch}"`);
64
+ let memText = `machine available ${formatNumber(available)}B/${formatNumber(total)}B`;
65
+ let lowMemory = total && available < total * LOW_MEMORY_FRACTION;
66
+ let changed = pids !== lastPids
67
+ || Math.abs(rss - lastRss) > total * SIGNIFICANT_MEMORY_CHANGE_FRACTION
68
+ || Math.abs(available - lastAvailable) > total * SIGNIFICANT_MEMORY_CHANGE_FRACTION
69
+ || lowMemory
70
+ || Date.now() - lastPrintTime >= HEARTBEAT_INTERVAL;
71
+ if (changed) {
72
+ console.log(`[${new Date().toLocaleTimeString()}] ${host} ${session}: ${processText}; ${lowMemory && red(memText) || memText}`);
73
+ lastPids = pids;
74
+ lastRss = rss;
75
+ lastAvailable = available;
76
+ lastPrintTime = Date.now();
77
+ }
78
+
79
+ let lines = pane.split("\n").map(x => x.trim()).filter(x => x);
80
+ let fresh = lines.filter(line => !seenLines.has(line) && pattern.test(line));
81
+ for (let line of lines) seenLines.add(line);
82
+ for (let line of firstPoll && fresh.slice(-10) || fresh) {
83
+ let progressKey = PROGRESS_LINE.exec(line)?.[1];
84
+ if (progressKey) {
85
+ latestProgress.set(progressKey, line);
86
+ continue;
87
+ }
88
+ if (BATCH_LINE.test(line)) continue;
89
+ console.log(yellow(` ${line}`));
90
+ }
91
+ if (latestProgress.size && Date.now() - lastProgressPrint >= PROGRESS_PRINT_INTERVAL) {
92
+ for (let line of latestProgress.values()) {
93
+ console.log(yellow(` ${line}`));
94
+ }
95
+ latestProgress.clear();
96
+ lastProgressPrint = Date.now();
97
+ }
98
+ firstPoll = false;
99
+ await delay(args.interval);
100
+ }
101
+ }
102
+
103
+ main().catch(e => {
104
+ console.error((e as Error).stack || e);
105
+ process.exit(1);
106
+ });
package/src/fs.ts CHANGED
@@ -5,6 +5,9 @@ import debugbreak from "debugbreak";
5
5
  import { isNode } from "socket-function/src/misc";
6
6
  import path from "path";
7
7
  import { magenta } from "socket-function/src/formatting/logColors";
8
+ import { moveFolder } from "./misc/moveFolderContents";
9
+
10
+ const SHARED_FOLDER_NAME = "querysub-shared";
8
11
 
9
12
  let rootDir = "";
10
13
  if (isNode()) {
@@ -57,6 +60,32 @@ export const getSubFolder = cache((path: string) => {
57
60
  });
58
61
  export const getStorageFolder = getSubFolder;
59
62
 
63
+ export function getSubFolderPath(name: string): string {
64
+ let dir = getStorageDir() + name;
65
+ if (!dir.endsWith("/")) {
66
+ dir += "/";
67
+ }
68
+ return dir;
69
+ }
70
+
71
+ export function getSharedFolderPath(name: string): string {
72
+ let root = path.parse(path.resolve(".")).root;
73
+ let dir = path.join(root, SHARED_FOLDER_NAME, name).replaceAll("\\", "/");
74
+ if (!dir.endsWith("/")) {
75
+ dir += "/";
76
+ }
77
+ return dir;
78
+ }
79
+
80
+ let movedFolders = new Map<string, Promise<void>>();
81
+ export function moveIntoFolder(config: { oldFolder: string; folder: string }): Promise<void> {
82
+ let existing = movedFolders.get(config.folder);
83
+ if (existing) return existing;
84
+ let promise = moveFolder({ from: config.oldFolder, to: config.folder });
85
+ movedFolders.set(config.folder, promise);
86
+ return promise;
87
+ }
88
+
60
89
  export async function* readDirRecursive(dir: string): AsyncGenerator<string> {
61
90
  try {
62
91
  for (let file of await fs.promises.readdir(dir)) {
@@ -0,0 +1,67 @@
1
+ import fs from "fs";
2
+ import path from "path";
3
+ import { pathToFileURL } from "url";
4
+ import { formatNumber, formatTime } from "socket-function/src/formatting/format";
5
+ import { green, magenta } from "socket-function/src/formatting/logColors";
6
+ import { timeInSecond } from "socket-function/src/misc";
7
+
8
+ const PARALLEL_MOVES = 16;
9
+ const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
10
+ const TEMP_FOLDER_NAME = "querysub-move-temp";
11
+
12
+ export async function moveFolder(config: { from: string; to: string }): Promise<void> {
13
+ let from = path.resolve(config.from);
14
+ let to = path.resolve(config.to);
15
+ if (!await fs.promises.stat(from).catch(() => undefined)) return;
16
+ await fs.promises.mkdir(to, { recursive: true });
17
+ let tempFolder = path.join(path.parse(to).root, TEMP_FOLDER_NAME);
18
+ await fs.promises.mkdir(tempFolder, { recursive: true });
19
+
20
+ let start = Date.now();
21
+ let entries = await fs.promises.readdir(from, { recursive: true, withFileTypes: true });
22
+ let files = entries.filter(x => x.isFile()).map(x => path.relative(from, path.join(x.parentPath, x.name)));
23
+ console.log(magenta(`Moving ${formatNumber(files.length)} files from ${pathToFileURL(from).href} to ${pathToFileURL(to).href}`));
24
+
25
+ let moved = 0;
26
+ let alreadyThere = 0;
27
+ let bytesMoved = 0;
28
+ let failed: { file: string; error: Error }[] = [];
29
+ let lastProgressLog = Date.now();
30
+ let nextIndex = 0;
31
+ async function moveWorker() {
32
+ while (nextIndex < files.length) {
33
+ let fileIndex = nextIndex++;
34
+ let file = files[fileIndex];
35
+ let source = path.join(from, file);
36
+ let target = path.join(to, file);
37
+ try {
38
+ let size = (await fs.promises.stat(source)).size;
39
+ let targetSize = (await fs.promises.stat(target).catch(() => undefined))?.size;
40
+ if (targetSize === undefined || size > targetSize) {
41
+ let temp = path.join(tempFolder, `${process.pid}-${fileIndex}`);
42
+ await fs.promises.mkdir(path.dirname(target), { recursive: true });
43
+ await fs.promises.copyFile(source, temp);
44
+ await fs.promises.rename(temp, target);
45
+ moved++;
46
+ bytesMoved += size;
47
+ } else {
48
+ alreadyThere++;
49
+ }
50
+ await fs.promises.unlink(source);
51
+ } catch (e) {
52
+ failed.push({ file, error: e as Error });
53
+ }
54
+ if (Date.now() - lastProgressLog < PROGRESS_LOG_INTERVAL) continue;
55
+ lastProgressLog = Date.now();
56
+ let elapsed = Date.now() - start;
57
+ let done = moved + alreadyThere + failed.length;
58
+ console.log(magenta(`Moving to ${pathToFileURL(to).href}: ${done}/${files.length} files (${alreadyThere} already there, ${failed.length} failed), ${formatNumber(bytesMoved)}B moved, ${formatNumber(bytesMoved / (elapsed / 1000))}B/s, ${formatTime(elapsed)} so far, about ${formatTime(elapsed * (files.length - done) / Math.max(1, done))} left`));
59
+ }
60
+ }
61
+ await Promise.all(Array.from({ length: PARALLEL_MOVES }, () => moveWorker()));
62
+
63
+ console.log(green(`Moved ${moved} of ${files.length} files (${formatNumber(bytesMoved)}B) from ${pathToFileURL(from).href} to ${pathToFileURL(to).href}, ${alreadyThere} were already there, ${failed.length} failed, in ${formatTime(Date.now() - start)}`));
64
+ if (failed.length) {
65
+ console.warn(`Failed to move ${failed.length} of ${files.length} files from ${pathToFileURL(from).href}, they were left where they are. First failure, ${failed[0].file}:`, failed[0].error.stack || failed[0].error);
66
+ }
67
+ }