querysub 0.725.0 → 0.727.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/backfill-log-categories.js +7 -0
- package/package.json +8 -2
- package/src/-a-archives/archiveCache.ts +7 -7
- package/src/-a-archives/archivesDisk.ts +20 -7
- package/src/0-path-value-core/AuthorityLookup.ts +113 -19
- package/src/deployManager/components/DeployPage.tsx +2 -2
- package/src/deployManager/components/ServiceDetailPage.tsx +17 -4
- package/src/deployManager/components/ServicesListPage.tsx +61 -8
- package/src/deployManager/components/deployButtons.tsx +166 -101
- package/src/deployManager/machineSchema.ts +13 -2
- package/src/deployManager/publishQuerysub.ts +121 -49
- package/src/deployManager/serviceCategories.ts +90 -1
- package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +2 -0
- package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +2 -1
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +40 -1
- package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +6 -2
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +2 -6
- package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +34 -0
- package/src/diagnostics/logs/IndexedLogs/limitPublicLogsSize.ts +119 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +13 -5
- package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +2 -0
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +93 -21
- package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts +4 -2
- package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +1 -1
- package/src/diagnostics/logs/logCategories/LogBreakdownPage.tsx +421 -0
- package/src/diagnostics/logs/logCategories/logCategories.ts +273 -0
- package/src/diagnostics/managementPages.tsx +6 -0
- package/src/diagnostics/network/backfillLogCategoriesMain.ts +128 -0
- package/src/diagnostics/network/deleteOldLogsMain.ts +57 -0
- package/src/diagnostics/network/listLogFilesMain.ts +60 -0
- package/src/diagnostics/network/logFilesMain.ts +13 -15
- package/src/diagnostics/network/probeLogDeleteMain.ts +71 -0
- package/src/diagnostics/network/watchServiceMain.ts +106 -0
- package/src/fs.ts +29 -0
- package/src/misc/moveFolderContents.ts +67 -0
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
|
|
2
|
+
import { getArchives2 } from "../../../-a-archives/archives2";
|
|
3
|
+
import { Querysub } from "../../../4-querysub/Querysub";
|
|
4
|
+
import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
|
|
5
|
+
import { user_data } from "../../../user-implementation/userData";
|
|
6
|
+
import { nextId, timeInMinute } from "socket-function/src/misc";
|
|
7
|
+
import { formatTime } from "socket-function/src/formatting/format";
|
|
8
|
+
import { blue, magenta } from "socket-function/src/formatting/logColors";
|
|
9
|
+
import { LogDatum } from "../diskLogger";
|
|
10
|
+
import { LogStreamer } from "../IndexedLogs/LogStreamer";
|
|
11
|
+
import { callOpenRouterJSON } from "../errorNotifications2/openRouterHelper";
|
|
12
|
+
import { getSuppressionEntries } from "../errorNotifications2/errorNotifications";
|
|
13
|
+
import { PatternSetMatcher, createPatternSetMatcher, getMatchBuffer } from "../errorNotifications2/suppressionPatternMatcher";
|
|
14
|
+
|
|
15
|
+
const CATEGORY_LIST_REFRESH_INTERVAL = timeInMinute * 5;
|
|
16
|
+
const WARNING_THROTTLE_INTERVAL = timeInMinute * 5;
|
|
17
|
+
const MAX_LOGGED_MESSAGE_LENGTH = 500;
|
|
18
|
+
const MIN_PATTERN_LITERAL_LENGTH = 5;
|
|
19
|
+
const PATTERN_OPERATOR_CHARS = /[*|&]/g;
|
|
20
|
+
const UNESCAPED_QUOTE = /(?<!\\)"/g;
|
|
21
|
+
const MOVE_LOGS_AI_CALL_LIMIT = 100;
|
|
22
|
+
const MOVE_LOGS_AI_CALL_WINDOW = timeInMinute * 15;
|
|
23
|
+
const SUPPRESSION_CATEGORIZED_LOGGER = "logs/error";
|
|
24
|
+
|
|
25
|
+
export const LOG_CATEGORIES_EXTENSION = ".categories";
|
|
26
|
+
export const CATEGORY_AI_LIMIT_HIT_ID = "category-ai-limit-hit";
|
|
27
|
+
export const CATEGORY_NO_API_KEY_ID = "category-no-api-key";
|
|
28
|
+
export const CATEGORY_PATTERN_FAILED_ID = "category-pattern-failed";
|
|
29
|
+
|
|
30
|
+
export type LogCategoryEntry = {
|
|
31
|
+
id: string;
|
|
32
|
+
pattern: string;
|
|
33
|
+
loggerName: string;
|
|
34
|
+
createdTime: number;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export type LogCategoryCounts = {
|
|
38
|
+
[categoryId: string]: {
|
|
39
|
+
count: number;
|
|
40
|
+
bytes: number;
|
|
41
|
+
};
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
export type AICallLimiter = {
|
|
45
|
+
tryTake(): boolean;
|
|
46
|
+
describe(): string;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
export type CategorizeStats = {
|
|
50
|
+
entries: number;
|
|
51
|
+
unmatchedEntries: number;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
type CategoryList = {
|
|
55
|
+
loggerName: string;
|
|
56
|
+
categories: { id: string; pattern: string }[];
|
|
57
|
+
matcher: PatternSetMatcher;
|
|
58
|
+
loadTime: number;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const logCategories = archiveJSONT<LogCategoryEntry>(() => getArchives2("logs/log-categories/"), { findFallbacks: true });
|
|
62
|
+
|
|
63
|
+
export async function getLogCategories(): Promise<LogCategoryEntry[]> {
|
|
64
|
+
return await logCategories.values();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export async function deleteLogCategory(id: string) {
|
|
68
|
+
let entry = await logCategories.get(id);
|
|
69
|
+
if (!entry) {
|
|
70
|
+
throw new Error(`Expected log category ${id} to exist, it was not found`);
|
|
71
|
+
}
|
|
72
|
+
await logCategories.delete(id);
|
|
73
|
+
console.log(magenta(`Deleted log category ${id} for ${blue(entry.loggerName)}, pattern ${JSON.stringify(entry.pattern)}`));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function createAICallLimiter(config: { maxCalls: number; window: number }): AICallLimiter {
|
|
77
|
+
let { maxCalls, window } = config;
|
|
78
|
+
let callTimes: number[] = [];
|
|
79
|
+
return {
|
|
80
|
+
tryTake() {
|
|
81
|
+
let windowStart = Date.now() - window;
|
|
82
|
+
callTimes = callTimes.filter(time => time > windowStart);
|
|
83
|
+
if (callTimes.length >= maxCalls) return false;
|
|
84
|
+
callTimes.push(Date.now());
|
|
85
|
+
return true;
|
|
86
|
+
},
|
|
87
|
+
describe() {
|
|
88
|
+
let windowText = Number.isFinite(window) && ` in the last ${formatTime(window)}` || "";
|
|
89
|
+
return `${callTimes.length}/${maxCalls} AI calls used${windowText}`;
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export const moveLogsAICallLimiter = createAICallLimiter({ maxCalls: MOVE_LOGS_AI_CALL_LIMIT, window: MOVE_LOGS_AI_CALL_WINDOW });
|
|
95
|
+
|
|
96
|
+
export async function createLogCategoriesFile(config: {
|
|
97
|
+
loggerName: string;
|
|
98
|
+
entries: Buffer[];
|
|
99
|
+
aiCallLimiter: AICallLimiter;
|
|
100
|
+
stats?: CategorizeStats;
|
|
101
|
+
}): Promise<Buffer> {
|
|
102
|
+
return Buffer.from(JSON.stringify(await categorizeLogEntries(config)));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function parseLogCategoriesFile(buffer: Buffer): LogCategoryCounts {
|
|
106
|
+
let counts: LogCategoryCounts = JSON.parse(buffer.toString());
|
|
107
|
+
return counts;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function categorizeLogEntries(config: {
|
|
111
|
+
loggerName: string;
|
|
112
|
+
entries: Buffer[];
|
|
113
|
+
aiCallLimiter: AICallLimiter;
|
|
114
|
+
stats?: CategorizeStats;
|
|
115
|
+
}): Promise<LogCategoryCounts> {
|
|
116
|
+
let { loggerName, entries, aiCallLimiter, stats } = config;
|
|
117
|
+
let list = await getCategoryList(loggerName);
|
|
118
|
+
let openRouterAPIKey = await Querysub.commitAsync(() => atomic(user_data().secure.openRouterAPIKey)) || "";
|
|
119
|
+
let counts: LogCategoryCounts = {};
|
|
120
|
+
function add(id: string, bytes: number) {
|
|
121
|
+
let value = counts[id];
|
|
122
|
+
if (!value) {
|
|
123
|
+
value = { count: 0, bytes: 0 };
|
|
124
|
+
counts[id] = value;
|
|
125
|
+
}
|
|
126
|
+
value.count++;
|
|
127
|
+
value.bytes += bytes;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
for (let entry of entries) {
|
|
131
|
+
let datum = LogStreamer.deserialize<LogDatum>(entry);
|
|
132
|
+
if (!datum) continue;
|
|
133
|
+
if (stats) {
|
|
134
|
+
stats.entries++;
|
|
135
|
+
}
|
|
136
|
+
let matched = list.matcher(getMatchBuffer(datum));
|
|
137
|
+
if (matched.length) {
|
|
138
|
+
add(list.categories[matched[0]].id, entry.length);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (stats) {
|
|
142
|
+
stats.unmatchedEntries++;
|
|
143
|
+
}
|
|
144
|
+
if (!openRouterAPIKey) {
|
|
145
|
+
add(CATEGORY_NO_API_KEY_ID, entry.length);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (!aiCallLimiter.tryTake()) {
|
|
149
|
+
add(CATEGORY_AI_LIMIT_HIT_ID, entry.length);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
let created = await createLogCategory({ list, datum, openRouterAPIKey });
|
|
153
|
+
add(created && created.id || CATEGORY_PATTERN_FAILED_ID, entry.length);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
let noAPIKey = counts[CATEGORY_NO_API_KEY_ID];
|
|
157
|
+
if (noAPIKey) {
|
|
158
|
+
warnThrottled(CATEGORY_NO_API_KEY_ID, `${noAPIKey.count} ${loggerName} entries were left uncategorized because no OpenRouter API key is set (set it on the Security management page)`);
|
|
159
|
+
}
|
|
160
|
+
let limitHit = counts[CATEGORY_AI_LIMIT_HIT_ID];
|
|
161
|
+
if (limitHit) {
|
|
162
|
+
warnThrottled(CATEGORY_AI_LIMIT_HIT_ID, `${limitHit.count} ${loggerName} entries were left uncategorized because the AI categorization limit was hit (${aiCallLimiter.describe()})`);
|
|
163
|
+
}
|
|
164
|
+
return counts;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
let lastWarningTimes = new Map<string, number>();
|
|
168
|
+
function warnThrottled(key: string, message: string) {
|
|
169
|
+
let last = lastWarningTimes.get(key) || 0;
|
|
170
|
+
if (Date.now() - last < WARNING_THROTTLE_INTERVAL) return;
|
|
171
|
+
lastWarningTimes.set(key, Date.now());
|
|
172
|
+
console.warn(message);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
let categoryLists = new Map<string, Promise<CategoryList>>();
|
|
176
|
+
async function getCategoryList(loggerName: string): Promise<CategoryList> {
|
|
177
|
+
let existing = categoryLists.get(loggerName);
|
|
178
|
+
if (existing) {
|
|
179
|
+
let list = await existing;
|
|
180
|
+
if (Date.now() - list.loadTime < CATEGORY_LIST_REFRESH_INTERVAL) return list;
|
|
181
|
+
}
|
|
182
|
+
let loading = loadCategoryList(loggerName);
|
|
183
|
+
categoryLists.set(loggerName, loading);
|
|
184
|
+
return await loading;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async function loadCategoryList(loggerName: string): Promise<CategoryList> {
|
|
188
|
+
let [allCategories, suppressions] = await Promise.all([
|
|
189
|
+
getLogCategories(),
|
|
190
|
+
loggerName === SUPPRESSION_CATEGORIZED_LOGGER && getSuppressionEntries() || [],
|
|
191
|
+
]);
|
|
192
|
+
let categories = [
|
|
193
|
+
...suppressions.map(x => ({ id: x.id, pattern: x.pattern })),
|
|
194
|
+
...allCategories.filter(x => x.loggerName === loggerName).map(x => ({ id: x.id, pattern: x.pattern })),
|
|
195
|
+
];
|
|
196
|
+
return {
|
|
197
|
+
loggerName,
|
|
198
|
+
categories,
|
|
199
|
+
matcher: createPatternSetMatcher(categories.map(x => x.pattern)),
|
|
200
|
+
loadTime: Date.now(),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async function createLogCategory(config: { list: CategoryList; datum: LogDatum; openRouterAPIKey: string }): Promise<LogCategoryEntry | undefined> {
|
|
205
|
+
let { list, datum, openRouterAPIKey } = config;
|
|
206
|
+
let message = String(datum.param0 || "(no message)");
|
|
207
|
+
let pattern: string;
|
|
208
|
+
try {
|
|
209
|
+
pattern = await generateLogCategoryPattern({ message: JSON.stringify(message), openRouterAPIKey });
|
|
210
|
+
} catch (e) {
|
|
211
|
+
setImmediate(() => {
|
|
212
|
+
throw e;
|
|
213
|
+
});
|
|
214
|
+
return undefined;
|
|
215
|
+
}
|
|
216
|
+
let literalLength = pattern.replaceAll(PATTERN_OPERATOR_CHARS, "").trim().length;
|
|
217
|
+
if (literalLength < MIN_PATTERN_LITERAL_LENGTH) {
|
|
218
|
+
console.warn(`AI category pattern ${JSON.stringify(pattern)} for ${list.loggerName} has only ${literalLength} literal characters (minimum ${MIN_PATTERN_LITERAL_LENGTH}), so it was not saved. Entry: ${message.slice(0, MAX_LOGGED_MESSAGE_LENGTH)}`);
|
|
219
|
+
return undefined;
|
|
220
|
+
}
|
|
221
|
+
let matchBuffer = getMatchBuffer(datum);
|
|
222
|
+
let alreadyMatched = list.matcher(matchBuffer);
|
|
223
|
+
if (alreadyMatched.length) {
|
|
224
|
+
let existing = list.categories[alreadyMatched[0]];
|
|
225
|
+
return { id: existing.id, pattern: existing.pattern, loggerName: list.loggerName, createdTime: 0 };
|
|
226
|
+
}
|
|
227
|
+
if (!createPatternSetMatcher([pattern])(matchBuffer).length) {
|
|
228
|
+
console.warn(`AI category pattern ${JSON.stringify(pattern)} does not match the ${list.loggerName} entry it was generated for, so it was not saved. Entry: ${message.slice(0, MAX_LOGGED_MESSAGE_LENGTH)}`);
|
|
229
|
+
return undefined;
|
|
230
|
+
}
|
|
231
|
+
let entry: LogCategoryEntry = {
|
|
232
|
+
id: nextId(),
|
|
233
|
+
pattern,
|
|
234
|
+
loggerName: list.loggerName,
|
|
235
|
+
createdTime: Date.now(),
|
|
236
|
+
};
|
|
237
|
+
await logCategories.set(entry.id, entry);
|
|
238
|
+
list.categories.push({ id: entry.id, pattern: entry.pattern });
|
|
239
|
+
list.matcher = createPatternSetMatcher(list.categories.map(x => x.pattern));
|
|
240
|
+
console.log(magenta(`Created log category ${entry.id} for ${blue(list.loggerName)} (${list.categories.length} categories now), pattern ${JSON.stringify(pattern)}, from: ${message.slice(0, MAX_LOGGED_MESSAGE_LENGTH)}`));
|
|
241
|
+
return entry;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async function generateLogCategoryPattern(config: { message: string; openRouterAPIKey: string }): Promise<string> {
|
|
245
|
+
let { message, openRouterAPIKey } = config;
|
|
246
|
+
let prompt = `You are helping to categorize log messages. We categorize them by finding specific unique text inside of the log message which will be the same even if the variables and the instance changes.
|
|
247
|
+
|
|
248
|
+
Here is a log message, JSON encoded exactly the way it will be searched:
|
|
249
|
+
${message}
|
|
250
|
+
|
|
251
|
+
Please provide a SHORT search string that would match this type of log message. The string should:
|
|
252
|
+
- Be specific enough to identify this type of log message
|
|
253
|
+
- Be generic enough to match similar log messages (avoid dynamic values like timestamps, IDs, or specific numbers)
|
|
254
|
+
- Use * as a wildcard to skip over variable parts in the middle of the message
|
|
255
|
+
|
|
256
|
+
DO NOT include variables. If something is some dynamic path that includes numbers, etc., definitely do not include that in the match pattern. That is a very, very bad match pattern. Replace each variable part with *.
|
|
257
|
+
|
|
258
|
+
The pattern is split on *, and every piece between the * characters must be exactly included in the message, in that order. Each piece is checked with .includes() in javascript, so a piece can't skip anything: never join two pieces of the message together without a * between them. For example, for "LoRA H3-GalaxyAce.safetensors x0.25 merged into 258 layers" a good pattern is "LoRA * merged into * layers".
|
|
259
|
+
|
|
260
|
+
NEVER use the characters | or & in the pattern, they are operators in our pattern language.
|
|
261
|
+
|
|
262
|
+
Also, you will be searching the JSON stringified version of this string. So for example, if there's a ", you're going to have to search with it escaped. So you're going to have to search \\" quote.
|
|
263
|
+
|
|
264
|
+
The message may contain terminal color codes, which appear in the JSON as \\u001b followed by something like [35m or [0m. NEVER include a color code, or any part of one, in the pattern. Pick plain text that sits between the color codes instead.
|
|
265
|
+
|
|
266
|
+
Respond with a JSON object with a "pattern" field containing the search string.`;
|
|
267
|
+
let response = await callOpenRouterJSON<{ pattern: string }>(openRouterAPIKey, prompt);
|
|
268
|
+
let pattern = String(response.pattern || "").trim().replaceAll(UNESCAPED_QUOTE, "\\\"");
|
|
269
|
+
if (!pattern) {
|
|
270
|
+
throw new Error(`AI returned an empty log category pattern for: ${message.slice(0, MAX_LOGGED_MESSAGE_LENGTH)}`);
|
|
271
|
+
}
|
|
272
|
+
return pattern;
|
|
273
|
+
}
|
|
@@ -146,6 +146,12 @@ export async function registerManagementPages2(config: {
|
|
|
146
146
|
controllerName: "",
|
|
147
147
|
getModule: () => import("./logs/errorNotifications2/ErrorNotificationPage"),
|
|
148
148
|
});
|
|
149
|
+
inputPages.push({
|
|
150
|
+
title: "Log Breakdown",
|
|
151
|
+
componentName: "LogBreakdownPage",
|
|
152
|
+
controllerName: "LogBreakdownController",
|
|
153
|
+
getModule: () => import("./logs/logCategories/LogBreakdownPage"),
|
|
154
|
+
});
|
|
149
155
|
|
|
150
156
|
// NOTE: Even if we never use this, it is needed if a server or site is bootstrapping. However, they should probably include this in their own header dynamically as well.
|
|
151
157
|
inputPages.push({
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
{
|
|
4
|
+
let argv = process.argv;
|
|
5
|
+
let cwdIdx = argv.indexOf("--cwd");
|
|
6
|
+
if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
|
|
7
|
+
process.chdir(argv[cwdIdx + 1]);
|
|
8
|
+
argv.splice(cwdIdx, 2);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
import "../../inject";
|
|
13
|
+
|
|
14
|
+
import yargs from "yargs";
|
|
15
|
+
import { runInParallel } from "socket-function/src/batching";
|
|
16
|
+
import { formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
17
|
+
import { green, magenta } from "socket-function/src/formatting/logColors";
|
|
18
|
+
import { sort, timeInDay, timeInSecond } from "socket-function/src/misc";
|
|
19
|
+
import { Querysub } from "../../4-querysub/Querysub";
|
|
20
|
+
import { atomic } from "../../2-proxy/PathValueProxyWatcher";
|
|
21
|
+
import { user_data } from "../../user-implementation/userData";
|
|
22
|
+
import { getLoggers2Async } from "../logs/diskLogger";
|
|
23
|
+
import { BufferIndex } from "../logs/IndexedLogs/BufferIndex";
|
|
24
|
+
import { TimeFilePath } from "../logs/IndexedLogs/TimeFileTree";
|
|
25
|
+
import { CategorizeStats, LOG_CATEGORIES_EXTENSION, createAICallLimiter, createLogCategoriesFile } from "../logs/logCategories/logCategories";
|
|
26
|
+
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
27
|
+
|
|
28
|
+
const PARALLEL_FILES = 8;
|
|
29
|
+
const PARALLEL_EXISTENCE_CHECKS = 32;
|
|
30
|
+
const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
|
|
31
|
+
const AI_CALL_LIMIT = 10000;
|
|
32
|
+
|
|
33
|
+
let args = yargs(process.argv.slice(2))
|
|
34
|
+
.option("logger", { type: "string", describe: "only this logger, ex logs/info (default: every logger)" })
|
|
35
|
+
.option("days", { type: "number", describe: "only files from the last N days (default: every file)" })
|
|
36
|
+
.option("public", { type: "boolean", describe: "connect to the production network, so the OpenRouter API key is read from the production database" })
|
|
37
|
+
.option("nobreak", { type: "boolean", describe: "don't break into the debugger on errors" })
|
|
38
|
+
.strict()
|
|
39
|
+
.argv;
|
|
40
|
+
|
|
41
|
+
function formatPercent(part: number, total: number): string {
|
|
42
|
+
return `${(total && part / total * 100 || 0).toFixed(2)}%`;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function main() {
|
|
46
|
+
await Querysub.hostService("BackfillLogCategories");
|
|
47
|
+
disablePathAuditer();
|
|
48
|
+
let loggers = Object.values(await getLoggers2Async());
|
|
49
|
+
if (args.logger) {
|
|
50
|
+
let names = loggers.map(x => x.config.name);
|
|
51
|
+
loggers = loggers.filter(x => x.config.name === args.logger);
|
|
52
|
+
if (!loggers.length) {
|
|
53
|
+
throw new Error(`Expected --logger to be one of ${names.join(", ")}, was ${JSON.stringify(args.logger)}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
let openRouterAPIKey = await Querysub.commitAsync(() => atomic(user_data().secure.openRouterAPIKey));
|
|
57
|
+
console.log(magenta(`OpenRouter API key is ${openRouterAPIKey && "set, so new categories can be created" || "NOT set, so every entry without an existing category will be counted as uncategorized"}`));
|
|
58
|
+
let startTime = args.days && Date.now() - args.days * timeInDay || 0;
|
|
59
|
+
let aiCallLimiter = createAICallLimiter({ maxCalls: AI_CALL_LIMIT, window: Infinity });
|
|
60
|
+
let overallStart = Date.now();
|
|
61
|
+
|
|
62
|
+
for (let logger of loggers) {
|
|
63
|
+
let name = logger.config.name;
|
|
64
|
+
let archives = logger.getPublicLogArchives();
|
|
65
|
+
let listStart = Date.now();
|
|
66
|
+
let paths = await logger.listPublicLogPaths({ startTime, endTime: Date.now() });
|
|
67
|
+
sort(paths, x => -x.startTime);
|
|
68
|
+
console.log(magenta(`${name}: listed ${formatNumber(paths.length)} log files in ${formatTime(Date.now() - listStart)}, checking which already have a ${LOG_CATEGORIES_EXTENSION} file`));
|
|
69
|
+
|
|
70
|
+
let missing: TimeFilePath[] = [];
|
|
71
|
+
let checkStart = Date.now();
|
|
72
|
+
let checked = 0;
|
|
73
|
+
let lastProgressLog = Date.now();
|
|
74
|
+
await Promise.all(paths.map(runInParallel({ parallelCount: PARALLEL_EXISTENCE_CHECKS }, async (path: TimeFilePath) => {
|
|
75
|
+
if (!await archives.getInfo(path.fullPath + LOG_CATEGORIES_EXTENSION)) {
|
|
76
|
+
missing.push(path);
|
|
77
|
+
}
|
|
78
|
+
checked++;
|
|
79
|
+
if (Date.now() - lastProgressLog < PROGRESS_LOG_INTERVAL) return;
|
|
80
|
+
lastProgressLog = Date.now();
|
|
81
|
+
console.log(magenta(`${name}: checked ${checked}/${paths.length} files, ${missing.length} have no categories so far, ${formatTime(Date.now() - checkStart)} elapsed`));
|
|
82
|
+
})));
|
|
83
|
+
sort(missing, x => -x.startTime);
|
|
84
|
+
let missingBytes = missing.reduce((acc, x) => acc + (x.compressedSize || 0), 0);
|
|
85
|
+
console.log(magenta(`${name}: ${missing.length} of ${paths.length} files have no categories (${formatNumber(missingBytes)}B compressed), categorizing them`));
|
|
86
|
+
|
|
87
|
+
let categorizeStart = Date.now();
|
|
88
|
+
let done = 0;
|
|
89
|
+
let skipped = 0;
|
|
90
|
+
let bytesDone = 0;
|
|
91
|
+
let stats: CategorizeStats = { entries: 0, unmatchedEntries: 0 };
|
|
92
|
+
let lastStats: CategorizeStats = { ...stats };
|
|
93
|
+
let progressTimer = setInterval(() => {
|
|
94
|
+
let elapsed = Date.now() - categorizeStart;
|
|
95
|
+
let windowEntries = stats.entries - lastStats.entries;
|
|
96
|
+
let windowUnmatched = stats.unmatchedEntries - lastStats.unmatchedEntries;
|
|
97
|
+
lastStats = { ...stats };
|
|
98
|
+
console.log(magenta(`${name}: ${done}/${missing.length} files categorized, ${formatPercent(windowUnmatched, windowEntries)} of the last ${formatNumber(windowEntries)} entries had no existing category (${formatPercent(stats.unmatchedEntries, stats.entries)} of ${formatNumber(stats.entries)} overall), ${formatNumber(bytesDone)}B/${formatNumber(missingBytes)}B (${formatNumber(bytesDone / (elapsed / 1000))}B/s), ${aiCallLimiter.describe()}, ${formatTime(elapsed)} so far, about ${formatTime(elapsed * (missingBytes - bytesDone) / Math.max(1, bytesDone))} left`));
|
|
99
|
+
}, PROGRESS_LOG_INTERVAL);
|
|
100
|
+
try {
|
|
101
|
+
await Promise.all(missing.map(runInParallel({ parallelCount: PARALLEL_FILES }, async (path: TimeFilePath) => {
|
|
102
|
+
let data = await archives.get(path.fullPath);
|
|
103
|
+
if (!data) {
|
|
104
|
+
skipped++;
|
|
105
|
+
console.warn(`${name}: ${path.fullPath} was listed but has no data (probably deleted while running), skipping it`);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
let entries = await BufferIndex.decodeAll(data);
|
|
109
|
+
let categories = await createLogCategoriesFile({ loggerName: name, entries, aiCallLimiter, stats });
|
|
110
|
+
await archives.set(path.fullPath + LOG_CATEGORIES_EXTENSION, categories);
|
|
111
|
+
done++;
|
|
112
|
+
bytesDone += path.compressedSize || 0;
|
|
113
|
+
})));
|
|
114
|
+
} finally {
|
|
115
|
+
clearInterval(progressTimer);
|
|
116
|
+
}
|
|
117
|
+
console.log(green(`${name}: wrote categories for ${done} files (${skipped} skipped), ${formatPercent(stats.unmatchedEntries, stats.entries)} of ${formatNumber(stats.entries)} entries had no existing category, ${aiCallLimiter.describe()}, in ${formatTime(Date.now() - categorizeStart)}`));
|
|
118
|
+
}
|
|
119
|
+
console.log(green(`Categorized ${loggers.length} logger(s) in ${formatTime(Date.now() - overallStart)}, ${aiCallLimiter.describe()}`));
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
main().then(
|
|
123
|
+
() => process.exit(),
|
|
124
|
+
e => {
|
|
125
|
+
console.error((e as Error).stack || e);
|
|
126
|
+
process.exit(1);
|
|
127
|
+
},
|
|
128
|
+
);
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
{
|
|
4
|
+
let argv = process.argv;
|
|
5
|
+
let cwdIdx = argv.indexOf("--cwd");
|
|
6
|
+
if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
|
|
7
|
+
process.chdir(argv[cwdIdx + 1]);
|
|
8
|
+
argv.splice(cwdIdx, 2);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
import "../../inject";
|
|
13
|
+
|
|
14
|
+
import yargs from "yargs";
|
|
15
|
+
import { formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
16
|
+
import { green, magenta } from "socket-function/src/formatting/logColors";
|
|
17
|
+
import { isPublic } from "../../config";
|
|
18
|
+
import { getLoggers2Async } from "../logs/diskLogger";
|
|
19
|
+
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
20
|
+
|
|
21
|
+
let args = yargs(process.argv.slice(2))
|
|
22
|
+
.option("logger", { type: "string", describe: "only this logger, ex logs/info (default: every logger)" })
|
|
23
|
+
.option("dry-run", { type: "boolean", default: false, describe: "size everything and report what would be deleted, delete nothing" })
|
|
24
|
+
.option("public", { type: "boolean", describe: "required, so the limit runs against public storage" })
|
|
25
|
+
.strict()
|
|
26
|
+
.argv;
|
|
27
|
+
|
|
28
|
+
async function main() {
|
|
29
|
+
disablePathAuditer();
|
|
30
|
+
if (!isPublic()) {
|
|
31
|
+
throw new Error(`delete-old-logs must run with --public, or it would limit a folder on this machine's own disk instead of public storage`);
|
|
32
|
+
}
|
|
33
|
+
let loggers = Object.values(await getLoggers2Async());
|
|
34
|
+
if (args.logger) {
|
|
35
|
+
let names = loggers.map(x => x.config.name);
|
|
36
|
+
loggers = loggers.filter(x => x.config.name === args.logger);
|
|
37
|
+
if (!loggers.length) {
|
|
38
|
+
throw new Error(`Expected --logger to be one of ${names.join(", ")}, was ${JSON.stringify(args.logger)}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
let start = Date.now();
|
|
42
|
+
for (let logger of loggers) {
|
|
43
|
+
let loggerStart = Date.now();
|
|
44
|
+
console.log(magenta(`Limiting public ${logger.config.name} logs${args["dry-run"] && " (dry run)" || ""}`));
|
|
45
|
+
let result = await logger.limitPublicLogsSize({ dryRun: args["dry-run"] });
|
|
46
|
+
console.log(green(`${logger.config.name}: kept ${result.keptFiles} files (${formatNumber(result.keptBytes)}B), ${args["dry-run"] && "would delete" || "deleted"} ${result.deletedFiles} files (${formatNumber(result.deletedBytes)}B), ${result.failedFiles} failed, in ${formatTime(Date.now() - loggerStart)}`));
|
|
47
|
+
}
|
|
48
|
+
console.log(green(`Done with ${loggers.length} logger(s) in ${formatTime(Date.now() - start)}`));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
main().then(
|
|
52
|
+
() => process.exit(),
|
|
53
|
+
e => {
|
|
54
|
+
console.error((e as Error).stack || e);
|
|
55
|
+
process.exit(1);
|
|
56
|
+
},
|
|
57
|
+
);
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
{
|
|
4
|
+
let argv = process.argv;
|
|
5
|
+
let cwdIdx = argv.indexOf("--cwd");
|
|
6
|
+
if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
|
|
7
|
+
process.chdir(argv[cwdIdx + 1]);
|
|
8
|
+
argv.splice(cwdIdx, 2);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
import "../../inject";
|
|
13
|
+
|
|
14
|
+
import { formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
15
|
+
import { green, magenta } from "socket-function/src/formatting/logColors";
|
|
16
|
+
import { timeInSecond } from "socket-function/src/misc";
|
|
17
|
+
import { getLoggers2Async } from "../logs/diskLogger";
|
|
18
|
+
import { TimeFileTree } from "../logs/IndexedLogs/TimeFileTree";
|
|
19
|
+
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
20
|
+
|
|
21
|
+
const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
|
|
22
|
+
|
|
23
|
+
async function main() {
|
|
24
|
+
disablePathAuditer();
|
|
25
|
+
let loggers = Object.values(await getLoggers2Async());
|
|
26
|
+
let overallStart = Date.now();
|
|
27
|
+
let totalFiles = 0;
|
|
28
|
+
for (let logger of loggers) {
|
|
29
|
+
let name = logger.config.name;
|
|
30
|
+
let archives = logger.getPublicLogArchives();
|
|
31
|
+
let start = Date.now();
|
|
32
|
+
let dayFolders = 0;
|
|
33
|
+
let files = 0;
|
|
34
|
+
let lastLog = Date.now();
|
|
35
|
+
console.log(magenta(`Listing every ${name} file through ${archives.getDebugName()}`));
|
|
36
|
+
let paths = await new TimeFileTree(archives).findAllPaths({
|
|
37
|
+
startTime: 0,
|
|
38
|
+
endTime: Date.now(),
|
|
39
|
+
onDayFolderListed: ({ dayFolder, fileCount }) => {
|
|
40
|
+
dayFolders++;
|
|
41
|
+
files += fileCount;
|
|
42
|
+
if (Date.now() - lastLog < PROGRESS_LOG_INTERVAL) return;
|
|
43
|
+
lastLog = Date.now();
|
|
44
|
+
console.log(magenta(`${name}: ${dayFolders} day folders, ${formatNumber(files)} file names so far, last ${dayFolder}, ${formatTime(Date.now() - start)} elapsed`));
|
|
45
|
+
},
|
|
46
|
+
});
|
|
47
|
+
let bytes = paths.reduce((acc, x) => acc + (x.compressedSize || 0), 0);
|
|
48
|
+
totalFiles += paths.length;
|
|
49
|
+
console.log(green(`${name}: ${formatNumber(paths.length)} log files (${formatNumber(bytes)}B) in ${dayFolders} day folders, listed in ${formatTime(Date.now() - start)}`));
|
|
50
|
+
}
|
|
51
|
+
console.log(green(`All loggers: ${formatNumber(totalFiles)} log files listed in ${formatTime(Date.now() - overallStart)}`));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
main().then(
|
|
55
|
+
() => process.exit(),
|
|
56
|
+
e => {
|
|
57
|
+
console.error((e as Error).stack || e);
|
|
58
|
+
process.exit(1);
|
|
59
|
+
},
|
|
60
|
+
);
|
|
@@ -14,7 +14,7 @@ import "../../inject";
|
|
|
14
14
|
import yargs from "yargs";
|
|
15
15
|
import { formatTime } from "socket-function/src/formatting/format";
|
|
16
16
|
import { magenta } from "socket-function/src/formatting/logColors";
|
|
17
|
-
import {
|
|
17
|
+
import { getLoggers2Async } from "../logs/diskLogger";
|
|
18
18
|
import { decodeLogFilePath } from "../logs/IndexedLogs/TimeFileTree";
|
|
19
19
|
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
20
20
|
|
|
@@ -24,9 +24,18 @@ let args = yargs(process.argv.slice(2))
|
|
|
24
24
|
.strict()
|
|
25
25
|
.argv;
|
|
26
26
|
|
|
27
|
-
async function
|
|
27
|
+
async function main() {
|
|
28
|
+
disablePathAuditer();
|
|
29
|
+
let loggers = Object.values(await getLoggers2Async());
|
|
30
|
+
let logger = loggers.find(x => x.config.name === args.logger);
|
|
31
|
+
if (!logger) {
|
|
32
|
+
throw new Error(`Expected --logger to be one of ${loggers.map(x => x.config.name).join(", ")}, was ${JSON.stringify(args.logger)}`);
|
|
33
|
+
}
|
|
34
|
+
let archives = logger.getPublicLogArchives();
|
|
35
|
+
let folder = args.day.endsWith("/") ? args.day : args.day + "/";
|
|
36
|
+
console.log(magenta(`Listing ${archives.getDebugName()} ${folder}`));
|
|
28
37
|
let start = Date.now();
|
|
29
|
-
let files = await
|
|
38
|
+
let files = await archives.find(folder, { shallow: true, type: "files" });
|
|
30
39
|
let byMachine = new Map<string, number>();
|
|
31
40
|
for (let file of files) {
|
|
32
41
|
let decoded = decodeLogFilePath(file);
|
|
@@ -34,18 +43,7 @@ async function listByMachine(config: { label: string; list: () => Promise<string
|
|
|
34
43
|
let machineId = decoded.machineId || "(none)";
|
|
35
44
|
byMachine.set(machineId, (byMachine.get(machineId) || 0) + 1);
|
|
36
45
|
}
|
|
37
|
-
console.log(magenta(`${
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
async function main() {
|
|
41
|
-
disablePathAuditer();
|
|
42
|
-
let archives = getArchives2(`final-indexed-logs/${args.logger}`);
|
|
43
|
-
let folder = args.day.endsWith("/") ? args.day : args.day + "/";
|
|
44
|
-
console.log(magenta(`Listing ${archives.getDebugName()}${folder}`));
|
|
45
|
-
await Promise.all([
|
|
46
|
-
listByMachine({ label: "find", list: () => archives.find(folder, { shallow: true, type: "files" }) }),
|
|
47
|
-
listByMachine({ label: "find with fallbacks", list: () => archives.find(folder, { shallow: true, type: "files", fallbacks: true }) }),
|
|
48
|
-
]);
|
|
46
|
+
console.log(magenta(`${files.length} files in ${formatTime(Date.now() - start)}`), [...byMachine].map(([id, count]) => `${id}=${count}`).join(", "));
|
|
49
47
|
}
|
|
50
48
|
|
|
51
49
|
main().then(
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
{
|
|
4
|
+
let argv = process.argv;
|
|
5
|
+
let cwdIdx = argv.indexOf("--cwd");
|
|
6
|
+
if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
|
|
7
|
+
process.chdir(argv[cwdIdx + 1]);
|
|
8
|
+
argv.splice(cwdIdx, 2);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
import "../../inject";
|
|
13
|
+
|
|
14
|
+
import yargs from "yargs";
|
|
15
|
+
import { delay } from "socket-function/src/batching";
|
|
16
|
+
import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
|
|
17
|
+
import { magenta, yellow } from "socket-function/src/formatting/logColors";
|
|
18
|
+
import { sort, timeInDay, timeInSecond } from "socket-function/src/misc";
|
|
19
|
+
import { isPublic } from "../../config";
|
|
20
|
+
import { getLoggers2Async } from "../logs/diskLogger";
|
|
21
|
+
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
22
|
+
|
|
23
|
+
const RECHECK_DELAY = timeInSecond * 60;
|
|
24
|
+
|
|
25
|
+
let args = yargs(process.argv.slice(2))
|
|
26
|
+
.option("logger", { type: "string", default: "logs/info", describe: "the logger whose oldest public file is deleted" })
|
|
27
|
+
.option("public", { type: "boolean", describe: "required, so the delete runs against public storage" })
|
|
28
|
+
.strict()
|
|
29
|
+
.argv;
|
|
30
|
+
|
|
31
|
+
async function main() {
|
|
32
|
+
disablePathAuditer();
|
|
33
|
+
if (!isPublic()) {
|
|
34
|
+
throw new Error(`probe-log-delete must run with --public, or it would delete from this machine's own disk instead of public storage`);
|
|
35
|
+
}
|
|
36
|
+
let loggers = Object.values(await getLoggers2Async());
|
|
37
|
+
let logger = loggers.find(x => x.config.name === args.logger);
|
|
38
|
+
if (!logger) {
|
|
39
|
+
throw new Error(`Expected --logger to be one of ${loggers.map(x => x.config.name).join(", ")}, was ${JSON.stringify(args.logger)}`);
|
|
40
|
+
}
|
|
41
|
+
let archives = logger.getPublicLogArchives();
|
|
42
|
+
let paths = await logger.listPublicLogPaths({ startTime: 0, endTime: Date.now() });
|
|
43
|
+
sort(paths, x => x.startTime);
|
|
44
|
+
let target = paths[0];
|
|
45
|
+
if (!target) {
|
|
46
|
+
throw new Error(`Expected ${args.logger} to have at least one public file, it has none`);
|
|
47
|
+
}
|
|
48
|
+
let dayStart = Math.floor(target.startTime / timeInDay) * timeInDay;
|
|
49
|
+
async function report(label: string) {
|
|
50
|
+
let info = await archives.getInfo(target.fullPath);
|
|
51
|
+
let listed = (await logger!.listPublicLogPaths({ startTime: dayStart, endTime: dayStart + timeInDay - 1 })).some(x => x.fullPath === target.fullPath);
|
|
52
|
+
console.log(magenta(`${label}: getInfo ${info && `size ${info.size}, writeTime ${formatDateTime(info.writeTime)}` || "undefined"}, listed ${listed}`));
|
|
53
|
+
}
|
|
54
|
+
console.log(magenta(`Oldest ${args.logger} file, started ${formatDateTime(target.startTime)}: ${target.fullPath}`));
|
|
55
|
+
await report("Before delete");
|
|
56
|
+
let deleteStart = Date.now();
|
|
57
|
+
await logger.deletePublicLogFile(target);
|
|
58
|
+
console.log(magenta(`Delete returned after ${formatTime(Date.now() - deleteStart)}`));
|
|
59
|
+
await report("Right after delete");
|
|
60
|
+
await delay(RECHECK_DELAY);
|
|
61
|
+
await report(`${formatTime(RECHECK_DELAY)} after delete`);
|
|
62
|
+
console.log(yellow(`Done`));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
main().then(
|
|
66
|
+
() => process.exit(),
|
|
67
|
+
e => {
|
|
68
|
+
console.error((e as Error).stack || e);
|
|
69
|
+
process.exit(1);
|
|
70
|
+
},
|
|
71
|
+
);
|