rigscope 0.1.8 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.next/BUILD_ID +1 -1
- package/.next/app-path-routes-manifest.json +1 -1
- package/.next/build-manifest.json +2 -2
- package/.next/server/app/_global-error/page_client-reference-manifest.js +1 -1
- package/.next/server/app/_global-error.html +1 -1
- package/.next/server/app/_global-error.rsc +1 -1
- package/.next/server/app/_global-error.segments/_full.segment.rsc +1 -1
- package/.next/server/app/_global-error.segments/_global-error/__PAGE__.segment.rsc +1 -1
- package/.next/server/app/_global-error.segments/_tree.segment.rsc +1 -1
- package/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/.next/server/app/_not-found.html +1 -1
- package/.next/server/app/_not-found.rsc +3 -3
- package/.next/server/app/_not-found.segments/_full.segment.rsc +3 -3
- package/.next/server/app/_not-found.segments/_not-found/__PAGE__.segment.rsc +3 -3
- package/.next/server/app/_not-found.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/administration/page_client-reference-manifest.js +1 -1
- package/.next/server/app/administration.html +1 -1
- package/.next/server/app/administration.rsc +3 -3
- package/.next/server/app/administration.segments/_full.segment.rsc +3 -3
- package/.next/server/app/administration.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/administration.segments/administration/__PAGE__.segment.rsc +3 -3
- package/.next/server/app/alerts/page_client-reference-manifest.js +1 -1
- package/.next/server/app/alerts.html +1 -1
- package/.next/server/app/alerts.rsc +3 -3
- package/.next/server/app/alerts.segments/_full.segment.rsc +3 -3
- package/.next/server/app/alerts.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/alerts.segments/alerts/__PAGE__.segment.rsc +3 -3
- package/.next/server/app/connections/page.js +1 -1
- package/.next/server/app/connections/page_client-reference-manifest.js +1 -1
- package/.next/server/app/connections.html +1 -1
- package/.next/server/app/connections.rsc +4 -4
- package/.next/server/app/connections.segments/_full.segment.rsc +4 -4
- package/.next/server/app/connections.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/connections.segments/connections/__PAGE__.segment.rsc +4 -4
- package/.next/server/app/history/page.js +2 -2
- package/.next/server/app/history/page_client-reference-manifest.js +1 -1
- package/.next/server/app/history.html +1 -1
- package/.next/server/app/history.rsc +6 -6
- package/.next/server/app/history.segments/_full.segment.rsc +6 -6
- package/.next/server/app/history.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/history.segments/history/__PAGE__.segment.rsc +6 -6
- package/.next/server/app/index.html +1 -1
- package/.next/server/app/index.rsc +3 -3
- package/.next/server/app/index.segments/__PAGE__.segment.rsc +3 -3
- package/.next/server/app/index.segments/_full.segment.rsc +3 -3
- package/.next/server/app/index.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/login/page_client-reference-manifest.js +1 -1
- package/.next/server/app/login.html +1 -1
- package/.next/server/app/login.rsc +3 -3
- package/.next/server/app/login.segments/_full.segment.rsc +3 -3
- package/.next/server/app/login.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/login.segments/login/__PAGE__.segment.rsc +3 -3
- package/.next/server/app/machines/[machineId]/page_client-reference-manifest.js +1 -1
- package/.next/server/app/page_client-reference-manifest.js +1 -1
- package/.next/server/app/processes/page_client-reference-manifest.js +1 -1
- package/.next/server/app/processes.html +1 -1
- package/.next/server/app/processes.rsc +3 -3
- package/.next/server/app/processes.segments/_full.segment.rsc +3 -3
- package/.next/server/app/processes.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/processes.segments/processes/__PAGE__.segment.rsc +3 -3
- package/.next/server/app-paths-manifest.json +1 -1
- package/.next/server/chunks/161.js +1 -1
- package/.next/server/chunks/168.js +1 -1
- package/.next/server/middleware-build-manifest.js +1 -1
- package/.next/server/pages/404.html +1 -1
- package/.next/server/pages/500.html +1 -1
- package/.next/static/chunks/408-351fe4a1e9a2d556.js +1 -0
- package/.next/static/chunks/939-92930fde48061e20.js +1 -0
- package/.next/static/chunks/app/connections/page-7d18b901f2338f20.js +1 -0
- package/.next/static/chunks/app/history/page-adc25194b30c689d.js +3 -0
- package/.next/static/css/{f5d3162842dcac31.css → 2abef6d49c1c2970.css} +1 -1
- package/ARCHITECTURE.md +43 -15
- package/CHANGELOG.md +62 -0
- package/README.md +10 -3
- package/dist/cli.cjs +868 -511
- package/dist/collector.cjs +603 -120
- package/dist/hub.cjs +3429 -1451
- package/dist/ops.cjs +1178 -888
- package/dist/system.cjs +1322 -363
- package/docs/configuration.md +34 -1
- package/docs/installation.md +12 -6
- package/docs/monitoring.md +60 -17
- package/docs/operations.md +78 -22
- package/docs/system-installation.md +11 -4
- package/docs/telemetry-storage.md +258 -0
- package/package.json +3 -1
- package/packaging/collector.env.example +6 -0
- package/packaging/system/rigscope-collector.service.example +3 -0
- package/.next/static/chunks/408-9fdba9e4e956ef08.js +0 -1
- package/.next/static/chunks/939-514e554087419c71.js +0 -1
- package/.next/static/chunks/app/connections/page-75abd16fc495fcbd.js +0 -1
- package/.next/static/chunks/app/history/page-a25ba8fab7bd7cb9.js +0 -3
- /package/.next/static/{ip43TIgmaByEBn2GFtvfO → JbaLjudQSxrMUD_VHPrnD}/_buildManifest.js +0 -0
- /package/.next/static/{ip43TIgmaByEBn2GFtvfO → JbaLjudQSxrMUD_VHPrnD}/_ssgManifest.js +0 -0
package/dist/ops.cjs
CHANGED
|
@@ -35,6 +35,7 @@ __export(main_exports, {
|
|
|
35
35
|
});
|
|
36
36
|
module.exports = __toCommonJS(main_exports);
|
|
37
37
|
var import_node_util = require("node:util");
|
|
38
|
+
var import_node_path8 = require("node:path");
|
|
38
39
|
|
|
39
40
|
// src/operations/database.ts
|
|
40
41
|
var import_node_fs3 = require("node:fs");
|
|
@@ -50,8 +51,17 @@ var import_node_path = require("node:path");
|
|
|
50
51
|
var import_node_sqlite = require("node:sqlite");
|
|
51
52
|
var sqlite = __toESM(require("node:sqlite"));
|
|
52
53
|
|
|
54
|
+
// src/contracts/telemetry.ts
|
|
55
|
+
var MAX_CORES = 1024;
|
|
56
|
+
var MIN_EPOCH = Date.parse("2000-01-01T00:00:00Z");
|
|
57
|
+
var MAX_EPOCH = Date.parse("2200-01-01T00:00:00Z");
|
|
58
|
+
|
|
53
59
|
// src/hub/config.ts
|
|
54
60
|
var MAX_HUB_COLLECTORS_BYTES = 1024 * 1024;
|
|
61
|
+
var DEFAULT_MEMORY_BUDGET_BYTES = 128 * 1024 * 1024;
|
|
62
|
+
var DEFAULT_RECORDING_MAX_DURATION_MS = 60 * 6e4;
|
|
63
|
+
var DEFAULT_RECORDING_MAX_BYTES = 64 * 1024 * 1024;
|
|
64
|
+
var DEFAULT_RECORDING_BACKLOG_MS = 10 * 6e4;
|
|
55
65
|
|
|
56
66
|
// src/hub/auth/password.ts
|
|
57
67
|
var import_node_crypto = require("node:crypto");
|
|
@@ -418,8 +428,45 @@ var MIGRATIONS = [
|
|
|
418
428
|
},
|
|
419
429
|
{ version: 3, name: "admin", up: adminSchema },
|
|
420
430
|
{ version: 4, name: "alerts", up: installAlertSchema },
|
|
421
|
-
{ version: 5, name: "collector_registry", up: installRegistrySchema }
|
|
431
|
+
{ version: 5, name: "collector_registry", up: installRegistrySchema },
|
|
432
|
+
{ version: 6, name: "telemetry_modes_and_recordings", up: telemetryModesAndRecordings }
|
|
422
433
|
];
|
|
434
|
+
function telemetryModesAndRecordings(db) {
|
|
435
|
+
db.exec(`
|
|
436
|
+
CREATE TABLE IF NOT EXISTS points_10s (
|
|
437
|
+
series_id INTEGER NOT NULL,
|
|
438
|
+
bucket_ms INTEGER NOT NULL,
|
|
439
|
+
min REAL,
|
|
440
|
+
max REAL,
|
|
441
|
+
sum REAL NOT NULL DEFAULT 0,
|
|
442
|
+
count INTEGER NOT NULL DEFAULT 0,
|
|
443
|
+
null_count INTEGER NOT NULL DEFAULT 0,
|
|
444
|
+
PRIMARY KEY (series_id, bucket_ms)
|
|
445
|
+
) WITHOUT ROWID;
|
|
446
|
+
|
|
447
|
+
CREATE TABLE IF NOT EXISTS recordings (
|
|
448
|
+
id TEXT PRIMARY KEY,
|
|
449
|
+
created_by TEXT NOT NULL,
|
|
450
|
+
status TEXT NOT NULL,
|
|
451
|
+
machines_json TEXT NOT NULL,
|
|
452
|
+
metrics_json TEXT NOT NULL,
|
|
453
|
+
resolution_ms INTEGER NOT NULL,
|
|
454
|
+
created_at_ms INTEGER NOT NULL,
|
|
455
|
+
started_at_ms INTEGER NOT NULL,
|
|
456
|
+
stopped_at_ms INTEGER,
|
|
457
|
+
deadline_ms INTEGER NOT NULL,
|
|
458
|
+
max_bytes INTEGER NOT NULL,
|
|
459
|
+
bytes_written INTEGER NOT NULL DEFAULT 0,
|
|
460
|
+
file_name TEXT NOT NULL,
|
|
461
|
+
include_backlog INTEGER NOT NULL DEFAULT 0,
|
|
462
|
+
reason TEXT,
|
|
463
|
+
error TEXT
|
|
464
|
+
);
|
|
465
|
+
|
|
466
|
+
CREATE INDEX IF NOT EXISTS recordings_created
|
|
467
|
+
ON recordings (created_at_ms DESC);
|
|
468
|
+
`);
|
|
469
|
+
}
|
|
423
470
|
|
|
424
471
|
// src/hub/storage/db.ts
|
|
425
472
|
var BUSY_TIMEOUT_MS = 5e3;
|
|
@@ -496,7 +543,7 @@ var SqliteHubDatabase = class {
|
|
|
496
543
|
const walBudgetBytes = Math.max(64 * 1024, Math.min(HUB_WAL_RESERVE_BYTES, Math.floor(limitBytes / 16)));
|
|
497
544
|
const mainBudgetBytes = Math.max(pageSize, limitBytes - walBudgetBytes);
|
|
498
545
|
const requestedPages = Math.max(1, Math.floor(mainBudgetBytes / pageSize));
|
|
499
|
-
const checkpointPages = Math.max(16, Math.min(
|
|
546
|
+
const checkpointPages = Math.max(16, Math.min(16384, Math.floor(walBudgetBytes / pageSize)));
|
|
500
547
|
this.db.exec(`PRAGMA journal_size_limit = ${walBudgetBytes}`);
|
|
501
548
|
this.db.exec(`PRAGMA wal_autocheckpoint = ${checkpointPages}`);
|
|
502
549
|
let maxPageCount = requestedPages;
|
|
@@ -1035,7 +1082,9 @@ var COUNTED_TABLES = /* @__PURE__ */ new Set([
|
|
|
1035
1082
|
"machine_meta",
|
|
1036
1083
|
"machine_latest",
|
|
1037
1084
|
"inventory",
|
|
1085
|
+
"points_10s",
|
|
1038
1086
|
"gaps",
|
|
1087
|
+
"recordings",
|
|
1039
1088
|
"alert_rules",
|
|
1040
1089
|
"alert_incidents",
|
|
1041
1090
|
"alert_jobs",
|
|
@@ -1505,6 +1554,63 @@ async function migrateDatabase(options, now = options.now ?? Date.now) {
|
|
|
1505
1554
|
integrity: after.integrity
|
|
1506
1555
|
};
|
|
1507
1556
|
}
|
|
1557
|
+
function resetTelemetryHistory(options) {
|
|
1558
|
+
const database = resolvePath(options.database, "database");
|
|
1559
|
+
assertPrivateRegularFile(database, "database");
|
|
1560
|
+
if (!options.confirmStopped) {
|
|
1561
|
+
throw new OperationsError(
|
|
1562
|
+
"confirmation-required",
|
|
1563
|
+
"resetting telemetry history requires --confirm-stopped after the hub is stopped"
|
|
1564
|
+
);
|
|
1565
|
+
}
|
|
1566
|
+
assertNoActiveHubLock(database);
|
|
1567
|
+
const db = openHubDatabase(database);
|
|
1568
|
+
const tables = [
|
|
1569
|
+
"points_raw",
|
|
1570
|
+
"points_10s",
|
|
1571
|
+
"points_1m",
|
|
1572
|
+
"points_1h",
|
|
1573
|
+
"sample_receipts",
|
|
1574
|
+
"gaps",
|
|
1575
|
+
"machine_latest",
|
|
1576
|
+
"series"
|
|
1577
|
+
];
|
|
1578
|
+
const deleted = {};
|
|
1579
|
+
try {
|
|
1580
|
+
db.db.exec("BEGIN IMMEDIATE");
|
|
1581
|
+
try {
|
|
1582
|
+
for (const table of tables) {
|
|
1583
|
+
const result = db.db.prepare(`DELETE FROM ${table}`).run();
|
|
1584
|
+
deleted[table] = Number(result.changes);
|
|
1585
|
+
}
|
|
1586
|
+
db.db.exec("COMMIT");
|
|
1587
|
+
} catch (error) {
|
|
1588
|
+
try {
|
|
1589
|
+
db.db.exec("ROLLBACK");
|
|
1590
|
+
} catch {
|
|
1591
|
+
}
|
|
1592
|
+
throw error;
|
|
1593
|
+
}
|
|
1594
|
+
db.reclaimPages(1024);
|
|
1595
|
+
db.checkpoint();
|
|
1596
|
+
} finally {
|
|
1597
|
+
db.close();
|
|
1598
|
+
}
|
|
1599
|
+
const info = inspectDatabase(database);
|
|
1600
|
+
return {
|
|
1601
|
+
database,
|
|
1602
|
+
deleted,
|
|
1603
|
+
integrity: info.integrity,
|
|
1604
|
+
preserved: [
|
|
1605
|
+
"users",
|
|
1606
|
+
"collector registry/grants/audit",
|
|
1607
|
+
"alert rules/incidents/jobs/webhook",
|
|
1608
|
+
"machine metadata",
|
|
1609
|
+
"inventory",
|
|
1610
|
+
"recordings"
|
|
1611
|
+
]
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1508
1614
|
function schemaVersionOf(database) {
|
|
1509
1615
|
if (!(0, import_node_fs3.existsSync)(database)) return null;
|
|
1510
1616
|
const info = inspectDatabase(database);
|
|
@@ -3051,27 +3157,29 @@ var HOUR_MS = 36e5;
|
|
|
3051
3157
|
var DAY_MS = 24 * HOUR_MS;
|
|
3052
3158
|
var DEFAULT_RETENTION_POLICY = {
|
|
3053
3159
|
rawMs: 2 * DAY_MS,
|
|
3160
|
+
tenSecondMs: 2 * HOUR_MS,
|
|
3054
3161
|
oneMinuteMs: 30 * DAY_MS,
|
|
3055
3162
|
oneHourMs: 400 * DAY_MS,
|
|
3056
|
-
receiptsMs:
|
|
3163
|
+
receiptsMs: 7 * DAY_MS,
|
|
3057
3164
|
gapsMs: 400 * DAY_MS
|
|
3058
3165
|
};
|
|
3059
3166
|
function resolveRetentionPolicy(partial) {
|
|
3060
3167
|
const rawMs = checkedMs(partial?.rawMs, DEFAULT_RETENTION_POLICY.rawMs, "rawMs");
|
|
3168
|
+
const tenSecondMs = checkedMs(partial?.tenSecondMs, DEFAULT_RETENTION_POLICY.tenSecondMs, "tenSecondMs");
|
|
3061
3169
|
const oneMinuteMs = checkedMs(partial?.oneMinuteMs, DEFAULT_RETENTION_POLICY.oneMinuteMs, "oneMinuteMs");
|
|
3062
3170
|
const oneHourMs = checkedMs(partial?.oneHourMs, DEFAULT_RETENTION_POLICY.oneHourMs, "oneHourMs");
|
|
3063
|
-
const receiptsDefault = partial?.rawMs !== void 0 ? rawMs : DEFAULT_RETENTION_POLICY.receiptsMs;
|
|
3064
|
-
const gapsDefault = partial?.oneHourMs !== void 0 ? oneHourMs : DEFAULT_RETENTION_POLICY.gapsMs;
|
|
3065
3171
|
return {
|
|
3066
3172
|
rawMs,
|
|
3173
|
+
tenSecondMs,
|
|
3067
3174
|
oneMinuteMs,
|
|
3068
3175
|
oneHourMs,
|
|
3069
|
-
receiptsMs: checkedMs(partial?.receiptsMs,
|
|
3070
|
-
gapsMs: checkedMs(partial?.gapsMs,
|
|
3176
|
+
receiptsMs: checkedMs(partial?.receiptsMs, DEFAULT_RETENTION_POLICY.receiptsMs, "receiptsMs"),
|
|
3177
|
+
gapsMs: checkedMs(partial?.gapsMs, DEFAULT_RETENTION_POLICY.gapsMs, "gapsMs")
|
|
3071
3178
|
};
|
|
3072
3179
|
}
|
|
3073
3180
|
var RETENTION_DELETE_SQL = {
|
|
3074
3181
|
raw: "DELETE FROM points_raw WHERE (series_id, ts_ms) IN (SELECT series_id, ts_ms FROM points_raw WHERE series_id = ? AND ts_ms < ? LIMIT ?)",
|
|
3182
|
+
tenSecond: "DELETE FROM points_10s WHERE (series_id, bucket_ms) IN (SELECT series_id, bucket_ms FROM points_10s WHERE series_id = ? AND bucket_ms < ? LIMIT ?)",
|
|
3075
3183
|
oneMinute: "DELETE FROM points_1m WHERE (series_id, bucket_ms) IN (SELECT series_id, bucket_ms FROM points_1m WHERE series_id = ? AND bucket_ms < ? LIMIT ?)",
|
|
3076
3184
|
oneHour: "DELETE FROM points_1h WHERE (series_id, bucket_ms) IN (SELECT series_id, bucket_ms FROM points_1h WHERE series_id = ? AND bucket_ms < ? LIMIT ?)"
|
|
3077
3185
|
};
|
|
@@ -3098,6 +3206,7 @@ var RetentionManager = class {
|
|
|
3098
3206
|
walBytes: 0
|
|
3099
3207
|
};
|
|
3100
3208
|
deleteRaw;
|
|
3209
|
+
delete10s;
|
|
3101
3210
|
delete1m;
|
|
3102
3211
|
delete1h;
|
|
3103
3212
|
deleteReceipts;
|
|
@@ -3126,6 +3235,7 @@ var RetentionManager = class {
|
|
|
3126
3235
|
this.chunkSize = positiveInt(options.chunkSize, 5e3, "chunkSize");
|
|
3127
3236
|
this.maxChunksPerPass = positiveInt(options.maxChunksPerPass, 1e4, "maxChunksPerPass");
|
|
3128
3237
|
this.deleteRaw = database.db.prepare(RETENTION_DELETE_SQL.raw);
|
|
3238
|
+
this.delete10s = database.db.prepare(RETENTION_DELETE_SQL.tenSecond);
|
|
3129
3239
|
this.delete1m = database.db.prepare(RETENTION_DELETE_SQL.oneMinute);
|
|
3130
3240
|
this.delete1h = database.db.prepare(RETENTION_DELETE_SQL.oneHour);
|
|
3131
3241
|
this.deleteReceipts = database.db.prepare(
|
|
@@ -3161,7 +3271,7 @@ var RetentionManager = class {
|
|
|
3161
3271
|
}
|
|
3162
3272
|
/**
|
|
3163
3273
|
* Synchronous, bounded pressure relief for the write path. Deletes only
|
|
3164
|
-
* oldest raw/one-minute rows outside the most-recent-hour floor, then asks
|
|
3274
|
+
* oldest raw/ten-second/one-minute rows outside the most-recent-hour floor, then asks
|
|
3165
3275
|
* SQLite for bounded incremental reclaim. It never deletes one-hour rollups
|
|
3166
3276
|
* to pretend a hard cap fits.
|
|
3167
3277
|
*/
|
|
@@ -3191,6 +3301,9 @@ var RetentionManager = class {
|
|
|
3191
3301
|
const rawCutoff = nowMs - this.policy.rawMs;
|
|
3192
3302
|
const seriesIds = this.seriesIds();
|
|
3193
3303
|
let deleted = this.deleteTierChunk(this.deleteRaw, seriesIds, rawCutoff);
|
|
3304
|
+
if (deleted === 0) {
|
|
3305
|
+
deleted = this.deleteTierChunk(this.delete10s, seriesIds, nowMs - this.policy.tenSecondMs);
|
|
3306
|
+
}
|
|
3194
3307
|
if (deleted === 0) {
|
|
3195
3308
|
deleted = this.deleteTierChunk(this.delete1m, seriesIds, nowMs - this.policy.oneMinuteMs);
|
|
3196
3309
|
}
|
|
@@ -3204,6 +3317,7 @@ var RetentionManager = class {
|
|
|
3204
3317
|
const floorCutoff = nowMs - HOUR_MS;
|
|
3205
3318
|
effectiveRawCutoff = Math.max(rawCutoff, floorCutoff);
|
|
3206
3319
|
deleted = this.deleteTierChunk(this.deleteRaw, seriesIds, effectiveRawCutoff);
|
|
3320
|
+
if (deleted === 0) deleted = this.deleteTierChunk(this.delete10s, seriesIds, floorCutoff);
|
|
3207
3321
|
if (deleted === 0) deleted = this.deleteTierChunk(this.delete1m, seriesIds, floorCutoff);
|
|
3208
3322
|
if (deleted > 0) {
|
|
3209
3323
|
this.db.reclaimPages(64);
|
|
@@ -3291,7 +3405,7 @@ var RetentionManager = class {
|
|
|
3291
3405
|
}
|
|
3292
3406
|
};
|
|
3293
3407
|
function yieldToEventLoop() {
|
|
3294
|
-
return new Promise((
|
|
3408
|
+
return new Promise((resolve7) => setImmediate(resolve7));
|
|
3295
3409
|
}
|
|
3296
3410
|
function checkedMs(value, fallback, name) {
|
|
3297
3411
|
const resolved = value ?? fallback;
|
|
@@ -3554,958 +3668,1097 @@ function seriesKey(machineId, metric, resourceId) {
|
|
|
3554
3668
|
return `${machineId}\0${metric}\0${resourceId}`;
|
|
3555
3669
|
}
|
|
3556
3670
|
|
|
3557
|
-
// src/
|
|
3671
|
+
// src/contracts/index.ts
|
|
3672
|
+
var MIN_TIMESTAMP_EPOCH = Date.parse("2000-01-01T00:00:00Z");
|
|
3673
|
+
var MAX_TIMESTAMP_EPOCH = Date.parse("2200-01-01T00:00:00Z");
|
|
3674
|
+
|
|
3675
|
+
// src/hub/storage/history-query.ts
|
|
3558
3676
|
var DAY_MS2 = 864e5;
|
|
3559
|
-
var
|
|
3560
|
-
var
|
|
3561
|
-
var
|
|
3562
|
-
var
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
queue = [];
|
|
3572
|
-
timer = null;
|
|
3573
|
-
driver = null;
|
|
3574
|
-
closed = false;
|
|
3575
|
-
committedRecords = 0;
|
|
3576
|
-
duplicateRecords = 0;
|
|
3577
|
-
rejectedRecords = 0;
|
|
3578
|
-
writeFailures = 0;
|
|
3579
|
-
diskFullFailures = 0;
|
|
3580
|
-
lastWriteError = null;
|
|
3581
|
-
lastFlushMs = null;
|
|
3582
|
-
lastFlushDurationMs = null;
|
|
3583
|
-
receiptInsert;
|
|
3584
|
-
rawInsert;
|
|
3585
|
-
upsert1m;
|
|
3586
|
-
upsert1h;
|
|
3587
|
-
gapInsert;
|
|
3588
|
-
latestSelect;
|
|
3589
|
-
latestUpsert;
|
|
3590
|
-
inventoryUpsert;
|
|
3591
|
-
inventorySelect;
|
|
3592
|
-
metaUpsertBoth;
|
|
3593
|
-
metaUpsertDisplay;
|
|
3594
|
-
metaUpsertTags;
|
|
3595
|
-
metaTouch;
|
|
3596
|
-
metaSelect;
|
|
3597
|
-
metaSelectAll;
|
|
3598
|
-
latestSelectAll;
|
|
3599
|
-
constructor(options) {
|
|
3600
|
-
this.database = options.database;
|
|
3601
|
-
this.now = options.now ?? (() => Date.now());
|
|
3602
|
-
this.maxQueuedRecords = positiveInt2(options.maxQueuedRecords, 4096, "maxQueuedRecords");
|
|
3603
|
-
this.flushIntervalMs = nonnegativeInt(options.flushIntervalMs, 20, "flushIntervalMs");
|
|
3604
|
-
this.maxRecordsPerTransaction = positiveInt2(
|
|
3605
|
-
options.maxRecordsPerTransaction,
|
|
3606
|
-
512,
|
|
3607
|
-
"maxRecordsPerTransaction"
|
|
3608
|
-
);
|
|
3609
|
-
this.maxPhysicalBytes = resolvePhysicalLimit(options);
|
|
3610
|
-
const db = options.database.db;
|
|
3611
|
-
this.series = new SeriesCatalog(db);
|
|
3612
|
-
this.receiptInsert = db.prepare(
|
|
3613
|
-
"INSERT OR IGNORE INTO sample_receipts (collector_id, run_id, seq, machine_id, kind, mode, captured_at_ms, received_at_ms, expected_interval_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
3614
|
-
);
|
|
3615
|
-
this.rawInsert = db.prepare(
|
|
3616
|
-
"INSERT OR IGNORE INTO points_raw (series_id, ts_ms, value) VALUES (?, ?, ?)"
|
|
3617
|
-
);
|
|
3618
|
-
this.upsert1m = prepareRollupUpsert(db, "points_1m");
|
|
3619
|
-
this.upsert1h = prepareRollupUpsert(db, "points_1h");
|
|
3620
|
-
this.gapInsert = db.prepare(
|
|
3621
|
-
"INSERT OR IGNORE INTO gaps (collector_id, run_id, seq, machine_id, from_ms, to_ms, dropped, reason, received_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
3622
|
-
);
|
|
3623
|
-
this.latestSelect = db.prepare(
|
|
3624
|
-
"SELECT run_id, seq, captured_at_ms FROM machine_latest WHERE machine_id = ?"
|
|
3625
|
-
);
|
|
3626
|
-
this.latestUpsert = db.prepare(
|
|
3627
|
-
"INSERT INTO machine_latest (machine_id, collector_id, run_id, boot_id, seq, captured_at_ms, received_at_ms, expected_interval_ms, hostname, payload_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET collector_id = excluded.collector_id, run_id = excluded.run_id, boot_id = excluded.boot_id, seq = excluded.seq, captured_at_ms = excluded.captured_at_ms, received_at_ms = excluded.received_at_ms, expected_interval_ms = COALESCE(excluded.expected_interval_ms, expected_interval_ms), hostname = COALESCE(excluded.hostname, hostname), payload_json = excluded.payload_json"
|
|
3628
|
-
);
|
|
3629
|
-
this.latestSelectAll = db.prepare(
|
|
3630
|
-
"SELECT machine_id, collector_id, run_id, boot_id, seq, captured_at_ms, received_at_ms, expected_interval_ms, hostname, payload_json FROM machine_latest ORDER BY machine_id"
|
|
3631
|
-
);
|
|
3632
|
-
this.inventoryUpsert = db.prepare(
|
|
3633
|
-
"INSERT INTO inventory (machine_id, captured_at_ms, received_at_ms, inventory_json) VALUES (?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET captured_at_ms = excluded.captured_at_ms, received_at_ms = excluded.received_at_ms, inventory_json = excluded.inventory_json"
|
|
3634
|
-
);
|
|
3635
|
-
this.inventorySelect = db.prepare(
|
|
3636
|
-
"SELECT captured_at_ms, received_at_ms, inventory_json FROM inventory WHERE machine_id = ?"
|
|
3637
|
-
);
|
|
3638
|
-
this.metaUpsertBoth = db.prepare(
|
|
3639
|
-
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET display_name = excluded.display_name, tags_json = excluded.tags_json, updated_at_ms = excluded.updated_at_ms"
|
|
3640
|
-
);
|
|
3641
|
-
this.metaUpsertDisplay = db.prepare(
|
|
3642
|
-
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, ?, '[]', ?) ON CONFLICT(machine_id) DO UPDATE SET display_name = excluded.display_name, updated_at_ms = excluded.updated_at_ms"
|
|
3643
|
-
);
|
|
3644
|
-
this.metaUpsertTags = db.prepare(
|
|
3645
|
-
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, NULL, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET tags_json = excluded.tags_json, updated_at_ms = excluded.updated_at_ms"
|
|
3646
|
-
);
|
|
3647
|
-
this.metaTouch = db.prepare(
|
|
3648
|
-
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, NULL, '[]', ?) ON CONFLICT(machine_id) DO UPDATE SET updated_at_ms = excluded.updated_at_ms"
|
|
3649
|
-
);
|
|
3650
|
-
this.metaSelect = db.prepare(
|
|
3651
|
-
"SELECT display_name, tags_json, updated_at_ms FROM machine_meta WHERE machine_id = ?"
|
|
3652
|
-
);
|
|
3653
|
-
this.metaSelectAll = db.prepare(
|
|
3654
|
-
"SELECT machine_id, display_name, tags_json FROM machine_meta ORDER BY machine_id"
|
|
3655
|
-
);
|
|
3656
|
-
this.retention = new RetentionManager(options.database, {
|
|
3657
|
-
policy: options.retention,
|
|
3658
|
-
maxDatabaseBytes: options.maxDatabaseBytes ?? null,
|
|
3659
|
-
maxPhysicalBytes: this.maxPhysicalBytes,
|
|
3660
|
-
now: this.now
|
|
3661
|
-
});
|
|
3662
|
-
this.retention.start();
|
|
3663
|
-
}
|
|
3664
|
-
/**
|
|
3665
|
-
* Queues one batch. The promise resolves only after the transaction that
|
|
3666
|
-
* writes it has committed, or immediately as `busy` when the bounded queue
|
|
3667
|
-
* cannot hold it (no database work is done in that case).
|
|
3668
|
-
*/
|
|
3669
|
-
enqueue(batch) {
|
|
3670
|
-
return new Promise((resolve6) => {
|
|
3671
|
-
if (this.closed) {
|
|
3672
|
-
resolve6({ status: "busy", retryAfterMs: FAILURE_RETRY_MS, reason: "storage is closed" });
|
|
3673
|
-
return;
|
|
3674
|
-
}
|
|
3675
|
-
validateBatch(batch);
|
|
3676
|
-
const queued = this.queuedRecords();
|
|
3677
|
-
if (queued + batch.records.length > this.maxQueuedRecords) {
|
|
3678
|
-
const ratio = this.maxQueuedRecords === 0 ? 1 : queued / this.maxQueuedRecords;
|
|
3679
|
-
const retryAfterMs = Math.min(5e3, Math.max(1e3, Math.round(1e3 + ratio * 4e3)));
|
|
3680
|
-
resolve6({ status: "busy", retryAfterMs, reason: "ingest queue is full" });
|
|
3681
|
-
return;
|
|
3682
|
-
}
|
|
3683
|
-
this.queue.push({
|
|
3684
|
-
batch,
|
|
3685
|
-
receivedAtMs: this.now(),
|
|
3686
|
-
resolve: resolve6,
|
|
3687
|
-
settled: false,
|
|
3688
|
-
accepted: [],
|
|
3689
|
-
rejected: [],
|
|
3690
|
-
newLive: null
|
|
3691
|
-
});
|
|
3692
|
-
this.scheduleFlush(this.flushIntervalMs);
|
|
3693
|
-
});
|
|
3694
|
-
}
|
|
3695
|
-
/** Persisted latest LIVE metrics per machine, for restart recovery. */
|
|
3696
|
-
latestLive() {
|
|
3697
|
-
const rows = this.latestSelectAll.all();
|
|
3698
|
-
return rows.map((row) => ({
|
|
3699
|
-
machineId: row.machine_id,
|
|
3700
|
-
collectorId: row.collector_id,
|
|
3701
|
-
runId: row.run_id,
|
|
3702
|
-
bootId: row.boot_id,
|
|
3703
|
-
sequence: Number(row.seq),
|
|
3704
|
-
capturedAt: new Date(Number(row.captured_at_ms)).toISOString(),
|
|
3705
|
-
receivedAt: new Date(Number(row.received_at_ms)).toISOString(),
|
|
3706
|
-
hostname: row.hostname,
|
|
3707
|
-
expectedIntervalMs: row.expected_interval_ms === null ? null : Number(row.expected_interval_ms),
|
|
3708
|
-
payload: JSON.parse(row.payload_json)
|
|
3709
|
-
}));
|
|
3677
|
+
var MAX_RANGE_MS = 400 * DAY_MS2;
|
|
3678
|
+
var MAX_METRICS = Object.keys(METRICS).length;
|
|
3679
|
+
var MAX_RESOURCE_IDS = 64;
|
|
3680
|
+
var DEFAULT_MAX_POINTS = 600;
|
|
3681
|
+
var MAX_POINTS = 2e3;
|
|
3682
|
+
var MAX_GAPS = 500;
|
|
3683
|
+
var TIER_RESOLUTION_MS = { raw: 100, "10s": 1e4, "1m": 6e4, "1h": 36e5 };
|
|
3684
|
+
var HistoryQueryError = class extends Error {
|
|
3685
|
+
code = "invalid-query";
|
|
3686
|
+
constructor(message) {
|
|
3687
|
+
super(message);
|
|
3688
|
+
this.name = "HistoryQueryError";
|
|
3710
3689
|
}
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3690
|
+
};
|
|
3691
|
+
var DEFAULT_TIERS = ["raw", "10s", "1m", "1h"];
|
|
3692
|
+
function queryHistory(db, q, budget = {}) {
|
|
3693
|
+
const started = performance.now();
|
|
3694
|
+
const maxPoints = validatedMaxPoints(q.maxPoints);
|
|
3695
|
+
const maxSeries = validatedBudget(budget.maxSeries, 64, "maxSeries");
|
|
3696
|
+
const maxRowsScanned = validatedBudget(budget.maxRowsScanned, 25e4, "maxRowsScanned");
|
|
3697
|
+
const maxElapsedMs = validatedBudget(budget.maxElapsedMs, 5e3, "maxElapsedMs");
|
|
3698
|
+
const retention = resolveRetentionPolicy(budget.retention);
|
|
3699
|
+
validatedQuery(q);
|
|
3700
|
+
const rangeMs = q.toMs - q.fromMs;
|
|
3701
|
+
const nowMs = budget.nowMs ?? Date.now();
|
|
3702
|
+
let allSeries = loadSeries(db, q);
|
|
3703
|
+
let truncated = false;
|
|
3704
|
+
if (allSeries.length > maxSeries) {
|
|
3705
|
+
allSeries = allSeries.slice(0, maxSeries);
|
|
3706
|
+
truncated = true;
|
|
3715
3707
|
}
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3708
|
+
const chosen = chooseTier(
|
|
3709
|
+
db,
|
|
3710
|
+
allSeries.map((row) => Number(row.series_id)),
|
|
3711
|
+
q.fromMs,
|
|
3712
|
+
q.toMs,
|
|
3713
|
+
nowMs,
|
|
3714
|
+
retention,
|
|
3715
|
+
budget.tiers,
|
|
3716
|
+
maxRowsScanned,
|
|
3717
|
+
maxElapsedMs,
|
|
3718
|
+
started
|
|
3719
|
+
);
|
|
3720
|
+
if (chosen.timeExceeded || chosen.overBudget) truncated = true;
|
|
3721
|
+
let seriesRows = allSeries;
|
|
3722
|
+
if (chosen.cutSeries < seriesRows.length) {
|
|
3723
|
+
seriesRows = seriesRows.slice(0, Math.max(0, chosen.cutSeries));
|
|
3724
|
+
truncated = true;
|
|
3724
3725
|
}
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
const hasTags = meta.tags !== void 0;
|
|
3733
|
-
const displayName = meta.displayName ?? null;
|
|
3734
|
-
const tags = hasTags ? validateTags(meta.tags) : [];
|
|
3735
|
-
if (hasDisplay && hasTags) {
|
|
3736
|
-
this.metaUpsertBoth.run(machineId, displayName, JSON.stringify(tags), updatedAt);
|
|
3737
|
-
} else if (hasDisplay) {
|
|
3738
|
-
this.metaUpsertDisplay.run(machineId, displayName, updatedAt);
|
|
3739
|
-
} else if (hasTags) {
|
|
3740
|
-
this.metaUpsertTags.run(machineId, JSON.stringify(tags), updatedAt);
|
|
3741
|
-
} else {
|
|
3742
|
-
this.metaTouch.run(machineId, updatedAt);
|
|
3726
|
+
const stepMs = Math.max(1, Math.ceil(rangeMs / maxPoints / chosen.resolutionMs)) * chosen.resolutionMs;
|
|
3727
|
+
const originMs = Math.floor(q.fromMs / stepMs) * stepMs;
|
|
3728
|
+
const series = [];
|
|
3729
|
+
for (const row of seriesRows) {
|
|
3730
|
+
if (performance.now() - started > maxElapsedMs) {
|
|
3731
|
+
truncated = true;
|
|
3732
|
+
break;
|
|
3743
3733
|
}
|
|
3734
|
+
const metric = row.metric;
|
|
3735
|
+
const definition = METRICS[metric];
|
|
3736
|
+
if (!definition) continue;
|
|
3737
|
+
const points = readSeries(
|
|
3738
|
+
db,
|
|
3739
|
+
chosen.tier,
|
|
3740
|
+
Number(row.series_id),
|
|
3741
|
+
fromBucketMs(chosen.tier, q.fromMs),
|
|
3742
|
+
q.toMs,
|
|
3743
|
+
originMs,
|
|
3744
|
+
stepMs,
|
|
3745
|
+
// An additional per-series guard. Tier selection already keeps the total
|
|
3746
|
+
// under budget; this makes a single query bounded even if a custom
|
|
3747
|
+
// retention policy forces the coarse fallback.
|
|
3748
|
+
maxRowsScanned
|
|
3749
|
+
);
|
|
3750
|
+
if (points.length === 0) continue;
|
|
3751
|
+
series.push({ metric: row.metric, resourceId: row.resource_id, unit: definition.unit, points });
|
|
3744
3752
|
}
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3753
|
+
const gaps = readGaps(db, q.machineId, q.fromMs, q.toMs);
|
|
3754
|
+
return {
|
|
3755
|
+
machineId: q.machineId,
|
|
3756
|
+
fromMs: q.fromMs,
|
|
3757
|
+
toMs: q.toMs,
|
|
3758
|
+
stepMs,
|
|
3759
|
+
tier: chosen.tier,
|
|
3760
|
+
series,
|
|
3761
|
+
gaps,
|
|
3762
|
+
truncated,
|
|
3763
|
+
elapsedMs: Math.max(0, Math.round(performance.now() - started))
|
|
3764
|
+
};
|
|
3765
|
+
}
|
|
3766
|
+
function loadSeries(db, q) {
|
|
3767
|
+
const placeholders = q.metrics.map(() => "?").join(", ");
|
|
3768
|
+
const params = [q.machineId, ...q.metrics];
|
|
3769
|
+
let sql = `SELECT series_id, metric, resource_id FROM series WHERE machine_id = ? AND metric IN (${placeholders})`;
|
|
3770
|
+
if (q.resourceIds && q.resourceIds.length > 0) {
|
|
3771
|
+
sql += ` AND resource_id IN (${q.resourceIds.map(() => "?").join(", ")})`;
|
|
3772
|
+
params.push(...q.resourceIds);
|
|
3749
3773
|
}
|
|
3750
|
-
|
|
3751
|
-
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
displayName: row.display_name,
|
|
3756
|
-
tags: parseTags(row.tags_json)
|
|
3757
|
-
});
|
|
3774
|
+
if (q.coreIds !== void 0) {
|
|
3775
|
+
if (q.coreIds.length === 0) sql += " AND metric <> 'cpu.core'";
|
|
3776
|
+
else {
|
|
3777
|
+
sql += ` AND (metric <> 'cpu.core' OR resource_id IN (${q.coreIds.map(() => "?").join(", ")}))`;
|
|
3778
|
+
params.push(...q.coreIds);
|
|
3758
3779
|
}
|
|
3759
|
-
return result;
|
|
3760
|
-
}
|
|
3761
|
-
stats() {
|
|
3762
|
-
return {
|
|
3763
|
-
queuedRecords: this.queuedRecords(),
|
|
3764
|
-
queuedBatches: this.queue.length,
|
|
3765
|
-
lastFlushMs: this.lastFlushMs,
|
|
3766
|
-
lastFlushDurationMs: this.lastFlushDurationMs,
|
|
3767
|
-
committedRecords: this.committedRecords,
|
|
3768
|
-
duplicateRecords: this.duplicateRecords,
|
|
3769
|
-
rejectedRecords: this.rejectedRecords,
|
|
3770
|
-
writeFailures: this.writeFailures,
|
|
3771
|
-
lastWriteError: this.lastWriteError,
|
|
3772
|
-
diskFullFailures: this.diskFullFailures,
|
|
3773
|
-
dbBytes: this.database.sizeBytes(),
|
|
3774
|
-
dbWalBytes: this.database.walBytes(),
|
|
3775
|
-
dbBytesLimit: this.maxPhysicalBytes,
|
|
3776
|
-
retention: this.retention.status()
|
|
3777
|
-
};
|
|
3778
3780
|
}
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3782
|
-
|
|
3783
|
-
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3781
|
+
sql += " ORDER BY metric, resource_id";
|
|
3782
|
+
return db.db.prepare(sql).all(...params);
|
|
3783
|
+
}
|
|
3784
|
+
var TIER_CANDIDATES = [
|
|
3785
|
+
{ tier: "raw", resolutionMs: TIER_RESOLUTION_MS.raw, retentionMs: 0 },
|
|
3786
|
+
{ tier: "10s", resolutionMs: TIER_RESOLUTION_MS["10s"], retentionMs: 0 },
|
|
3787
|
+
{ tier: "1m", resolutionMs: TIER_RESOLUTION_MS["1m"], retentionMs: 0 },
|
|
3788
|
+
{ tier: "1h", resolutionMs: TIER_RESOLUTION_MS["1h"], retentionMs: 0 }
|
|
3789
|
+
];
|
|
3790
|
+
function chooseTier(db, seriesIds, fromMs, toMs, nowMs, retention, tiers, maxRowsScanned, maxElapsedMs, started) {
|
|
3791
|
+
const allowed = new Set(
|
|
3792
|
+
(tiers ?? DEFAULT_TIERS).filter((tier) => tier !== "1s")
|
|
3793
|
+
);
|
|
3794
|
+
const candidates = TIER_CANDIDATES.filter(
|
|
3795
|
+
(candidate) => allowed.has(candidate.tier)
|
|
3796
|
+
).map((candidate) => ({
|
|
3797
|
+
...candidate,
|
|
3798
|
+
retentionMs: candidate.tier === "raw" ? retention.rawMs : candidate.tier === "10s" ? retention.tenSecondMs : candidate.tier === "1m" ? retention.oneMinuteMs : retention.oneHourMs
|
|
3799
|
+
}));
|
|
3800
|
+
if (candidates.length === 0) {
|
|
3801
|
+
throw new HistoryQueryError("no history storage tiers are available");
|
|
3792
3802
|
}
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
this.retention.stop();
|
|
3801
|
-
if (this.timer !== null) {
|
|
3802
|
-
clearTimeout(this.timer);
|
|
3803
|
-
this.timer = null;
|
|
3803
|
+
const available = candidates.filter((candidate) => fromMs >= nowMs - candidate.retentionMs);
|
|
3804
|
+
const pool = available.length > 0 ? available : [candidates[candidates.length - 1]];
|
|
3805
|
+
let timeExceeded = false;
|
|
3806
|
+
for (const candidate of pool) {
|
|
3807
|
+
if (performance.now() - started > maxElapsedMs) {
|
|
3808
|
+
timeExceeded = true;
|
|
3809
|
+
break;
|
|
3804
3810
|
}
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3811
|
+
const rows = countRowsBounded(
|
|
3812
|
+
db,
|
|
3813
|
+
candidate.tier,
|
|
3814
|
+
seriesIds,
|
|
3815
|
+
fromBucketMs(candidate.tier, fromMs),
|
|
3816
|
+
toMs,
|
|
3817
|
+
maxRowsScanned
|
|
3818
|
+
);
|
|
3819
|
+
if (rows <= maxRowsScanned) {
|
|
3820
|
+
if (rows === 0 && candidate !== pool[pool.length - 1]) continue;
|
|
3821
|
+
return {
|
|
3822
|
+
tier: candidate.tier,
|
|
3823
|
+
resolutionMs: candidate.resolutionMs,
|
|
3824
|
+
cutSeries: seriesIds.length,
|
|
3825
|
+
timeExceeded,
|
|
3826
|
+
overBudget: false
|
|
3827
|
+
};
|
|
3810
3828
|
}
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
if (!ok) break;
|
|
3815
|
-
if (this.queue.length > 0) await yieldToEventLoop2();
|
|
3829
|
+
if (performance.now() - started > maxElapsedMs) {
|
|
3830
|
+
timeExceeded = true;
|
|
3831
|
+
break;
|
|
3816
3832
|
}
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3833
|
+
}
|
|
3834
|
+
const fallback = pool[pool.length - 1];
|
|
3835
|
+
let cut = seriesIds.length;
|
|
3836
|
+
let fits = seriesIds.length === 0;
|
|
3837
|
+
if (timeExceeded) cut = Math.max(1, Math.floor(cut / 2));
|
|
3838
|
+
while (!fits && cut >= 1) {
|
|
3839
|
+
if (performance.now() - started > maxElapsedMs) {
|
|
3840
|
+
timeExceeded = true;
|
|
3841
|
+
break;
|
|
3820
3842
|
}
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3843
|
+
const rows = countRowsBounded(
|
|
3844
|
+
db,
|
|
3845
|
+
fallback.tier,
|
|
3846
|
+
seriesIds.slice(0, cut),
|
|
3847
|
+
fromBucketMs(fallback.tier, fromMs),
|
|
3848
|
+
toMs,
|
|
3849
|
+
maxRowsScanned
|
|
3850
|
+
);
|
|
3851
|
+
if (rows <= maxRowsScanned) {
|
|
3852
|
+
fits = true;
|
|
3853
|
+
break;
|
|
3824
3854
|
}
|
|
3855
|
+
if (cut === 1) break;
|
|
3856
|
+
cut = Math.max(1, Math.floor(cut / 2));
|
|
3825
3857
|
}
|
|
3826
|
-
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
return promise;
|
|
3849
|
-
}
|
|
3850
|
-
async runDriver() {
|
|
3851
|
-
while (!this.closed && this.queue.length > 0) {
|
|
3852
|
-
const ok = this.flushTransaction();
|
|
3853
|
-
if (!ok) return;
|
|
3854
|
-
if (this.queue.length > 0) await yieldToEventLoop2();
|
|
3855
|
-
}
|
|
3858
|
+
return {
|
|
3859
|
+
tier: fallback.tier,
|
|
3860
|
+
resolutionMs: fallback.resolutionMs,
|
|
3861
|
+
cutSeries: Math.max(1, Math.min(cut, seriesIds.length)),
|
|
3862
|
+
timeExceeded,
|
|
3863
|
+
overBudget: !fits
|
|
3864
|
+
};
|
|
3865
|
+
}
|
|
3866
|
+
function countRowsBounded(db, tier, seriesIds, fromMs, toMs, budget) {
|
|
3867
|
+
if (seriesIds.length === 0) return 0;
|
|
3868
|
+
const column = tier === "raw" ? "ts_ms" : "bucket_ms";
|
|
3869
|
+
const table = tableForTier(tier);
|
|
3870
|
+
const statement = db.db.prepare(
|
|
3871
|
+
`SELECT COUNT(*) AS n FROM (SELECT 1 FROM ${table} WHERE series_id = ? AND ${column} >= ? AND ${column} <= ? LIMIT ?)`
|
|
3872
|
+
);
|
|
3873
|
+
let total = 0;
|
|
3874
|
+
for (const seriesId of seriesIds) {
|
|
3875
|
+
const remaining = budget + 1 - total;
|
|
3876
|
+
if (remaining <= 0) return total;
|
|
3877
|
+
const row = statement.get(seriesId, fromMs, toMs, remaining);
|
|
3878
|
+
total += Number(row?.n ?? 0);
|
|
3879
|
+
if (total > budget) return total;
|
|
3856
3880
|
}
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
}
|
|
3883
|
-
const stillFull = physical !== null && physical.pageCount >= physical.maxPageCount && physical.freePages === 0;
|
|
3884
|
-
if (bytes >= this.maxPhysicalBytes || stillFull) {
|
|
3885
|
-
this.diskFullFailures += 1;
|
|
3886
|
-
this.lastWriteError = "storage disk budget reached";
|
|
3887
|
-
for (const pending of selected) {
|
|
3888
|
-
this.resolveBusy(pending, FAILURE_RETRY_MS, "storage disk budget reached");
|
|
3889
|
-
}
|
|
3890
|
-
return false;
|
|
3891
|
-
}
|
|
3892
|
-
} catch (error) {
|
|
3893
|
-
const full = isStorageFullError(error);
|
|
3894
|
-
if (full) this.diskFullFailures += 1;
|
|
3895
|
-
this.lastWriteError = full ? "storage disk budget reached" : sanitizeWriteError(error);
|
|
3896
|
-
for (const pending of selected) {
|
|
3897
|
-
this.resolveBusy(
|
|
3898
|
-
pending,
|
|
3899
|
-
FAILURE_RETRY_MS,
|
|
3900
|
-
full ? "storage disk budget reached" : "storage write failed"
|
|
3901
|
-
);
|
|
3902
|
-
}
|
|
3903
|
-
return false;
|
|
3904
|
-
}
|
|
3905
|
-
}
|
|
3906
|
-
const started = import_node_perf_hooks.performance.now();
|
|
3907
|
-
const counters = { committed: 0, duplicates: 0, rejected: 0 };
|
|
3908
|
-
const cursors = /* @__PURE__ */ new Map();
|
|
3909
|
-
const upserts = /* @__PURE__ */ new Map();
|
|
3910
|
-
try {
|
|
3911
|
-
this.database.db.exec("BEGIN IMMEDIATE");
|
|
3912
|
-
for (const pending of selected) {
|
|
3913
|
-
this.processBatch(pending, counters, cursors, upserts);
|
|
3914
|
-
}
|
|
3915
|
-
for (const [machineId, live] of upserts) {
|
|
3916
|
-
this.latestUpsert.run(
|
|
3917
|
-
machineId,
|
|
3918
|
-
live.record.collectorId,
|
|
3919
|
-
live.record.runId,
|
|
3920
|
-
live.record.bootId,
|
|
3921
|
-
live.record.sequence,
|
|
3922
|
-
Date.parse(live.record.capturedAt),
|
|
3923
|
-
live.receivedAtMs,
|
|
3924
|
-
live.expectedIntervalMs,
|
|
3925
|
-
live.hostname,
|
|
3926
|
-
JSON.stringify(live.record.payload)
|
|
3927
|
-
);
|
|
3928
|
-
}
|
|
3929
|
-
this.database.db.exec("COMMIT");
|
|
3930
|
-
} catch (error) {
|
|
3931
|
-
try {
|
|
3932
|
-
this.database.db.exec("ROLLBACK");
|
|
3933
|
-
} catch {
|
|
3934
|
-
}
|
|
3935
|
-
this.writeFailures += 1;
|
|
3936
|
-
const full = isStorageFullError(error);
|
|
3937
|
-
if (full) this.diskFullFailures += 1;
|
|
3938
|
-
this.lastWriteError = full ? "storage disk budget reached" : sanitizeWriteError(error);
|
|
3939
|
-
for (const pending of selected) {
|
|
3940
|
-
this.resolveBusy(
|
|
3941
|
-
pending,
|
|
3942
|
-
FAILURE_RETRY_MS,
|
|
3943
|
-
full ? "storage disk budget reached" : "storage write failed"
|
|
3944
|
-
);
|
|
3945
|
-
}
|
|
3946
|
-
return false;
|
|
3947
|
-
}
|
|
3948
|
-
this.committedRecords += counters.committed;
|
|
3949
|
-
this.duplicateRecords += counters.duplicates;
|
|
3950
|
-
this.rejectedRecords += counters.rejected;
|
|
3951
|
-
this.lastFlushMs = this.now();
|
|
3952
|
-
this.lastFlushDurationMs = Math.max(0, Math.round(import_node_perf_hooks.performance.now() - started));
|
|
3953
|
-
this.checkpointWALIfNeeded();
|
|
3954
|
-
if (this.maxPhysicalBytes !== null && this.database.sizeBytes() > this.maxPhysicalBytes) {
|
|
3955
|
-
try {
|
|
3956
|
-
this.retention.relievePressure(4);
|
|
3957
|
-
} catch {
|
|
3958
|
-
}
|
|
3959
|
-
}
|
|
3960
|
-
for (const pending of selected) this.resolveCommitted(pending);
|
|
3961
|
-
return true;
|
|
3962
|
-
}
|
|
3963
|
-
checkpointWALIfNeeded() {
|
|
3964
|
-
if (this.maxPhysicalBytes === null) return;
|
|
3965
|
-
try {
|
|
3966
|
-
const physical = this.database.physicalLimit();
|
|
3967
|
-
if (physical !== null && this.database.walBytes() > physical.walBudgetBytes) {
|
|
3968
|
-
this.database.checkpoint();
|
|
3969
|
-
}
|
|
3970
|
-
} catch {
|
|
3971
|
-
}
|
|
3972
|
-
}
|
|
3973
|
-
processBatch(pending, counters, cursors, upserts) {
|
|
3974
|
-
const batch = pending.batch;
|
|
3975
|
-
let best = null;
|
|
3976
|
-
let bestCapturedMs = 0;
|
|
3977
|
-
let bestSequence = -1;
|
|
3978
|
-
for (const record of batch.records) {
|
|
3979
|
-
const ref = { runId: record.runId, sequence: record.sequence };
|
|
3980
|
-
const reason = rejectionReason(record, batch, pending.receivedAtMs);
|
|
3981
|
-
if (reason !== null) {
|
|
3982
|
-
pending.rejected.push({ ...ref, reason });
|
|
3983
|
-
counters.rejected += 1;
|
|
3984
|
-
continue;
|
|
3985
|
-
}
|
|
3986
|
-
const capturedMs = Date.parse(record.capturedAt);
|
|
3987
|
-
const inserted = Number(
|
|
3988
|
-
this.receiptInsert.run(
|
|
3989
|
-
batch.collectorId,
|
|
3990
|
-
record.runId,
|
|
3991
|
-
record.sequence,
|
|
3992
|
-
batch.machineId,
|
|
3993
|
-
record.kind,
|
|
3994
|
-
batch.mode,
|
|
3995
|
-
capturedMs,
|
|
3996
|
-
pending.receivedAtMs,
|
|
3997
|
-
batch.expectedIntervalMs
|
|
3998
|
-
).changes
|
|
3999
|
-
) === 1;
|
|
4000
|
-
if (!inserted) {
|
|
4001
|
-
pending.accepted.push(ref);
|
|
4002
|
-
counters.duplicates += 1;
|
|
4003
|
-
continue;
|
|
4004
|
-
}
|
|
4005
|
-
pending.accepted.push(ref);
|
|
4006
|
-
counters.committed += 1;
|
|
4007
|
-
if (record.kind === "metrics") {
|
|
4008
|
-
const known = needsKnownSeries(record.payload) ? this.series.knownFor(batch.machineId) : void 0;
|
|
4009
|
-
for (const point of flattenMetrics(record.payload, known)) {
|
|
4010
|
-
const seriesId = this.series.idFor(batch.machineId, point.metric, point.resourceId);
|
|
4011
|
-
this.insertPoint(seriesId, capturedMs, point.value);
|
|
4012
|
-
}
|
|
4013
|
-
if (batch.mode === "live") {
|
|
4014
|
-
if (best === null || capturedMs > bestCapturedMs || capturedMs === bestCapturedMs && record.sequence > bestSequence) {
|
|
4015
|
-
best = record;
|
|
4016
|
-
bestCapturedMs = capturedMs;
|
|
4017
|
-
bestSequence = record.sequence;
|
|
4018
|
-
}
|
|
4019
|
-
}
|
|
4020
|
-
} else {
|
|
4021
|
-
this.gapInsert.run(
|
|
4022
|
-
batch.collectorId,
|
|
4023
|
-
record.runId,
|
|
4024
|
-
record.sequence,
|
|
4025
|
-
batch.machineId,
|
|
4026
|
-
Date.parse(record.payload.fromCapturedAt),
|
|
4027
|
-
Date.parse(record.payload.toCapturedAt),
|
|
4028
|
-
record.payload.droppedRecords,
|
|
4029
|
-
record.payload.reason,
|
|
4030
|
-
pending.receivedAtMs
|
|
4031
|
-
);
|
|
4032
|
-
}
|
|
4033
|
-
}
|
|
4034
|
-
if (batch.mode !== "live" || best === null) return;
|
|
4035
|
-
const cursor = this.latestCursor(batch.machineId, cursors);
|
|
4036
|
-
const candidate = {
|
|
4037
|
-
runId: best.runId,
|
|
4038
|
-
sequence: best.sequence,
|
|
4039
|
-
capturedAtMs: bestCapturedMs
|
|
3881
|
+
return total;
|
|
3882
|
+
}
|
|
3883
|
+
function readSeries(db, tier, seriesId, fromBucketMs2, toMs, originMs, stepMs, rowLimit) {
|
|
3884
|
+
const statement = tier === "raw" ? db.db.prepare(
|
|
3885
|
+
"SELECT ? + CAST((ts_ms - ?) / ? AS INTEGER) * ? AS bucket, MIN(value) AS min_value, MAX(value) AS max_value, SUM(value) AS sum_value, COUNT(value) AS count_value, COUNT(*) AS row_count FROM (SELECT ts_ms, value FROM points_raw WHERE series_id = ? AND ts_ms >= ? AND ts_ms <= ? ORDER BY ts_ms LIMIT ?) GROUP BY bucket ORDER BY bucket"
|
|
3886
|
+
) : db.db.prepare(
|
|
3887
|
+
`SELECT ? + CAST((bucket_ms - ?) / ? AS INTEGER) * ? AS bucket, MIN(min) AS min_value, MAX(max) AS max_value, SUM(sum) AS sum_value, SUM(count) AS count_value, COUNT(*) AS row_count FROM (SELECT bucket_ms, min, max, sum, count FROM ${tableForTier(tier)} WHERE series_id = ? AND bucket_ms >= ? AND bucket_ms <= ? ORDER BY bucket_ms LIMIT ?) GROUP BY bucket ORDER BY bucket`
|
|
3888
|
+
);
|
|
3889
|
+
const rows = statement.all(
|
|
3890
|
+
originMs,
|
|
3891
|
+
originMs,
|
|
3892
|
+
stepMs,
|
|
3893
|
+
stepMs,
|
|
3894
|
+
seriesId,
|
|
3895
|
+
fromBucketMs2,
|
|
3896
|
+
toMs,
|
|
3897
|
+
rowLimit
|
|
3898
|
+
);
|
|
3899
|
+
return rows.map((row) => {
|
|
3900
|
+
const count = Number(row.count_value ?? 0);
|
|
3901
|
+
return {
|
|
3902
|
+
t: Number(row.bucket),
|
|
3903
|
+
min: row.min_value === null ? null : Number(row.min_value),
|
|
3904
|
+
max: row.max_value === null ? null : Number(row.max_value),
|
|
3905
|
+
avg: count > 0 && row.sum_value !== null ? Number(row.sum_value) / count : null
|
|
4040
3906
|
};
|
|
4041
|
-
|
|
4042
|
-
|
|
4043
|
-
|
|
4044
|
-
|
|
4045
|
-
|
|
4046
|
-
|
|
4047
|
-
|
|
4048
|
-
|
|
4049
|
-
|
|
4050
|
-
|
|
3907
|
+
});
|
|
3908
|
+
}
|
|
3909
|
+
function fromBucketMs(tier, fromMs) {
|
|
3910
|
+
if (tier === "raw") return fromMs;
|
|
3911
|
+
const span = tier === "10s" ? 1e4 : tier === "1m" ? 6e4 : 36e5;
|
|
3912
|
+
return Math.floor(fromMs / span) * span;
|
|
3913
|
+
}
|
|
3914
|
+
function tableForTier(tier) {
|
|
3915
|
+
return tier === "raw" ? "points_raw" : `points_${tier}`;
|
|
3916
|
+
}
|
|
3917
|
+
function readGaps(db, machineId, fromMs, toMs) {
|
|
3918
|
+
const dropped = db.db.prepare(
|
|
3919
|
+
"SELECT from_ms, to_ms, dropped, reason FROM gaps WHERE machine_id = ? AND to_ms >= ? AND from_ms <= ? ORDER BY from_ms LIMIT ?"
|
|
3920
|
+
).all(machineId, fromMs, toMs, MAX_GAPS);
|
|
3921
|
+
const missing = db.db.prepare(
|
|
3922
|
+
"WITH before_window AS ( SELECT captured_at_ms, expected_interval_ms FROM sample_receipts WHERE machine_id = ? AND captured_at_ms < ? ORDER BY captured_at_ms DESC LIMIT 1 ), in_window AS ( SELECT captured_at_ms, expected_interval_ms FROM sample_receipts WHERE machine_id = ? AND captured_at_ms >= ? AND captured_at_ms <= ? ), ordered AS ( SELECT captured_at_ms, expected_interval_ms, LAG(captured_at_ms) OVER (ORDER BY captured_at_ms) AS prev_ms FROM (SELECT * FROM before_window UNION ALL SELECT * FROM in_window) ) SELECT MAX(prev_ms, ?) AS from_ms, captured_at_ms AS to_ms FROM ordered WHERE prev_ms IS NOT NULL AND captured_at_ms > ? AND captured_at_ms - prev_ms > MAX(15000, 3 * COALESCE(expected_interval_ms, 5000)) ORDER BY captured_at_ms LIMIT ?"
|
|
3923
|
+
).all(machineId, fromMs, machineId, fromMs, toMs, fromMs, fromMs, MAX_GAPS);
|
|
3924
|
+
const gaps = [
|
|
3925
|
+
...dropped.map((row) => ({
|
|
3926
|
+
fromMs: Number(row.from_ms),
|
|
3927
|
+
toMs: Number(row.to_ms),
|
|
3928
|
+
kind: "dropped",
|
|
3929
|
+
reason: row.reason,
|
|
3930
|
+
droppedRecords: Number(row.dropped)
|
|
3931
|
+
})),
|
|
3932
|
+
...missing.map((row) => ({
|
|
3933
|
+
fromMs: Number(row.from_ms),
|
|
3934
|
+
toMs: Number(row.to_ms),
|
|
3935
|
+
kind: "missing",
|
|
3936
|
+
reason: "expected interval missed"
|
|
3937
|
+
}))
|
|
3938
|
+
];
|
|
3939
|
+
gaps.sort((a, b) => a.fromMs - b.fromMs || a.toMs - b.toMs);
|
|
3940
|
+
return gaps.slice(0, MAX_GAPS);
|
|
3941
|
+
}
|
|
3942
|
+
function validatedQuery(q) {
|
|
3943
|
+
if (q === null || typeof q !== "object") throw new HistoryQueryError("query must be an object");
|
|
3944
|
+
if (typeof q.machineId !== "string" || q.machineId.length === 0 || q.machineId.length > 256) {
|
|
3945
|
+
throw new HistoryQueryError("machineId must be a nonempty string");
|
|
4051
3946
|
}
|
|
4052
|
-
|
|
4053
|
-
|
|
4054
|
-
const row = this.latestSelect.get(machineId);
|
|
4055
|
-
const cursor = row ? { runId: row.run_id, sequence: Number(row.seq), capturedAtMs: Number(row.captured_at_ms) } : null;
|
|
4056
|
-
cache.set(machineId, cursor);
|
|
4057
|
-
return cursor;
|
|
3947
|
+
if (!Array.isArray(q.metrics) || q.metrics.length === 0 || q.metrics.length > MAX_METRICS) {
|
|
3948
|
+
throw new HistoryQueryError(`metrics must contain 1..${MAX_METRICS} metric names`);
|
|
4058
3949
|
}
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
const minute = Math.floor(tsMs / 6e4) * 6e4;
|
|
4067
|
-
const hour = Math.floor(tsMs / 36e5) * 36e5;
|
|
4068
|
-
this.applyRollup(this.upsert1m, seriesId, minute, value);
|
|
4069
|
-
this.applyRollup(this.upsert1h, seriesId, hour, value);
|
|
3950
|
+
const seenMetrics = /* @__PURE__ */ new Set();
|
|
3951
|
+
for (const metric of q.metrics) {
|
|
3952
|
+
if (typeof metric !== "string" || !Object.prototype.hasOwnProperty.call(METRICS, metric)) {
|
|
3953
|
+
throw new HistoryQueryError(`unknown metric ${JSON.stringify(metric)}`);
|
|
3954
|
+
}
|
|
3955
|
+
if (seenMetrics.has(metric)) throw new HistoryQueryError(`duplicate metric ${JSON.stringify(metric)}`);
|
|
3956
|
+
seenMetrics.add(metric);
|
|
4070
3957
|
}
|
|
4071
|
-
|
|
4072
|
-
if (
|
|
4073
|
-
|
|
4074
|
-
}
|
|
4075
|
-
|
|
3958
|
+
if (q.resourceIds !== void 0) {
|
|
3959
|
+
if (!Array.isArray(q.resourceIds) || q.resourceIds.length > MAX_RESOURCE_IDS) {
|
|
3960
|
+
throw new HistoryQueryError(`resourceIds must contain at most ${MAX_RESOURCE_IDS} ids`);
|
|
3961
|
+
}
|
|
3962
|
+
const seenResources = /* @__PURE__ */ new Set();
|
|
3963
|
+
for (const id of q.resourceIds) {
|
|
3964
|
+
if (typeof id !== "string" || id.length === 0 || id.length > 256) {
|
|
3965
|
+
throw new HistoryQueryError("resourceIds must be nonempty strings");
|
|
3966
|
+
}
|
|
3967
|
+
if (seenResources.has(id)) throw new HistoryQueryError(`duplicate resource id ${JSON.stringify(id)}`);
|
|
3968
|
+
seenResources.add(id);
|
|
4076
3969
|
}
|
|
4077
3970
|
}
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
pending.settled = true;
|
|
4081
|
-
pending.resolve({
|
|
4082
|
-
status: "committed",
|
|
4083
|
-
accepted: pending.accepted,
|
|
4084
|
-
rejected: pending.rejected,
|
|
4085
|
-
newLive: pending.newLive,
|
|
4086
|
-
receivedAtMs: pending.receivedAtMs
|
|
4087
|
-
});
|
|
3971
|
+
if (q.coreIds !== void 0 && (!Array.isArray(q.coreIds) || q.coreIds.length > MAX_CORES || q.coreIds.some((id) => typeof id !== "string" || !/^core:(?:0|[1-9]\d{0,3})$/.test(id) || Number(id.slice(5)) >= MAX_CORES) || new Set(q.coreIds).size !== q.coreIds.length)) {
|
|
3972
|
+
throw new HistoryQueryError(`coreIds must contain at most ${MAX_CORES} unique core identifiers`);
|
|
4088
3973
|
}
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
3974
|
+
if (!Number.isSafeInteger(q.fromMs) || !Number.isSafeInteger(q.toMs)) {
|
|
3975
|
+
throw new HistoryQueryError("fromMs and toMs must be safe integer epoch milliseconds");
|
|
3976
|
+
}
|
|
3977
|
+
if (q.fromMs >= q.toMs) throw new HistoryQueryError("fromMs must be before toMs");
|
|
3978
|
+
if (q.toMs - q.fromMs > MAX_RANGE_MS) {
|
|
3979
|
+
throw new HistoryQueryError("range exceeds 400 days");
|
|
4093
3980
|
}
|
|
4094
|
-
};
|
|
4095
|
-
function prepareRollupUpsert(db, table) {
|
|
4096
|
-
return db.prepare(
|
|
4097
|
-
`INSERT INTO ${table} (series_id, bucket_ms, min, max, sum, count, null_count) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(series_id, bucket_ms) DO UPDATE SET min = CASE WHEN excluded.count = 0 THEN min WHEN min IS NULL THEN excluded.min ELSE MIN(min, excluded.min) END, max = CASE WHEN excluded.count = 0 THEN max WHEN max IS NULL THEN excluded.max ELSE MAX(max, excluded.max) END, sum = sum + excluded.sum, count = count + excluded.count, null_count = null_count + excluded.null_count`
|
|
4098
|
-
);
|
|
4099
3981
|
}
|
|
4100
|
-
function
|
|
4101
|
-
if (
|
|
4102
|
-
|
|
3982
|
+
function validatedMaxPoints(value) {
|
|
3983
|
+
if (value === void 0) return DEFAULT_MAX_POINTS;
|
|
3984
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
3985
|
+
throw new HistoryQueryError("maxPoints must be a positive safe integer");
|
|
3986
|
+
}
|
|
3987
|
+
return Math.min(value, MAX_POINTS);
|
|
4103
3988
|
}
|
|
4104
|
-
function
|
|
4105
|
-
|
|
3989
|
+
function validatedBudget(value, fallback, name) {
|
|
3990
|
+
if (value === void 0) return fallback;
|
|
3991
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
3992
|
+
throw new HistoryQueryError(`${name} must be a positive safe integer`);
|
|
3993
|
+
}
|
|
3994
|
+
return value;
|
|
4106
3995
|
}
|
|
4107
|
-
|
|
4108
|
-
|
|
4109
|
-
|
|
3996
|
+
|
|
3997
|
+
// src/hub/storage/telemetry-store.ts
|
|
3998
|
+
var DAY_MS3 = 864e5;
|
|
3999
|
+
var FUTURE_TOLERANCE_MS = DAY_MS3;
|
|
4000
|
+
var FAILURE_RETRY_MS = 5e3;
|
|
4001
|
+
var MIN_DERIVED_PHYSICAL_BYTES = 8 * 1024 * 1024;
|
|
4002
|
+
var TelemetryStore = class {
|
|
4003
|
+
historyMode = "persistent";
|
|
4004
|
+
database;
|
|
4005
|
+
now;
|
|
4006
|
+
maxQueuedRecords;
|
|
4007
|
+
flushIntervalMs;
|
|
4008
|
+
maxRecordsPerTransaction;
|
|
4009
|
+
maxPhysicalBytes;
|
|
4010
|
+
storageTier;
|
|
4011
|
+
series;
|
|
4012
|
+
retention;
|
|
4013
|
+
queue = [];
|
|
4014
|
+
timer = null;
|
|
4015
|
+
driver = null;
|
|
4016
|
+
closed = false;
|
|
4017
|
+
committedRecords = 0;
|
|
4018
|
+
duplicateRecords = 0;
|
|
4019
|
+
rejectedRecords = 0;
|
|
4020
|
+
writeFailures = 0;
|
|
4021
|
+
diskFullFailures = 0;
|
|
4022
|
+
lastWriteError = null;
|
|
4023
|
+
lastFlushMs = null;
|
|
4024
|
+
lastFlushDurationMs = null;
|
|
4025
|
+
receiptInsert;
|
|
4026
|
+
rawInsert;
|
|
4027
|
+
upsert10s;
|
|
4028
|
+
upsert1m;
|
|
4029
|
+
upsert1h;
|
|
4030
|
+
gapInsert;
|
|
4031
|
+
latestSelect;
|
|
4032
|
+
latestUpsert;
|
|
4033
|
+
inventoryUpsert;
|
|
4034
|
+
inventorySelect;
|
|
4035
|
+
metaUpsertBoth;
|
|
4036
|
+
metaUpsertDisplay;
|
|
4037
|
+
metaUpsertTags;
|
|
4038
|
+
metaTouch;
|
|
4039
|
+
metaSelect;
|
|
4040
|
+
metaSelectAll;
|
|
4041
|
+
latestSelectAll;
|
|
4042
|
+
constructor(options) {
|
|
4043
|
+
this.database = options.database;
|
|
4044
|
+
this.now = options.now ?? (() => Date.now());
|
|
4045
|
+
this.maxQueuedRecords = positiveInt2(options.maxQueuedRecords, 4096, "maxQueuedRecords");
|
|
4046
|
+
this.flushIntervalMs = nonnegativeInt(options.flushIntervalMs, 20, "flushIntervalMs");
|
|
4047
|
+
this.maxRecordsPerTransaction = positiveInt2(
|
|
4048
|
+
options.maxRecordsPerTransaction,
|
|
4049
|
+
512,
|
|
4050
|
+
"maxRecordsPerTransaction"
|
|
4051
|
+
);
|
|
4052
|
+
this.maxPhysicalBytes = resolvePhysicalLimit(options);
|
|
4053
|
+
this.storageTier = options.storageTier ?? "raw";
|
|
4054
|
+
if (this.storageTier !== "raw" && this.storageTier !== "summary") {
|
|
4055
|
+
throw new Error("storageTier must be raw or summary");
|
|
4056
|
+
}
|
|
4057
|
+
const db = options.database.db;
|
|
4058
|
+
this.series = new SeriesCatalog(db);
|
|
4059
|
+
this.receiptInsert = db.prepare(
|
|
4060
|
+
"INSERT OR IGNORE INTO sample_receipts (collector_id, run_id, seq, machine_id, kind, mode, captured_at_ms, received_at_ms, expected_interval_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
4061
|
+
);
|
|
4062
|
+
this.rawInsert = db.prepare(
|
|
4063
|
+
"INSERT OR IGNORE INTO points_raw (series_id, ts_ms, value) VALUES (?, ?, ?)"
|
|
4064
|
+
);
|
|
4065
|
+
this.upsert10s = prepareRollupUpsert(db, "points_10s");
|
|
4066
|
+
this.upsert1m = prepareRollupUpsert(db, "points_1m");
|
|
4067
|
+
this.upsert1h = prepareRollupUpsert(db, "points_1h");
|
|
4068
|
+
this.gapInsert = db.prepare(
|
|
4069
|
+
"INSERT OR IGNORE INTO gaps (collector_id, run_id, seq, machine_id, from_ms, to_ms, dropped, reason, received_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
4070
|
+
);
|
|
4071
|
+
this.latestSelect = db.prepare(
|
|
4072
|
+
"SELECT run_id, seq, captured_at_ms FROM machine_latest WHERE machine_id = ?"
|
|
4073
|
+
);
|
|
4074
|
+
this.latestUpsert = db.prepare(
|
|
4075
|
+
"INSERT INTO machine_latest (machine_id, collector_id, run_id, boot_id, seq, captured_at_ms, received_at_ms, expected_interval_ms, hostname, payload_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET collector_id = excluded.collector_id, run_id = excluded.run_id, boot_id = excluded.boot_id, seq = excluded.seq, captured_at_ms = excluded.captured_at_ms, received_at_ms = excluded.received_at_ms, expected_interval_ms = COALESCE(excluded.expected_interval_ms, expected_interval_ms), hostname = COALESCE(excluded.hostname, hostname), payload_json = excluded.payload_json"
|
|
4076
|
+
);
|
|
4077
|
+
this.latestSelectAll = db.prepare(
|
|
4078
|
+
"SELECT machine_id, collector_id, run_id, boot_id, seq, captured_at_ms, received_at_ms, expected_interval_ms, hostname, payload_json FROM machine_latest ORDER BY machine_id"
|
|
4079
|
+
);
|
|
4080
|
+
this.inventoryUpsert = db.prepare(
|
|
4081
|
+
"INSERT INTO inventory (machine_id, captured_at_ms, received_at_ms, inventory_json) VALUES (?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET captured_at_ms = excluded.captured_at_ms, received_at_ms = excluded.received_at_ms, inventory_json = excluded.inventory_json"
|
|
4082
|
+
);
|
|
4083
|
+
this.inventorySelect = db.prepare(
|
|
4084
|
+
"SELECT captured_at_ms, received_at_ms, inventory_json FROM inventory WHERE machine_id = ?"
|
|
4085
|
+
);
|
|
4086
|
+
this.metaUpsertBoth = db.prepare(
|
|
4087
|
+
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET display_name = excluded.display_name, tags_json = excluded.tags_json, updated_at_ms = excluded.updated_at_ms"
|
|
4088
|
+
);
|
|
4089
|
+
this.metaUpsertDisplay = db.prepare(
|
|
4090
|
+
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, ?, '[]', ?) ON CONFLICT(machine_id) DO UPDATE SET display_name = excluded.display_name, updated_at_ms = excluded.updated_at_ms"
|
|
4091
|
+
);
|
|
4092
|
+
this.metaUpsertTags = db.prepare(
|
|
4093
|
+
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, NULL, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET tags_json = excluded.tags_json, updated_at_ms = excluded.updated_at_ms"
|
|
4094
|
+
);
|
|
4095
|
+
this.metaTouch = db.prepare(
|
|
4096
|
+
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, NULL, '[]', ?) ON CONFLICT(machine_id) DO UPDATE SET updated_at_ms = excluded.updated_at_ms"
|
|
4097
|
+
);
|
|
4098
|
+
this.metaSelect = db.prepare(
|
|
4099
|
+
"SELECT display_name, tags_json, updated_at_ms FROM machine_meta WHERE machine_id = ?"
|
|
4100
|
+
);
|
|
4101
|
+
this.metaSelectAll = db.prepare(
|
|
4102
|
+
"SELECT machine_id, display_name, tags_json FROM machine_meta ORDER BY machine_id"
|
|
4103
|
+
);
|
|
4104
|
+
this.retention = new RetentionManager(options.database, {
|
|
4105
|
+
policy: options.retention,
|
|
4106
|
+
maxDatabaseBytes: options.maxDatabaseBytes ?? null,
|
|
4107
|
+
maxPhysicalBytes: this.maxPhysicalBytes,
|
|
4108
|
+
now: this.now
|
|
4109
|
+
});
|
|
4110
|
+
this.retention.start();
|
|
4110
4111
|
}
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4112
|
+
/**
|
|
4113
|
+
* Queues one batch. The promise resolves only after the transaction that
|
|
4114
|
+
* writes it has committed, or immediately as `busy` when the bounded queue
|
|
4115
|
+
* cannot hold it (no database work is done in that case).
|
|
4116
|
+
*/
|
|
4117
|
+
enqueue(batch) {
|
|
4118
|
+
return new Promise((resolve7) => {
|
|
4119
|
+
if (this.closed) {
|
|
4120
|
+
resolve7({ status: "busy", retryAfterMs: FAILURE_RETRY_MS, reason: "storage is closed" });
|
|
4121
|
+
return;
|
|
4122
|
+
}
|
|
4123
|
+
validateBatch(batch);
|
|
4124
|
+
const queued = this.queuedRecords();
|
|
4125
|
+
if (queued + batch.records.length > this.maxQueuedRecords) {
|
|
4126
|
+
const ratio = this.maxQueuedRecords === 0 ? 1 : queued / this.maxQueuedRecords;
|
|
4127
|
+
const retryAfterMs = Math.min(5e3, Math.max(1e3, Math.round(1e3 + ratio * 4e3)));
|
|
4128
|
+
resolve7({ status: "busy", retryAfterMs, reason: "ingest queue is full" });
|
|
4129
|
+
return;
|
|
4130
|
+
}
|
|
4131
|
+
this.queue.push({
|
|
4132
|
+
batch,
|
|
4133
|
+
receivedAtMs: this.now(),
|
|
4134
|
+
resolve: resolve7,
|
|
4135
|
+
settled: false,
|
|
4136
|
+
accepted: [],
|
|
4137
|
+
rejected: [],
|
|
4138
|
+
newLive: null
|
|
4139
|
+
});
|
|
4140
|
+
this.scheduleFlush(this.flushIntervalMs);
|
|
4141
|
+
});
|
|
4115
4142
|
}
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4143
|
+
/** Persisted latest LIVE metrics per machine, for restart recovery. */
|
|
4144
|
+
latestLive() {
|
|
4145
|
+
const rows = this.latestSelectAll.all();
|
|
4146
|
+
return rows.map((row) => ({
|
|
4147
|
+
machineId: row.machine_id,
|
|
4148
|
+
collectorId: row.collector_id,
|
|
4149
|
+
runId: row.run_id,
|
|
4150
|
+
bootId: row.boot_id,
|
|
4151
|
+
sequence: Number(row.seq),
|
|
4152
|
+
capturedAt: new Date(Number(row.captured_at_ms)).toISOString(),
|
|
4153
|
+
receivedAt: new Date(Number(row.received_at_ms)).toISOString(),
|
|
4154
|
+
hostname: row.hostname,
|
|
4155
|
+
expectedIntervalMs: row.expected_interval_ms === null ? null : Number(row.expected_interval_ms),
|
|
4156
|
+
payload: JSON.parse(row.payload_json)
|
|
4157
|
+
}));
|
|
4122
4158
|
}
|
|
4123
|
-
|
|
4124
|
-
|
|
4159
|
+
saveInventory(machineId, capturedAt, inventory) {
|
|
4160
|
+
const capturedMs = Date.parse(capturedAt);
|
|
4161
|
+
if (!Number.isFinite(capturedMs)) throw new Error("inventory capturedAt must be a parseable timestamp");
|
|
4162
|
+
this.inventoryUpsert.run(machineId, capturedMs, this.now(), JSON.stringify(inventory));
|
|
4125
4163
|
}
|
|
4126
|
-
|
|
4127
|
-
|
|
4164
|
+
loadInventory(machineId) {
|
|
4165
|
+
const row = this.inventorySelect.get(machineId);
|
|
4166
|
+
if (!row) return null;
|
|
4167
|
+
return {
|
|
4168
|
+
capturedAt: new Date(Number(row.captured_at_ms)).toISOString(),
|
|
4169
|
+
receivedAt: new Date(Number(row.received_at_ms)).toISOString(),
|
|
4170
|
+
inventory: JSON.parse(row.inventory_json)
|
|
4171
|
+
};
|
|
4128
4172
|
}
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4173
|
+
/**
|
|
4174
|
+
* Merges presentation metadata. Omitted fields are preserved; an explicit
|
|
4175
|
+
* `displayName: null` clears it.
|
|
4176
|
+
*/
|
|
4177
|
+
setMachineMeta(machineId, meta) {
|
|
4178
|
+
const updatedAt = this.now();
|
|
4179
|
+
const hasDisplay = meta.displayName !== void 0;
|
|
4180
|
+
const hasTags = meta.tags !== void 0;
|
|
4181
|
+
const displayName = meta.displayName ?? null;
|
|
4182
|
+
const tags = hasTags ? validateTags(meta.tags) : [];
|
|
4183
|
+
if (hasDisplay && hasTags) {
|
|
4184
|
+
this.metaUpsertBoth.run(machineId, displayName, JSON.stringify(tags), updatedAt);
|
|
4185
|
+
} else if (hasDisplay) {
|
|
4186
|
+
this.metaUpsertDisplay.run(machineId, displayName, updatedAt);
|
|
4187
|
+
} else if (hasTags) {
|
|
4188
|
+
this.metaUpsertTags.run(machineId, JSON.stringify(tags), updatedAt);
|
|
4189
|
+
} else {
|
|
4190
|
+
this.metaTouch.run(machineId, updatedAt);
|
|
4191
|
+
}
|
|
4132
4192
|
}
|
|
4133
|
-
|
|
4134
|
-
|
|
4193
|
+
getMachineMeta(machineId) {
|
|
4194
|
+
const row = this.metaSelect.get(machineId);
|
|
4195
|
+
if (!row) return { displayName: null, tags: [] };
|
|
4196
|
+
return { displayName: row.display_name, tags: parseTags(row.tags_json) };
|
|
4135
4197
|
}
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4198
|
+
listMachineMeta() {
|
|
4199
|
+
const rows = this.metaSelectAll.all();
|
|
4200
|
+
const result = /* @__PURE__ */ new Map();
|
|
4201
|
+
for (const row of rows) {
|
|
4202
|
+
result.set(row.machine_id, {
|
|
4203
|
+
displayName: row.display_name,
|
|
4204
|
+
tags: parseTags(row.tags_json)
|
|
4205
|
+
});
|
|
4144
4206
|
}
|
|
4145
|
-
|
|
4146
|
-
seen.add(tag);
|
|
4147
|
-
result.push(tag);
|
|
4207
|
+
return result;
|
|
4148
4208
|
}
|
|
4149
|
-
|
|
4150
|
-
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
|
|
4157
|
-
|
|
4209
|
+
stats() {
|
|
4210
|
+
return {
|
|
4211
|
+
queuedRecords: this.queuedRecords(),
|
|
4212
|
+
queuedBatches: this.queue.length,
|
|
4213
|
+
lastFlushMs: this.lastFlushMs,
|
|
4214
|
+
lastFlushDurationMs: this.lastFlushDurationMs,
|
|
4215
|
+
committedRecords: this.committedRecords,
|
|
4216
|
+
duplicateRecords: this.duplicateRecords,
|
|
4217
|
+
rejectedRecords: this.rejectedRecords,
|
|
4218
|
+
writeFailures: this.writeFailures,
|
|
4219
|
+
lastWriteError: this.lastWriteError,
|
|
4220
|
+
diskFullFailures: this.diskFullFailures,
|
|
4221
|
+
dbBytes: this.database.sizeBytes(),
|
|
4222
|
+
dbWalBytes: this.database.walBytes(),
|
|
4223
|
+
dbBytesLimit: this.maxPhysicalBytes,
|
|
4224
|
+
retention: this.retention.status()
|
|
4225
|
+
};
|
|
4158
4226
|
}
|
|
4159
|
-
}
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
}
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
|
|
4167
|
-
throw new Error(`${name} must be a positive safe integer`);
|
|
4227
|
+
queryHistory(query, budget = {}) {
|
|
4228
|
+
return queryHistory(this.database, query, {
|
|
4229
|
+
...budget,
|
|
4230
|
+
nowMs: budget.nowMs ?? this.now(),
|
|
4231
|
+
retention: { ...this.retention.policy, ...budget.retention },
|
|
4232
|
+
tiers: budget.tiers ?? (this.storageTier === "summary" ? ["10s", "1m", "1h"] : void 0)
|
|
4233
|
+
});
|
|
4168
4234
|
}
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4235
|
+
historyStatus() {
|
|
4236
|
+
const stats = this.stats();
|
|
4237
|
+
const physical = this.database.physicalLimit();
|
|
4238
|
+
const now = this.now();
|
|
4239
|
+
const retention = this.retention.policy;
|
|
4240
|
+
return {
|
|
4241
|
+
mode: "persistent",
|
|
4242
|
+
persistentHistory: true,
|
|
4243
|
+
availableFromMs: this.storageTier === "summary" ? now - Math.max(retention.tenSecondMs, retention.oneMinuteMs, retention.oneHourMs) : now - Math.max(retention.rawMs, retention.oneMinuteMs, retention.oneHourMs),
|
|
4244
|
+
availableToMs: now,
|
|
4245
|
+
memory: null,
|
|
4246
|
+
persistence: {
|
|
4247
|
+
enabled: true,
|
|
4248
|
+
storageTier: this.storageTier,
|
|
4249
|
+
flushIntervalMs: this.flushIntervalMs,
|
|
4250
|
+
queuedRecords: stats.queuedRecords,
|
|
4251
|
+
queuedBatches: stats.queuedBatches,
|
|
4252
|
+
lastFlushMs: stats.lastFlushMs,
|
|
4253
|
+
lastFlushDurationMs: stats.lastFlushDurationMs,
|
|
4254
|
+
writeFailures: stats.writeFailures,
|
|
4255
|
+
lastWriteError: stats.lastWriteError,
|
|
4256
|
+
dbBytes: stats.dbBytes,
|
|
4257
|
+
dbWalBytes: stats.dbWalBytes,
|
|
4258
|
+
dbBytesLimit: physical?.limitBytes ?? this.maxPhysicalBytes,
|
|
4259
|
+
tiers: this.storageTier === "summary" ? [
|
|
4260
|
+
{ tier: "10s", stepMs: 1e4, retentionMs: retention.tenSecondMs, bytes: stats.dbBytes },
|
|
4261
|
+
{ tier: "1m", stepMs: 6e4, retentionMs: retention.oneMinuteMs, bytes: 0 },
|
|
4262
|
+
{ tier: "1h", stepMs: 36e5, retentionMs: retention.oneHourMs, bytes: 0 }
|
|
4263
|
+
] : [
|
|
4264
|
+
{ tier: "raw", stepMs: 100, retentionMs: retention.rawMs, bytes: stats.dbBytes },
|
|
4265
|
+
{ tier: "1m", stepMs: 6e4, retentionMs: retention.oneMinuteMs, bytes: 0 },
|
|
4266
|
+
{ tier: "1h", stepMs: 36e5, retentionMs: retention.oneHourMs, bytes: 0 }
|
|
4267
|
+
]
|
|
4268
|
+
},
|
|
4269
|
+
gaps: 0,
|
|
4270
|
+
acceptedRecords: stats.committedRecords,
|
|
4271
|
+
generatedAt: new Date(now).toISOString()
|
|
4272
|
+
};
|
|
4273
|
+
}
|
|
4274
|
+
/** Drains the queue immediately; intended for tests and shutdown. */
|
|
4275
|
+
async flushNow() {
|
|
4276
|
+
if (this.closed) return;
|
|
4277
|
+
if (this.timer !== null) {
|
|
4278
|
+
clearTimeout(this.timer);
|
|
4279
|
+
this.timer = null;
|
|
4280
|
+
}
|
|
4281
|
+
await this.drive();
|
|
4282
|
+
while (!this.closed && this.queue.length > 0 && this.driver === null) {
|
|
4283
|
+
const failures = this.writeFailures;
|
|
4284
|
+
await this.drive();
|
|
4285
|
+
if (this.writeFailures > failures) break;
|
|
4176
4286
|
}
|
|
4177
|
-
return options.maxPhysicalBytes;
|
|
4178
4287
|
}
|
|
4179
|
-
|
|
4180
|
-
|
|
4288
|
+
/**
|
|
4289
|
+
* Drains for up to ~2s, then resolves anything left as busy and closes the
|
|
4290
|
+
* database. Batches that were already committed have resolved normally.
|
|
4291
|
+
*/
|
|
4292
|
+
async close() {
|
|
4293
|
+
if (this.closed) return;
|
|
4294
|
+
this.closed = true;
|
|
4295
|
+
this.retention.stop();
|
|
4296
|
+
if (this.timer !== null) {
|
|
4297
|
+
clearTimeout(this.timer);
|
|
4298
|
+
this.timer = null;
|
|
4299
|
+
}
|
|
4300
|
+
if (this.driver !== null) {
|
|
4301
|
+
try {
|
|
4302
|
+
await this.driver;
|
|
4303
|
+
} catch {
|
|
4304
|
+
}
|
|
4305
|
+
}
|
|
4306
|
+
const deadline = Date.now() + 2e3;
|
|
4307
|
+
while (this.queue.length > 0 && Date.now() < deadline) {
|
|
4308
|
+
const ok = this.flushTransaction();
|
|
4309
|
+
if (!ok) break;
|
|
4310
|
+
if (this.queue.length > 0) await yieldToEventLoop2();
|
|
4311
|
+
}
|
|
4312
|
+
while (this.queue.length > 0) {
|
|
4313
|
+
const pending = this.queue.shift();
|
|
4314
|
+
this.resolveBusy(pending, FAILURE_RETRY_MS, "storage is shutting down");
|
|
4315
|
+
}
|
|
4316
|
+
try {
|
|
4317
|
+
this.database.close();
|
|
4318
|
+
} catch {
|
|
4319
|
+
}
|
|
4181
4320
|
}
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
if (!Number.isSafeInteger(resolved) || resolved < 0) {
|
|
4187
|
-
throw new Error(`${name} must be a nonnegative safe integer`);
|
|
4321
|
+
queuedRecords() {
|
|
4322
|
+
let total = 0;
|
|
4323
|
+
for (const pending of this.queue) total += pending.batch.records.length;
|
|
4324
|
+
return total;
|
|
4188
4325
|
}
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
var MAX_CORES = 1024;
|
|
4197
|
-
var MIN_EPOCH = Date.parse("2000-01-01T00:00:00Z");
|
|
4198
|
-
var MAX_EPOCH = Date.parse("2200-01-01T00:00:00Z");
|
|
4199
|
-
|
|
4200
|
-
// src/contracts/index.ts
|
|
4201
|
-
var MIN_TIMESTAMP_EPOCH = Date.parse("2000-01-01T00:00:00Z");
|
|
4202
|
-
var MAX_TIMESTAMP_EPOCH = Date.parse("2200-01-01T00:00:00Z");
|
|
4203
|
-
|
|
4204
|
-
// src/hub/storage/history-query.ts
|
|
4205
|
-
var DAY_MS3 = 864e5;
|
|
4206
|
-
var MAX_RANGE_MS = 400 * DAY_MS3;
|
|
4207
|
-
var MAX_METRICS = Object.keys(METRICS).length;
|
|
4208
|
-
var MAX_RESOURCE_IDS = 64;
|
|
4209
|
-
var DEFAULT_MAX_POINTS = 600;
|
|
4210
|
-
var MAX_POINTS = 2e3;
|
|
4211
|
-
var MAX_GAPS = 500;
|
|
4212
|
-
var TIER_RESOLUTION_MS = { raw: 100, "1m": 6e4, "1h": 36e5 };
|
|
4213
|
-
var HistoryQueryError = class extends Error {
|
|
4214
|
-
code = "invalid-query";
|
|
4215
|
-
constructor(message) {
|
|
4216
|
-
super(message);
|
|
4217
|
-
this.name = "HistoryQueryError";
|
|
4326
|
+
scheduleFlush(delayMs) {
|
|
4327
|
+
if (this.closed || this.timer !== null || this.driver !== null) return;
|
|
4328
|
+
this.timer = setTimeout(() => {
|
|
4329
|
+
this.timer = null;
|
|
4330
|
+
void this.drive();
|
|
4331
|
+
}, delayMs);
|
|
4332
|
+
this.timer.unref?.();
|
|
4218
4333
|
}
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
const nowMs = Date.now();
|
|
4230
|
-
let allSeries = loadSeries(db, q);
|
|
4231
|
-
let truncated = false;
|
|
4232
|
-
if (allSeries.length > maxSeries) {
|
|
4233
|
-
allSeries = allSeries.slice(0, maxSeries);
|
|
4234
|
-
truncated = true;
|
|
4334
|
+
drive() {
|
|
4335
|
+
if (this.driver !== null) return this.driver;
|
|
4336
|
+
const promise = this.runDriver().finally(() => {
|
|
4337
|
+
if (this.driver === promise) this.driver = null;
|
|
4338
|
+
if (!this.closed && this.queue.length > 0 && this.timer === null) {
|
|
4339
|
+
this.scheduleFlush(FAILURE_RETRY_MS);
|
|
4340
|
+
}
|
|
4341
|
+
});
|
|
4342
|
+
this.driver = promise;
|
|
4343
|
+
return promise;
|
|
4235
4344
|
}
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
retention,
|
|
4243
|
-
maxRowsScanned,
|
|
4244
|
-
maxElapsedMs,
|
|
4245
|
-
started
|
|
4246
|
-
);
|
|
4247
|
-
if (chosen.timeExceeded || chosen.overBudget) truncated = true;
|
|
4248
|
-
let seriesRows = allSeries;
|
|
4249
|
-
if (chosen.cutSeries < seriesRows.length) {
|
|
4250
|
-
seriesRows = seriesRows.slice(0, Math.max(0, chosen.cutSeries));
|
|
4251
|
-
truncated = true;
|
|
4345
|
+
async runDriver() {
|
|
4346
|
+
while (!this.closed && this.queue.length > 0) {
|
|
4347
|
+
const ok = this.flushTransaction();
|
|
4348
|
+
if (!ok) return;
|
|
4349
|
+
if (this.queue.length > 0) await yieldToEventLoop2();
|
|
4350
|
+
}
|
|
4252
4351
|
}
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4352
|
+
/**
|
|
4353
|
+
* Commits the front of the queue as one transaction. Returns false when the
|
|
4354
|
+
* transaction failed; every batch in it has then been resolved as busy.
|
|
4355
|
+
*/
|
|
4356
|
+
flushTransaction() {
|
|
4357
|
+
if (this.queue.length === 0) return true;
|
|
4358
|
+
const selected = [];
|
|
4359
|
+
let recordCount = 0;
|
|
4360
|
+
for (const pending of this.queue) {
|
|
4361
|
+
const size = pending.batch.records.length;
|
|
4362
|
+
if (selected.length > 0 && recordCount + size > this.maxRecordsPerTransaction) break;
|
|
4363
|
+
selected.push(pending);
|
|
4364
|
+
recordCount += size;
|
|
4365
|
+
if (recordCount >= this.maxRecordsPerTransaction) break;
|
|
4366
|
+
}
|
|
4367
|
+
this.queue.splice(0, selected.length);
|
|
4368
|
+
if (this.maxPhysicalBytes !== null) {
|
|
4369
|
+
try {
|
|
4370
|
+
let physical = this.database.physicalLimit();
|
|
4371
|
+
let bytes = this.database.sizeBytes();
|
|
4372
|
+
const pageFull = physical !== null && physical.pageCount >= physical.maxPageCount && physical.freePages === 0;
|
|
4373
|
+
if (bytes >= this.maxPhysicalBytes || pageFull) {
|
|
4374
|
+
this.retention.relievePressure(4);
|
|
4375
|
+
physical = this.database.physicalLimit();
|
|
4376
|
+
bytes = this.database.sizeBytes();
|
|
4377
|
+
}
|
|
4378
|
+
const stillFull = physical !== null && physical.pageCount >= physical.maxPageCount && physical.freePages === 0;
|
|
4379
|
+
if (bytes >= this.maxPhysicalBytes || stillFull) {
|
|
4380
|
+
this.diskFullFailures += 1;
|
|
4381
|
+
this.lastWriteError = "storage disk budget reached";
|
|
4382
|
+
for (const pending of selected) {
|
|
4383
|
+
this.resolveBusy(pending, FAILURE_RETRY_MS, "storage disk budget reached");
|
|
4384
|
+
}
|
|
4385
|
+
return false;
|
|
4386
|
+
}
|
|
4387
|
+
} catch (error) {
|
|
4388
|
+
const full = isStorageFullError(error);
|
|
4389
|
+
if (full) this.diskFullFailures += 1;
|
|
4390
|
+
this.lastWriteError = full ? "storage disk budget reached" : sanitizeWriteError(error);
|
|
4391
|
+
for (const pending of selected) {
|
|
4392
|
+
this.resolveBusy(
|
|
4393
|
+
pending,
|
|
4394
|
+
FAILURE_RETRY_MS,
|
|
4395
|
+
full ? "storage disk budget reached" : "storage write failed"
|
|
4396
|
+
);
|
|
4397
|
+
}
|
|
4398
|
+
return false;
|
|
4399
|
+
}
|
|
4260
4400
|
}
|
|
4261
|
-
const
|
|
4262
|
-
const
|
|
4263
|
-
|
|
4264
|
-
const
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
}
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4401
|
+
const started = import_node_perf_hooks.performance.now();
|
|
4402
|
+
const counters = { committed: 0, duplicates: 0, rejected: 0 };
|
|
4403
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
4404
|
+
const upserts = /* @__PURE__ */ new Map();
|
|
4405
|
+
const summaries = /* @__PURE__ */ new Map();
|
|
4406
|
+
try {
|
|
4407
|
+
this.database.db.exec("BEGIN IMMEDIATE");
|
|
4408
|
+
for (const pending of selected) {
|
|
4409
|
+
this.processBatch(pending, counters, cursors, upserts, summaries);
|
|
4410
|
+
}
|
|
4411
|
+
if (this.storageTier === "summary" && summaries.size > 0) {
|
|
4412
|
+
this.writeSummaries(summaries);
|
|
4413
|
+
}
|
|
4414
|
+
for (const [machineId, live] of upserts) {
|
|
4415
|
+
this.latestUpsert.run(
|
|
4416
|
+
machineId,
|
|
4417
|
+
live.record.collectorId,
|
|
4418
|
+
live.record.runId,
|
|
4419
|
+
live.record.bootId,
|
|
4420
|
+
live.record.sequence,
|
|
4421
|
+
Date.parse(live.record.capturedAt),
|
|
4422
|
+
live.receivedAtMs,
|
|
4423
|
+
live.expectedIntervalMs,
|
|
4424
|
+
live.hostname,
|
|
4425
|
+
JSON.stringify(live.record.payload)
|
|
4426
|
+
);
|
|
4427
|
+
}
|
|
4428
|
+
this.database.db.exec("COMMIT");
|
|
4429
|
+
} catch (error) {
|
|
4430
|
+
try {
|
|
4431
|
+
this.database.db.exec("ROLLBACK");
|
|
4432
|
+
} catch {
|
|
4433
|
+
}
|
|
4434
|
+
this.writeFailures += 1;
|
|
4435
|
+
const full = isStorageFullError(error);
|
|
4436
|
+
if (full) this.diskFullFailures += 1;
|
|
4437
|
+
this.lastWriteError = full ? "storage disk budget reached" : sanitizeWriteError(error);
|
|
4438
|
+
for (const pending of selected) {
|
|
4439
|
+
this.resolveBusy(
|
|
4440
|
+
pending,
|
|
4441
|
+
FAILURE_RETRY_MS,
|
|
4442
|
+
full ? "storage disk budget reached" : "storage write failed"
|
|
4443
|
+
);
|
|
4444
|
+
}
|
|
4445
|
+
return false;
|
|
4446
|
+
}
|
|
4447
|
+
this.committedRecords += counters.committed;
|
|
4448
|
+
this.duplicateRecords += counters.duplicates;
|
|
4449
|
+
this.rejectedRecords += counters.rejected;
|
|
4450
|
+
this.lastFlushMs = this.now();
|
|
4451
|
+
this.lastFlushDurationMs = Math.max(0, Math.round(import_node_perf_hooks.performance.now() - started));
|
|
4452
|
+
this.checkpointWALIfNeeded();
|
|
4453
|
+
if (this.maxPhysicalBytes !== null && this.database.sizeBytes() > this.maxPhysicalBytes) {
|
|
4454
|
+
try {
|
|
4455
|
+
this.retention.relievePressure(4);
|
|
4456
|
+
} catch {
|
|
4457
|
+
}
|
|
4458
|
+
}
|
|
4459
|
+
for (const pending of selected) this.resolveCommitted(pending);
|
|
4460
|
+
return true;
|
|
4300
4461
|
}
|
|
4301
|
-
|
|
4302
|
-
if (
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4462
|
+
checkpointWALIfNeeded() {
|
|
4463
|
+
if (this.maxPhysicalBytes === null) return;
|
|
4464
|
+
try {
|
|
4465
|
+
const physical = this.database.physicalLimit();
|
|
4466
|
+
if (physical !== null && this.database.walBytes() > physical.walBudgetBytes) {
|
|
4467
|
+
this.database.checkpoint();
|
|
4468
|
+
}
|
|
4469
|
+
} catch {
|
|
4306
4470
|
}
|
|
4307
4471
|
}
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4472
|
+
processBatch(pending, counters, cursors, upserts, summaries) {
|
|
4473
|
+
const batch = pending.batch;
|
|
4474
|
+
let best = null;
|
|
4475
|
+
let bestCapturedMs = 0;
|
|
4476
|
+
let bestSequence = -1;
|
|
4477
|
+
for (const record of batch.records) {
|
|
4478
|
+
const ref = { runId: record.runId, sequence: record.sequence };
|
|
4479
|
+
const reason = rejectionReason(record, batch, pending.receivedAtMs);
|
|
4480
|
+
if (reason !== null) {
|
|
4481
|
+
pending.rejected.push({ ...ref, reason });
|
|
4482
|
+
counters.rejected += 1;
|
|
4483
|
+
continue;
|
|
4484
|
+
}
|
|
4485
|
+
const capturedMs = Date.parse(record.capturedAt);
|
|
4486
|
+
const inserted = Number(
|
|
4487
|
+
this.receiptInsert.run(
|
|
4488
|
+
batch.collectorId,
|
|
4489
|
+
record.runId,
|
|
4490
|
+
record.sequence,
|
|
4491
|
+
batch.machineId,
|
|
4492
|
+
record.kind,
|
|
4493
|
+
batch.mode,
|
|
4494
|
+
capturedMs,
|
|
4495
|
+
pending.receivedAtMs,
|
|
4496
|
+
batch.expectedIntervalMs
|
|
4497
|
+
).changes
|
|
4498
|
+
) === 1;
|
|
4499
|
+
if (!inserted) {
|
|
4500
|
+
pending.accepted.push(ref);
|
|
4501
|
+
counters.duplicates += 1;
|
|
4502
|
+
continue;
|
|
4503
|
+
}
|
|
4504
|
+
pending.accepted.push(ref);
|
|
4505
|
+
counters.committed += 1;
|
|
4506
|
+
if (record.kind === "metrics") {
|
|
4507
|
+
const known = needsKnownSeries(record.payload) ? this.series.knownFor(batch.machineId) : void 0;
|
|
4508
|
+
for (const point of flattenMetrics(record.payload, known)) {
|
|
4509
|
+
const seriesId = this.series.idFor(batch.machineId, point.metric, point.resourceId);
|
|
4510
|
+
if (this.storageTier === "summary") {
|
|
4511
|
+
addSummaryPoint(summaries, seriesId, capturedMs, point.value);
|
|
4512
|
+
} else {
|
|
4513
|
+
this.insertPoint(seriesId, capturedMs, point.value);
|
|
4514
|
+
}
|
|
4515
|
+
}
|
|
4516
|
+
if (batch.mode === "live") {
|
|
4517
|
+
if (best === null || capturedMs > bestCapturedMs || capturedMs === bestCapturedMs && record.sequence > bestSequence) {
|
|
4518
|
+
best = record;
|
|
4519
|
+
bestCapturedMs = capturedMs;
|
|
4520
|
+
bestSequence = record.sequence;
|
|
4521
|
+
}
|
|
4522
|
+
}
|
|
4523
|
+
} else {
|
|
4524
|
+
this.gapInsert.run(
|
|
4525
|
+
batch.collectorId,
|
|
4526
|
+
record.runId,
|
|
4527
|
+
record.sequence,
|
|
4528
|
+
batch.machineId,
|
|
4529
|
+
Date.parse(record.payload.fromCapturedAt),
|
|
4530
|
+
Date.parse(record.payload.toCapturedAt),
|
|
4531
|
+
record.payload.droppedRecords,
|
|
4532
|
+
record.payload.reason,
|
|
4533
|
+
pending.receivedAtMs
|
|
4534
|
+
);
|
|
4535
|
+
}
|
|
4328
4536
|
}
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
)
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
};
|
|
4537
|
+
if (batch.mode !== "live" || best === null) return;
|
|
4538
|
+
const cursor = this.latestCursor(batch.machineId, cursors);
|
|
4539
|
+
const candidate = {
|
|
4540
|
+
runId: best.runId,
|
|
4541
|
+
sequence: best.sequence,
|
|
4542
|
+
capturedAtMs: bestCapturedMs
|
|
4543
|
+
};
|
|
4544
|
+
if (cursor === null || isNewer(candidate, cursor)) {
|
|
4545
|
+
pending.newLive = best;
|
|
4546
|
+
cursors.set(batch.machineId, candidate);
|
|
4547
|
+
upserts.set(batch.machineId, {
|
|
4548
|
+
record: best,
|
|
4549
|
+
receivedAtMs: pending.receivedAtMs,
|
|
4550
|
+
expectedIntervalMs: batch.expectedIntervalMs,
|
|
4551
|
+
hostname: batch.hostname ?? null
|
|
4552
|
+
});
|
|
4345
4553
|
}
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4554
|
+
}
|
|
4555
|
+
latestCursor(machineId, cache) {
|
|
4556
|
+
if (cache.has(machineId)) return cache.get(machineId);
|
|
4557
|
+
const row = this.latestSelect.get(machineId);
|
|
4558
|
+
const cursor = row ? { runId: row.run_id, sequence: Number(row.seq), capturedAtMs: Number(row.captured_at_ms) } : null;
|
|
4559
|
+
cache.set(machineId, cursor);
|
|
4560
|
+
return cursor;
|
|
4561
|
+
}
|
|
4562
|
+
/**
|
|
4563
|
+
* Writes one compact ten-second summary per series/bucket and folds the same
|
|
4564
|
+
* aggregate into both larger rollups. The ten-second row is upserted because
|
|
4565
|
+
* late or replayed samples may extend a bucket across flushes; the one-minute
|
|
4566
|
+
* and one-hour rows only receive the new delta, so a sample is counted once.
|
|
4567
|
+
*/
|
|
4568
|
+
writeSummaries(summaries) {
|
|
4569
|
+
for (const [seriesId, buckets] of summaries) {
|
|
4570
|
+
for (const [bucketMs, aggregate] of buckets) {
|
|
4571
|
+
this.upsert10s.run(
|
|
4572
|
+
seriesId,
|
|
4573
|
+
bucketMs,
|
|
4574
|
+
aggregate.min,
|
|
4575
|
+
aggregate.max,
|
|
4576
|
+
aggregate.sum,
|
|
4577
|
+
aggregate.count,
|
|
4578
|
+
aggregate.nullCount
|
|
4579
|
+
);
|
|
4580
|
+
this.applyRollupAggregate(
|
|
4581
|
+
this.upsert1m,
|
|
4582
|
+
seriesId,
|
|
4583
|
+
Math.floor(bucketMs / 6e4) * 6e4,
|
|
4584
|
+
aggregate
|
|
4585
|
+
);
|
|
4586
|
+
this.applyRollupAggregate(
|
|
4587
|
+
this.upsert1h,
|
|
4588
|
+
seriesId,
|
|
4589
|
+
Math.floor(bucketMs / 36e5) * 36e5,
|
|
4590
|
+
aggregate
|
|
4591
|
+
);
|
|
4592
|
+
}
|
|
4349
4593
|
}
|
|
4350
4594
|
}
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4595
|
+
/**
|
|
4596
|
+
* Writes one raw point and incrementally updates both rollup tiers. The raw
|
|
4597
|
+
* INSERT OR IGNORE guard prevents a collision (same series/timestamp) from
|
|
4598
|
+
* double-counting a rollup.
|
|
4599
|
+
*/
|
|
4600
|
+
insertPoint(seriesId, tsMs, value) {
|
|
4601
|
+
if (Number(this.rawInsert.run(seriesId, tsMs, value).changes) !== 1) return;
|
|
4602
|
+
const minute = Math.floor(tsMs / 6e4) * 6e4;
|
|
4603
|
+
const hour = Math.floor(tsMs / 36e5) * 36e5;
|
|
4604
|
+
this.applyRollup(this.upsert1m, seriesId, minute, value);
|
|
4605
|
+
this.applyRollup(this.upsert1h, seriesId, hour, value);
|
|
4606
|
+
}
|
|
4607
|
+
applyRollup(statement, seriesId, bucketMs, value) {
|
|
4608
|
+
if (value === null) {
|
|
4609
|
+
statement.run(seriesId, bucketMs, null, null, 0, 0, 1);
|
|
4610
|
+
} else {
|
|
4611
|
+
statement.run(seriesId, bucketMs, value, value, value, 1, 0);
|
|
4359
4612
|
}
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4613
|
+
}
|
|
4614
|
+
applyRollupAggregate(statement, seriesId, bucketMs, aggregate) {
|
|
4615
|
+
statement.run(
|
|
4616
|
+
seriesId,
|
|
4617
|
+
bucketMs,
|
|
4618
|
+
aggregate.min,
|
|
4619
|
+
aggregate.max,
|
|
4620
|
+
aggregate.sum,
|
|
4621
|
+
aggregate.count,
|
|
4622
|
+
aggregate.nullCount
|
|
4367
4623
|
);
|
|
4368
|
-
if (rows <= maxRowsScanned) {
|
|
4369
|
-
fits = true;
|
|
4370
|
-
break;
|
|
4371
|
-
}
|
|
4372
|
-
if (cut === 1) break;
|
|
4373
|
-
cut = Math.max(1, Math.floor(cut / 2));
|
|
4374
4624
|
}
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
const column = tier === "raw" ? "ts_ms" : "bucket_ms";
|
|
4386
|
-
const table = tier === "raw" ? "points_raw" : `points_${tier}`;
|
|
4387
|
-
const statement = db.db.prepare(
|
|
4388
|
-
`SELECT COUNT(*) AS n FROM (SELECT 1 FROM ${table} WHERE series_id = ? AND ${column} >= ? AND ${column} <= ? LIMIT ?)`
|
|
4389
|
-
);
|
|
4390
|
-
let total = 0;
|
|
4391
|
-
for (const seriesId of seriesIds) {
|
|
4392
|
-
const remaining = budget + 1 - total;
|
|
4393
|
-
if (remaining <= 0) return total;
|
|
4394
|
-
const row = statement.get(seriesId, fromMs, toMs, remaining);
|
|
4395
|
-
total += Number(row?.n ?? 0);
|
|
4396
|
-
if (total > budget) return total;
|
|
4625
|
+
resolveCommitted(pending) {
|
|
4626
|
+
if (pending.settled) return;
|
|
4627
|
+
pending.settled = true;
|
|
4628
|
+
pending.resolve({
|
|
4629
|
+
status: "committed",
|
|
4630
|
+
accepted: pending.accepted,
|
|
4631
|
+
rejected: pending.rejected,
|
|
4632
|
+
newLive: pending.newLive,
|
|
4633
|
+
receivedAtMs: pending.receivedAtMs
|
|
4634
|
+
});
|
|
4397
4635
|
}
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
originMs,
|
|
4408
|
-
originMs,
|
|
4409
|
-
stepMs,
|
|
4410
|
-
stepMs,
|
|
4411
|
-
seriesId,
|
|
4412
|
-
fromBucketMs2,
|
|
4413
|
-
toMs,
|
|
4414
|
-
rowLimit
|
|
4636
|
+
resolveBusy(pending, retryAfterMs, reason) {
|
|
4637
|
+
if (pending.settled) return;
|
|
4638
|
+
pending.settled = true;
|
|
4639
|
+
pending.resolve({ status: "busy", retryAfterMs, reason });
|
|
4640
|
+
}
|
|
4641
|
+
};
|
|
4642
|
+
function prepareRollupUpsert(db, table) {
|
|
4643
|
+
return db.prepare(
|
|
4644
|
+
`INSERT INTO ${table} (series_id, bucket_ms, min, max, sum, count, null_count) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(series_id, bucket_ms) DO UPDATE SET min = CASE WHEN excluded.count = 0 THEN min WHEN min IS NULL THEN excluded.min ELSE MIN(min, excluded.min) END, max = CASE WHEN excluded.count = 0 THEN max WHEN max IS NULL THEN excluded.max ELSE MAX(max, excluded.max) END, sum = sum + excluded.sum, count = count + excluded.count, null_count = null_count + excluded.null_count`
|
|
4415
4645
|
);
|
|
4416
|
-
return rows.map((row) => {
|
|
4417
|
-
const count = Number(row.count_value ?? 0);
|
|
4418
|
-
return {
|
|
4419
|
-
t: Number(row.bucket),
|
|
4420
|
-
min: row.min_value === null ? null : Number(row.min_value),
|
|
4421
|
-
max: row.max_value === null ? null : Number(row.max_value),
|
|
4422
|
-
avg: count > 0 && row.sum_value !== null ? Number(row.sum_value) / count : null
|
|
4423
|
-
};
|
|
4424
|
-
});
|
|
4425
4646
|
}
|
|
4426
|
-
function
|
|
4427
|
-
if (
|
|
4428
|
-
|
|
4429
|
-
return Math.floor(fromMs / span) * span;
|
|
4647
|
+
function isNewer(candidate, latest) {
|
|
4648
|
+
if (candidate.runId === latest.runId) return candidate.sequence > latest.sequence;
|
|
4649
|
+
return candidate.capturedAtMs > latest.capturedAtMs;
|
|
4430
4650
|
}
|
|
4431
|
-
function
|
|
4432
|
-
|
|
4433
|
-
"SELECT from_ms, to_ms, dropped, reason FROM gaps WHERE machine_id = ? AND to_ms >= ? AND from_ms <= ? ORDER BY from_ms LIMIT ?"
|
|
4434
|
-
).all(machineId, fromMs, toMs, MAX_GAPS);
|
|
4435
|
-
const missing = db.db.prepare(
|
|
4436
|
-
"WITH before_window AS ( SELECT captured_at_ms, expected_interval_ms FROM sample_receipts WHERE machine_id = ? AND captured_at_ms < ? ORDER BY captured_at_ms DESC LIMIT 1 ), in_window AS ( SELECT captured_at_ms, expected_interval_ms FROM sample_receipts WHERE machine_id = ? AND captured_at_ms >= ? AND captured_at_ms <= ? ), ordered AS ( SELECT captured_at_ms, expected_interval_ms, LAG(captured_at_ms) OVER (ORDER BY captured_at_ms) AS prev_ms FROM (SELECT * FROM before_window UNION ALL SELECT * FROM in_window) ) SELECT MAX(prev_ms, ?) AS from_ms, captured_at_ms AS to_ms FROM ordered WHERE prev_ms IS NOT NULL AND captured_at_ms > ? AND captured_at_ms - prev_ms > MAX(15000, 3 * COALESCE(expected_interval_ms, 5000)) ORDER BY captured_at_ms LIMIT ?"
|
|
4437
|
-
).all(machineId, fromMs, machineId, fromMs, toMs, fromMs, fromMs, MAX_GAPS);
|
|
4438
|
-
const gaps = [
|
|
4439
|
-
...dropped.map((row) => ({
|
|
4440
|
-
fromMs: Number(row.from_ms),
|
|
4441
|
-
toMs: Number(row.to_ms),
|
|
4442
|
-
kind: "dropped",
|
|
4443
|
-
reason: row.reason,
|
|
4444
|
-
droppedRecords: Number(row.dropped)
|
|
4445
|
-
})),
|
|
4446
|
-
...missing.map((row) => ({
|
|
4447
|
-
fromMs: Number(row.from_ms),
|
|
4448
|
-
toMs: Number(row.to_ms),
|
|
4449
|
-
kind: "missing",
|
|
4450
|
-
reason: "expected interval missed"
|
|
4451
|
-
}))
|
|
4452
|
-
];
|
|
4453
|
-
gaps.sort((a, b) => a.fromMs - b.fromMs || a.toMs - b.toMs);
|
|
4454
|
-
return gaps.slice(0, MAX_GAPS);
|
|
4651
|
+
function needsKnownSeries(payload) {
|
|
4652
|
+
return payload.cpu.status === "unavailable" || payload.gpus.status === "unavailable" || payload.disks.status === "unavailable" || payload.network.status === "unavailable";
|
|
4455
4653
|
}
|
|
4456
|
-
function
|
|
4457
|
-
if (
|
|
4458
|
-
|
|
4459
|
-
throw new HistoryQueryError("machineId must be a nonempty string");
|
|
4654
|
+
function rejectionReason(record, batch, receivedAtMs) {
|
|
4655
|
+
if (record.collectorId !== batch.collectorId) {
|
|
4656
|
+
return "collectorId does not match the authenticated batch";
|
|
4460
4657
|
}
|
|
4461
|
-
|
|
4462
|
-
|
|
4658
|
+
const capturedMs = Date.parse(record.capturedAt);
|
|
4659
|
+
if (!Number.isFinite(capturedMs)) return "capturedAt is not a parseable timestamp";
|
|
4660
|
+
if (capturedMs > receivedAtMs + FUTURE_TOLERANCE_MS) {
|
|
4661
|
+
return "capturedAt is more than 24h in the future";
|
|
4463
4662
|
}
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
seenMetrics.add(metric);
|
|
4663
|
+
return null;
|
|
4664
|
+
}
|
|
4665
|
+
function validateBatch(batch) {
|
|
4666
|
+
if (batch === null || typeof batch !== "object") throw new Error("ingest batch must be an object");
|
|
4667
|
+
if (typeof batch.machineId !== "string" || batch.machineId.length === 0) {
|
|
4668
|
+
throw new Error("ingest batch machineId must be a nonempty string");
|
|
4471
4669
|
}
|
|
4472
|
-
if (
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4670
|
+
if (typeof batch.collectorId !== "string" || batch.collectorId.length === 0) {
|
|
4671
|
+
throw new Error("ingest batch collectorId must be a nonempty string");
|
|
4672
|
+
}
|
|
4673
|
+
if (batch.mode !== "live" && batch.mode !== "replay") {
|
|
4674
|
+
throw new Error("ingest batch mode must be live or replay");
|
|
4675
|
+
}
|
|
4676
|
+
if (!Array.isArray(batch.records)) throw new Error("ingest batch records must be an array");
|
|
4677
|
+
if (batch.expectedIntervalMs !== null && !Number.isSafeInteger(batch.expectedIntervalMs)) {
|
|
4678
|
+
throw new Error("ingest batch expectedIntervalMs must be a safe integer or null");
|
|
4679
|
+
}
|
|
4680
|
+
if (batch.hostname !== void 0 && batch.hostname !== null && (typeof batch.hostname !== "string" || batch.hostname.length > 255)) {
|
|
4681
|
+
throw new Error("ingest batch hostname must be a string of at most 255 characters or null");
|
|
4682
|
+
}
|
|
4683
|
+
}
|
|
4684
|
+
function validateTags(tags) {
|
|
4685
|
+
if (!Array.isArray(tags) || tags.length > 64) throw new Error("tags must be an array of at most 64 strings");
|
|
4686
|
+
const result = [];
|
|
4687
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4688
|
+
for (const tag of tags) {
|
|
4689
|
+
if (typeof tag !== "string" || tag.length === 0 || tag.length > 64) {
|
|
4690
|
+
throw new Error("tags must be nonempty strings of at most 64 characters");
|
|
4483
4691
|
}
|
|
4692
|
+
if (seen.has(tag)) throw new Error("tags must be unique");
|
|
4693
|
+
seen.add(tag);
|
|
4694
|
+
result.push(tag);
|
|
4484
4695
|
}
|
|
4485
|
-
|
|
4486
|
-
|
|
4696
|
+
return result;
|
|
4697
|
+
}
|
|
4698
|
+
function parseTags(raw) {
|
|
4699
|
+
try {
|
|
4700
|
+
const parsed = JSON.parse(raw);
|
|
4701
|
+
if (!Array.isArray(parsed)) return [];
|
|
4702
|
+
return parsed.filter((tag) => typeof tag === "string");
|
|
4703
|
+
} catch {
|
|
4704
|
+
return [];
|
|
4487
4705
|
}
|
|
4488
|
-
|
|
4489
|
-
|
|
4706
|
+
}
|
|
4707
|
+
function sanitizeWriteError(error) {
|
|
4708
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4709
|
+
return message.replace(/\s+/g, " ").slice(0, 200);
|
|
4710
|
+
}
|
|
4711
|
+
function positiveInt2(value, fallback, name) {
|
|
4712
|
+
const resolved = value ?? fallback;
|
|
4713
|
+
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
|
|
4714
|
+
throw new Error(`${name} must be a positive safe integer`);
|
|
4490
4715
|
}
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4716
|
+
return resolved;
|
|
4717
|
+
}
|
|
4718
|
+
function resolvePhysicalLimit(options) {
|
|
4719
|
+
if (options.maxPhysicalBytes === null) return null;
|
|
4720
|
+
if (options.maxPhysicalBytes !== void 0) {
|
|
4721
|
+
if (!Number.isSafeInteger(options.maxPhysicalBytes) || options.maxPhysicalBytes < 16 * 1024) {
|
|
4722
|
+
throw new Error("maxPhysicalBytes must be a safe integer of at least 16 KiB or null");
|
|
4723
|
+
}
|
|
4724
|
+
return options.maxPhysicalBytes;
|
|
4494
4725
|
}
|
|
4726
|
+
if (options.maxDatabaseBytes !== null && options.maxDatabaseBytes !== void 0 && options.maxDatabaseBytes >= MIN_DERIVED_PHYSICAL_BYTES) {
|
|
4727
|
+
return options.maxDatabaseBytes;
|
|
4728
|
+
}
|
|
4729
|
+
return DEFAULT_MAX_DATABASE_DISK_BYTES;
|
|
4495
4730
|
}
|
|
4496
|
-
function
|
|
4497
|
-
|
|
4498
|
-
if (!Number.isSafeInteger(
|
|
4499
|
-
throw new
|
|
4731
|
+
function nonnegativeInt(value, fallback, name) {
|
|
4732
|
+
const resolved = value ?? fallback;
|
|
4733
|
+
if (!Number.isSafeInteger(resolved) || resolved < 0) {
|
|
4734
|
+
throw new Error(`${name} must be a nonnegative safe integer`);
|
|
4500
4735
|
}
|
|
4501
|
-
return
|
|
4736
|
+
return resolved;
|
|
4502
4737
|
}
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4738
|
+
var SUMMARY_STEP_MS = 1e4;
|
|
4739
|
+
function addSummaryPoint(summaries, seriesId, tsMs, value) {
|
|
4740
|
+
const bucketMs = Math.floor(tsMs / SUMMARY_STEP_MS) * SUMMARY_STEP_MS;
|
|
4741
|
+
let buckets = summaries.get(seriesId);
|
|
4742
|
+
if (buckets === void 0) {
|
|
4743
|
+
buckets = /* @__PURE__ */ new Map();
|
|
4744
|
+
summaries.set(seriesId, buckets);
|
|
4745
|
+
}
|
|
4746
|
+
let aggregate = buckets.get(bucketMs);
|
|
4747
|
+
if (aggregate === void 0) {
|
|
4748
|
+
aggregate = { min: null, max: null, sum: 0, count: 0, nullCount: 0 };
|
|
4749
|
+
buckets.set(bucketMs, aggregate);
|
|
4750
|
+
}
|
|
4751
|
+
if (value === null) {
|
|
4752
|
+
aggregate.nullCount += 1;
|
|
4753
|
+
} else {
|
|
4754
|
+
aggregate.min = aggregate.min === null ? value : Math.min(aggregate.min, value);
|
|
4755
|
+
aggregate.max = aggregate.max === null ? value : Math.max(aggregate.max, value);
|
|
4756
|
+
aggregate.sum += value;
|
|
4757
|
+
aggregate.count += 1;
|
|
4507
4758
|
}
|
|
4508
|
-
|
|
4759
|
+
}
|
|
4760
|
+
function yieldToEventLoop2() {
|
|
4761
|
+
return new Promise((resolve7) => setImmediate(resolve7));
|
|
4509
4762
|
}
|
|
4510
4763
|
|
|
4511
4764
|
// src/operations/measure.ts
|
|
@@ -4646,7 +4899,7 @@ async function measure(options = {}) {
|
|
|
4646
4899
|
const historyWallStart = import_node_perf_hooks2.performance.now();
|
|
4647
4900
|
const historyHolder = { value: null };
|
|
4648
4901
|
const runHistoryQuery = async () => {
|
|
4649
|
-
await new Promise((
|
|
4902
|
+
await new Promise((resolve7) => setImmediate(resolve7));
|
|
4650
4903
|
const historyStart = import_node_perf_hooks2.performance.now();
|
|
4651
4904
|
const result = queryHistory(
|
|
4652
4905
|
database,
|
|
@@ -4765,6 +5018,7 @@ Usage:
|
|
|
4765
5018
|
[--database PATH] [--activate --confirm-stopped] [--install-deps] [--json]
|
|
4766
5019
|
ops activate --prefix DIR --state-dir DIR --version V [--confirm-stopped] [--json]
|
|
4767
5020
|
ops rollback --prefix DIR --state-dir DIR --confirm-stopped [--json]
|
|
5021
|
+
ops reset-telemetry --database PATH [--backup-dir DIR] --confirm-stopped [--json]
|
|
4768
5022
|
ops recover-admin --database PATH --password-file PATH [--user NAME] [--enable] [--json]
|
|
4769
5023
|
ops measure [--machines N] [--batches N] [--records-per-batch N] [--gpus N]
|
|
4770
5024
|
[--viewers N] [--history-concurrency N] [--history-points N] [--mode live|replay|mixed]
|
|
@@ -5051,6 +5305,42 @@ async function run() {
|
|
|
5051
5305
|
}
|
|
5052
5306
|
return 0;
|
|
5053
5307
|
}
|
|
5308
|
+
if (command === "reset-telemetry") {
|
|
5309
|
+
const parsed = (0, import_node_util.parseArgs)({
|
|
5310
|
+
args: options,
|
|
5311
|
+
options: {
|
|
5312
|
+
database: { type: "string" },
|
|
5313
|
+
"backup-dir": { type: "string" },
|
|
5314
|
+
"confirm-stopped": { type: "boolean" },
|
|
5315
|
+
json: { type: "boolean" }
|
|
5316
|
+
},
|
|
5317
|
+
allowPositionals: false
|
|
5318
|
+
});
|
|
5319
|
+
const database = optionString(parsed.values, "database", true);
|
|
5320
|
+
const backupDir = optionString(parsed.values, "backup-dir");
|
|
5321
|
+
let backupPath = null;
|
|
5322
|
+
if (backupDir !== void 0) {
|
|
5323
|
+
const report2 = await backupDatabase({
|
|
5324
|
+
database,
|
|
5325
|
+
output: (0, import_node_path8.resolve)(backupDir, `telemetry-pre-reset-${Date.now()}.sqlite`)
|
|
5326
|
+
});
|
|
5327
|
+
backupPath = report2.output;
|
|
5328
|
+
}
|
|
5329
|
+
const report = resetTelemetryHistory({
|
|
5330
|
+
database,
|
|
5331
|
+
confirmStopped: optionBoolean(parsed.values, "confirm-stopped")
|
|
5332
|
+
});
|
|
5333
|
+
if (optionBoolean(parsed.values, "json")) {
|
|
5334
|
+
console.log(JSON.stringify({ ...report, backupPath }, null, 2));
|
|
5335
|
+
} else {
|
|
5336
|
+
console.log(`telemetry history reset: ${report.database}`);
|
|
5337
|
+
console.log(`deleted rows: ${Object.entries(report.deleted).map(([table, rows]) => `${table}=${rows}`).join(", ")}`);
|
|
5338
|
+
console.log(`preserved: ${report.preserved.join(", ")}`);
|
|
5339
|
+
console.log(`integrity: ${report.integrity}`);
|
|
5340
|
+
if (backupPath !== null) console.log(`pre-reset backup: ${backupPath}`);
|
|
5341
|
+
}
|
|
5342
|
+
return 0;
|
|
5343
|
+
}
|
|
5054
5344
|
if (command === "recover-admin") {
|
|
5055
5345
|
const parsed = (0, import_node_util.parseArgs)({
|
|
5056
5346
|
args: options,
|