rigscope 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.next/BUILD_ID +1 -1
- package/.next/app-path-routes-manifest.json +4 -4
- package/.next/build-manifest.json +2 -2
- package/.next/server/app/_global-error/page_client-reference-manifest.js +1 -1
- package/.next/server/app/_global-error.html +1 -1
- package/.next/server/app/_global-error.rsc +1 -1
- package/.next/server/app/_global-error.segments/_full.segment.rsc +1 -1
- package/.next/server/app/_global-error.segments/_global-error/__PAGE__.segment.rsc +1 -1
- package/.next/server/app/_global-error.segments/_tree.segment.rsc +1 -1
- package/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/.next/server/app/_not-found.html +1 -1
- package/.next/server/app/_not-found.rsc +2 -2
- package/.next/server/app/_not-found.segments/_full.segment.rsc +2 -2
- package/.next/server/app/_not-found.segments/_not-found/__PAGE__.segment.rsc +2 -2
- package/.next/server/app/_not-found.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/administration/page_client-reference-manifest.js +1 -1
- package/.next/server/app/administration.html +1 -1
- package/.next/server/app/administration.rsc +2 -2
- package/.next/server/app/administration.segments/_full.segment.rsc +2 -2
- package/.next/server/app/administration.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/administration.segments/administration/__PAGE__.segment.rsc +2 -2
- package/.next/server/app/alerts/page_client-reference-manifest.js +1 -1
- package/.next/server/app/alerts.html +1 -1
- package/.next/server/app/alerts.rsc +2 -2
- package/.next/server/app/alerts.segments/_full.segment.rsc +2 -2
- package/.next/server/app/alerts.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/alerts.segments/alerts/__PAGE__.segment.rsc +2 -2
- package/.next/server/app/connections/page_client-reference-manifest.js +1 -1
- package/.next/server/app/connections.html +1 -1
- package/.next/server/app/connections.rsc +2 -2
- package/.next/server/app/connections.segments/_full.segment.rsc +2 -2
- package/.next/server/app/connections.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/connections.segments/connections/__PAGE__.segment.rsc +2 -2
- package/.next/server/app/history/page.js +2 -2
- package/.next/server/app/history/page_client-reference-manifest.js +1 -1
- package/.next/server/app/history.html +1 -1
- package/.next/server/app/history.rsc +5 -5
- package/.next/server/app/history.segments/_full.segment.rsc +5 -5
- package/.next/server/app/history.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/history.segments/history/__PAGE__.segment.rsc +5 -5
- package/.next/server/app/index.html +1 -1
- package/.next/server/app/index.rsc +2 -2
- package/.next/server/app/index.segments/__PAGE__.segment.rsc +2 -2
- package/.next/server/app/index.segments/_full.segment.rsc +2 -2
- package/.next/server/app/index.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/login/page_client-reference-manifest.js +1 -1
- package/.next/server/app/login.html +1 -1
- package/.next/server/app/login.rsc +2 -2
- package/.next/server/app/login.segments/_full.segment.rsc +2 -2
- package/.next/server/app/login.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/login.segments/login/__PAGE__.segment.rsc +2 -2
- package/.next/server/app/machines/[machineId]/page_client-reference-manifest.js +1 -1
- package/.next/server/app/page_client-reference-manifest.js +1 -1
- package/.next/server/app/processes/page_client-reference-manifest.js +1 -1
- package/.next/server/app/processes.html +1 -1
- package/.next/server/app/processes.rsc +2 -2
- package/.next/server/app/processes.segments/_full.segment.rsc +2 -2
- package/.next/server/app/processes.segments/_tree.segment.rsc +2 -2
- package/.next/server/app/processes.segments/processes/__PAGE__.segment.rsc +2 -2
- package/.next/server/app-paths-manifest.json +4 -4
- package/.next/server/chunks/168.js +1 -1
- package/.next/server/middleware-build-manifest.js +1 -1
- package/.next/server/pages/404.html +1 -1
- package/.next/server/pages/500.html +1 -1
- package/.next/static/chunks/408-351fe4a1e9a2d556.js +1 -0
- package/.next/static/chunks/app/history/page-adc25194b30c689d.js +3 -0
- package/.next/static/css/{f5d3162842dcac31.css → 2abef6d49c1c2970.css} +1 -1
- package/ARCHITECTURE.md +25 -9
- package/CHANGELOG.md +35 -0
- package/README.md +10 -3
- package/dist/cli.cjs +106 -2
- package/dist/collector.cjs +23 -6
- package/dist/hub.cjs +3441 -1457
- package/dist/ops.cjs +1196 -896
- package/dist/system.cjs +65 -0
- package/docs/configuration.md +33 -1
- package/docs/installation.md +3 -3
- package/docs/monitoring.md +57 -17
- package/docs/operations.md +38 -9
- package/docs/releasing.md +12 -12
- package/docs/telemetry-storage.md +187 -0
- package/package.json +3 -1
- package/.next/static/chunks/408-9fdba9e4e956ef08.js +0 -1
- package/.next/static/chunks/app/history/page-a25ba8fab7bd7cb9.js +0 -3
- /package/.next/static/{9SHY6v87ulfunqR2e84n9 → Jlbibe2_iFCljUO4-BUK7}/_buildManifest.js +0 -0
- /package/.next/static/{9SHY6v87ulfunqR2e84n9 → Jlbibe2_iFCljUO4-BUK7}/_ssgManifest.js +0 -0
package/dist/ops.cjs
CHANGED
|
@@ -35,6 +35,7 @@ __export(main_exports, {
|
|
|
35
35
|
});
|
|
36
36
|
module.exports = __toCommonJS(main_exports);
|
|
37
37
|
var import_node_util = require("node:util");
|
|
38
|
+
var import_node_path8 = require("node:path");
|
|
38
39
|
|
|
39
40
|
// src/operations/database.ts
|
|
40
41
|
var import_node_fs3 = require("node:fs");
|
|
@@ -50,8 +51,17 @@ var import_node_path = require("node:path");
|
|
|
50
51
|
var import_node_sqlite = require("node:sqlite");
|
|
51
52
|
var sqlite = __toESM(require("node:sqlite"));
|
|
52
53
|
|
|
54
|
+
// src/contracts/telemetry.ts
|
|
55
|
+
var MAX_CORES = 1024;
|
|
56
|
+
var MIN_EPOCH = Date.parse("2000-01-01T00:00:00Z");
|
|
57
|
+
var MAX_EPOCH = Date.parse("2200-01-01T00:00:00Z");
|
|
58
|
+
|
|
53
59
|
// src/hub/config.ts
|
|
54
60
|
var MAX_HUB_COLLECTORS_BYTES = 1024 * 1024;
|
|
61
|
+
var DEFAULT_MEMORY_BUDGET_BYTES = 128 * 1024 * 1024;
|
|
62
|
+
var DEFAULT_RECORDING_MAX_DURATION_MS = 60 * 6e4;
|
|
63
|
+
var DEFAULT_RECORDING_MAX_BYTES = 64 * 1024 * 1024;
|
|
64
|
+
var DEFAULT_RECORDING_BACKLOG_MS = 10 * 6e4;
|
|
55
65
|
|
|
56
66
|
// src/hub/auth/password.ts
|
|
57
67
|
var import_node_crypto = require("node:crypto");
|
|
@@ -418,8 +428,45 @@ var MIGRATIONS = [
|
|
|
418
428
|
},
|
|
419
429
|
{ version: 3, name: "admin", up: adminSchema },
|
|
420
430
|
{ version: 4, name: "alerts", up: installAlertSchema },
|
|
421
|
-
{ version: 5, name: "collector_registry", up: installRegistrySchema }
|
|
431
|
+
{ version: 5, name: "collector_registry", up: installRegistrySchema },
|
|
432
|
+
{ version: 6, name: "telemetry_modes_and_recordings", up: telemetryModesAndRecordings }
|
|
422
433
|
];
|
|
434
|
+
function telemetryModesAndRecordings(db) {
|
|
435
|
+
db.exec(`
|
|
436
|
+
CREATE TABLE IF NOT EXISTS points_10s (
|
|
437
|
+
series_id INTEGER NOT NULL,
|
|
438
|
+
bucket_ms INTEGER NOT NULL,
|
|
439
|
+
min REAL,
|
|
440
|
+
max REAL,
|
|
441
|
+
sum REAL NOT NULL DEFAULT 0,
|
|
442
|
+
count INTEGER NOT NULL DEFAULT 0,
|
|
443
|
+
null_count INTEGER NOT NULL DEFAULT 0,
|
|
444
|
+
PRIMARY KEY (series_id, bucket_ms)
|
|
445
|
+
) WITHOUT ROWID;
|
|
446
|
+
|
|
447
|
+
CREATE TABLE IF NOT EXISTS recordings (
|
|
448
|
+
id TEXT PRIMARY KEY,
|
|
449
|
+
created_by TEXT NOT NULL,
|
|
450
|
+
status TEXT NOT NULL,
|
|
451
|
+
machines_json TEXT NOT NULL,
|
|
452
|
+
metrics_json TEXT NOT NULL,
|
|
453
|
+
resolution_ms INTEGER NOT NULL,
|
|
454
|
+
created_at_ms INTEGER NOT NULL,
|
|
455
|
+
started_at_ms INTEGER NOT NULL,
|
|
456
|
+
stopped_at_ms INTEGER,
|
|
457
|
+
deadline_ms INTEGER NOT NULL,
|
|
458
|
+
max_bytes INTEGER NOT NULL,
|
|
459
|
+
bytes_written INTEGER NOT NULL DEFAULT 0,
|
|
460
|
+
file_name TEXT NOT NULL,
|
|
461
|
+
include_backlog INTEGER NOT NULL DEFAULT 0,
|
|
462
|
+
reason TEXT,
|
|
463
|
+
error TEXT
|
|
464
|
+
);
|
|
465
|
+
|
|
466
|
+
CREATE INDEX IF NOT EXISTS recordings_created
|
|
467
|
+
ON recordings (created_at_ms DESC);
|
|
468
|
+
`);
|
|
469
|
+
}
|
|
423
470
|
|
|
424
471
|
// src/hub/storage/db.ts
|
|
425
472
|
var BUSY_TIMEOUT_MS = 5e3;
|
|
@@ -496,7 +543,7 @@ var SqliteHubDatabase = class {
|
|
|
496
543
|
const walBudgetBytes = Math.max(64 * 1024, Math.min(HUB_WAL_RESERVE_BYTES, Math.floor(limitBytes / 16)));
|
|
497
544
|
const mainBudgetBytes = Math.max(pageSize, limitBytes - walBudgetBytes);
|
|
498
545
|
const requestedPages = Math.max(1, Math.floor(mainBudgetBytes / pageSize));
|
|
499
|
-
const checkpointPages = Math.max(16, Math.min(
|
|
546
|
+
const checkpointPages = Math.max(16, Math.min(16384, Math.floor(walBudgetBytes / pageSize)));
|
|
500
547
|
this.db.exec(`PRAGMA journal_size_limit = ${walBudgetBytes}`);
|
|
501
548
|
this.db.exec(`PRAGMA wal_autocheckpoint = ${checkpointPages}`);
|
|
502
549
|
let maxPageCount = requestedPages;
|
|
@@ -1035,7 +1082,9 @@ var COUNTED_TABLES = /* @__PURE__ */ new Set([
|
|
|
1035
1082
|
"machine_meta",
|
|
1036
1083
|
"machine_latest",
|
|
1037
1084
|
"inventory",
|
|
1085
|
+
"points_10s",
|
|
1038
1086
|
"gaps",
|
|
1087
|
+
"recordings",
|
|
1039
1088
|
"alert_rules",
|
|
1040
1089
|
"alert_incidents",
|
|
1041
1090
|
"alert_jobs",
|
|
@@ -1505,6 +1554,63 @@ async function migrateDatabase(options, now = options.now ?? Date.now) {
|
|
|
1505
1554
|
integrity: after.integrity
|
|
1506
1555
|
};
|
|
1507
1556
|
}
|
|
1557
|
+
function resetTelemetryHistory(options) {
|
|
1558
|
+
const database = resolvePath(options.database, "database");
|
|
1559
|
+
assertPrivateRegularFile(database, "database");
|
|
1560
|
+
if (!options.confirmStopped) {
|
|
1561
|
+
throw new OperationsError(
|
|
1562
|
+
"confirmation-required",
|
|
1563
|
+
"resetting telemetry history requires --confirm-stopped after the hub is stopped"
|
|
1564
|
+
);
|
|
1565
|
+
}
|
|
1566
|
+
assertNoActiveHubLock(database);
|
|
1567
|
+
const db = openHubDatabase(database);
|
|
1568
|
+
const tables = [
|
|
1569
|
+
"points_raw",
|
|
1570
|
+
"points_10s",
|
|
1571
|
+
"points_1m",
|
|
1572
|
+
"points_1h",
|
|
1573
|
+
"sample_receipts",
|
|
1574
|
+
"gaps",
|
|
1575
|
+
"machine_latest",
|
|
1576
|
+
"series"
|
|
1577
|
+
];
|
|
1578
|
+
const deleted = {};
|
|
1579
|
+
try {
|
|
1580
|
+
db.db.exec("BEGIN IMMEDIATE");
|
|
1581
|
+
try {
|
|
1582
|
+
for (const table of tables) {
|
|
1583
|
+
const result = db.db.prepare(`DELETE FROM ${table}`).run();
|
|
1584
|
+
deleted[table] = Number(result.changes);
|
|
1585
|
+
}
|
|
1586
|
+
db.db.exec("COMMIT");
|
|
1587
|
+
} catch (error) {
|
|
1588
|
+
try {
|
|
1589
|
+
db.db.exec("ROLLBACK");
|
|
1590
|
+
} catch {
|
|
1591
|
+
}
|
|
1592
|
+
throw error;
|
|
1593
|
+
}
|
|
1594
|
+
db.reclaimPages(1024);
|
|
1595
|
+
db.checkpoint();
|
|
1596
|
+
} finally {
|
|
1597
|
+
db.close();
|
|
1598
|
+
}
|
|
1599
|
+
const info = inspectDatabase(database);
|
|
1600
|
+
return {
|
|
1601
|
+
database,
|
|
1602
|
+
deleted,
|
|
1603
|
+
integrity: info.integrity,
|
|
1604
|
+
preserved: [
|
|
1605
|
+
"users",
|
|
1606
|
+
"collector registry/grants/audit",
|
|
1607
|
+
"alert rules/incidents/jobs/webhook",
|
|
1608
|
+
"machine metadata",
|
|
1609
|
+
"inventory",
|
|
1610
|
+
"recordings"
|
|
1611
|
+
]
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1508
1614
|
function schemaVersionOf(database) {
|
|
1509
1615
|
if (!(0, import_node_fs3.existsSync)(database)) return null;
|
|
1510
1616
|
const info = inspectDatabase(database);
|
|
@@ -3051,27 +3157,29 @@ var HOUR_MS = 36e5;
|
|
|
3051
3157
|
var DAY_MS = 24 * HOUR_MS;
|
|
3052
3158
|
var DEFAULT_RETENTION_POLICY = {
|
|
3053
3159
|
rawMs: 2 * DAY_MS,
|
|
3160
|
+
tenSecondMs: 2 * HOUR_MS,
|
|
3054
3161
|
oneMinuteMs: 30 * DAY_MS,
|
|
3055
3162
|
oneHourMs: 400 * DAY_MS,
|
|
3056
|
-
receiptsMs:
|
|
3163
|
+
receiptsMs: 7 * DAY_MS,
|
|
3057
3164
|
gapsMs: 400 * DAY_MS
|
|
3058
3165
|
};
|
|
3059
3166
|
function resolveRetentionPolicy(partial) {
|
|
3060
3167
|
const rawMs = checkedMs(partial?.rawMs, DEFAULT_RETENTION_POLICY.rawMs, "rawMs");
|
|
3168
|
+
const tenSecondMs = checkedMs(partial?.tenSecondMs, DEFAULT_RETENTION_POLICY.tenSecondMs, "tenSecondMs");
|
|
3061
3169
|
const oneMinuteMs = checkedMs(partial?.oneMinuteMs, DEFAULT_RETENTION_POLICY.oneMinuteMs, "oneMinuteMs");
|
|
3062
3170
|
const oneHourMs = checkedMs(partial?.oneHourMs, DEFAULT_RETENTION_POLICY.oneHourMs, "oneHourMs");
|
|
3063
|
-
const receiptsDefault = partial?.rawMs !== void 0 ? rawMs : DEFAULT_RETENTION_POLICY.receiptsMs;
|
|
3064
|
-
const gapsDefault = partial?.oneHourMs !== void 0 ? oneHourMs : DEFAULT_RETENTION_POLICY.gapsMs;
|
|
3065
3171
|
return {
|
|
3066
3172
|
rawMs,
|
|
3173
|
+
tenSecondMs,
|
|
3067
3174
|
oneMinuteMs,
|
|
3068
3175
|
oneHourMs,
|
|
3069
|
-
receiptsMs: checkedMs(partial?.receiptsMs,
|
|
3070
|
-
gapsMs: checkedMs(partial?.gapsMs,
|
|
3176
|
+
receiptsMs: checkedMs(partial?.receiptsMs, DEFAULT_RETENTION_POLICY.receiptsMs, "receiptsMs"),
|
|
3177
|
+
gapsMs: checkedMs(partial?.gapsMs, DEFAULT_RETENTION_POLICY.gapsMs, "gapsMs")
|
|
3071
3178
|
};
|
|
3072
3179
|
}
|
|
3073
3180
|
var RETENTION_DELETE_SQL = {
|
|
3074
3181
|
raw: "DELETE FROM points_raw WHERE (series_id, ts_ms) IN (SELECT series_id, ts_ms FROM points_raw WHERE series_id = ? AND ts_ms < ? LIMIT ?)",
|
|
3182
|
+
tenSecond: "DELETE FROM points_10s WHERE (series_id, bucket_ms) IN (SELECT series_id, bucket_ms FROM points_10s WHERE series_id = ? AND bucket_ms < ? LIMIT ?)",
|
|
3075
3183
|
oneMinute: "DELETE FROM points_1m WHERE (series_id, bucket_ms) IN (SELECT series_id, bucket_ms FROM points_1m WHERE series_id = ? AND bucket_ms < ? LIMIT ?)",
|
|
3076
3184
|
oneHour: "DELETE FROM points_1h WHERE (series_id, bucket_ms) IN (SELECT series_id, bucket_ms FROM points_1h WHERE series_id = ? AND bucket_ms < ? LIMIT ?)"
|
|
3077
3185
|
};
|
|
@@ -3098,6 +3206,7 @@ var RetentionManager = class {
|
|
|
3098
3206
|
walBytes: 0
|
|
3099
3207
|
};
|
|
3100
3208
|
deleteRaw;
|
|
3209
|
+
delete10s;
|
|
3101
3210
|
delete1m;
|
|
3102
3211
|
delete1h;
|
|
3103
3212
|
deleteReceipts;
|
|
@@ -3126,6 +3235,7 @@ var RetentionManager = class {
|
|
|
3126
3235
|
this.chunkSize = positiveInt(options.chunkSize, 5e3, "chunkSize");
|
|
3127
3236
|
this.maxChunksPerPass = positiveInt(options.maxChunksPerPass, 1e4, "maxChunksPerPass");
|
|
3128
3237
|
this.deleteRaw = database.db.prepare(RETENTION_DELETE_SQL.raw);
|
|
3238
|
+
this.delete10s = database.db.prepare(RETENTION_DELETE_SQL.tenSecond);
|
|
3129
3239
|
this.delete1m = database.db.prepare(RETENTION_DELETE_SQL.oneMinute);
|
|
3130
3240
|
this.delete1h = database.db.prepare(RETENTION_DELETE_SQL.oneHour);
|
|
3131
3241
|
this.deleteReceipts = database.db.prepare(
|
|
@@ -3161,7 +3271,7 @@ var RetentionManager = class {
|
|
|
3161
3271
|
}
|
|
3162
3272
|
/**
|
|
3163
3273
|
* Synchronous, bounded pressure relief for the write path. Deletes only
|
|
3164
|
-
* oldest raw/one-minute rows outside the most-recent-hour floor, then asks
|
|
3274
|
+
* oldest raw/ten-second/one-minute rows outside the most-recent-hour floor, then asks
|
|
3165
3275
|
* SQLite for bounded incremental reclaim. It never deletes one-hour rollups
|
|
3166
3276
|
* to pretend a hard cap fits.
|
|
3167
3277
|
*/
|
|
@@ -3191,6 +3301,9 @@ var RetentionManager = class {
|
|
|
3191
3301
|
const rawCutoff = nowMs - this.policy.rawMs;
|
|
3192
3302
|
const seriesIds = this.seriesIds();
|
|
3193
3303
|
let deleted = this.deleteTierChunk(this.deleteRaw, seriesIds, rawCutoff);
|
|
3304
|
+
if (deleted === 0) {
|
|
3305
|
+
deleted = this.deleteTierChunk(this.delete10s, seriesIds, nowMs - this.policy.tenSecondMs);
|
|
3306
|
+
}
|
|
3194
3307
|
if (deleted === 0) {
|
|
3195
3308
|
deleted = this.deleteTierChunk(this.delete1m, seriesIds, nowMs - this.policy.oneMinuteMs);
|
|
3196
3309
|
}
|
|
@@ -3204,6 +3317,7 @@ var RetentionManager = class {
|
|
|
3204
3317
|
const floorCutoff = nowMs - HOUR_MS;
|
|
3205
3318
|
effectiveRawCutoff = Math.max(rawCutoff, floorCutoff);
|
|
3206
3319
|
deleted = this.deleteTierChunk(this.deleteRaw, seriesIds, effectiveRawCutoff);
|
|
3320
|
+
if (deleted === 0) deleted = this.deleteTierChunk(this.delete10s, seriesIds, floorCutoff);
|
|
3207
3321
|
if (deleted === 0) deleted = this.deleteTierChunk(this.delete1m, seriesIds, floorCutoff);
|
|
3208
3322
|
if (deleted > 0) {
|
|
3209
3323
|
this.db.reclaimPages(64);
|
|
@@ -3235,19 +3349,29 @@ var RetentionManager = class {
|
|
|
3235
3349
|
}
|
|
3236
3350
|
/**
|
|
3237
3351
|
* Deletes up to `chunkSize` rows older than `cutoffMs`, one series at a
|
|
3238
|
-
* time.
|
|
3239
|
-
* row budget, so a series with no expired rows costs one seek rather than a
|
|
3352
|
+
* time. The statements share one transaction per chunk. Each is a primary-key
|
|
3353
|
+
* range seek bounded by the remaining row budget, so a series with no expired rows costs one seek rather than a
|
|
3240
3354
|
* table scan. The series catalog is append-only and ingestion is its only
|
|
3241
3355
|
* writer, so every stored point has a catalog row.
|
|
3242
3356
|
*/
|
|
3243
3357
|
deleteTierChunk(statement, seriesIds, cutoffMs) {
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3358
|
+
this.db.db.exec("BEGIN IMMEDIATE");
|
|
3359
|
+
try {
|
|
3360
|
+
let deleted = 0;
|
|
3361
|
+
for (const seriesId of seriesIds) {
|
|
3362
|
+
const remaining = this.chunkSize - deleted;
|
|
3363
|
+
if (remaining <= 0) break;
|
|
3364
|
+
deleted += Number(statement.run(seriesId, cutoffMs, remaining).changes);
|
|
3365
|
+
}
|
|
3366
|
+
this.db.db.exec("COMMIT");
|
|
3367
|
+
return deleted;
|
|
3368
|
+
} catch (error) {
|
|
3369
|
+
try {
|
|
3370
|
+
this.db.db.exec("ROLLBACK");
|
|
3371
|
+
} catch {
|
|
3372
|
+
}
|
|
3373
|
+
throw error;
|
|
3249
3374
|
}
|
|
3250
|
-
return deleted;
|
|
3251
3375
|
}
|
|
3252
3376
|
seriesIds() {
|
|
3253
3377
|
const rows = this.selectSeriesIds.all();
|
|
@@ -3281,7 +3405,7 @@ var RetentionManager = class {
|
|
|
3281
3405
|
}
|
|
3282
3406
|
};
|
|
3283
3407
|
function yieldToEventLoop() {
|
|
3284
|
-
return new Promise((
|
|
3408
|
+
return new Promise((resolve7) => setImmediate(resolve7));
|
|
3285
3409
|
}
|
|
3286
3410
|
function checkedMs(value, fallback, name) {
|
|
3287
3411
|
const resolved = value ?? fallback;
|
|
@@ -3544,958 +3668,1097 @@ function seriesKey(machineId, metric, resourceId) {
|
|
|
3544
3668
|
return `${machineId}\0${metric}\0${resourceId}`;
|
|
3545
3669
|
}
|
|
3546
3670
|
|
|
3547
|
-
// src/
|
|
3671
|
+
// src/contracts/index.ts
|
|
3672
|
+
var MIN_TIMESTAMP_EPOCH = Date.parse("2000-01-01T00:00:00Z");
|
|
3673
|
+
var MAX_TIMESTAMP_EPOCH = Date.parse("2200-01-01T00:00:00Z");
|
|
3674
|
+
|
|
3675
|
+
// src/hub/storage/history-query.ts
|
|
3548
3676
|
var DAY_MS2 = 864e5;
|
|
3549
|
-
var
|
|
3550
|
-
var
|
|
3551
|
-
var
|
|
3552
|
-
var
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
queue = [];
|
|
3562
|
-
timer = null;
|
|
3563
|
-
driver = null;
|
|
3564
|
-
closed = false;
|
|
3565
|
-
committedRecords = 0;
|
|
3566
|
-
duplicateRecords = 0;
|
|
3567
|
-
rejectedRecords = 0;
|
|
3568
|
-
writeFailures = 0;
|
|
3569
|
-
diskFullFailures = 0;
|
|
3570
|
-
lastWriteError = null;
|
|
3571
|
-
lastFlushMs = null;
|
|
3572
|
-
lastFlushDurationMs = null;
|
|
3573
|
-
receiptInsert;
|
|
3574
|
-
rawInsert;
|
|
3575
|
-
upsert1m;
|
|
3576
|
-
upsert1h;
|
|
3577
|
-
gapInsert;
|
|
3578
|
-
latestSelect;
|
|
3579
|
-
latestUpsert;
|
|
3580
|
-
inventoryUpsert;
|
|
3581
|
-
inventorySelect;
|
|
3582
|
-
metaUpsertBoth;
|
|
3583
|
-
metaUpsertDisplay;
|
|
3584
|
-
metaUpsertTags;
|
|
3585
|
-
metaTouch;
|
|
3586
|
-
metaSelect;
|
|
3587
|
-
metaSelectAll;
|
|
3588
|
-
latestSelectAll;
|
|
3589
|
-
constructor(options) {
|
|
3590
|
-
this.database = options.database;
|
|
3591
|
-
this.now = options.now ?? (() => Date.now());
|
|
3592
|
-
this.maxQueuedRecords = positiveInt2(options.maxQueuedRecords, 4096, "maxQueuedRecords");
|
|
3593
|
-
this.flushIntervalMs = nonnegativeInt(options.flushIntervalMs, 20, "flushIntervalMs");
|
|
3594
|
-
this.maxRecordsPerTransaction = positiveInt2(
|
|
3595
|
-
options.maxRecordsPerTransaction,
|
|
3596
|
-
512,
|
|
3597
|
-
"maxRecordsPerTransaction"
|
|
3598
|
-
);
|
|
3599
|
-
this.maxPhysicalBytes = resolvePhysicalLimit(options);
|
|
3600
|
-
const db = options.database.db;
|
|
3601
|
-
this.series = new SeriesCatalog(db);
|
|
3602
|
-
this.receiptInsert = db.prepare(
|
|
3603
|
-
"INSERT OR IGNORE INTO sample_receipts (collector_id, run_id, seq, machine_id, kind, mode, captured_at_ms, received_at_ms, expected_interval_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
3604
|
-
);
|
|
3605
|
-
this.rawInsert = db.prepare(
|
|
3606
|
-
"INSERT OR IGNORE INTO points_raw (series_id, ts_ms, value) VALUES (?, ?, ?)"
|
|
3607
|
-
);
|
|
3608
|
-
this.upsert1m = prepareRollupUpsert(db, "points_1m");
|
|
3609
|
-
this.upsert1h = prepareRollupUpsert(db, "points_1h");
|
|
3610
|
-
this.gapInsert = db.prepare(
|
|
3611
|
-
"INSERT OR IGNORE INTO gaps (collector_id, run_id, seq, machine_id, from_ms, to_ms, dropped, reason, received_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
3612
|
-
);
|
|
3613
|
-
this.latestSelect = db.prepare(
|
|
3614
|
-
"SELECT run_id, seq, captured_at_ms FROM machine_latest WHERE machine_id = ?"
|
|
3615
|
-
);
|
|
3616
|
-
this.latestUpsert = db.prepare(
|
|
3617
|
-
"INSERT INTO machine_latest (machine_id, collector_id, run_id, boot_id, seq, captured_at_ms, received_at_ms, expected_interval_ms, hostname, payload_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET collector_id = excluded.collector_id, run_id = excluded.run_id, boot_id = excluded.boot_id, seq = excluded.seq, captured_at_ms = excluded.captured_at_ms, received_at_ms = excluded.received_at_ms, expected_interval_ms = COALESCE(excluded.expected_interval_ms, expected_interval_ms), hostname = COALESCE(excluded.hostname, hostname), payload_json = excluded.payload_json"
|
|
3618
|
-
);
|
|
3619
|
-
this.latestSelectAll = db.prepare(
|
|
3620
|
-
"SELECT machine_id, collector_id, run_id, boot_id, seq, captured_at_ms, received_at_ms, expected_interval_ms, hostname, payload_json FROM machine_latest ORDER BY machine_id"
|
|
3621
|
-
);
|
|
3622
|
-
this.inventoryUpsert = db.prepare(
|
|
3623
|
-
"INSERT INTO inventory (machine_id, captured_at_ms, received_at_ms, inventory_json) VALUES (?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET captured_at_ms = excluded.captured_at_ms, received_at_ms = excluded.received_at_ms, inventory_json = excluded.inventory_json"
|
|
3624
|
-
);
|
|
3625
|
-
this.inventorySelect = db.prepare(
|
|
3626
|
-
"SELECT captured_at_ms, received_at_ms, inventory_json FROM inventory WHERE machine_id = ?"
|
|
3627
|
-
);
|
|
3628
|
-
this.metaUpsertBoth = db.prepare(
|
|
3629
|
-
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET display_name = excluded.display_name, tags_json = excluded.tags_json, updated_at_ms = excluded.updated_at_ms"
|
|
3630
|
-
);
|
|
3631
|
-
this.metaUpsertDisplay = db.prepare(
|
|
3632
|
-
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, ?, '[]', ?) ON CONFLICT(machine_id) DO UPDATE SET display_name = excluded.display_name, updated_at_ms = excluded.updated_at_ms"
|
|
3633
|
-
);
|
|
3634
|
-
this.metaUpsertTags = db.prepare(
|
|
3635
|
-
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, NULL, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET tags_json = excluded.tags_json, updated_at_ms = excluded.updated_at_ms"
|
|
3636
|
-
);
|
|
3637
|
-
this.metaTouch = db.prepare(
|
|
3638
|
-
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, NULL, '[]', ?) ON CONFLICT(machine_id) DO UPDATE SET updated_at_ms = excluded.updated_at_ms"
|
|
3639
|
-
);
|
|
3640
|
-
this.metaSelect = db.prepare(
|
|
3641
|
-
"SELECT display_name, tags_json, updated_at_ms FROM machine_meta WHERE machine_id = ?"
|
|
3642
|
-
);
|
|
3643
|
-
this.metaSelectAll = db.prepare(
|
|
3644
|
-
"SELECT machine_id, display_name, tags_json FROM machine_meta ORDER BY machine_id"
|
|
3645
|
-
);
|
|
3646
|
-
this.retention = new RetentionManager(options.database, {
|
|
3647
|
-
policy: options.retention,
|
|
3648
|
-
maxDatabaseBytes: options.maxDatabaseBytes ?? null,
|
|
3649
|
-
maxPhysicalBytes: this.maxPhysicalBytes,
|
|
3650
|
-
now: this.now
|
|
3651
|
-
});
|
|
3652
|
-
this.retention.start();
|
|
3653
|
-
}
|
|
3654
|
-
/**
|
|
3655
|
-
* Queues one batch. The promise resolves only after the transaction that
|
|
3656
|
-
* writes it has committed, or immediately as `busy` when the bounded queue
|
|
3657
|
-
* cannot hold it (no database work is done in that case).
|
|
3658
|
-
*/
|
|
3659
|
-
enqueue(batch) {
|
|
3660
|
-
return new Promise((resolve6) => {
|
|
3661
|
-
if (this.closed) {
|
|
3662
|
-
resolve6({ status: "busy", retryAfterMs: FAILURE_RETRY_MS, reason: "storage is closed" });
|
|
3663
|
-
return;
|
|
3664
|
-
}
|
|
3665
|
-
validateBatch(batch);
|
|
3666
|
-
const queued = this.queuedRecords();
|
|
3667
|
-
if (queued + batch.records.length > this.maxQueuedRecords) {
|
|
3668
|
-
const ratio = this.maxQueuedRecords === 0 ? 1 : queued / this.maxQueuedRecords;
|
|
3669
|
-
const retryAfterMs = Math.min(5e3, Math.max(1e3, Math.round(1e3 + ratio * 4e3)));
|
|
3670
|
-
resolve6({ status: "busy", retryAfterMs, reason: "ingest queue is full" });
|
|
3671
|
-
return;
|
|
3672
|
-
}
|
|
3673
|
-
this.queue.push({
|
|
3674
|
-
batch,
|
|
3675
|
-
receivedAtMs: this.now(),
|
|
3676
|
-
resolve: resolve6,
|
|
3677
|
-
settled: false,
|
|
3678
|
-
accepted: [],
|
|
3679
|
-
rejected: [],
|
|
3680
|
-
newLive: null
|
|
3681
|
-
});
|
|
3682
|
-
this.scheduleFlush(this.flushIntervalMs);
|
|
3683
|
-
});
|
|
3684
|
-
}
|
|
3685
|
-
/** Persisted latest LIVE metrics per machine, for restart recovery. */
|
|
3686
|
-
latestLive() {
|
|
3687
|
-
const rows = this.latestSelectAll.all();
|
|
3688
|
-
return rows.map((row) => ({
|
|
3689
|
-
machineId: row.machine_id,
|
|
3690
|
-
collectorId: row.collector_id,
|
|
3691
|
-
runId: row.run_id,
|
|
3692
|
-
bootId: row.boot_id,
|
|
3693
|
-
sequence: Number(row.seq),
|
|
3694
|
-
capturedAt: new Date(Number(row.captured_at_ms)).toISOString(),
|
|
3695
|
-
receivedAt: new Date(Number(row.received_at_ms)).toISOString(),
|
|
3696
|
-
hostname: row.hostname,
|
|
3697
|
-
expectedIntervalMs: row.expected_interval_ms === null ? null : Number(row.expected_interval_ms),
|
|
3698
|
-
payload: JSON.parse(row.payload_json)
|
|
3699
|
-
}));
|
|
3677
|
+
var MAX_RANGE_MS = 400 * DAY_MS2;
|
|
3678
|
+
var MAX_METRICS = Object.keys(METRICS).length;
|
|
3679
|
+
var MAX_RESOURCE_IDS = 64;
|
|
3680
|
+
var DEFAULT_MAX_POINTS = 600;
|
|
3681
|
+
var MAX_POINTS = 2e3;
|
|
3682
|
+
var MAX_GAPS = 500;
|
|
3683
|
+
var TIER_RESOLUTION_MS = { raw: 100, "10s": 1e4, "1m": 6e4, "1h": 36e5 };
|
|
3684
|
+
var HistoryQueryError = class extends Error {
|
|
3685
|
+
code = "invalid-query";
|
|
3686
|
+
constructor(message) {
|
|
3687
|
+
super(message);
|
|
3688
|
+
this.name = "HistoryQueryError";
|
|
3700
3689
|
}
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3690
|
+
};
|
|
3691
|
+
var DEFAULT_TIERS = ["raw", "10s", "1m", "1h"];
|
|
3692
|
+
function queryHistory(db, q, budget = {}) {
|
|
3693
|
+
const started = performance.now();
|
|
3694
|
+
const maxPoints = validatedMaxPoints(q.maxPoints);
|
|
3695
|
+
const maxSeries = validatedBudget(budget.maxSeries, 64, "maxSeries");
|
|
3696
|
+
const maxRowsScanned = validatedBudget(budget.maxRowsScanned, 25e4, "maxRowsScanned");
|
|
3697
|
+
const maxElapsedMs = validatedBudget(budget.maxElapsedMs, 5e3, "maxElapsedMs");
|
|
3698
|
+
const retention = resolveRetentionPolicy(budget.retention);
|
|
3699
|
+
validatedQuery(q);
|
|
3700
|
+
const rangeMs = q.toMs - q.fromMs;
|
|
3701
|
+
const nowMs = budget.nowMs ?? Date.now();
|
|
3702
|
+
let allSeries = loadSeries(db, q);
|
|
3703
|
+
let truncated = false;
|
|
3704
|
+
if (allSeries.length > maxSeries) {
|
|
3705
|
+
allSeries = allSeries.slice(0, maxSeries);
|
|
3706
|
+
truncated = true;
|
|
3705
3707
|
}
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3708
|
+
const chosen = chooseTier(
|
|
3709
|
+
db,
|
|
3710
|
+
allSeries.map((row) => Number(row.series_id)),
|
|
3711
|
+
q.fromMs,
|
|
3712
|
+
q.toMs,
|
|
3713
|
+
nowMs,
|
|
3714
|
+
retention,
|
|
3715
|
+
budget.tiers,
|
|
3716
|
+
maxRowsScanned,
|
|
3717
|
+
maxElapsedMs,
|
|
3718
|
+
started
|
|
3719
|
+
);
|
|
3720
|
+
if (chosen.timeExceeded || chosen.overBudget) truncated = true;
|
|
3721
|
+
let seriesRows = allSeries;
|
|
3722
|
+
if (chosen.cutSeries < seriesRows.length) {
|
|
3723
|
+
seriesRows = seriesRows.slice(0, Math.max(0, chosen.cutSeries));
|
|
3724
|
+
truncated = true;
|
|
3714
3725
|
}
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
const hasTags = meta.tags !== void 0;
|
|
3723
|
-
const displayName = meta.displayName ?? null;
|
|
3724
|
-
const tags = hasTags ? validateTags(meta.tags) : [];
|
|
3725
|
-
if (hasDisplay && hasTags) {
|
|
3726
|
-
this.metaUpsertBoth.run(machineId, displayName, JSON.stringify(tags), updatedAt);
|
|
3727
|
-
} else if (hasDisplay) {
|
|
3728
|
-
this.metaUpsertDisplay.run(machineId, displayName, updatedAt);
|
|
3729
|
-
} else if (hasTags) {
|
|
3730
|
-
this.metaUpsertTags.run(machineId, JSON.stringify(tags), updatedAt);
|
|
3731
|
-
} else {
|
|
3732
|
-
this.metaTouch.run(machineId, updatedAt);
|
|
3726
|
+
const stepMs = Math.max(1, Math.ceil(rangeMs / maxPoints / chosen.resolutionMs)) * chosen.resolutionMs;
|
|
3727
|
+
const originMs = Math.floor(q.fromMs / stepMs) * stepMs;
|
|
3728
|
+
const series = [];
|
|
3729
|
+
for (const row of seriesRows) {
|
|
3730
|
+
if (performance.now() - started > maxElapsedMs) {
|
|
3731
|
+
truncated = true;
|
|
3732
|
+
break;
|
|
3733
3733
|
}
|
|
3734
|
+
const metric = row.metric;
|
|
3735
|
+
const definition = METRICS[metric];
|
|
3736
|
+
if (!definition) continue;
|
|
3737
|
+
const points = readSeries(
|
|
3738
|
+
db,
|
|
3739
|
+
chosen.tier,
|
|
3740
|
+
Number(row.series_id),
|
|
3741
|
+
fromBucketMs(chosen.tier, q.fromMs),
|
|
3742
|
+
q.toMs,
|
|
3743
|
+
originMs,
|
|
3744
|
+
stepMs,
|
|
3745
|
+
// An additional per-series guard. Tier selection already keeps the total
|
|
3746
|
+
// under budget; this makes a single query bounded even if a custom
|
|
3747
|
+
// retention policy forces the coarse fallback.
|
|
3748
|
+
maxRowsScanned
|
|
3749
|
+
);
|
|
3750
|
+
if (points.length === 0) continue;
|
|
3751
|
+
series.push({ metric: row.metric, resourceId: row.resource_id, unit: definition.unit, points });
|
|
3734
3752
|
}
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3753
|
+
const gaps = readGaps(db, q.machineId, q.fromMs, q.toMs);
|
|
3754
|
+
return {
|
|
3755
|
+
machineId: q.machineId,
|
|
3756
|
+
fromMs: q.fromMs,
|
|
3757
|
+
toMs: q.toMs,
|
|
3758
|
+
stepMs,
|
|
3759
|
+
tier: chosen.tier,
|
|
3760
|
+
series,
|
|
3761
|
+
gaps,
|
|
3762
|
+
truncated,
|
|
3763
|
+
elapsedMs: Math.max(0, Math.round(performance.now() - started))
|
|
3764
|
+
};
|
|
3765
|
+
}
|
|
3766
|
+
function loadSeries(db, q) {
|
|
3767
|
+
const placeholders = q.metrics.map(() => "?").join(", ");
|
|
3768
|
+
const params = [q.machineId, ...q.metrics];
|
|
3769
|
+
let sql = `SELECT series_id, metric, resource_id FROM series WHERE machine_id = ? AND metric IN (${placeholders})`;
|
|
3770
|
+
if (q.resourceIds && q.resourceIds.length > 0) {
|
|
3771
|
+
sql += ` AND resource_id IN (${q.resourceIds.map(() => "?").join(", ")})`;
|
|
3772
|
+
params.push(...q.resourceIds);
|
|
3739
3773
|
}
|
|
3740
|
-
|
|
3741
|
-
|
|
3742
|
-
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
displayName: row.display_name,
|
|
3746
|
-
tags: parseTags(row.tags_json)
|
|
3747
|
-
});
|
|
3774
|
+
if (q.coreIds !== void 0) {
|
|
3775
|
+
if (q.coreIds.length === 0) sql += " AND metric <> 'cpu.core'";
|
|
3776
|
+
else {
|
|
3777
|
+
sql += ` AND (metric <> 'cpu.core' OR resource_id IN (${q.coreIds.map(() => "?").join(", ")}))`;
|
|
3778
|
+
params.push(...q.coreIds);
|
|
3748
3779
|
}
|
|
3749
|
-
return result;
|
|
3750
|
-
}
|
|
3751
|
-
stats() {
|
|
3752
|
-
return {
|
|
3753
|
-
queuedRecords: this.queuedRecords(),
|
|
3754
|
-
queuedBatches: this.queue.length,
|
|
3755
|
-
lastFlushMs: this.lastFlushMs,
|
|
3756
|
-
lastFlushDurationMs: this.lastFlushDurationMs,
|
|
3757
|
-
committedRecords: this.committedRecords,
|
|
3758
|
-
duplicateRecords: this.duplicateRecords,
|
|
3759
|
-
rejectedRecords: this.rejectedRecords,
|
|
3760
|
-
writeFailures: this.writeFailures,
|
|
3761
|
-
lastWriteError: this.lastWriteError,
|
|
3762
|
-
diskFullFailures: this.diskFullFailures,
|
|
3763
|
-
dbBytes: this.database.sizeBytes(),
|
|
3764
|
-
dbWalBytes: this.database.walBytes(),
|
|
3765
|
-
dbBytesLimit: this.maxPhysicalBytes,
|
|
3766
|
-
retention: this.retention.status()
|
|
3767
|
-
};
|
|
3768
3780
|
}
|
|
3769
|
-
|
|
3770
|
-
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3781
|
+
sql += " ORDER BY metric, resource_id";
|
|
3782
|
+
return db.db.prepare(sql).all(...params);
|
|
3783
|
+
}
|
|
3784
|
+
var TIER_CANDIDATES = [
|
|
3785
|
+
{ tier: "raw", resolutionMs: TIER_RESOLUTION_MS.raw, retentionMs: 0 },
|
|
3786
|
+
{ tier: "10s", resolutionMs: TIER_RESOLUTION_MS["10s"], retentionMs: 0 },
|
|
3787
|
+
{ tier: "1m", resolutionMs: TIER_RESOLUTION_MS["1m"], retentionMs: 0 },
|
|
3788
|
+
{ tier: "1h", resolutionMs: TIER_RESOLUTION_MS["1h"], retentionMs: 0 }
|
|
3789
|
+
];
|
|
3790
|
+
function chooseTier(db, seriesIds, fromMs, toMs, nowMs, retention, tiers, maxRowsScanned, maxElapsedMs, started) {
|
|
3791
|
+
const allowed = new Set(
|
|
3792
|
+
(tiers ?? DEFAULT_TIERS).filter((tier) => tier !== "1s")
|
|
3793
|
+
);
|
|
3794
|
+
const candidates = TIER_CANDIDATES.filter(
|
|
3795
|
+
(candidate) => allowed.has(candidate.tier)
|
|
3796
|
+
).map((candidate) => ({
|
|
3797
|
+
...candidate,
|
|
3798
|
+
retentionMs: candidate.tier === "raw" ? retention.rawMs : candidate.tier === "10s" ? retention.tenSecondMs : candidate.tier === "1m" ? retention.oneMinuteMs : retention.oneHourMs
|
|
3799
|
+
}));
|
|
3800
|
+
if (candidates.length === 0) {
|
|
3801
|
+
throw new HistoryQueryError("no history storage tiers are available");
|
|
3782
3802
|
}
|
|
3783
|
-
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
this.retention.stop();
|
|
3791
|
-
if (this.timer !== null) {
|
|
3792
|
-
clearTimeout(this.timer);
|
|
3793
|
-
this.timer = null;
|
|
3803
|
+
const available = candidates.filter((candidate) => fromMs >= nowMs - candidate.retentionMs);
|
|
3804
|
+
const pool = available.length > 0 ? available : [candidates[candidates.length - 1]];
|
|
3805
|
+
let timeExceeded = false;
|
|
3806
|
+
for (const candidate of pool) {
|
|
3807
|
+
if (performance.now() - started > maxElapsedMs) {
|
|
3808
|
+
timeExceeded = true;
|
|
3809
|
+
break;
|
|
3794
3810
|
}
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3811
|
+
const rows = countRowsBounded(
|
|
3812
|
+
db,
|
|
3813
|
+
candidate.tier,
|
|
3814
|
+
seriesIds,
|
|
3815
|
+
fromBucketMs(candidate.tier, fromMs),
|
|
3816
|
+
toMs,
|
|
3817
|
+
maxRowsScanned
|
|
3818
|
+
);
|
|
3819
|
+
if (rows <= maxRowsScanned) {
|
|
3820
|
+
if (rows === 0 && candidate !== pool[pool.length - 1]) continue;
|
|
3821
|
+
return {
|
|
3822
|
+
tier: candidate.tier,
|
|
3823
|
+
resolutionMs: candidate.resolutionMs,
|
|
3824
|
+
cutSeries: seriesIds.length,
|
|
3825
|
+
timeExceeded,
|
|
3826
|
+
overBudget: false
|
|
3827
|
+
};
|
|
3800
3828
|
}
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
if (!ok) break;
|
|
3805
|
-
if (this.queue.length > 0) await yieldToEventLoop2();
|
|
3829
|
+
if (performance.now() - started > maxElapsedMs) {
|
|
3830
|
+
timeExceeded = true;
|
|
3831
|
+
break;
|
|
3806
3832
|
}
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3833
|
+
}
|
|
3834
|
+
const fallback = pool[pool.length - 1];
|
|
3835
|
+
let cut = seriesIds.length;
|
|
3836
|
+
let fits = seriesIds.length === 0;
|
|
3837
|
+
if (timeExceeded) cut = Math.max(1, Math.floor(cut / 2));
|
|
3838
|
+
while (!fits && cut >= 1) {
|
|
3839
|
+
if (performance.now() - started > maxElapsedMs) {
|
|
3840
|
+
timeExceeded = true;
|
|
3841
|
+
break;
|
|
3810
3842
|
}
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3843
|
+
const rows = countRowsBounded(
|
|
3844
|
+
db,
|
|
3845
|
+
fallback.tier,
|
|
3846
|
+
seriesIds.slice(0, cut),
|
|
3847
|
+
fromBucketMs(fallback.tier, fromMs),
|
|
3848
|
+
toMs,
|
|
3849
|
+
maxRowsScanned
|
|
3850
|
+
);
|
|
3851
|
+
if (rows <= maxRowsScanned) {
|
|
3852
|
+
fits = true;
|
|
3853
|
+
break;
|
|
3814
3854
|
}
|
|
3855
|
+
if (cut === 1) break;
|
|
3856
|
+
cut = Math.max(1, Math.floor(cut / 2));
|
|
3815
3857
|
}
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
return promise;
|
|
3839
|
-
}
|
|
3840
|
-
async runDriver() {
|
|
3841
|
-
while (!this.closed && this.queue.length > 0) {
|
|
3842
|
-
const ok = this.flushTransaction();
|
|
3843
|
-
if (!ok) return;
|
|
3844
|
-
if (this.queue.length > 0) await yieldToEventLoop2();
|
|
3845
|
-
}
|
|
3858
|
+
return {
|
|
3859
|
+
tier: fallback.tier,
|
|
3860
|
+
resolutionMs: fallback.resolutionMs,
|
|
3861
|
+
cutSeries: Math.max(1, Math.min(cut, seriesIds.length)),
|
|
3862
|
+
timeExceeded,
|
|
3863
|
+
overBudget: !fits
|
|
3864
|
+
};
|
|
3865
|
+
}
|
|
3866
|
+
function countRowsBounded(db, tier, seriesIds, fromMs, toMs, budget) {
|
|
3867
|
+
if (seriesIds.length === 0) return 0;
|
|
3868
|
+
const column = tier === "raw" ? "ts_ms" : "bucket_ms";
|
|
3869
|
+
const table = tableForTier(tier);
|
|
3870
|
+
const statement = db.db.prepare(
|
|
3871
|
+
`SELECT COUNT(*) AS n FROM (SELECT 1 FROM ${table} WHERE series_id = ? AND ${column} >= ? AND ${column} <= ? LIMIT ?)`
|
|
3872
|
+
);
|
|
3873
|
+
let total = 0;
|
|
3874
|
+
for (const seriesId of seriesIds) {
|
|
3875
|
+
const remaining = budget + 1 - total;
|
|
3876
|
+
if (remaining <= 0) return total;
|
|
3877
|
+
const row = statement.get(seriesId, fromMs, toMs, remaining);
|
|
3878
|
+
total += Number(row?.n ?? 0);
|
|
3879
|
+
if (total > budget) return total;
|
|
3846
3880
|
}
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
}
|
|
3873
|
-
const stillFull = physical !== null && physical.pageCount >= physical.maxPageCount && physical.freePages === 0;
|
|
3874
|
-
if (bytes >= this.maxPhysicalBytes || stillFull) {
|
|
3875
|
-
this.diskFullFailures += 1;
|
|
3876
|
-
this.lastWriteError = "storage disk budget reached";
|
|
3877
|
-
for (const pending of selected) {
|
|
3878
|
-
this.resolveBusy(pending, FAILURE_RETRY_MS, "storage disk budget reached");
|
|
3879
|
-
}
|
|
3880
|
-
return false;
|
|
3881
|
-
}
|
|
3882
|
-
} catch (error) {
|
|
3883
|
-
const full = isStorageFullError(error);
|
|
3884
|
-
if (full) this.diskFullFailures += 1;
|
|
3885
|
-
this.lastWriteError = full ? "storage disk budget reached" : sanitizeWriteError(error);
|
|
3886
|
-
for (const pending of selected) {
|
|
3887
|
-
this.resolveBusy(
|
|
3888
|
-
pending,
|
|
3889
|
-
FAILURE_RETRY_MS,
|
|
3890
|
-
full ? "storage disk budget reached" : "storage write failed"
|
|
3891
|
-
);
|
|
3892
|
-
}
|
|
3893
|
-
return false;
|
|
3894
|
-
}
|
|
3895
|
-
}
|
|
3896
|
-
const started = import_node_perf_hooks.performance.now();
|
|
3897
|
-
const counters = { committed: 0, duplicates: 0, rejected: 0 };
|
|
3898
|
-
const cursors = /* @__PURE__ */ new Map();
|
|
3899
|
-
const upserts = /* @__PURE__ */ new Map();
|
|
3900
|
-
try {
|
|
3901
|
-
this.database.db.exec("BEGIN IMMEDIATE");
|
|
3902
|
-
for (const pending of selected) {
|
|
3903
|
-
this.processBatch(pending, counters, cursors, upserts);
|
|
3904
|
-
}
|
|
3905
|
-
for (const [machineId, live] of upserts) {
|
|
3906
|
-
this.latestUpsert.run(
|
|
3907
|
-
machineId,
|
|
3908
|
-
live.record.collectorId,
|
|
3909
|
-
live.record.runId,
|
|
3910
|
-
live.record.bootId,
|
|
3911
|
-
live.record.sequence,
|
|
3912
|
-
Date.parse(live.record.capturedAt),
|
|
3913
|
-
live.receivedAtMs,
|
|
3914
|
-
live.expectedIntervalMs,
|
|
3915
|
-
live.hostname,
|
|
3916
|
-
JSON.stringify(live.record.payload)
|
|
3917
|
-
);
|
|
3918
|
-
}
|
|
3919
|
-
this.database.db.exec("COMMIT");
|
|
3920
|
-
} catch (error) {
|
|
3921
|
-
try {
|
|
3922
|
-
this.database.db.exec("ROLLBACK");
|
|
3923
|
-
} catch {
|
|
3924
|
-
}
|
|
3925
|
-
this.writeFailures += 1;
|
|
3926
|
-
const full = isStorageFullError(error);
|
|
3927
|
-
if (full) this.diskFullFailures += 1;
|
|
3928
|
-
this.lastWriteError = full ? "storage disk budget reached" : sanitizeWriteError(error);
|
|
3929
|
-
for (const pending of selected) {
|
|
3930
|
-
this.resolveBusy(
|
|
3931
|
-
pending,
|
|
3932
|
-
FAILURE_RETRY_MS,
|
|
3933
|
-
full ? "storage disk budget reached" : "storage write failed"
|
|
3934
|
-
);
|
|
3935
|
-
}
|
|
3936
|
-
return false;
|
|
3937
|
-
}
|
|
3938
|
-
this.committedRecords += counters.committed;
|
|
3939
|
-
this.duplicateRecords += counters.duplicates;
|
|
3940
|
-
this.rejectedRecords += counters.rejected;
|
|
3941
|
-
this.lastFlushMs = this.now();
|
|
3942
|
-
this.lastFlushDurationMs = Math.max(0, Math.round(import_node_perf_hooks.performance.now() - started));
|
|
3943
|
-
this.checkpointWALIfNeeded();
|
|
3944
|
-
if (this.maxPhysicalBytes !== null && this.database.sizeBytes() > this.maxPhysicalBytes) {
|
|
3945
|
-
try {
|
|
3946
|
-
this.retention.relievePressure(4);
|
|
3947
|
-
} catch {
|
|
3948
|
-
}
|
|
3949
|
-
}
|
|
3950
|
-
for (const pending of selected) this.resolveCommitted(pending);
|
|
3951
|
-
return true;
|
|
3952
|
-
}
|
|
3953
|
-
checkpointWALIfNeeded() {
|
|
3954
|
-
if (this.maxPhysicalBytes === null) return;
|
|
3955
|
-
try {
|
|
3956
|
-
const physical = this.database.physicalLimit();
|
|
3957
|
-
if (physical !== null && this.database.walBytes() > physical.walBudgetBytes) {
|
|
3958
|
-
this.database.checkpoint();
|
|
3959
|
-
}
|
|
3960
|
-
} catch {
|
|
3961
|
-
}
|
|
3962
|
-
}
|
|
3963
|
-
processBatch(pending, counters, cursors, upserts) {
|
|
3964
|
-
const batch = pending.batch;
|
|
3965
|
-
let best = null;
|
|
3966
|
-
let bestCapturedMs = 0;
|
|
3967
|
-
let bestSequence = -1;
|
|
3968
|
-
for (const record of batch.records) {
|
|
3969
|
-
const ref = { runId: record.runId, sequence: record.sequence };
|
|
3970
|
-
const reason = rejectionReason(record, batch, pending.receivedAtMs);
|
|
3971
|
-
if (reason !== null) {
|
|
3972
|
-
pending.rejected.push({ ...ref, reason });
|
|
3973
|
-
counters.rejected += 1;
|
|
3974
|
-
continue;
|
|
3975
|
-
}
|
|
3976
|
-
const capturedMs = Date.parse(record.capturedAt);
|
|
3977
|
-
const inserted = Number(
|
|
3978
|
-
this.receiptInsert.run(
|
|
3979
|
-
batch.collectorId,
|
|
3980
|
-
record.runId,
|
|
3981
|
-
record.sequence,
|
|
3982
|
-
batch.machineId,
|
|
3983
|
-
record.kind,
|
|
3984
|
-
batch.mode,
|
|
3985
|
-
capturedMs,
|
|
3986
|
-
pending.receivedAtMs,
|
|
3987
|
-
batch.expectedIntervalMs
|
|
3988
|
-
).changes
|
|
3989
|
-
) === 1;
|
|
3990
|
-
if (!inserted) {
|
|
3991
|
-
pending.accepted.push(ref);
|
|
3992
|
-
counters.duplicates += 1;
|
|
3993
|
-
continue;
|
|
3994
|
-
}
|
|
3995
|
-
pending.accepted.push(ref);
|
|
3996
|
-
counters.committed += 1;
|
|
3997
|
-
if (record.kind === "metrics") {
|
|
3998
|
-
const known = needsKnownSeries(record.payload) ? this.series.knownFor(batch.machineId) : void 0;
|
|
3999
|
-
for (const point of flattenMetrics(record.payload, known)) {
|
|
4000
|
-
const seriesId = this.series.idFor(batch.machineId, point.metric, point.resourceId);
|
|
4001
|
-
this.insertPoint(seriesId, capturedMs, point.value);
|
|
4002
|
-
}
|
|
4003
|
-
if (batch.mode === "live") {
|
|
4004
|
-
if (best === null || capturedMs > bestCapturedMs || capturedMs === bestCapturedMs && record.sequence > bestSequence) {
|
|
4005
|
-
best = record;
|
|
4006
|
-
bestCapturedMs = capturedMs;
|
|
4007
|
-
bestSequence = record.sequence;
|
|
4008
|
-
}
|
|
4009
|
-
}
|
|
4010
|
-
} else {
|
|
4011
|
-
this.gapInsert.run(
|
|
4012
|
-
batch.collectorId,
|
|
4013
|
-
record.runId,
|
|
4014
|
-
record.sequence,
|
|
4015
|
-
batch.machineId,
|
|
4016
|
-
Date.parse(record.payload.fromCapturedAt),
|
|
4017
|
-
Date.parse(record.payload.toCapturedAt),
|
|
4018
|
-
record.payload.droppedRecords,
|
|
4019
|
-
record.payload.reason,
|
|
4020
|
-
pending.receivedAtMs
|
|
4021
|
-
);
|
|
4022
|
-
}
|
|
4023
|
-
}
|
|
4024
|
-
if (batch.mode !== "live" || best === null) return;
|
|
4025
|
-
const cursor = this.latestCursor(batch.machineId, cursors);
|
|
4026
|
-
const candidate = {
|
|
4027
|
-
runId: best.runId,
|
|
4028
|
-
sequence: best.sequence,
|
|
4029
|
-
capturedAtMs: bestCapturedMs
|
|
3881
|
+
return total;
|
|
3882
|
+
}
|
|
3883
|
+
function readSeries(db, tier, seriesId, fromBucketMs2, toMs, originMs, stepMs, rowLimit) {
|
|
3884
|
+
const statement = tier === "raw" ? db.db.prepare(
|
|
3885
|
+
"SELECT ? + CAST((ts_ms - ?) / ? AS INTEGER) * ? AS bucket, MIN(value) AS min_value, MAX(value) AS max_value, SUM(value) AS sum_value, COUNT(value) AS count_value, COUNT(*) AS row_count FROM (SELECT ts_ms, value FROM points_raw WHERE series_id = ? AND ts_ms >= ? AND ts_ms <= ? ORDER BY ts_ms LIMIT ?) GROUP BY bucket ORDER BY bucket"
|
|
3886
|
+
) : db.db.prepare(
|
|
3887
|
+
`SELECT ? + CAST((bucket_ms - ?) / ? AS INTEGER) * ? AS bucket, MIN(min) AS min_value, MAX(max) AS max_value, SUM(sum) AS sum_value, SUM(count) AS count_value, COUNT(*) AS row_count FROM (SELECT bucket_ms, min, max, sum, count FROM ${tableForTier(tier)} WHERE series_id = ? AND bucket_ms >= ? AND bucket_ms <= ? ORDER BY bucket_ms LIMIT ?) GROUP BY bucket ORDER BY bucket`
|
|
3888
|
+
);
|
|
3889
|
+
const rows = statement.all(
|
|
3890
|
+
originMs,
|
|
3891
|
+
originMs,
|
|
3892
|
+
stepMs,
|
|
3893
|
+
stepMs,
|
|
3894
|
+
seriesId,
|
|
3895
|
+
fromBucketMs2,
|
|
3896
|
+
toMs,
|
|
3897
|
+
rowLimit
|
|
3898
|
+
);
|
|
3899
|
+
return rows.map((row) => {
|
|
3900
|
+
const count = Number(row.count_value ?? 0);
|
|
3901
|
+
return {
|
|
3902
|
+
t: Number(row.bucket),
|
|
3903
|
+
min: row.min_value === null ? null : Number(row.min_value),
|
|
3904
|
+
max: row.max_value === null ? null : Number(row.max_value),
|
|
3905
|
+
avg: count > 0 && row.sum_value !== null ? Number(row.sum_value) / count : null
|
|
4030
3906
|
};
|
|
4031
|
-
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
|
|
4037
|
-
|
|
4038
|
-
|
|
4039
|
-
|
|
4040
|
-
|
|
3907
|
+
});
|
|
3908
|
+
}
|
|
3909
|
+
function fromBucketMs(tier, fromMs) {
|
|
3910
|
+
if (tier === "raw") return fromMs;
|
|
3911
|
+
const span = tier === "10s" ? 1e4 : tier === "1m" ? 6e4 : 36e5;
|
|
3912
|
+
return Math.floor(fromMs / span) * span;
|
|
3913
|
+
}
|
|
3914
|
+
function tableForTier(tier) {
|
|
3915
|
+
return tier === "raw" ? "points_raw" : `points_${tier}`;
|
|
3916
|
+
}
|
|
3917
|
+
function readGaps(db, machineId, fromMs, toMs) {
|
|
3918
|
+
const dropped = db.db.prepare(
|
|
3919
|
+
"SELECT from_ms, to_ms, dropped, reason FROM gaps WHERE machine_id = ? AND to_ms >= ? AND from_ms <= ? ORDER BY from_ms LIMIT ?"
|
|
3920
|
+
).all(machineId, fromMs, toMs, MAX_GAPS);
|
|
3921
|
+
const missing = db.db.prepare(
|
|
3922
|
+
"WITH before_window AS ( SELECT captured_at_ms, expected_interval_ms FROM sample_receipts WHERE machine_id = ? AND captured_at_ms < ? ORDER BY captured_at_ms DESC LIMIT 1 ), in_window AS ( SELECT captured_at_ms, expected_interval_ms FROM sample_receipts WHERE machine_id = ? AND captured_at_ms >= ? AND captured_at_ms <= ? ), ordered AS ( SELECT captured_at_ms, expected_interval_ms, LAG(captured_at_ms) OVER (ORDER BY captured_at_ms) AS prev_ms FROM (SELECT * FROM before_window UNION ALL SELECT * FROM in_window) ) SELECT MAX(prev_ms, ?) AS from_ms, captured_at_ms AS to_ms FROM ordered WHERE prev_ms IS NOT NULL AND captured_at_ms > ? AND captured_at_ms - prev_ms > MAX(15000, 3 * COALESCE(expected_interval_ms, 5000)) ORDER BY captured_at_ms LIMIT ?"
|
|
3923
|
+
).all(machineId, fromMs, machineId, fromMs, toMs, fromMs, fromMs, MAX_GAPS);
|
|
3924
|
+
const gaps = [
|
|
3925
|
+
...dropped.map((row) => ({
|
|
3926
|
+
fromMs: Number(row.from_ms),
|
|
3927
|
+
toMs: Number(row.to_ms),
|
|
3928
|
+
kind: "dropped",
|
|
3929
|
+
reason: row.reason,
|
|
3930
|
+
droppedRecords: Number(row.dropped)
|
|
3931
|
+
})),
|
|
3932
|
+
...missing.map((row) => ({
|
|
3933
|
+
fromMs: Number(row.from_ms),
|
|
3934
|
+
toMs: Number(row.to_ms),
|
|
3935
|
+
kind: "missing",
|
|
3936
|
+
reason: "expected interval missed"
|
|
3937
|
+
}))
|
|
3938
|
+
];
|
|
3939
|
+
gaps.sort((a, b) => a.fromMs - b.fromMs || a.toMs - b.toMs);
|
|
3940
|
+
return gaps.slice(0, MAX_GAPS);
|
|
3941
|
+
}
|
|
3942
|
+
function validatedQuery(q) {
|
|
3943
|
+
if (q === null || typeof q !== "object") throw new HistoryQueryError("query must be an object");
|
|
3944
|
+
if (typeof q.machineId !== "string" || q.machineId.length === 0 || q.machineId.length > 256) {
|
|
3945
|
+
throw new HistoryQueryError("machineId must be a nonempty string");
|
|
4041
3946
|
}
|
|
4042
|
-
|
|
4043
|
-
|
|
4044
|
-
const row = this.latestSelect.get(machineId);
|
|
4045
|
-
const cursor = row ? { runId: row.run_id, sequence: Number(row.seq), capturedAtMs: Number(row.captured_at_ms) } : null;
|
|
4046
|
-
cache.set(machineId, cursor);
|
|
4047
|
-
return cursor;
|
|
3947
|
+
if (!Array.isArray(q.metrics) || q.metrics.length === 0 || q.metrics.length > MAX_METRICS) {
|
|
3948
|
+
throw new HistoryQueryError(`metrics must contain 1..${MAX_METRICS} metric names`);
|
|
4048
3949
|
}
|
|
4049
|
-
|
|
4050
|
-
|
|
4051
|
-
|
|
4052
|
-
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
const minute = Math.floor(tsMs / 6e4) * 6e4;
|
|
4057
|
-
const hour = Math.floor(tsMs / 36e5) * 36e5;
|
|
4058
|
-
this.applyRollup(this.upsert1m, seriesId, minute, value);
|
|
4059
|
-
this.applyRollup(this.upsert1h, seriesId, hour, value);
|
|
3950
|
+
const seenMetrics = /* @__PURE__ */ new Set();
|
|
3951
|
+
for (const metric of q.metrics) {
|
|
3952
|
+
if (typeof metric !== "string" || !Object.prototype.hasOwnProperty.call(METRICS, metric)) {
|
|
3953
|
+
throw new HistoryQueryError(`unknown metric ${JSON.stringify(metric)}`);
|
|
3954
|
+
}
|
|
3955
|
+
if (seenMetrics.has(metric)) throw new HistoryQueryError(`duplicate metric ${JSON.stringify(metric)}`);
|
|
3956
|
+
seenMetrics.add(metric);
|
|
4060
3957
|
}
|
|
4061
|
-
|
|
4062
|
-
if (
|
|
4063
|
-
|
|
4064
|
-
}
|
|
4065
|
-
|
|
3958
|
+
if (q.resourceIds !== void 0) {
|
|
3959
|
+
if (!Array.isArray(q.resourceIds) || q.resourceIds.length > MAX_RESOURCE_IDS) {
|
|
3960
|
+
throw new HistoryQueryError(`resourceIds must contain at most ${MAX_RESOURCE_IDS} ids`);
|
|
3961
|
+
}
|
|
3962
|
+
const seenResources = /* @__PURE__ */ new Set();
|
|
3963
|
+
for (const id of q.resourceIds) {
|
|
3964
|
+
if (typeof id !== "string" || id.length === 0 || id.length > 256) {
|
|
3965
|
+
throw new HistoryQueryError("resourceIds must be nonempty strings");
|
|
3966
|
+
}
|
|
3967
|
+
if (seenResources.has(id)) throw new HistoryQueryError(`duplicate resource id ${JSON.stringify(id)}`);
|
|
3968
|
+
seenResources.add(id);
|
|
4066
3969
|
}
|
|
4067
3970
|
}
|
|
4068
|
-
|
|
4069
|
-
|
|
4070
|
-
pending.settled = true;
|
|
4071
|
-
pending.resolve({
|
|
4072
|
-
status: "committed",
|
|
4073
|
-
accepted: pending.accepted,
|
|
4074
|
-
rejected: pending.rejected,
|
|
4075
|
-
newLive: pending.newLive,
|
|
4076
|
-
receivedAtMs: pending.receivedAtMs
|
|
4077
|
-
});
|
|
3971
|
+
if (q.coreIds !== void 0 && (!Array.isArray(q.coreIds) || q.coreIds.length > MAX_CORES || q.coreIds.some((id) => typeof id !== "string" || !/^core:(?:0|[1-9]\d{0,3})$/.test(id) || Number(id.slice(5)) >= MAX_CORES) || new Set(q.coreIds).size !== q.coreIds.length)) {
|
|
3972
|
+
throw new HistoryQueryError(`coreIds must contain at most ${MAX_CORES} unique core identifiers`);
|
|
4078
3973
|
}
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
3974
|
+
if (!Number.isSafeInteger(q.fromMs) || !Number.isSafeInteger(q.toMs)) {
|
|
3975
|
+
throw new HistoryQueryError("fromMs and toMs must be safe integer epoch milliseconds");
|
|
3976
|
+
}
|
|
3977
|
+
if (q.fromMs >= q.toMs) throw new HistoryQueryError("fromMs must be before toMs");
|
|
3978
|
+
if (q.toMs - q.fromMs > MAX_RANGE_MS) {
|
|
3979
|
+
throw new HistoryQueryError("range exceeds 400 days");
|
|
4083
3980
|
}
|
|
4084
|
-
};
|
|
4085
|
-
function prepareRollupUpsert(db, table) {
|
|
4086
|
-
return db.prepare(
|
|
4087
|
-
`INSERT INTO ${table} (series_id, bucket_ms, min, max, sum, count, null_count) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(series_id, bucket_ms) DO UPDATE SET min = CASE WHEN excluded.count = 0 THEN min WHEN min IS NULL THEN excluded.min ELSE MIN(min, excluded.min) END, max = CASE WHEN excluded.count = 0 THEN max WHEN max IS NULL THEN excluded.max ELSE MAX(max, excluded.max) END, sum = sum + excluded.sum, count = count + excluded.count, null_count = null_count + excluded.null_count`
|
|
4088
|
-
);
|
|
4089
3981
|
}
|
|
4090
|
-
function
|
|
4091
|
-
if (
|
|
4092
|
-
|
|
3982
|
+
function validatedMaxPoints(value) {
|
|
3983
|
+
if (value === void 0) return DEFAULT_MAX_POINTS;
|
|
3984
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
3985
|
+
throw new HistoryQueryError("maxPoints must be a positive safe integer");
|
|
3986
|
+
}
|
|
3987
|
+
return Math.min(value, MAX_POINTS);
|
|
4093
3988
|
}
|
|
4094
|
-
function
|
|
4095
|
-
|
|
3989
|
+
function validatedBudget(value, fallback, name) {
|
|
3990
|
+
if (value === void 0) return fallback;
|
|
3991
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
3992
|
+
throw new HistoryQueryError(`${name} must be a positive safe integer`);
|
|
3993
|
+
}
|
|
3994
|
+
return value;
|
|
4096
3995
|
}
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
3996
|
+
|
|
3997
|
+
// src/hub/storage/telemetry-store.ts
|
|
3998
|
+
var DAY_MS3 = 864e5;
|
|
3999
|
+
var FUTURE_TOLERANCE_MS = DAY_MS3;
|
|
4000
|
+
var FAILURE_RETRY_MS = 5e3;
|
|
4001
|
+
var MIN_DERIVED_PHYSICAL_BYTES = 8 * 1024 * 1024;
|
|
4002
|
+
var TelemetryStore = class {
|
|
4003
|
+
historyMode = "persistent";
|
|
4004
|
+
database;
|
|
4005
|
+
now;
|
|
4006
|
+
maxQueuedRecords;
|
|
4007
|
+
flushIntervalMs;
|
|
4008
|
+
maxRecordsPerTransaction;
|
|
4009
|
+
maxPhysicalBytes;
|
|
4010
|
+
storageTier;
|
|
4011
|
+
series;
|
|
4012
|
+
retention;
|
|
4013
|
+
queue = [];
|
|
4014
|
+
timer = null;
|
|
4015
|
+
driver = null;
|
|
4016
|
+
closed = false;
|
|
4017
|
+
committedRecords = 0;
|
|
4018
|
+
duplicateRecords = 0;
|
|
4019
|
+
rejectedRecords = 0;
|
|
4020
|
+
writeFailures = 0;
|
|
4021
|
+
diskFullFailures = 0;
|
|
4022
|
+
lastWriteError = null;
|
|
4023
|
+
lastFlushMs = null;
|
|
4024
|
+
lastFlushDurationMs = null;
|
|
4025
|
+
receiptInsert;
|
|
4026
|
+
rawInsert;
|
|
4027
|
+
upsert10s;
|
|
4028
|
+
upsert1m;
|
|
4029
|
+
upsert1h;
|
|
4030
|
+
gapInsert;
|
|
4031
|
+
latestSelect;
|
|
4032
|
+
latestUpsert;
|
|
4033
|
+
inventoryUpsert;
|
|
4034
|
+
inventorySelect;
|
|
4035
|
+
metaUpsertBoth;
|
|
4036
|
+
metaUpsertDisplay;
|
|
4037
|
+
metaUpsertTags;
|
|
4038
|
+
metaTouch;
|
|
4039
|
+
metaSelect;
|
|
4040
|
+
metaSelectAll;
|
|
4041
|
+
latestSelectAll;
|
|
4042
|
+
constructor(options) {
|
|
4043
|
+
this.database = options.database;
|
|
4044
|
+
this.now = options.now ?? (() => Date.now());
|
|
4045
|
+
this.maxQueuedRecords = positiveInt2(options.maxQueuedRecords, 4096, "maxQueuedRecords");
|
|
4046
|
+
this.flushIntervalMs = nonnegativeInt(options.flushIntervalMs, 20, "flushIntervalMs");
|
|
4047
|
+
this.maxRecordsPerTransaction = positiveInt2(
|
|
4048
|
+
options.maxRecordsPerTransaction,
|
|
4049
|
+
512,
|
|
4050
|
+
"maxRecordsPerTransaction"
|
|
4051
|
+
);
|
|
4052
|
+
this.maxPhysicalBytes = resolvePhysicalLimit(options);
|
|
4053
|
+
this.storageTier = options.storageTier ?? "raw";
|
|
4054
|
+
if (this.storageTier !== "raw" && this.storageTier !== "summary") {
|
|
4055
|
+
throw new Error("storageTier must be raw or summary");
|
|
4056
|
+
}
|
|
4057
|
+
const db = options.database.db;
|
|
4058
|
+
this.series = new SeriesCatalog(db);
|
|
4059
|
+
this.receiptInsert = db.prepare(
|
|
4060
|
+
"INSERT OR IGNORE INTO sample_receipts (collector_id, run_id, seq, machine_id, kind, mode, captured_at_ms, received_at_ms, expected_interval_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
4061
|
+
);
|
|
4062
|
+
this.rawInsert = db.prepare(
|
|
4063
|
+
"INSERT OR IGNORE INTO points_raw (series_id, ts_ms, value) VALUES (?, ?, ?)"
|
|
4064
|
+
);
|
|
4065
|
+
this.upsert10s = prepareRollupUpsert(db, "points_10s");
|
|
4066
|
+
this.upsert1m = prepareRollupUpsert(db, "points_1m");
|
|
4067
|
+
this.upsert1h = prepareRollupUpsert(db, "points_1h");
|
|
4068
|
+
this.gapInsert = db.prepare(
|
|
4069
|
+
"INSERT OR IGNORE INTO gaps (collector_id, run_id, seq, machine_id, from_ms, to_ms, dropped, reason, received_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
4070
|
+
);
|
|
4071
|
+
this.latestSelect = db.prepare(
|
|
4072
|
+
"SELECT run_id, seq, captured_at_ms FROM machine_latest WHERE machine_id = ?"
|
|
4073
|
+
);
|
|
4074
|
+
this.latestUpsert = db.prepare(
|
|
4075
|
+
"INSERT INTO machine_latest (machine_id, collector_id, run_id, boot_id, seq, captured_at_ms, received_at_ms, expected_interval_ms, hostname, payload_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET collector_id = excluded.collector_id, run_id = excluded.run_id, boot_id = excluded.boot_id, seq = excluded.seq, captured_at_ms = excluded.captured_at_ms, received_at_ms = excluded.received_at_ms, expected_interval_ms = COALESCE(excluded.expected_interval_ms, expected_interval_ms), hostname = COALESCE(excluded.hostname, hostname), payload_json = excluded.payload_json"
|
|
4076
|
+
);
|
|
4077
|
+
this.latestSelectAll = db.prepare(
|
|
4078
|
+
"SELECT machine_id, collector_id, run_id, boot_id, seq, captured_at_ms, received_at_ms, expected_interval_ms, hostname, payload_json FROM machine_latest ORDER BY machine_id"
|
|
4079
|
+
);
|
|
4080
|
+
this.inventoryUpsert = db.prepare(
|
|
4081
|
+
"INSERT INTO inventory (machine_id, captured_at_ms, received_at_ms, inventory_json) VALUES (?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET captured_at_ms = excluded.captured_at_ms, received_at_ms = excluded.received_at_ms, inventory_json = excluded.inventory_json"
|
|
4082
|
+
);
|
|
4083
|
+
this.inventorySelect = db.prepare(
|
|
4084
|
+
"SELECT captured_at_ms, received_at_ms, inventory_json FROM inventory WHERE machine_id = ?"
|
|
4085
|
+
);
|
|
4086
|
+
this.metaUpsertBoth = db.prepare(
|
|
4087
|
+
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, ?, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET display_name = excluded.display_name, tags_json = excluded.tags_json, updated_at_ms = excluded.updated_at_ms"
|
|
4088
|
+
);
|
|
4089
|
+
this.metaUpsertDisplay = db.prepare(
|
|
4090
|
+
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, ?, '[]', ?) ON CONFLICT(machine_id) DO UPDATE SET display_name = excluded.display_name, updated_at_ms = excluded.updated_at_ms"
|
|
4091
|
+
);
|
|
4092
|
+
this.metaUpsertTags = db.prepare(
|
|
4093
|
+
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, NULL, ?, ?) ON CONFLICT(machine_id) DO UPDATE SET tags_json = excluded.tags_json, updated_at_ms = excluded.updated_at_ms"
|
|
4094
|
+
);
|
|
4095
|
+
this.metaTouch = db.prepare(
|
|
4096
|
+
"INSERT INTO machine_meta (machine_id, display_name, tags_json, updated_at_ms) VALUES (?, NULL, '[]', ?) ON CONFLICT(machine_id) DO UPDATE SET updated_at_ms = excluded.updated_at_ms"
|
|
4097
|
+
);
|
|
4098
|
+
this.metaSelect = db.prepare(
|
|
4099
|
+
"SELECT display_name, tags_json, updated_at_ms FROM machine_meta WHERE machine_id = ?"
|
|
4100
|
+
);
|
|
4101
|
+
this.metaSelectAll = db.prepare(
|
|
4102
|
+
"SELECT machine_id, display_name, tags_json FROM machine_meta ORDER BY machine_id"
|
|
4103
|
+
);
|
|
4104
|
+
this.retention = new RetentionManager(options.database, {
|
|
4105
|
+
policy: options.retention,
|
|
4106
|
+
maxDatabaseBytes: options.maxDatabaseBytes ?? null,
|
|
4107
|
+
maxPhysicalBytes: this.maxPhysicalBytes,
|
|
4108
|
+
now: this.now
|
|
4109
|
+
});
|
|
4110
|
+
this.retention.start();
|
|
4100
4111
|
}
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4112
|
+
/**
|
|
4113
|
+
* Queues one batch. The promise resolves only after the transaction that
|
|
4114
|
+
* writes it has committed, or immediately as `busy` when the bounded queue
|
|
4115
|
+
* cannot hold it (no database work is done in that case).
|
|
4116
|
+
*/
|
|
4117
|
+
enqueue(batch) {
|
|
4118
|
+
return new Promise((resolve7) => {
|
|
4119
|
+
if (this.closed) {
|
|
4120
|
+
resolve7({ status: "busy", retryAfterMs: FAILURE_RETRY_MS, reason: "storage is closed" });
|
|
4121
|
+
return;
|
|
4122
|
+
}
|
|
4123
|
+
validateBatch(batch);
|
|
4124
|
+
const queued = this.queuedRecords();
|
|
4125
|
+
if (queued + batch.records.length > this.maxQueuedRecords) {
|
|
4126
|
+
const ratio = this.maxQueuedRecords === 0 ? 1 : queued / this.maxQueuedRecords;
|
|
4127
|
+
const retryAfterMs = Math.min(5e3, Math.max(1e3, Math.round(1e3 + ratio * 4e3)));
|
|
4128
|
+
resolve7({ status: "busy", retryAfterMs, reason: "ingest queue is full" });
|
|
4129
|
+
return;
|
|
4130
|
+
}
|
|
4131
|
+
this.queue.push({
|
|
4132
|
+
batch,
|
|
4133
|
+
receivedAtMs: this.now(),
|
|
4134
|
+
resolve: resolve7,
|
|
4135
|
+
settled: false,
|
|
4136
|
+
accepted: [],
|
|
4137
|
+
rejected: [],
|
|
4138
|
+
newLive: null
|
|
4139
|
+
});
|
|
4140
|
+
this.scheduleFlush(this.flushIntervalMs);
|
|
4141
|
+
});
|
|
4105
4142
|
}
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4143
|
+
/** Persisted latest LIVE metrics per machine, for restart recovery. */
|
|
4144
|
+
latestLive() {
|
|
4145
|
+
const rows = this.latestSelectAll.all();
|
|
4146
|
+
return rows.map((row) => ({
|
|
4147
|
+
machineId: row.machine_id,
|
|
4148
|
+
collectorId: row.collector_id,
|
|
4149
|
+
runId: row.run_id,
|
|
4150
|
+
bootId: row.boot_id,
|
|
4151
|
+
sequence: Number(row.seq),
|
|
4152
|
+
capturedAt: new Date(Number(row.captured_at_ms)).toISOString(),
|
|
4153
|
+
receivedAt: new Date(Number(row.received_at_ms)).toISOString(),
|
|
4154
|
+
hostname: row.hostname,
|
|
4155
|
+
expectedIntervalMs: row.expected_interval_ms === null ? null : Number(row.expected_interval_ms),
|
|
4156
|
+
payload: JSON.parse(row.payload_json)
|
|
4157
|
+
}));
|
|
4112
4158
|
}
|
|
4113
|
-
|
|
4114
|
-
|
|
4159
|
+
saveInventory(machineId, capturedAt, inventory) {
|
|
4160
|
+
const capturedMs = Date.parse(capturedAt);
|
|
4161
|
+
if (!Number.isFinite(capturedMs)) throw new Error("inventory capturedAt must be a parseable timestamp");
|
|
4162
|
+
this.inventoryUpsert.run(machineId, capturedMs, this.now(), JSON.stringify(inventory));
|
|
4115
4163
|
}
|
|
4116
|
-
|
|
4117
|
-
|
|
4164
|
+
loadInventory(machineId) {
|
|
4165
|
+
const row = this.inventorySelect.get(machineId);
|
|
4166
|
+
if (!row) return null;
|
|
4167
|
+
return {
|
|
4168
|
+
capturedAt: new Date(Number(row.captured_at_ms)).toISOString(),
|
|
4169
|
+
receivedAt: new Date(Number(row.received_at_ms)).toISOString(),
|
|
4170
|
+
inventory: JSON.parse(row.inventory_json)
|
|
4171
|
+
};
|
|
4118
4172
|
}
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4173
|
+
/**
|
|
4174
|
+
* Merges presentation metadata. Omitted fields are preserved; an explicit
|
|
4175
|
+
* `displayName: null` clears it.
|
|
4176
|
+
*/
|
|
4177
|
+
setMachineMeta(machineId, meta) {
|
|
4178
|
+
const updatedAt = this.now();
|
|
4179
|
+
const hasDisplay = meta.displayName !== void 0;
|
|
4180
|
+
const hasTags = meta.tags !== void 0;
|
|
4181
|
+
const displayName = meta.displayName ?? null;
|
|
4182
|
+
const tags = hasTags ? validateTags(meta.tags) : [];
|
|
4183
|
+
if (hasDisplay && hasTags) {
|
|
4184
|
+
this.metaUpsertBoth.run(machineId, displayName, JSON.stringify(tags), updatedAt);
|
|
4185
|
+
} else if (hasDisplay) {
|
|
4186
|
+
this.metaUpsertDisplay.run(machineId, displayName, updatedAt);
|
|
4187
|
+
} else if (hasTags) {
|
|
4188
|
+
this.metaUpsertTags.run(machineId, JSON.stringify(tags), updatedAt);
|
|
4189
|
+
} else {
|
|
4190
|
+
this.metaTouch.run(machineId, updatedAt);
|
|
4191
|
+
}
|
|
4122
4192
|
}
|
|
4123
|
-
|
|
4124
|
-
|
|
4193
|
+
getMachineMeta(machineId) {
|
|
4194
|
+
const row = this.metaSelect.get(machineId);
|
|
4195
|
+
if (!row) return { displayName: null, tags: [] };
|
|
4196
|
+
return { displayName: row.display_name, tags: parseTags(row.tags_json) };
|
|
4125
4197
|
}
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4133
|
-
|
|
4198
|
+
listMachineMeta() {
|
|
4199
|
+
const rows = this.metaSelectAll.all();
|
|
4200
|
+
const result = /* @__PURE__ */ new Map();
|
|
4201
|
+
for (const row of rows) {
|
|
4202
|
+
result.set(row.machine_id, {
|
|
4203
|
+
displayName: row.display_name,
|
|
4204
|
+
tags: parseTags(row.tags_json)
|
|
4205
|
+
});
|
|
4134
4206
|
}
|
|
4135
|
-
|
|
4136
|
-
seen.add(tag);
|
|
4137
|
-
result.push(tag);
|
|
4207
|
+
return result;
|
|
4138
4208
|
}
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
|
|
4209
|
+
stats() {
|
|
4210
|
+
return {
|
|
4211
|
+
queuedRecords: this.queuedRecords(),
|
|
4212
|
+
queuedBatches: this.queue.length,
|
|
4213
|
+
lastFlushMs: this.lastFlushMs,
|
|
4214
|
+
lastFlushDurationMs: this.lastFlushDurationMs,
|
|
4215
|
+
committedRecords: this.committedRecords,
|
|
4216
|
+
duplicateRecords: this.duplicateRecords,
|
|
4217
|
+
rejectedRecords: this.rejectedRecords,
|
|
4218
|
+
writeFailures: this.writeFailures,
|
|
4219
|
+
lastWriteError: this.lastWriteError,
|
|
4220
|
+
diskFullFailures: this.diskFullFailures,
|
|
4221
|
+
dbBytes: this.database.sizeBytes(),
|
|
4222
|
+
dbWalBytes: this.database.walBytes(),
|
|
4223
|
+
dbBytesLimit: this.maxPhysicalBytes,
|
|
4224
|
+
retention: this.retention.status()
|
|
4225
|
+
};
|
|
4148
4226
|
}
|
|
4149
|
-
}
|
|
4150
|
-
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
}
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
|
|
4157
|
-
throw new Error(`${name} must be a positive safe integer`);
|
|
4227
|
+
queryHistory(query, budget = {}) {
|
|
4228
|
+
return queryHistory(this.database, query, {
|
|
4229
|
+
...budget,
|
|
4230
|
+
nowMs: budget.nowMs ?? this.now(),
|
|
4231
|
+
retention: { ...this.retention.policy, ...budget.retention },
|
|
4232
|
+
tiers: budget.tiers ?? (this.storageTier === "summary" ? ["10s", "1m", "1h"] : void 0)
|
|
4233
|
+
});
|
|
4158
4234
|
}
|
|
4159
|
-
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4235
|
+
historyStatus() {
|
|
4236
|
+
const stats = this.stats();
|
|
4237
|
+
const physical = this.database.physicalLimit();
|
|
4238
|
+
const now = this.now();
|
|
4239
|
+
const retention = this.retention.policy;
|
|
4240
|
+
return {
|
|
4241
|
+
mode: "persistent",
|
|
4242
|
+
persistentHistory: true,
|
|
4243
|
+
availableFromMs: this.storageTier === "summary" ? now - Math.max(retention.tenSecondMs, retention.oneMinuteMs, retention.oneHourMs) : now - Math.max(retention.rawMs, retention.oneMinuteMs, retention.oneHourMs),
|
|
4244
|
+
availableToMs: now,
|
|
4245
|
+
memory: null,
|
|
4246
|
+
persistence: {
|
|
4247
|
+
enabled: true,
|
|
4248
|
+
storageTier: this.storageTier,
|
|
4249
|
+
flushIntervalMs: this.flushIntervalMs,
|
|
4250
|
+
queuedRecords: stats.queuedRecords,
|
|
4251
|
+
queuedBatches: stats.queuedBatches,
|
|
4252
|
+
lastFlushMs: stats.lastFlushMs,
|
|
4253
|
+
lastFlushDurationMs: stats.lastFlushDurationMs,
|
|
4254
|
+
writeFailures: stats.writeFailures,
|
|
4255
|
+
lastWriteError: stats.lastWriteError,
|
|
4256
|
+
dbBytes: stats.dbBytes,
|
|
4257
|
+
dbWalBytes: stats.dbWalBytes,
|
|
4258
|
+
dbBytesLimit: physical?.limitBytes ?? this.maxPhysicalBytes,
|
|
4259
|
+
tiers: this.storageTier === "summary" ? [
|
|
4260
|
+
{ tier: "10s", stepMs: 1e4, retentionMs: retention.tenSecondMs, bytes: stats.dbBytes },
|
|
4261
|
+
{ tier: "1m", stepMs: 6e4, retentionMs: retention.oneMinuteMs, bytes: 0 },
|
|
4262
|
+
{ tier: "1h", stepMs: 36e5, retentionMs: retention.oneHourMs, bytes: 0 }
|
|
4263
|
+
] : [
|
|
4264
|
+
{ tier: "raw", stepMs: 100, retentionMs: retention.rawMs, bytes: stats.dbBytes },
|
|
4265
|
+
{ tier: "1m", stepMs: 6e4, retentionMs: retention.oneMinuteMs, bytes: 0 },
|
|
4266
|
+
{ tier: "1h", stepMs: 36e5, retentionMs: retention.oneHourMs, bytes: 0 }
|
|
4267
|
+
]
|
|
4268
|
+
},
|
|
4269
|
+
gaps: 0,
|
|
4270
|
+
acceptedRecords: stats.committedRecords,
|
|
4271
|
+
generatedAt: new Date(now).toISOString()
|
|
4272
|
+
};
|
|
4273
|
+
}
|
|
4274
|
+
/** Drains the queue immediately; intended for tests and shutdown. */
|
|
4275
|
+
async flushNow() {
|
|
4276
|
+
if (this.closed) return;
|
|
4277
|
+
if (this.timer !== null) {
|
|
4278
|
+
clearTimeout(this.timer);
|
|
4279
|
+
this.timer = null;
|
|
4280
|
+
}
|
|
4281
|
+
await this.drive();
|
|
4282
|
+
while (!this.closed && this.queue.length > 0 && this.driver === null) {
|
|
4283
|
+
const failures = this.writeFailures;
|
|
4284
|
+
await this.drive();
|
|
4285
|
+
if (this.writeFailures > failures) break;
|
|
4166
4286
|
}
|
|
4167
|
-
return options.maxPhysicalBytes;
|
|
4168
4287
|
}
|
|
4169
|
-
|
|
4170
|
-
|
|
4288
|
+
/**
|
|
4289
|
+
* Drains for up to ~2s, then resolves anything left as busy and closes the
|
|
4290
|
+
* database. Batches that were already committed have resolved normally.
|
|
4291
|
+
*/
|
|
4292
|
+
async close() {
|
|
4293
|
+
if (this.closed) return;
|
|
4294
|
+
this.closed = true;
|
|
4295
|
+
this.retention.stop();
|
|
4296
|
+
if (this.timer !== null) {
|
|
4297
|
+
clearTimeout(this.timer);
|
|
4298
|
+
this.timer = null;
|
|
4299
|
+
}
|
|
4300
|
+
if (this.driver !== null) {
|
|
4301
|
+
try {
|
|
4302
|
+
await this.driver;
|
|
4303
|
+
} catch {
|
|
4304
|
+
}
|
|
4305
|
+
}
|
|
4306
|
+
const deadline = Date.now() + 2e3;
|
|
4307
|
+
while (this.queue.length > 0 && Date.now() < deadline) {
|
|
4308
|
+
const ok = this.flushTransaction();
|
|
4309
|
+
if (!ok) break;
|
|
4310
|
+
if (this.queue.length > 0) await yieldToEventLoop2();
|
|
4311
|
+
}
|
|
4312
|
+
while (this.queue.length > 0) {
|
|
4313
|
+
const pending = this.queue.shift();
|
|
4314
|
+
this.resolveBusy(pending, FAILURE_RETRY_MS, "storage is shutting down");
|
|
4315
|
+
}
|
|
4316
|
+
try {
|
|
4317
|
+
this.database.close();
|
|
4318
|
+
} catch {
|
|
4319
|
+
}
|
|
4171
4320
|
}
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
if (!Number.isSafeInteger(resolved) || resolved < 0) {
|
|
4177
|
-
throw new Error(`${name} must be a nonnegative safe integer`);
|
|
4321
|
+
queuedRecords() {
|
|
4322
|
+
let total = 0;
|
|
4323
|
+
for (const pending of this.queue) total += pending.batch.records.length;
|
|
4324
|
+
return total;
|
|
4178
4325
|
}
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
var MAX_CORES = 1024;
|
|
4187
|
-
var MIN_EPOCH = Date.parse("2000-01-01T00:00:00Z");
|
|
4188
|
-
var MAX_EPOCH = Date.parse("2200-01-01T00:00:00Z");
|
|
4189
|
-
|
|
4190
|
-
// src/contracts/index.ts
|
|
4191
|
-
var MIN_TIMESTAMP_EPOCH = Date.parse("2000-01-01T00:00:00Z");
|
|
4192
|
-
var MAX_TIMESTAMP_EPOCH = Date.parse("2200-01-01T00:00:00Z");
|
|
4193
|
-
|
|
4194
|
-
// src/hub/storage/history-query.ts
|
|
4195
|
-
var DAY_MS3 = 864e5;
|
|
4196
|
-
var MAX_RANGE_MS = 400 * DAY_MS3;
|
|
4197
|
-
var MAX_METRICS = Object.keys(METRICS).length;
|
|
4198
|
-
var MAX_RESOURCE_IDS = 64;
|
|
4199
|
-
var DEFAULT_MAX_POINTS = 600;
|
|
4200
|
-
var MAX_POINTS = 2e3;
|
|
4201
|
-
var MAX_GAPS = 500;
|
|
4202
|
-
var TIER_RESOLUTION_MS = { raw: 100, "1m": 6e4, "1h": 36e5 };
|
|
4203
|
-
var HistoryQueryError = class extends Error {
|
|
4204
|
-
code = "invalid-query";
|
|
4205
|
-
constructor(message) {
|
|
4206
|
-
super(message);
|
|
4207
|
-
this.name = "HistoryQueryError";
|
|
4326
|
+
scheduleFlush(delayMs) {
|
|
4327
|
+
if (this.closed || this.timer !== null || this.driver !== null) return;
|
|
4328
|
+
this.timer = setTimeout(() => {
|
|
4329
|
+
this.timer = null;
|
|
4330
|
+
void this.drive();
|
|
4331
|
+
}, delayMs);
|
|
4332
|
+
this.timer.unref?.();
|
|
4208
4333
|
}
|
|
4209
|
-
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
const nowMs = Date.now();
|
|
4220
|
-
let allSeries = loadSeries(db, q);
|
|
4221
|
-
let truncated = false;
|
|
4222
|
-
if (allSeries.length > maxSeries) {
|
|
4223
|
-
allSeries = allSeries.slice(0, maxSeries);
|
|
4224
|
-
truncated = true;
|
|
4334
|
+
drive() {
|
|
4335
|
+
if (this.driver !== null) return this.driver;
|
|
4336
|
+
const promise = this.runDriver().finally(() => {
|
|
4337
|
+
if (this.driver === promise) this.driver = null;
|
|
4338
|
+
if (!this.closed && this.queue.length > 0 && this.timer === null) {
|
|
4339
|
+
this.scheduleFlush(FAILURE_RETRY_MS);
|
|
4340
|
+
}
|
|
4341
|
+
});
|
|
4342
|
+
this.driver = promise;
|
|
4343
|
+
return promise;
|
|
4225
4344
|
}
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4232
|
-
retention,
|
|
4233
|
-
maxRowsScanned,
|
|
4234
|
-
maxElapsedMs,
|
|
4235
|
-
started
|
|
4236
|
-
);
|
|
4237
|
-
if (chosen.timeExceeded || chosen.overBudget) truncated = true;
|
|
4238
|
-
let seriesRows = allSeries;
|
|
4239
|
-
if (chosen.cutSeries < seriesRows.length) {
|
|
4240
|
-
seriesRows = seriesRows.slice(0, Math.max(0, chosen.cutSeries));
|
|
4241
|
-
truncated = true;
|
|
4345
|
+
async runDriver() {
|
|
4346
|
+
while (!this.closed && this.queue.length > 0) {
|
|
4347
|
+
const ok = this.flushTransaction();
|
|
4348
|
+
if (!ok) return;
|
|
4349
|
+
if (this.queue.length > 0) await yieldToEventLoop2();
|
|
4350
|
+
}
|
|
4242
4351
|
}
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4352
|
+
/**
|
|
4353
|
+
* Commits the front of the queue as one transaction. Returns false when the
|
|
4354
|
+
* transaction failed; every batch in it has then been resolved as busy.
|
|
4355
|
+
*/
|
|
4356
|
+
flushTransaction() {
|
|
4357
|
+
if (this.queue.length === 0) return true;
|
|
4358
|
+
const selected = [];
|
|
4359
|
+
let recordCount = 0;
|
|
4360
|
+
for (const pending of this.queue) {
|
|
4361
|
+
const size = pending.batch.records.length;
|
|
4362
|
+
if (selected.length > 0 && recordCount + size > this.maxRecordsPerTransaction) break;
|
|
4363
|
+
selected.push(pending);
|
|
4364
|
+
recordCount += size;
|
|
4365
|
+
if (recordCount >= this.maxRecordsPerTransaction) break;
|
|
4366
|
+
}
|
|
4367
|
+
this.queue.splice(0, selected.length);
|
|
4368
|
+
if (this.maxPhysicalBytes !== null) {
|
|
4369
|
+
try {
|
|
4370
|
+
let physical = this.database.physicalLimit();
|
|
4371
|
+
let bytes = this.database.sizeBytes();
|
|
4372
|
+
const pageFull = physical !== null && physical.pageCount >= physical.maxPageCount && physical.freePages === 0;
|
|
4373
|
+
if (bytes >= this.maxPhysicalBytes || pageFull) {
|
|
4374
|
+
this.retention.relievePressure(4);
|
|
4375
|
+
physical = this.database.physicalLimit();
|
|
4376
|
+
bytes = this.database.sizeBytes();
|
|
4377
|
+
}
|
|
4378
|
+
const stillFull = physical !== null && physical.pageCount >= physical.maxPageCount && physical.freePages === 0;
|
|
4379
|
+
if (bytes >= this.maxPhysicalBytes || stillFull) {
|
|
4380
|
+
this.diskFullFailures += 1;
|
|
4381
|
+
this.lastWriteError = "storage disk budget reached";
|
|
4382
|
+
for (const pending of selected) {
|
|
4383
|
+
this.resolveBusy(pending, FAILURE_RETRY_MS, "storage disk budget reached");
|
|
4384
|
+
}
|
|
4385
|
+
return false;
|
|
4386
|
+
}
|
|
4387
|
+
} catch (error) {
|
|
4388
|
+
const full = isStorageFullError(error);
|
|
4389
|
+
if (full) this.diskFullFailures += 1;
|
|
4390
|
+
this.lastWriteError = full ? "storage disk budget reached" : sanitizeWriteError(error);
|
|
4391
|
+
for (const pending of selected) {
|
|
4392
|
+
this.resolveBusy(
|
|
4393
|
+
pending,
|
|
4394
|
+
FAILURE_RETRY_MS,
|
|
4395
|
+
full ? "storage disk budget reached" : "storage write failed"
|
|
4396
|
+
);
|
|
4397
|
+
}
|
|
4398
|
+
return false;
|
|
4399
|
+
}
|
|
4250
4400
|
}
|
|
4251
|
-
const
|
|
4252
|
-
const
|
|
4253
|
-
|
|
4254
|
-
const
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
}
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4401
|
+
const started = import_node_perf_hooks.performance.now();
|
|
4402
|
+
const counters = { committed: 0, duplicates: 0, rejected: 0 };
|
|
4403
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
4404
|
+
const upserts = /* @__PURE__ */ new Map();
|
|
4405
|
+
const summaries = /* @__PURE__ */ new Map();
|
|
4406
|
+
try {
|
|
4407
|
+
this.database.db.exec("BEGIN IMMEDIATE");
|
|
4408
|
+
for (const pending of selected) {
|
|
4409
|
+
this.processBatch(pending, counters, cursors, upserts, summaries);
|
|
4410
|
+
}
|
|
4411
|
+
if (this.storageTier === "summary" && summaries.size > 0) {
|
|
4412
|
+
this.writeSummaries(summaries);
|
|
4413
|
+
}
|
|
4414
|
+
for (const [machineId, live] of upserts) {
|
|
4415
|
+
this.latestUpsert.run(
|
|
4416
|
+
machineId,
|
|
4417
|
+
live.record.collectorId,
|
|
4418
|
+
live.record.runId,
|
|
4419
|
+
live.record.bootId,
|
|
4420
|
+
live.record.sequence,
|
|
4421
|
+
Date.parse(live.record.capturedAt),
|
|
4422
|
+
live.receivedAtMs,
|
|
4423
|
+
live.expectedIntervalMs,
|
|
4424
|
+
live.hostname,
|
|
4425
|
+
JSON.stringify(live.record.payload)
|
|
4426
|
+
);
|
|
4427
|
+
}
|
|
4428
|
+
this.database.db.exec("COMMIT");
|
|
4429
|
+
} catch (error) {
|
|
4430
|
+
try {
|
|
4431
|
+
this.database.db.exec("ROLLBACK");
|
|
4432
|
+
} catch {
|
|
4433
|
+
}
|
|
4434
|
+
this.writeFailures += 1;
|
|
4435
|
+
const full = isStorageFullError(error);
|
|
4436
|
+
if (full) this.diskFullFailures += 1;
|
|
4437
|
+
this.lastWriteError = full ? "storage disk budget reached" : sanitizeWriteError(error);
|
|
4438
|
+
for (const pending of selected) {
|
|
4439
|
+
this.resolveBusy(
|
|
4440
|
+
pending,
|
|
4441
|
+
FAILURE_RETRY_MS,
|
|
4442
|
+
full ? "storage disk budget reached" : "storage write failed"
|
|
4443
|
+
);
|
|
4444
|
+
}
|
|
4445
|
+
return false;
|
|
4446
|
+
}
|
|
4447
|
+
this.committedRecords += counters.committed;
|
|
4448
|
+
this.duplicateRecords += counters.duplicates;
|
|
4449
|
+
this.rejectedRecords += counters.rejected;
|
|
4450
|
+
this.lastFlushMs = this.now();
|
|
4451
|
+
this.lastFlushDurationMs = Math.max(0, Math.round(import_node_perf_hooks.performance.now() - started));
|
|
4452
|
+
this.checkpointWALIfNeeded();
|
|
4453
|
+
if (this.maxPhysicalBytes !== null && this.database.sizeBytes() > this.maxPhysicalBytes) {
|
|
4454
|
+
try {
|
|
4455
|
+
this.retention.relievePressure(4);
|
|
4456
|
+
} catch {
|
|
4457
|
+
}
|
|
4458
|
+
}
|
|
4459
|
+
for (const pending of selected) this.resolveCommitted(pending);
|
|
4460
|
+
return true;
|
|
4290
4461
|
}
|
|
4291
|
-
|
|
4292
|
-
if (
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4462
|
+
checkpointWALIfNeeded() {
|
|
4463
|
+
if (this.maxPhysicalBytes === null) return;
|
|
4464
|
+
try {
|
|
4465
|
+
const physical = this.database.physicalLimit();
|
|
4466
|
+
if (physical !== null && this.database.walBytes() > physical.walBudgetBytes) {
|
|
4467
|
+
this.database.checkpoint();
|
|
4468
|
+
}
|
|
4469
|
+
} catch {
|
|
4296
4470
|
}
|
|
4297
4471
|
}
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
|
|
4301
|
-
|
|
4302
|
-
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4472
|
+
processBatch(pending, counters, cursors, upserts, summaries) {
|
|
4473
|
+
const batch = pending.batch;
|
|
4474
|
+
let best = null;
|
|
4475
|
+
let bestCapturedMs = 0;
|
|
4476
|
+
let bestSequence = -1;
|
|
4477
|
+
for (const record of batch.records) {
|
|
4478
|
+
const ref = { runId: record.runId, sequence: record.sequence };
|
|
4479
|
+
const reason = rejectionReason(record, batch, pending.receivedAtMs);
|
|
4480
|
+
if (reason !== null) {
|
|
4481
|
+
pending.rejected.push({ ...ref, reason });
|
|
4482
|
+
counters.rejected += 1;
|
|
4483
|
+
continue;
|
|
4484
|
+
}
|
|
4485
|
+
const capturedMs = Date.parse(record.capturedAt);
|
|
4486
|
+
const inserted = Number(
|
|
4487
|
+
this.receiptInsert.run(
|
|
4488
|
+
batch.collectorId,
|
|
4489
|
+
record.runId,
|
|
4490
|
+
record.sequence,
|
|
4491
|
+
batch.machineId,
|
|
4492
|
+
record.kind,
|
|
4493
|
+
batch.mode,
|
|
4494
|
+
capturedMs,
|
|
4495
|
+
pending.receivedAtMs,
|
|
4496
|
+
batch.expectedIntervalMs
|
|
4497
|
+
).changes
|
|
4498
|
+
) === 1;
|
|
4499
|
+
if (!inserted) {
|
|
4500
|
+
pending.accepted.push(ref);
|
|
4501
|
+
counters.duplicates += 1;
|
|
4502
|
+
continue;
|
|
4503
|
+
}
|
|
4504
|
+
pending.accepted.push(ref);
|
|
4505
|
+
counters.committed += 1;
|
|
4506
|
+
if (record.kind === "metrics") {
|
|
4507
|
+
const known = needsKnownSeries(record.payload) ? this.series.knownFor(batch.machineId) : void 0;
|
|
4508
|
+
for (const point of flattenMetrics(record.payload, known)) {
|
|
4509
|
+
const seriesId = this.series.idFor(batch.machineId, point.metric, point.resourceId);
|
|
4510
|
+
if (this.storageTier === "summary") {
|
|
4511
|
+
addSummaryPoint(summaries, seriesId, capturedMs, point.value);
|
|
4512
|
+
} else {
|
|
4513
|
+
this.insertPoint(seriesId, capturedMs, point.value);
|
|
4514
|
+
}
|
|
4515
|
+
}
|
|
4516
|
+
if (batch.mode === "live") {
|
|
4517
|
+
if (best === null || capturedMs > bestCapturedMs || capturedMs === bestCapturedMs && record.sequence > bestSequence) {
|
|
4518
|
+
best = record;
|
|
4519
|
+
bestCapturedMs = capturedMs;
|
|
4520
|
+
bestSequence = record.sequence;
|
|
4521
|
+
}
|
|
4522
|
+
}
|
|
4523
|
+
} else {
|
|
4524
|
+
this.gapInsert.run(
|
|
4525
|
+
batch.collectorId,
|
|
4526
|
+
record.runId,
|
|
4527
|
+
record.sequence,
|
|
4528
|
+
batch.machineId,
|
|
4529
|
+
Date.parse(record.payload.fromCapturedAt),
|
|
4530
|
+
Date.parse(record.payload.toCapturedAt),
|
|
4531
|
+
record.payload.droppedRecords,
|
|
4532
|
+
record.payload.reason,
|
|
4533
|
+
pending.receivedAtMs
|
|
4534
|
+
);
|
|
4535
|
+
}
|
|
4318
4536
|
}
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
)
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
};
|
|
4537
|
+
if (batch.mode !== "live" || best === null) return;
|
|
4538
|
+
const cursor = this.latestCursor(batch.machineId, cursors);
|
|
4539
|
+
const candidate = {
|
|
4540
|
+
runId: best.runId,
|
|
4541
|
+
sequence: best.sequence,
|
|
4542
|
+
capturedAtMs: bestCapturedMs
|
|
4543
|
+
};
|
|
4544
|
+
if (cursor === null || isNewer(candidate, cursor)) {
|
|
4545
|
+
pending.newLive = best;
|
|
4546
|
+
cursors.set(batch.machineId, candidate);
|
|
4547
|
+
upserts.set(batch.machineId, {
|
|
4548
|
+
record: best,
|
|
4549
|
+
receivedAtMs: pending.receivedAtMs,
|
|
4550
|
+
expectedIntervalMs: batch.expectedIntervalMs,
|
|
4551
|
+
hostname: batch.hostname ?? null
|
|
4552
|
+
});
|
|
4335
4553
|
}
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4554
|
+
}
|
|
4555
|
+
latestCursor(machineId, cache) {
|
|
4556
|
+
if (cache.has(machineId)) return cache.get(machineId);
|
|
4557
|
+
const row = this.latestSelect.get(machineId);
|
|
4558
|
+
const cursor = row ? { runId: row.run_id, sequence: Number(row.seq), capturedAtMs: Number(row.captured_at_ms) } : null;
|
|
4559
|
+
cache.set(machineId, cursor);
|
|
4560
|
+
return cursor;
|
|
4561
|
+
}
|
|
4562
|
+
/**
|
|
4563
|
+
* Writes one compact ten-second summary per series/bucket and folds the same
|
|
4564
|
+
* aggregate into both larger rollups. The ten-second row is upserted because
|
|
4565
|
+
* late or replayed samples may extend a bucket across flushes; the one-minute
|
|
4566
|
+
* and one-hour rows only receive the new delta, so a sample is counted once.
|
|
4567
|
+
*/
|
|
4568
|
+
writeSummaries(summaries) {
|
|
4569
|
+
for (const [seriesId, buckets] of summaries) {
|
|
4570
|
+
for (const [bucketMs, aggregate] of buckets) {
|
|
4571
|
+
this.upsert10s.run(
|
|
4572
|
+
seriesId,
|
|
4573
|
+
bucketMs,
|
|
4574
|
+
aggregate.min,
|
|
4575
|
+
aggregate.max,
|
|
4576
|
+
aggregate.sum,
|
|
4577
|
+
aggregate.count,
|
|
4578
|
+
aggregate.nullCount
|
|
4579
|
+
);
|
|
4580
|
+
this.applyRollupAggregate(
|
|
4581
|
+
this.upsert1m,
|
|
4582
|
+
seriesId,
|
|
4583
|
+
Math.floor(bucketMs / 6e4) * 6e4,
|
|
4584
|
+
aggregate
|
|
4585
|
+
);
|
|
4586
|
+
this.applyRollupAggregate(
|
|
4587
|
+
this.upsert1h,
|
|
4588
|
+
seriesId,
|
|
4589
|
+
Math.floor(bucketMs / 36e5) * 36e5,
|
|
4590
|
+
aggregate
|
|
4591
|
+
);
|
|
4592
|
+
}
|
|
4339
4593
|
}
|
|
4340
4594
|
}
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4595
|
+
/**
|
|
4596
|
+
* Writes one raw point and incrementally updates both rollup tiers. The raw
|
|
4597
|
+
* INSERT OR IGNORE guard prevents a collision (same series/timestamp) from
|
|
4598
|
+
* double-counting a rollup.
|
|
4599
|
+
*/
|
|
4600
|
+
insertPoint(seriesId, tsMs, value) {
|
|
4601
|
+
if (Number(this.rawInsert.run(seriesId, tsMs, value).changes) !== 1) return;
|
|
4602
|
+
const minute = Math.floor(tsMs / 6e4) * 6e4;
|
|
4603
|
+
const hour = Math.floor(tsMs / 36e5) * 36e5;
|
|
4604
|
+
this.applyRollup(this.upsert1m, seriesId, minute, value);
|
|
4605
|
+
this.applyRollup(this.upsert1h, seriesId, hour, value);
|
|
4606
|
+
}
|
|
4607
|
+
applyRollup(statement, seriesId, bucketMs, value) {
|
|
4608
|
+
if (value === null) {
|
|
4609
|
+
statement.run(seriesId, bucketMs, null, null, 0, 0, 1);
|
|
4610
|
+
} else {
|
|
4611
|
+
statement.run(seriesId, bucketMs, value, value, value, 1, 0);
|
|
4349
4612
|
}
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4613
|
+
}
|
|
4614
|
+
applyRollupAggregate(statement, seriesId, bucketMs, aggregate) {
|
|
4615
|
+
statement.run(
|
|
4616
|
+
seriesId,
|
|
4617
|
+
bucketMs,
|
|
4618
|
+
aggregate.min,
|
|
4619
|
+
aggregate.max,
|
|
4620
|
+
aggregate.sum,
|
|
4621
|
+
aggregate.count,
|
|
4622
|
+
aggregate.nullCount
|
|
4357
4623
|
);
|
|
4358
|
-
if (rows <= maxRowsScanned) {
|
|
4359
|
-
fits = true;
|
|
4360
|
-
break;
|
|
4361
|
-
}
|
|
4362
|
-
if (cut === 1) break;
|
|
4363
|
-
cut = Math.max(1, Math.floor(cut / 2));
|
|
4364
4624
|
}
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
const column = tier === "raw" ? "ts_ms" : "bucket_ms";
|
|
4376
|
-
const table = tier === "raw" ? "points_raw" : `points_${tier}`;
|
|
4377
|
-
const statement = db.db.prepare(
|
|
4378
|
-
`SELECT COUNT(*) AS n FROM (SELECT 1 FROM ${table} WHERE series_id = ? AND ${column} >= ? AND ${column} <= ? LIMIT ?)`
|
|
4379
|
-
);
|
|
4380
|
-
let total = 0;
|
|
4381
|
-
for (const seriesId of seriesIds) {
|
|
4382
|
-
const remaining = budget + 1 - total;
|
|
4383
|
-
if (remaining <= 0) return total;
|
|
4384
|
-
const row = statement.get(seriesId, fromMs, toMs, remaining);
|
|
4385
|
-
total += Number(row?.n ?? 0);
|
|
4386
|
-
if (total > budget) return total;
|
|
4625
|
+
resolveCommitted(pending) {
|
|
4626
|
+
if (pending.settled) return;
|
|
4627
|
+
pending.settled = true;
|
|
4628
|
+
pending.resolve({
|
|
4629
|
+
status: "committed",
|
|
4630
|
+
accepted: pending.accepted,
|
|
4631
|
+
rejected: pending.rejected,
|
|
4632
|
+
newLive: pending.newLive,
|
|
4633
|
+
receivedAtMs: pending.receivedAtMs
|
|
4634
|
+
});
|
|
4387
4635
|
}
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
originMs,
|
|
4398
|
-
originMs,
|
|
4399
|
-
stepMs,
|
|
4400
|
-
stepMs,
|
|
4401
|
-
seriesId,
|
|
4402
|
-
fromBucketMs2,
|
|
4403
|
-
toMs,
|
|
4404
|
-
rowLimit
|
|
4636
|
+
resolveBusy(pending, retryAfterMs, reason) {
|
|
4637
|
+
if (pending.settled) return;
|
|
4638
|
+
pending.settled = true;
|
|
4639
|
+
pending.resolve({ status: "busy", retryAfterMs, reason });
|
|
4640
|
+
}
|
|
4641
|
+
};
|
|
4642
|
+
function prepareRollupUpsert(db, table) {
|
|
4643
|
+
return db.prepare(
|
|
4644
|
+
`INSERT INTO ${table} (series_id, bucket_ms, min, max, sum, count, null_count) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(series_id, bucket_ms) DO UPDATE SET min = CASE WHEN excluded.count = 0 THEN min WHEN min IS NULL THEN excluded.min ELSE MIN(min, excluded.min) END, max = CASE WHEN excluded.count = 0 THEN max WHEN max IS NULL THEN excluded.max ELSE MAX(max, excluded.max) END, sum = sum + excluded.sum, count = count + excluded.count, null_count = null_count + excluded.null_count`
|
|
4405
4645
|
);
|
|
4406
|
-
return rows.map((row) => {
|
|
4407
|
-
const count = Number(row.count_value ?? 0);
|
|
4408
|
-
return {
|
|
4409
|
-
t: Number(row.bucket),
|
|
4410
|
-
min: row.min_value === null ? null : Number(row.min_value),
|
|
4411
|
-
max: row.max_value === null ? null : Number(row.max_value),
|
|
4412
|
-
avg: count > 0 && row.sum_value !== null ? Number(row.sum_value) / count : null
|
|
4413
|
-
};
|
|
4414
|
-
});
|
|
4415
4646
|
}
|
|
4416
|
-
function
|
|
4417
|
-
if (
|
|
4418
|
-
|
|
4419
|
-
return Math.floor(fromMs / span) * span;
|
|
4647
|
+
function isNewer(candidate, latest) {
|
|
4648
|
+
if (candidate.runId === latest.runId) return candidate.sequence > latest.sequence;
|
|
4649
|
+
return candidate.capturedAtMs > latest.capturedAtMs;
|
|
4420
4650
|
}
|
|
4421
|
-
function
|
|
4422
|
-
|
|
4423
|
-
"SELECT from_ms, to_ms, dropped, reason FROM gaps WHERE machine_id = ? AND to_ms >= ? AND from_ms <= ? ORDER BY from_ms LIMIT ?"
|
|
4424
|
-
).all(machineId, fromMs, toMs, MAX_GAPS);
|
|
4425
|
-
const missing = db.db.prepare(
|
|
4426
|
-
"WITH before_window AS ( SELECT captured_at_ms, expected_interval_ms FROM sample_receipts WHERE machine_id = ? AND captured_at_ms < ? ORDER BY captured_at_ms DESC LIMIT 1 ), in_window AS ( SELECT captured_at_ms, expected_interval_ms FROM sample_receipts WHERE machine_id = ? AND captured_at_ms >= ? AND captured_at_ms <= ? ), ordered AS ( SELECT captured_at_ms, expected_interval_ms, LAG(captured_at_ms) OVER (ORDER BY captured_at_ms) AS prev_ms FROM (SELECT * FROM before_window UNION ALL SELECT * FROM in_window) ) SELECT MAX(prev_ms, ?) AS from_ms, captured_at_ms AS to_ms FROM ordered WHERE prev_ms IS NOT NULL AND captured_at_ms > ? AND captured_at_ms - prev_ms > MAX(15000, 3 * COALESCE(expected_interval_ms, 5000)) ORDER BY captured_at_ms LIMIT ?"
|
|
4427
|
-
).all(machineId, fromMs, machineId, fromMs, toMs, fromMs, fromMs, MAX_GAPS);
|
|
4428
|
-
const gaps = [
|
|
4429
|
-
...dropped.map((row) => ({
|
|
4430
|
-
fromMs: Number(row.from_ms),
|
|
4431
|
-
toMs: Number(row.to_ms),
|
|
4432
|
-
kind: "dropped",
|
|
4433
|
-
reason: row.reason,
|
|
4434
|
-
droppedRecords: Number(row.dropped)
|
|
4435
|
-
})),
|
|
4436
|
-
...missing.map((row) => ({
|
|
4437
|
-
fromMs: Number(row.from_ms),
|
|
4438
|
-
toMs: Number(row.to_ms),
|
|
4439
|
-
kind: "missing",
|
|
4440
|
-
reason: "expected interval missed"
|
|
4441
|
-
}))
|
|
4442
|
-
];
|
|
4443
|
-
gaps.sort((a, b) => a.fromMs - b.fromMs || a.toMs - b.toMs);
|
|
4444
|
-
return gaps.slice(0, MAX_GAPS);
|
|
4651
|
+
function needsKnownSeries(payload) {
|
|
4652
|
+
return payload.cpu.status === "unavailable" || payload.gpus.status === "unavailable" || payload.disks.status === "unavailable" || payload.network.status === "unavailable";
|
|
4445
4653
|
}
|
|
4446
|
-
function
|
|
4447
|
-
if (
|
|
4448
|
-
|
|
4449
|
-
throw new HistoryQueryError("machineId must be a nonempty string");
|
|
4654
|
+
function rejectionReason(record, batch, receivedAtMs) {
|
|
4655
|
+
if (record.collectorId !== batch.collectorId) {
|
|
4656
|
+
return "collectorId does not match the authenticated batch";
|
|
4450
4657
|
}
|
|
4451
|
-
|
|
4452
|
-
|
|
4658
|
+
const capturedMs = Date.parse(record.capturedAt);
|
|
4659
|
+
if (!Number.isFinite(capturedMs)) return "capturedAt is not a parseable timestamp";
|
|
4660
|
+
if (capturedMs > receivedAtMs + FUTURE_TOLERANCE_MS) {
|
|
4661
|
+
return "capturedAt is more than 24h in the future";
|
|
4453
4662
|
}
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
seenMetrics.add(metric);
|
|
4663
|
+
return null;
|
|
4664
|
+
}
|
|
4665
|
+
function validateBatch(batch) {
|
|
4666
|
+
if (batch === null || typeof batch !== "object") throw new Error("ingest batch must be an object");
|
|
4667
|
+
if (typeof batch.machineId !== "string" || batch.machineId.length === 0) {
|
|
4668
|
+
throw new Error("ingest batch machineId must be a nonempty string");
|
|
4461
4669
|
}
|
|
4462
|
-
if (
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4670
|
+
if (typeof batch.collectorId !== "string" || batch.collectorId.length === 0) {
|
|
4671
|
+
throw new Error("ingest batch collectorId must be a nonempty string");
|
|
4672
|
+
}
|
|
4673
|
+
if (batch.mode !== "live" && batch.mode !== "replay") {
|
|
4674
|
+
throw new Error("ingest batch mode must be live or replay");
|
|
4675
|
+
}
|
|
4676
|
+
if (!Array.isArray(batch.records)) throw new Error("ingest batch records must be an array");
|
|
4677
|
+
if (batch.expectedIntervalMs !== null && !Number.isSafeInteger(batch.expectedIntervalMs)) {
|
|
4678
|
+
throw new Error("ingest batch expectedIntervalMs must be a safe integer or null");
|
|
4679
|
+
}
|
|
4680
|
+
if (batch.hostname !== void 0 && batch.hostname !== null && (typeof batch.hostname !== "string" || batch.hostname.length > 255)) {
|
|
4681
|
+
throw new Error("ingest batch hostname must be a string of at most 255 characters or null");
|
|
4682
|
+
}
|
|
4683
|
+
}
|
|
4684
|
+
function validateTags(tags) {
|
|
4685
|
+
if (!Array.isArray(tags) || tags.length > 64) throw new Error("tags must be an array of at most 64 strings");
|
|
4686
|
+
const result = [];
|
|
4687
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4688
|
+
for (const tag of tags) {
|
|
4689
|
+
if (typeof tag !== "string" || tag.length === 0 || tag.length > 64) {
|
|
4690
|
+
throw new Error("tags must be nonempty strings of at most 64 characters");
|
|
4473
4691
|
}
|
|
4692
|
+
if (seen.has(tag)) throw new Error("tags must be unique");
|
|
4693
|
+
seen.add(tag);
|
|
4694
|
+
result.push(tag);
|
|
4474
4695
|
}
|
|
4475
|
-
|
|
4476
|
-
|
|
4696
|
+
return result;
|
|
4697
|
+
}
|
|
4698
|
+
function parseTags(raw) {
|
|
4699
|
+
try {
|
|
4700
|
+
const parsed = JSON.parse(raw);
|
|
4701
|
+
if (!Array.isArray(parsed)) return [];
|
|
4702
|
+
return parsed.filter((tag) => typeof tag === "string");
|
|
4703
|
+
} catch {
|
|
4704
|
+
return [];
|
|
4477
4705
|
}
|
|
4478
|
-
|
|
4479
|
-
|
|
4706
|
+
}
|
|
4707
|
+
function sanitizeWriteError(error) {
|
|
4708
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4709
|
+
return message.replace(/\s+/g, " ").slice(0, 200);
|
|
4710
|
+
}
|
|
4711
|
+
function positiveInt2(value, fallback, name) {
|
|
4712
|
+
const resolved = value ?? fallback;
|
|
4713
|
+
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
|
|
4714
|
+
throw new Error(`${name} must be a positive safe integer`);
|
|
4480
4715
|
}
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4716
|
+
return resolved;
|
|
4717
|
+
}
|
|
4718
|
+
function resolvePhysicalLimit(options) {
|
|
4719
|
+
if (options.maxPhysicalBytes === null) return null;
|
|
4720
|
+
if (options.maxPhysicalBytes !== void 0) {
|
|
4721
|
+
if (!Number.isSafeInteger(options.maxPhysicalBytes) || options.maxPhysicalBytes < 16 * 1024) {
|
|
4722
|
+
throw new Error("maxPhysicalBytes must be a safe integer of at least 16 KiB or null");
|
|
4723
|
+
}
|
|
4724
|
+
return options.maxPhysicalBytes;
|
|
4484
4725
|
}
|
|
4726
|
+
if (options.maxDatabaseBytes !== null && options.maxDatabaseBytes !== void 0 && options.maxDatabaseBytes >= MIN_DERIVED_PHYSICAL_BYTES) {
|
|
4727
|
+
return options.maxDatabaseBytes;
|
|
4728
|
+
}
|
|
4729
|
+
return DEFAULT_MAX_DATABASE_DISK_BYTES;
|
|
4485
4730
|
}
|
|
4486
|
-
function
|
|
4487
|
-
|
|
4488
|
-
if (!Number.isSafeInteger(
|
|
4489
|
-
throw new
|
|
4731
|
+
function nonnegativeInt(value, fallback, name) {
|
|
4732
|
+
const resolved = value ?? fallback;
|
|
4733
|
+
if (!Number.isSafeInteger(resolved) || resolved < 0) {
|
|
4734
|
+
throw new Error(`${name} must be a nonnegative safe integer`);
|
|
4490
4735
|
}
|
|
4491
|
-
return
|
|
4736
|
+
return resolved;
|
|
4492
4737
|
}
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4738
|
+
var SUMMARY_STEP_MS = 1e4;
|
|
4739
|
+
function addSummaryPoint(summaries, seriesId, tsMs, value) {
|
|
4740
|
+
const bucketMs = Math.floor(tsMs / SUMMARY_STEP_MS) * SUMMARY_STEP_MS;
|
|
4741
|
+
let buckets = summaries.get(seriesId);
|
|
4742
|
+
if (buckets === void 0) {
|
|
4743
|
+
buckets = /* @__PURE__ */ new Map();
|
|
4744
|
+
summaries.set(seriesId, buckets);
|
|
4745
|
+
}
|
|
4746
|
+
let aggregate = buckets.get(bucketMs);
|
|
4747
|
+
if (aggregate === void 0) {
|
|
4748
|
+
aggregate = { min: null, max: null, sum: 0, count: 0, nullCount: 0 };
|
|
4749
|
+
buckets.set(bucketMs, aggregate);
|
|
4750
|
+
}
|
|
4751
|
+
if (value === null) {
|
|
4752
|
+
aggregate.nullCount += 1;
|
|
4753
|
+
} else {
|
|
4754
|
+
aggregate.min = aggregate.min === null ? value : Math.min(aggregate.min, value);
|
|
4755
|
+
aggregate.max = aggregate.max === null ? value : Math.max(aggregate.max, value);
|
|
4756
|
+
aggregate.sum += value;
|
|
4757
|
+
aggregate.count += 1;
|
|
4497
4758
|
}
|
|
4498
|
-
|
|
4759
|
+
}
|
|
4760
|
+
function yieldToEventLoop2() {
|
|
4761
|
+
return new Promise((resolve7) => setImmediate(resolve7));
|
|
4499
4762
|
}
|
|
4500
4763
|
|
|
4501
4764
|
// src/operations/measure.ts
|
|
@@ -4636,7 +4899,7 @@ async function measure(options = {}) {
|
|
|
4636
4899
|
const historyWallStart = import_node_perf_hooks2.performance.now();
|
|
4637
4900
|
const historyHolder = { value: null };
|
|
4638
4901
|
const runHistoryQuery = async () => {
|
|
4639
|
-
await new Promise((
|
|
4902
|
+
await new Promise((resolve7) => setImmediate(resolve7));
|
|
4640
4903
|
const historyStart = import_node_perf_hooks2.performance.now();
|
|
4641
4904
|
const result = queryHistory(
|
|
4642
4905
|
database,
|
|
@@ -4755,6 +5018,7 @@ Usage:
|
|
|
4755
5018
|
[--database PATH] [--activate --confirm-stopped] [--install-deps] [--json]
|
|
4756
5019
|
ops activate --prefix DIR --state-dir DIR --version V [--confirm-stopped] [--json]
|
|
4757
5020
|
ops rollback --prefix DIR --state-dir DIR --confirm-stopped [--json]
|
|
5021
|
+
ops reset-telemetry --database PATH [--backup-dir DIR] --confirm-stopped [--json]
|
|
4758
5022
|
ops recover-admin --database PATH --password-file PATH [--user NAME] [--enable] [--json]
|
|
4759
5023
|
ops measure [--machines N] [--batches N] [--records-per-batch N] [--gpus N]
|
|
4760
5024
|
[--viewers N] [--history-concurrency N] [--history-points N] [--mode live|replay|mixed]
|
|
@@ -5041,6 +5305,42 @@ async function run() {
|
|
|
5041
5305
|
}
|
|
5042
5306
|
return 0;
|
|
5043
5307
|
}
|
|
5308
|
+
if (command === "reset-telemetry") {
|
|
5309
|
+
const parsed = (0, import_node_util.parseArgs)({
|
|
5310
|
+
args: options,
|
|
5311
|
+
options: {
|
|
5312
|
+
database: { type: "string" },
|
|
5313
|
+
"backup-dir": { type: "string" },
|
|
5314
|
+
"confirm-stopped": { type: "boolean" },
|
|
5315
|
+
json: { type: "boolean" }
|
|
5316
|
+
},
|
|
5317
|
+
allowPositionals: false
|
|
5318
|
+
});
|
|
5319
|
+
const database = optionString(parsed.values, "database", true);
|
|
5320
|
+
const backupDir = optionString(parsed.values, "backup-dir");
|
|
5321
|
+
let backupPath = null;
|
|
5322
|
+
if (backupDir !== void 0) {
|
|
5323
|
+
const report2 = await backupDatabase({
|
|
5324
|
+
database,
|
|
5325
|
+
output: (0, import_node_path8.resolve)(backupDir, `telemetry-pre-reset-${Date.now()}.sqlite`)
|
|
5326
|
+
});
|
|
5327
|
+
backupPath = report2.output;
|
|
5328
|
+
}
|
|
5329
|
+
const report = resetTelemetryHistory({
|
|
5330
|
+
database,
|
|
5331
|
+
confirmStopped: optionBoolean(parsed.values, "confirm-stopped")
|
|
5332
|
+
});
|
|
5333
|
+
if (optionBoolean(parsed.values, "json")) {
|
|
5334
|
+
console.log(JSON.stringify({ ...report, backupPath }, null, 2));
|
|
5335
|
+
} else {
|
|
5336
|
+
console.log(`telemetry history reset: ${report.database}`);
|
|
5337
|
+
console.log(`deleted rows: ${Object.entries(report.deleted).map(([table, rows]) => `${table}=${rows}`).join(", ")}`);
|
|
5338
|
+
console.log(`preserved: ${report.preserved.join(", ")}`);
|
|
5339
|
+
console.log(`integrity: ${report.integrity}`);
|
|
5340
|
+
if (backupPath !== null) console.log(`pre-reset backup: ${backupPath}`);
|
|
5341
|
+
}
|
|
5342
|
+
return 0;
|
|
5343
|
+
}
|
|
5044
5344
|
if (command === "recover-admin") {
|
|
5045
5345
|
const parsed = (0, import_node_util.parseArgs)({
|
|
5046
5346
|
args: options,
|