peekinside 1.2.5 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/index.d.ts +41 -12
- package/package.json +10 -1
- package/src/PeekInside.js +70 -14
- package/src/capture/exchange-recorder.js +5 -12
- package/src/capture/http-message.js +17 -0
- package/src/dumps/dump-capture.js +100 -0
- package/src/dumps/dump-record.js +57 -0
- package/src/dumps/dump-value.js +428 -0
- package/src/exceptions/exception-capture.js +5 -4
- package/src/exceptions/request-context.js +118 -10
- package/src/hub/ingest-payload.js +66 -6
- package/src/hub/project-log.js +71 -4
- package/src/hub/server.js +18 -7
- package/src/hub/spool-ingester.js +1 -0
- package/src/hub/store.js +44 -1
- package/src/instrumentation/index.js +48 -3
- package/src/instrumentation/jobs.js +93 -2
- package/src/instrumentation/loggers.js +78 -4
- package/src/integrations/fastify.js +18 -1
- package/src/integrations/hono.js +209 -0
- package/src/integrations/nest.js +277 -0
- package/src/integrations/server-binder.js +9 -6
- package/src/logs/log-capture.js +5 -5
- package/src/logs/log-record.js +2 -2
- package/src/record.js +6 -2
- package/src/timeline/timeline.js +24 -3
- package/src/transport/batch.js +4 -1
- package/src/transport/cloud.js +12 -0
- package/src/transport/local.js +7 -5
|
@@ -13,9 +13,10 @@ import { PROJECT_ID } from './store.js';
|
|
|
13
13
|
export const MAX_BATCH = 1000;
|
|
14
14
|
// Exceptions per batch.
|
|
15
15
|
export const MAX_EXCEPTIONS_BATCH = 500;
|
|
16
|
-
// Job runs
|
|
16
|
+
// Job runs, log lines and dumps per batch.
|
|
17
17
|
export const MAX_JOBS_BATCH = 1000;
|
|
18
18
|
export const MAX_LOGS_BATCH = 2000;
|
|
19
|
+
export const MAX_DUMPS_BATCH = 1000;
|
|
19
20
|
// Request ids are UUIDs in practice. They appear in URLs and index files, so only safe
|
|
20
21
|
// characters are accepted.
|
|
21
22
|
export const REQUEST_ID = /^[A-Za-z0-9_-]{1,64}$/;
|
|
@@ -27,13 +28,20 @@ const FINGERPRINT = /^[0-9a-f]{16}$/;
|
|
|
27
28
|
const STAT_KEYS = ['queries', 'queryTime', 'slowQueries', 'nPlusOne', 'duplicates', 'http', 'httpTime', 'logs', 'loopBlock'];
|
|
28
29
|
const STAT_TIMES = new Set(['queryTime', 'httpTime', 'loopBlock']);
|
|
29
30
|
const MAX_STAT = 1e9;
|
|
30
|
-
const MECHANISMS = new Set(['express', 'koa', 'fastify', 'http', 'uncaught', 'unhandledRejection', 'report', 'job']);
|
|
31
|
+
const MECHANISMS = new Set(['express', 'koa', 'fastify', 'nest', 'hono', 'http', 'uncaught', 'unhandledRejection', 'report', 'job']);
|
|
31
32
|
const JOB_STATUSES = new Set(['ok', 'failed']);
|
|
32
33
|
const JOB_TRIGGERS = new Set(['cron', 'queue', 'manual']);
|
|
33
34
|
const LOG_LEVELS = new Set(['debug', 'info', 'warn', 'error']);
|
|
34
|
-
const LOG_SOURCES = new Set(['console', 'pino', 'winston']);
|
|
35
|
+
const LOG_SOURCES = new Set(['console', 'pino', 'winston', 'nest']);
|
|
36
|
+
// What the SDK keeps of a route pattern and a handler name.
|
|
37
|
+
const MAX_ROUTE = 512;
|
|
38
|
+
const MAX_HANDLER = 256;
|
|
35
39
|
// What the index keeps of a log line; the line in full is its detail.
|
|
36
40
|
const LOG_PREVIEW = 500;
|
|
41
|
+
// What the index keeps of a dump: its label, the type and a one-line preview of its value.
|
|
42
|
+
const DUMP_LABEL = 120;
|
|
43
|
+
const DUMP_TYPE = 64;
|
|
44
|
+
const DUMP_PREVIEW = 200;
|
|
37
45
|
// What release.js produces.
|
|
38
46
|
const RELEASE = /^[A-Za-z0-9._:@/+-]{1,64}$/;
|
|
39
47
|
// Process samples per batch (the SDK sends one every 10 seconds, more after the hub was down).
|
|
@@ -77,10 +85,14 @@ export class IngestPayload {
|
|
|
77
85
|
}
|
|
78
86
|
const jobs = body.jobs != null ? body.jobs : [];
|
|
79
87
|
const logs = body.logs != null ? body.logs : [];
|
|
88
|
+
const dumps = body.dumps != null ? body.dumps : [];
|
|
80
89
|
if (!Array.isArray(jobs) || jobs.length > MAX_JOBS_BATCH || !Array.isArray(logs) || logs.length > MAX_LOGS_BATCH) {
|
|
81
90
|
return null;
|
|
82
91
|
}
|
|
83
|
-
|
|
92
|
+
if (!Array.isArray(dumps) || dumps.length > MAX_DUMPS_BATCH) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
return new IngestPayload({ project, instance, requests, exceptions, jobs, logs, dumps, samples: body.samples, blocked: body.blocked, hasExceptions: Array.isArray(body.exceptions) });
|
|
84
96
|
}
|
|
85
97
|
|
|
86
98
|
/**
|
|
@@ -93,12 +105,13 @@ export class IngestPayload {
|
|
|
93
105
|
* @param {Array} [parts.exceptions] - Exception records as sent
|
|
94
106
|
* @param {Array} [parts.jobs] - Job runs as sent
|
|
95
107
|
* @param {Array} [parts.logs] - Log lines as sent
|
|
108
|
+
* @param {Array} [parts.dumps] - Dumps as sent
|
|
96
109
|
* @param {*} [parts.samples] - Process samples (memory, CPU, event-loop delay), as sent
|
|
97
110
|
* @param {*} parts.blocked - Counts of unrecorded server requests, as sent
|
|
98
111
|
* @param {boolean} [parts.hasExceptions=false] - Whether the batch had an exception list (SDKs
|
|
99
112
|
* from before exceptions don't send one)
|
|
100
113
|
*/
|
|
101
|
-
constructor({ project, instance, requests, exceptions = [], jobs = [], logs = [], samples, blocked, hasExceptions = false }) {
|
|
114
|
+
constructor({ project, instance, requests, exceptions = [], jobs = [], logs = [], dumps = [], samples, blocked, hasExceptions = false }) {
|
|
102
115
|
this.project = project;
|
|
103
116
|
this.instance = instance;
|
|
104
117
|
/**
|
|
@@ -139,6 +152,12 @@ export class IngestPayload {
|
|
|
139
152
|
this.logs = logs.map(readLog).filter(Boolean);
|
|
140
153
|
this.rejectedJobs = jobs.length - this.jobs.length;
|
|
141
154
|
this.rejectedLogs = logs.length - this.logs.length;
|
|
155
|
+
/**
|
|
156
|
+
* The valid dumps, flattened into what the store keeps (the value's tree is the detail)
|
|
157
|
+
* @type {Object[]}
|
|
158
|
+
*/
|
|
159
|
+
this.dumps = dumps.map(readDump).filter(Boolean);
|
|
160
|
+
this.rejectedDumps = dumps.length - this.dumps.length;
|
|
142
161
|
/**
|
|
143
162
|
* The valid counts of server requests the SDK didn't record
|
|
144
163
|
* @type {{ kind: 'host'|'client', value: string, count: number }[]}
|
|
@@ -196,6 +215,8 @@ function readRecord(value) {
|
|
|
196
215
|
url: text(summary.url, 16384),
|
|
197
216
|
path: text(summary.path, 16384),
|
|
198
217
|
contentType: text(summary.contentType, 256),
|
|
218
|
+
route: text(summary.route, MAX_ROUTE) || null,
|
|
219
|
+
handler: text(summary.handler, MAX_HANDLER) || null,
|
|
199
220
|
user: readUser(summary.user),
|
|
200
221
|
detail
|
|
201
222
|
};
|
|
@@ -299,6 +320,43 @@ function readLog(value) {
|
|
|
299
320
|
};
|
|
300
321
|
}
|
|
301
322
|
|
|
323
|
+
// A dump: its fields, the start of its value for the list, and the value's tree as its detail. The
|
|
324
|
+
// tree is the dashboard's to read; the hub only keeps it.
|
|
325
|
+
function readDump(value) {
|
|
326
|
+
if (!value || typeof value !== 'object') return null;
|
|
327
|
+
if (typeof value.id !== 'string' || !REQUEST_ID.test(value.id) || !Number.isFinite(value.occurredAt)) return null;
|
|
328
|
+
if (!optionalId(value.requestId) || !optionalId(value.jobRunId)) return null;
|
|
329
|
+
const summary = value.summary && typeof value.summary === 'object' ? value.summary : {};
|
|
330
|
+
const detail = value.detail && typeof value.detail === 'object' && !Array.isArray(value.detail) ? value.detail : {};
|
|
331
|
+
const tree = detail.value && typeof detail.value === 'object' && !Array.isArray(detail.value) && typeof detail.value.t === 'string' ? detail.value : { t: 'undef' };
|
|
332
|
+
return {
|
|
333
|
+
id: value.id,
|
|
334
|
+
occurredAt: value.occurredAt,
|
|
335
|
+
environment: text(value.environment, 32) || 'development',
|
|
336
|
+
release: release(value.release),
|
|
337
|
+
requestId: value.requestId || null,
|
|
338
|
+
jobRunId: value.jobRunId || null,
|
|
339
|
+
label: text(value.label, DUMP_LABEL) || null,
|
|
340
|
+
location: readCaller(value.location),
|
|
341
|
+
type: text(summary.type, DUMP_TYPE) || 'value',
|
|
342
|
+
preview: text(summary.preview, DUMP_PREVIEW),
|
|
343
|
+
truncated: summary.truncated === true,
|
|
344
|
+
detail: { value: tree }
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// Where in the app something happened: an absolute file, the same relative to the project, a line.
|
|
349
|
+
function readCaller(value) {
|
|
350
|
+
if (!value || typeof value !== 'object' || typeof value.file !== 'string' || !value.file) return null;
|
|
351
|
+
return {
|
|
352
|
+
file: value.file.slice(0, 4096),
|
|
353
|
+
relativeFile: typeof value.relativeFile === 'string' ? value.relativeFile.slice(0, 4096) : null,
|
|
354
|
+
line: positiveInt(value.line),
|
|
355
|
+
column: positiveInt(value.column),
|
|
356
|
+
function: typeof value.function === 'string' ? value.function.slice(0, 1024) : null
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
302
360
|
function optionalId(value) {
|
|
303
361
|
return value === null || value === undefined || (typeof value === 'string' && REQUEST_ID.test(value));
|
|
304
362
|
}
|
|
@@ -351,7 +409,9 @@ function readUser(value) {
|
|
|
351
409
|
|
|
352
410
|
function readRequestLine(value) {
|
|
353
411
|
if (!value || typeof value !== 'object' || typeof value.method !== 'string' || typeof value.path !== 'string') return null;
|
|
354
|
-
|
|
412
|
+
const route = text(value.route, MAX_ROUTE);
|
|
413
|
+
const handler = text(value.handler, MAX_HANDLER);
|
|
414
|
+
return { method: value.method.slice(0, 16), path: value.path.slice(0, 16384), ...(route ? { route } : {}), ...(handler ? { handler } : {}) };
|
|
355
415
|
}
|
|
356
416
|
|
|
357
417
|
function positiveInt(value) {
|
package/src/hub/project-log.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* exception-state.json fingerprints the dashboard marked resolved
|
|
9
9
|
* jobs/00000001.* job runs, in the same format
|
|
10
10
|
* logs/00000001.* log lines: the index keeps the start of each line, the details the rest
|
|
11
|
+
* dumps/00000001.* dumps: the index keeps a preview of each value, the details its tree
|
|
11
12
|
*
|
|
12
13
|
* Process samples (memory, CPU, event-loop delay) are kept in memory only, for the Runtime page:
|
|
13
14
|
* the last six hours of each of the project's recent processes.
|
|
@@ -33,13 +34,15 @@ const MAX_BLOCKED_ENTRIES = 10;
|
|
|
33
34
|
const MAX_RUNTIME_SAMPLES = 2160;
|
|
34
35
|
const MAX_RUNTIME_PROCESSES = 8;
|
|
35
36
|
// Fields a summary may contain, in the order they are written.
|
|
36
|
-
const SUMMARY_FIELDS = ['seq', 'id', 'occurredAt', 'environment', 'release', 'method', 'status', 'duration', 'requestSize', 'responseSize', 'aborted', 'exceptionCount', 'stats', 'url', 'path', 'contentType', 'user'];
|
|
37
|
+
const SUMMARY_FIELDS = ['seq', 'id', 'occurredAt', 'environment', 'release', 'method', 'status', 'duration', 'requestSize', 'responseSize', 'aborted', 'exceptionCount', 'stats', 'url', 'path', 'contentType', 'route', 'handler', 'user'];
|
|
37
38
|
const EXCEPTION_FIELDS = ['seq', 'id', 'occurredAt', 'environment', 'release', 'handled', 'fatal', 'requestId', 'jobRunId', 'name', 'message', 'code', 'fingerprint', 'mechanism', 'location', 'request', 'job', 'user'];
|
|
38
39
|
const JOB_FIELDS = ['seq', 'id', 'occurredAt', 'environment', 'release', 'name', 'queue', 'schedule', 'timezone', 'trigger', 'status', 'duration', 'attempt', 'exceptionCount', 'stats', 'error', 'user'];
|
|
39
40
|
const LOG_FIELDS = ['seq', 'id', 'occurredAt', 'environment', 'release', 'level', 'source', 'requestId', 'jobRunId', 'message'];
|
|
41
|
+
const DUMP_FIELDS = ['seq', 'id', 'occurredAt', 'environment', 'release', 'requestId', 'jobRunId', 'label', 'location', 'type', 'preview', 'truncated'];
|
|
40
42
|
const EXCEPTIONS_FOLDER = 'exceptions';
|
|
41
43
|
const JOBS_FOLDER = 'jobs';
|
|
42
44
|
const LOGS_FOLDER = 'logs';
|
|
45
|
+
const DUMPS_FOLDER = 'dumps';
|
|
43
46
|
|
|
44
47
|
/**
|
|
45
48
|
* @typedef {Object} ProjectInfo
|
|
@@ -71,7 +74,8 @@ export class ProjectLog {
|
|
|
71
74
|
* @param {number} limits.exceptionSegmentSize - Exceptions per segment file
|
|
72
75
|
* @param {number} limits.maxJobs - Job runs kept
|
|
73
76
|
* @param {number} limits.maxLogs - Log lines kept
|
|
74
|
-
* @param {
|
|
77
|
+
* @param {number} limits.maxDumps - Dumps kept
|
|
78
|
+
* @param {{ requests?: SegmentLog, exceptions?: SegmentLog, jobs?: SegmentLog, logs?: SegmentLog, dumps?: SegmentLog }} [logs] - Logs
|
|
75
79
|
* read from disk
|
|
76
80
|
*/
|
|
77
81
|
constructor(dir, info, limits, logs = {}) {
|
|
@@ -82,6 +86,7 @@ export class ProjectLog {
|
|
|
82
86
|
this.exceptions = logs.exceptions || new SegmentLog(path.join(dir, EXCEPTIONS_FOLDER), ProjectLog._exceptionOptions(info, limits));
|
|
83
87
|
this.jobs = logs.jobs || new SegmentLog(path.join(dir, JOBS_FOLDER), ProjectLog._jobOptions(info, limits));
|
|
84
88
|
this.logs = logs.logs || new SegmentLog(path.join(dir, LOGS_FOLDER), ProjectLog._logOptions(info, limits));
|
|
89
|
+
this.dumps = logs.dumps || new SegmentLog(path.join(dir, DUMPS_FOLDER), ProjectLog._dumpOptions(info, limits));
|
|
85
90
|
this.exceptionState = new ExceptionState(dir);
|
|
86
91
|
// Processes running the project, by pid. In memory only: they check in again after a restart.
|
|
87
92
|
this.instances = new Map();
|
|
@@ -111,7 +116,8 @@ export class ProjectLog {
|
|
|
111
116
|
seqBase: 1,
|
|
112
117
|
exceptionSeqBase: 1,
|
|
113
118
|
jobSeqBase: 1,
|
|
114
|
-
logSeqBase: 1
|
|
119
|
+
logSeqBase: 1,
|
|
120
|
+
dumpSeqBase: 1
|
|
115
121
|
}, limits);
|
|
116
122
|
project.save();
|
|
117
123
|
return project;
|
|
@@ -138,13 +144,15 @@ export class ProjectLog {
|
|
|
138
144
|
exceptionSeqBase: Number(saved.exceptionSeqBase) || 1,
|
|
139
145
|
jobSeqBase: Number(saved.jobSeqBase) || 1,
|
|
140
146
|
logSeqBase: Number(saved.logSeqBase) || 1,
|
|
147
|
+
dumpSeqBase: Number(saved.dumpSeqBase) || 1,
|
|
141
148
|
...(isBlockedInfo(saved.blocked) ? { blocked: saved.blocked } : {})
|
|
142
149
|
};
|
|
143
150
|
return new ProjectLog(dir, info, limits, {
|
|
144
151
|
requests: SegmentLog.load(dir, ProjectLog._requestOptions(info, limits)),
|
|
145
152
|
exceptions: SegmentLog.load(path.join(dir, EXCEPTIONS_FOLDER), ProjectLog._exceptionOptions(info, limits)),
|
|
146
153
|
jobs: SegmentLog.load(path.join(dir, JOBS_FOLDER), ProjectLog._jobOptions(info, limits)),
|
|
147
|
-
logs: SegmentLog.load(path.join(dir, LOGS_FOLDER), ProjectLog._logOptions(info, limits))
|
|
154
|
+
logs: SegmentLog.load(path.join(dir, LOGS_FOLDER), ProjectLog._logOptions(info, limits)),
|
|
155
|
+
dumps: SegmentLog.load(path.join(dir, DUMPS_FOLDER), ProjectLog._dumpOptions(info, limits))
|
|
148
156
|
});
|
|
149
157
|
}
|
|
150
158
|
|
|
@@ -416,6 +424,48 @@ export class ProjectLog {
|
|
|
416
424
|
this.save();
|
|
417
425
|
}
|
|
418
426
|
|
|
427
|
+
/**
|
|
428
|
+
* Stores new dumps; ones already stored (same id) are skipped.
|
|
429
|
+
*
|
|
430
|
+
* @param {Object[]} records - Validated dumps, with their detail (the value's tree)
|
|
431
|
+
* @returns {Object[]}
|
|
432
|
+
*/
|
|
433
|
+
appendDumps(records) {
|
|
434
|
+
return this.dumps.append(records);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* Dump summaries, newest first; oldest first when `after` is given.
|
|
439
|
+
*
|
|
440
|
+
* @param {{ after?: number, before?: number, limit: number }} query
|
|
441
|
+
* @returns {{ dumps: Object[], hasMore: boolean }}
|
|
442
|
+
*/
|
|
443
|
+
listDumps(query) {
|
|
444
|
+
const { entries, hasMore } = this.dumps.list(query);
|
|
445
|
+
return { dumps: entries, hasMore };
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* One dump's value (its tree).
|
|
450
|
+
*
|
|
451
|
+
* @param {string} dumpId
|
|
452
|
+
* @returns {Object|null}
|
|
453
|
+
*/
|
|
454
|
+
dumpDetail(dumpId) {
|
|
455
|
+
return this.dumps.detail(dumpId);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* Deletes all dumps.
|
|
460
|
+
*
|
|
461
|
+
* @returns {void}
|
|
462
|
+
*/
|
|
463
|
+
clearDumps() {
|
|
464
|
+
this.dumps.clear();
|
|
465
|
+
this.info.dumpSeqBase = this.dumps.seqBase;
|
|
466
|
+
this.save();
|
|
467
|
+
}
|
|
468
|
+
|
|
419
469
|
/**
|
|
420
470
|
* Deletes the project's folder.
|
|
421
471
|
*
|
|
@@ -444,6 +494,7 @@ export class ProjectLog {
|
|
|
444
494
|
const newest = this.requests.newest;
|
|
445
495
|
const newestException = this.exceptions.newest;
|
|
446
496
|
const newestJob = this.jobs.newest;
|
|
497
|
+
const newestDump = this.dumps.newest;
|
|
447
498
|
return {
|
|
448
499
|
id: this.info.id,
|
|
449
500
|
name: this.info.name,
|
|
@@ -457,6 +508,8 @@ export class ProjectLog {
|
|
|
457
508
|
jobCount: this.jobs.size,
|
|
458
509
|
lastJobAt: newestJob ? newestJob.occurredAt : null,
|
|
459
510
|
logCount: this.logs.size,
|
|
511
|
+
dumpCount: this.dumps.size,
|
|
512
|
+
lastDumpAt: newestDump ? newestDump.occurredAt : null,
|
|
460
513
|
online: instances.length > 0,
|
|
461
514
|
instances,
|
|
462
515
|
blocked: this.info.blocked && now - this.info.blocked.lastAt < BLOCKED_NOTICE_MS ? this.info.blocked : null
|
|
@@ -495,6 +548,7 @@ export class ProjectLog {
|
|
|
495
548
|
this.exceptions.enforceRetention();
|
|
496
549
|
this.jobs.enforceRetention();
|
|
497
550
|
this.logs.enforceRetention();
|
|
551
|
+
this.dumps.enforceRetention();
|
|
498
552
|
}
|
|
499
553
|
|
|
500
554
|
static _requestOptions(info, limits) {
|
|
@@ -509,6 +563,10 @@ export class ProjectLog {
|
|
|
509
563
|
return { maxEntries: limits.maxLogs, segmentSize: Math.max(10, Math.ceil(limits.maxLogs / 4)), seqBase: info.logSeqBase || 1, pick: pickLog };
|
|
510
564
|
}
|
|
511
565
|
|
|
566
|
+
static _dumpOptions(info, limits) {
|
|
567
|
+
return { maxEntries: limits.maxDumps, segmentSize: Math.max(10, Math.ceil(limits.maxDumps / 4)), seqBase: info.dumpSeqBase || 1, pick: pickDump };
|
|
568
|
+
}
|
|
569
|
+
|
|
512
570
|
static _exceptionOptions(info, limits) {
|
|
513
571
|
return {
|
|
514
572
|
maxEntries: limits.maxExceptions,
|
|
@@ -533,6 +591,9 @@ function pickSummary(source) {
|
|
|
533
591
|
// Stored before releases (or users) were recorded.
|
|
534
592
|
if (summary.release === undefined) summary.release = null;
|
|
535
593
|
if (summary.user === undefined) summary.user = null;
|
|
594
|
+
// Stored before route patterns were recorded, or from a framework that doesn't name them.
|
|
595
|
+
if (summary.route === undefined) summary.route = null;
|
|
596
|
+
if (summary.handler === undefined) summary.handler = null;
|
|
536
597
|
return summary;
|
|
537
598
|
}
|
|
538
599
|
|
|
@@ -552,6 +613,12 @@ function pickJob(source) {
|
|
|
552
613
|
return summary;
|
|
553
614
|
}
|
|
554
615
|
|
|
616
|
+
function pickDump(source) {
|
|
617
|
+
const summary = {};
|
|
618
|
+
for (const field of DUMP_FIELDS) summary[field] = source[field] === undefined ? null : source[field];
|
|
619
|
+
return summary;
|
|
620
|
+
}
|
|
621
|
+
|
|
555
622
|
function pickLog(source) {
|
|
556
623
|
const summary = {};
|
|
557
624
|
for (const field of LOG_FIELDS) summary[field] = source[field] === undefined ? null : source[field];
|
package/src/hub/server.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* The local hub's HTTP server, on 127.0.0.1 only.
|
|
3
3
|
*
|
|
4
4
|
* GET / Dashboard page (also /p/<project id>, and its
|
|
5
|
-
* /exceptions, /routes, /queries, /logs, /jobs and /runtime)
|
|
5
|
+
* /exceptions, /routes, /queries, /logs, /jobs, /dumps and /runtime)
|
|
6
6
|
* POST /api/ingest Requests and exceptions from the SDK in each app, and
|
|
7
7
|
* counts of server requests it didn't record (free plan)
|
|
8
8
|
* GET /api/status Hub version and settings
|
|
@@ -20,6 +20,9 @@
|
|
|
20
20
|
* GET /api/projects/:id/logs Log lines (their start), paged like requests
|
|
21
21
|
* GET /api/projects/:id/logs/:lid One log line in full
|
|
22
22
|
* DELETE /api/projects/:id/logs Delete a project's log lines
|
|
23
|
+
* GET /api/projects/:id/dumps Dumps (a preview of each value), paged like requests
|
|
24
|
+
* GET /api/projects/:id/dumps/:did One dump's value
|
|
25
|
+
* DELETE /api/projects/:id/dumps Delete a project's dumps
|
|
23
26
|
* GET /api/projects/:id/exception-state Resolved issues, with a version
|
|
24
27
|
* PUT /api/projects/:id/exception-state Save them, if the version is still the current one
|
|
25
28
|
* DELETE /api/projects/:id Forget a project
|
|
@@ -38,7 +41,7 @@ import { readFileSync } from 'fs';
|
|
|
38
41
|
import { HUB_API_VERSION, HUB_APP_ID } from '../config.js';
|
|
39
42
|
import { ExceptionState } from './exception-state.js';
|
|
40
43
|
import { IngestPayload, REQUEST_ID } from './ingest-payload.js';
|
|
41
|
-
import { PROJECT_ID } from './store.js';
|
|
44
|
+
import { DEFAULT_MAX_DUMPS, PROJECT_ID } from './store.js';
|
|
42
45
|
|
|
43
46
|
const MAX_INGEST_BYTES = 32 * 1024 * 1024;
|
|
44
47
|
// 5000 fingerprints with their timestamps take about 200 KB.
|
|
@@ -46,9 +49,9 @@ const MAX_STATE_BYTES = 1024 * 1024;
|
|
|
46
49
|
const DEFAULT_PAGE = 500;
|
|
47
50
|
const MAX_PAGE = 1000;
|
|
48
51
|
// /api/projects/:id, then optionally a collection and an item.
|
|
49
|
-
const PROJECT_ROUTE = /^\/api\/projects\/([^/]+)(?:\/(requests|exceptions|exception-state|runtime|jobs|logs)(?:\/([^/]+))?)?$/;
|
|
52
|
+
const PROJECT_ROUTE = /^\/api\/projects\/([^/]+)(?:\/(requests|exceptions|exception-state|runtime|jobs|logs|dumps)(?:\/([^/]+))?)?$/;
|
|
50
53
|
// The dashboard page: the project list, a project's requests, its exceptions and its queries.
|
|
51
|
-
const SHELL_ROUTE = /^\/p\/[a-z0-9-]{1,64}(?:\/(?:exceptions|routes|queries|runtime|logs|jobs))?\/?$/;
|
|
54
|
+
const SHELL_ROUTE = /^\/p\/[a-z0-9-]{1,64}(?:\/(?:exceptions|routes|queries|runtime|logs|jobs|dumps))?\/?$/;
|
|
52
55
|
|
|
53
56
|
const BASE_HEADERS = {
|
|
54
57
|
'X-Content-Type-Options': 'nosniff',
|
|
@@ -150,7 +153,9 @@ export class HubServer {
|
|
|
150
153
|
dataDir: this.dataDir,
|
|
151
154
|
siteUrl: this.siteUrl,
|
|
152
155
|
maxRequests: this.store.maxRequests,
|
|
153
|
-
maxExceptions: this.store.maxExceptions
|
|
156
|
+
maxExceptions: this.store.maxExceptions,
|
|
157
|
+
// Also tells the dashboard this hub keeps dumps (older ones don't).
|
|
158
|
+
maxDumps: DEFAULT_MAX_DUMPS
|
|
154
159
|
};
|
|
155
160
|
}
|
|
156
161
|
|
|
@@ -243,6 +248,7 @@ export class HubServer {
|
|
|
243
248
|
this.store.addSamples(payload.project.id, payload.instance, payload.samples);
|
|
244
249
|
const addedJobs = this.store.appendJobs(payload.project.id, payload.jobs);
|
|
245
250
|
const addedLogs = this.store.appendLogs(payload.project.id, payload.logs);
|
|
251
|
+
const addedDumps = this.store.appendDumps(payload.project.id, payload.dumps);
|
|
246
252
|
// SDKs from before exceptions get the answer they know.
|
|
247
253
|
return sendJson(res, 200, payload.hasExceptions
|
|
248
254
|
? {
|
|
@@ -251,9 +257,10 @@ export class HubServer {
|
|
|
251
257
|
acceptedExceptions: addedExceptions.length,
|
|
252
258
|
rejectedExceptions: payload.rejectedExceptions,
|
|
253
259
|
acceptedJobs: addedJobs.length,
|
|
254
|
-
acceptedLogs: addedLogs.length
|
|
260
|
+
acceptedLogs: addedLogs.length,
|
|
261
|
+
acceptedDumps: addedDumps.length
|
|
255
262
|
}
|
|
256
|
-
: { accepted: added.length, rejected: payload.rejected, acceptedJobs: addedJobs.length, acceptedLogs: addedLogs.length });
|
|
263
|
+
: { accepted: added.length, rejected: payload.rejected, acceptedJobs: addedJobs.length, acceptedLogs: addedLogs.length, acceptedDumps: addedDumps.length });
|
|
257
264
|
}
|
|
258
265
|
|
|
259
266
|
// Every /api/ route except ingest; the caller has checked the origin.
|
|
@@ -299,6 +306,10 @@ export class HubServer {
|
|
|
299
306
|
if (item !== undefined) return this._item(req, res, decode(item), (id) => this.store.logDetail(projectId, id), 'This log line is no longer stored.');
|
|
300
307
|
return this._collection(req, res, query, () => this.store.listLogs(projectId, pageOf(query)), () => this.store.clearLogs(projectId));
|
|
301
308
|
}
|
|
309
|
+
if (collection === 'dumps') {
|
|
310
|
+
if (item !== undefined) return this._item(req, res, decode(item), (id) => this.store.dumpDetail(projectId, id), 'This dump is no longer stored.');
|
|
311
|
+
return this._collection(req, res, query, () => this.store.listDumps(projectId, pageOf(query)), () => this.store.clearDumps(projectId));
|
|
312
|
+
}
|
|
302
313
|
if (collection === 'runtime') {
|
|
303
314
|
if (item !== undefined) return sendJson(res, 404, { error: 'Not found' });
|
|
304
315
|
return read ? sendJson(res, 200, this.store.runtime(projectId)) : methodNotAllowed(res, 'GET, HEAD');
|
|
@@ -104,6 +104,7 @@ export class SpoolIngester {
|
|
|
104
104
|
const exceptions = this.store.appendExceptions(payload.project.id, payload.exceptions).length;
|
|
105
105
|
this.store.appendJobs(payload.project.id, payload.jobs);
|
|
106
106
|
this.store.appendLogs(payload.project.id, payload.logs);
|
|
107
|
+
this.store.appendDumps(payload.project.id, payload.dumps);
|
|
107
108
|
spool.remove(name);
|
|
108
109
|
return { requests, exceptions };
|
|
109
110
|
}
|
package/src/hub/store.js
CHANGED
|
@@ -16,6 +16,7 @@ export const DEFAULT_MAX_REQUESTS = 1000;
|
|
|
16
16
|
export const DEFAULT_MAX_EXCEPTIONS = 500;
|
|
17
17
|
export const DEFAULT_MAX_JOBS = 2000;
|
|
18
18
|
export const DEFAULT_MAX_LOGS = 10000;
|
|
19
|
+
export const DEFAULT_MAX_DUMPS = 1000;
|
|
19
20
|
|
|
20
21
|
/**
|
|
21
22
|
* All projects on this machine.
|
|
@@ -206,6 +207,47 @@ export class Store {
|
|
|
206
207
|
if (project) project.clearLogs();
|
|
207
208
|
}
|
|
208
209
|
|
|
210
|
+
/**
|
|
211
|
+
* Stores new dumps of a project.
|
|
212
|
+
*
|
|
213
|
+
* @param {string} projectId
|
|
214
|
+
* @param {Object[]} records
|
|
215
|
+
* @returns {Object[]}
|
|
216
|
+
*/
|
|
217
|
+
appendDumps(projectId, records) {
|
|
218
|
+
const project = this.projects.get(projectId);
|
|
219
|
+
return project && records.length ? project.appendDumps(records) : [];
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* @param {string} projectId
|
|
224
|
+
* @param {{ after?: number, before?: number, limit: number }} query
|
|
225
|
+
* @returns {{ dumps: Object[], hasMore: boolean }|null}
|
|
226
|
+
*/
|
|
227
|
+
listDumps(projectId, query) {
|
|
228
|
+
const project = this.projects.get(projectId);
|
|
229
|
+
return project ? project.listDumps(query) : null;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* @param {string} projectId
|
|
234
|
+
* @param {string} dumpId
|
|
235
|
+
* @returns {Object|null}
|
|
236
|
+
*/
|
|
237
|
+
dumpDetail(projectId, dumpId) {
|
|
238
|
+
const project = this.projects.get(projectId);
|
|
239
|
+
return project ? project.dumpDetail(dumpId) : null;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* @param {string} projectId
|
|
244
|
+
* @returns {void}
|
|
245
|
+
*/
|
|
246
|
+
clearDumps(projectId) {
|
|
247
|
+
const project = this.projects.get(projectId);
|
|
248
|
+
if (project) project.clearDumps();
|
|
249
|
+
}
|
|
250
|
+
|
|
209
251
|
/**
|
|
210
252
|
* Adds process samples of a project's process (kept in memory, for the Runtime page).
|
|
211
253
|
*
|
|
@@ -398,7 +440,8 @@ export class Store {
|
|
|
398
440
|
maxExceptions: this.maxExceptions,
|
|
399
441
|
exceptionSegmentSize: this.exceptionSegmentSize,
|
|
400
442
|
maxJobs: DEFAULT_MAX_JOBS,
|
|
401
|
-
maxLogs: DEFAULT_MAX_LOGS
|
|
443
|
+
maxLogs: DEFAULT_MAX_LOGS,
|
|
444
|
+
maxDumps: DEFAULT_MAX_DUMPS
|
|
402
445
|
};
|
|
403
446
|
}
|
|
404
447
|
}
|
|
@@ -12,12 +12,14 @@
|
|
|
12
12
|
*
|
|
13
13
|
* Outgoing HTTP and console capture are process-wide subscriptions; they record only for code
|
|
14
14
|
* running inside a request (log lines go to the Logs page from anywhere). Loggers (pino, winston)
|
|
15
|
-
* and job libraries (node-cron, cron, BullMQ) are found and patched like the drivers
|
|
15
|
+
* and job libraries (node-cron, cron, BullMQ) are found and patched like the drivers; NestJS's
|
|
16
|
+
* Logger and @nestjs/schedule are reached through the app instead (see integrations/nest.js).
|
|
16
17
|
*/
|
|
17
18
|
|
|
18
19
|
import { createRequire } from 'module';
|
|
19
20
|
import fs from 'fs';
|
|
20
21
|
import path from 'path';
|
|
22
|
+
import { pathToFileURL } from 'url';
|
|
21
23
|
import { ConsoleInstrumentation } from './console.js';
|
|
22
24
|
import { HttpInstrumentation } from './http.js';
|
|
23
25
|
import { JobInstrumentation } from './jobs.js';
|
|
@@ -71,6 +73,10 @@ export class Instrumentation {
|
|
|
71
73
|
if (!fromRoot) return patched;
|
|
72
74
|
if (patchWith(fromRoot, 'pino', LoggerInstrumentation.pino)) patched.push('pino');
|
|
73
75
|
if (patchWith(fromRoot, 'winston', LoggerInstrumentation.winston)) patched.push('winston');
|
|
76
|
+
// NestJS's ConsoleLogger, for a NestJS project.
|
|
77
|
+
if (declares(root, '@nestjs/common')) {
|
|
78
|
+
void importInstalled(fromRoot, '@nestjs/common', null, (common) => LoggerInstrumentation.nest(exported(common, 'ConsoleLogger')));
|
|
79
|
+
}
|
|
74
80
|
return patched;
|
|
75
81
|
}
|
|
76
82
|
|
|
@@ -92,6 +98,11 @@ export class Instrumentation {
|
|
|
92
98
|
const directory = packageDirectory(main, name);
|
|
93
99
|
if (directory) void JobInstrumentation.patchModuleBuild(directory, patch);
|
|
94
100
|
}
|
|
101
|
+
// @nestjs/schedule registers a NestJS app's scheduled methods as the app initializes, which a
|
|
102
|
+
// worker (NestFactory.createApplicationContext()) does before it can be attached.
|
|
103
|
+
if (declares(root, '@nestjs/schedule')) {
|
|
104
|
+
void importInstalled(fromRoot, '@nestjs/schedule', path.join('dist', 'schedule.explorer.js'), (explorer) => JobInstrumentation.nestSchedule(exported(explorer, 'ScheduleExplorer')));
|
|
105
|
+
}
|
|
95
106
|
return patched;
|
|
96
107
|
}
|
|
97
108
|
|
|
@@ -217,14 +228,48 @@ function resolve(fromSomewhere, name) {
|
|
|
217
228
|
}
|
|
218
229
|
}
|
|
219
230
|
|
|
220
|
-
// The folder of the package `name` that contains `file`.
|
|
231
|
+
// The folder of the package `name` (scoped or not) that contains `file`.
|
|
221
232
|
function packageDirectory(file, name) {
|
|
233
|
+
const [scope, base] = name.includes('/') ? name.split('/') : [null, name];
|
|
222
234
|
for (let directory = path.dirname(file); ; directory = path.dirname(directory)) {
|
|
223
|
-
|
|
235
|
+
const named = path.basename(directory) === base && (!scope || path.basename(path.dirname(directory)) === scope);
|
|
236
|
+
if (named && fs.existsSync(path.join(directory, 'package.json'))) return directory;
|
|
224
237
|
if (path.dirname(directory) === directory) return null;
|
|
225
238
|
}
|
|
226
239
|
}
|
|
227
240
|
|
|
241
|
+
// Whether the project's package.json depends on a package. Frameworks are only loaded for projects
|
|
242
|
+
// that use them: a monorepo's shared node_modules may hold another app's.
|
|
243
|
+
function declares(root, name) {
|
|
244
|
+
try {
|
|
245
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
|
|
246
|
+
return ['dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies'].some((field) => Boolean(pkg[field] && pkg[field][name]));
|
|
247
|
+
} catch (error) {
|
|
248
|
+
return false;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Loads a package installed in the project (or one of its files) the way an import does, which
|
|
253
|
+
// gives the module instance the app runs whether the package ships CommonJS or ES modules, and
|
|
254
|
+
// patches it.
|
|
255
|
+
async function importInstalled(fromRoot, name, file, patch) {
|
|
256
|
+
const main = resolve(fromRoot, name);
|
|
257
|
+
const directory = main && file ? packageDirectory(main, name) : null;
|
|
258
|
+
const target = file ? directory && path.join(directory, file) : main;
|
|
259
|
+
if (!target || !fs.existsSync(target)) return false;
|
|
260
|
+
try {
|
|
261
|
+
return patch(await import(pathToFileURL(target).href));
|
|
262
|
+
} catch (error) {
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// An export of a module, imported as an ES module or as CommonJS (under `default`).
|
|
268
|
+
function exported(module, name) {
|
|
269
|
+
if (!module) return undefined;
|
|
270
|
+
return module[name] || (module.default && module.default[name]);
|
|
271
|
+
}
|
|
272
|
+
|
|
228
273
|
// The file a module object was loaded from, found in the CommonJS cache.
|
|
229
274
|
function fileOf(exports) {
|
|
230
275
|
for (const module of Object.values(requireHere.cache)) {
|