vika-cli 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +37 -2
- package/dist/cli.js +32 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands/approve-and-create-tickets.js +51 -0
- package/dist/commands/approve-and-create-tickets.js.map +1 -0
- package/dist/commands/create-tickets-command.js +30 -0
- package/dist/commands/create-tickets-command.js.map +1 -0
- package/dist/commands/doctor-command.js +59 -0
- package/dist/commands/doctor-command.js.map +1 -0
- package/dist/commands/init-command.js +21 -0
- package/dist/commands/init-command.js.map +1 -0
- package/dist/commands/mark-deploy-command.js +59 -0
- package/dist/commands/mark-deploy-command.js.map +1 -0
- package/dist/commands/report-command.js +40 -0
- package/dist/commands/report-command.js.map +1 -0
- package/dist/commands/run-cli.js +197 -0
- package/dist/commands/run-cli.js.map +1 -0
- package/dist/commands/run-exit-code.js +11 -0
- package/dist/commands/run-exit-code.js.map +1 -0
- package/dist/commands/serve-command.js +49 -0
- package/dist/commands/serve-command.js.map +1 -0
- package/dist/configuration/configuration-errors.js +43 -0
- package/dist/configuration/configuration-errors.js.map +1 -0
- package/dist/configuration/configuration-schema.js +69 -0
- package/dist/configuration/configuration-schema.js.map +1 -0
- package/dist/configuration/database-url-checks.js +43 -0
- package/dist/configuration/database-url-checks.js.map +1 -0
- package/dist/configuration/describe-configuration-safely.js +20 -0
- package/dist/configuration/describe-configuration-safely.js.map +1 -0
- package/dist/configuration/hide-secrets.js +44 -0
- package/dist/configuration/hide-secrets.js.map +1 -0
- package/dist/configuration/load-configuration.js +97 -0
- package/dist/configuration/load-configuration.js.map +1 -0
- package/dist/configuration/read-environment-file.js +77 -0
- package/dist/configuration/read-environment-file.js.map +1 -0
- package/dist/dashboard/dashboard-page.js +267 -0
- package/dist/dashboard/dashboard-page.js.map +1 -0
- package/dist/dashboard/dashboard-server.js +144 -0
- package/dist/dashboard/dashboard-server.js.map +1 -0
- package/dist/dashboard/github-issue-link.js +23 -0
- package/dist/dashboard/github-issue-link.js.map +1 -0
- package/dist/dashboard/request-guard.js +34 -0
- package/dist/dashboard/request-guard.js.map +1 -0
- package/dist/database/database-store.js +425 -0
- package/dist/database/database-store.js.map +1 -0
- package/dist/database/store-types.js +2 -0
- package/dist/database/store-types.js.map +1 -0
- package/dist/deploys/check-sha-on-branch.js +21 -0
- package/dist/deploys/check-sha-on-branch.js.map +1 -0
- package/dist/deploys/commits-in-range.js +31 -0
- package/dist/deploys/commits-in-range.js.map +1 -0
- package/dist/deploys/marker-deploys.js +18 -0
- package/dist/deploys/marker-deploys.js.map +1 -0
- package/dist/deploys/merge-log.js +33 -0
- package/dist/deploys/merge-log.js.map +1 -0
- package/dist/deploys/pull-request-number.js +5 -0
- package/dist/deploys/pull-request-number.js.map +1 -0
- package/dist/deploys/read-merge-deploys.js +46 -0
- package/dist/deploys/read-merge-deploys.js.map +1 -0
- package/dist/deploys/record-deploys.js +12 -0
- package/dist/deploys/record-deploys.js.map +1 -0
- package/dist/deploys/run-git.js +49 -0
- package/dist/deploys/run-git.js.map +1 -0
- package/dist/deploys/starting-point.js +5 -0
- package/dist/deploys/starting-point.js.map +1 -0
- package/dist/deploys/utc-timestamp.js +6 -0
- package/dist/deploys/utc-timestamp.js.map +1 -0
- package/dist/doctor/check-configuration.js +46 -0
- package/dist/doctor/check-configuration.js.map +1 -0
- package/dist/doctor/check-database.js +30 -0
- package/dist/doctor/check-database.js.map +1 -0
- package/dist/doctor/check-deploys.js +22 -0
- package/dist/doctor/check-deploys.js.map +1 -0
- package/dist/doctor/check-service-logs.js +107 -0
- package/dist/doctor/check-service-logs.js.map +1 -0
- package/dist/doctor/check-service-repository.js +43 -0
- package/dist/doctor/check-service-repository.js.map +1 -0
- package/dist/doctor/check-tracker.js +20 -0
- package/dist/doctor/check-tracker.js.map +1 -0
- package/dist/doctor/check-webhook.js +13 -0
- package/dist/doctor/check-webhook.js.map +1 -0
- package/dist/doctor/doctor-check-types.js +4 -0
- package/dist/doctor/doctor-check-types.js.map +1 -0
- package/dist/doctor/format-doctor-report.js +15 -0
- package/dist/doctor/format-doctor-report.js.map +1 -0
- package/dist/doctor/suggest-path-map.js +35 -0
- package/dist/doctor/suggest-path-map.js.map +1 -0
- package/dist/error-grouping/build-clusters.js +77 -0
- package/dist/error-grouping/build-clusters.js.map +1 -0
- package/dist/error-grouping/cluster-types.js +3 -0
- package/dist/error-grouping/cluster-types.js.map +1 -0
- package/dist/error-grouping/exception-type.js +6 -0
- package/dist/error-grouping/exception-type.js.map +1 -0
- package/dist/error-grouping/fingerprints.js +18 -0
- package/dist/error-grouping/fingerprints.js.map +1 -0
- package/dist/error-grouping/normalize-message.js +29 -0
- package/dist/error-grouping/normalize-message.js.map +1 -0
- package/dist/error-grouping/template-miner.js +75 -0
- package/dist/error-grouping/template-miner.js.map +1 -0
- package/dist/log-parsing/assemble-log-entries.js +49 -0
- package/dist/log-parsing/assemble-log-entries.js.map +1 -0
- package/dist/log-parsing/build-log-entry.js +33 -0
- package/dist/log-parsing/build-log-entry.js.map +1 -0
- package/dist/log-parsing/detect-log-format.js +44 -0
- package/dist/log-parsing/detect-log-format.js.map +1 -0
- package/dist/log-parsing/json-log-format.js +56 -0
- package/dist/log-parsing/json-log-format.js.map +1 -0
- package/dist/log-parsing/line-readers.js +14 -0
- package/dist/log-parsing/line-readers.js.map +1 -0
- package/dist/log-parsing/log-entry-types.js +3 -0
- package/dist/log-parsing/log-entry-types.js.map +1 -0
- package/dist/log-parsing/log-levels.js +21 -0
- package/dist/log-parsing/log-levels.js.map +1 -0
- package/dist/log-parsing/nginx-log-format.js +56 -0
- package/dist/log-parsing/nginx-log-format.js.map +1 -0
- package/dist/log-parsing/parse-log-lines.js +11 -0
- package/dist/log-parsing/parse-log-lines.js.map +1 -0
- package/dist/log-parsing/read-line-timestamp.js +21 -0
- package/dist/log-parsing/read-line-timestamp.js.map +1 -0
- package/dist/log-parsing/read-log-command.js +68 -0
- package/dist/log-parsing/read-log-command.js.map +1 -0
- package/dist/log-parsing/read-log-file.js +32 -0
- package/dist/log-parsing/read-log-file.js.map +1 -0
- package/dist/log-parsing/stack-frames.js +10 -0
- package/dist/log-parsing/stack-frames.js.map +1 -0
- package/dist/log-parsing/text-log-format.js +24 -0
- package/dist/log-parsing/text-log-format.js.map +1 -0
- package/dist/log-parsing/timestamps.js +22 -0
- package/dist/log-parsing/timestamps.js.map +1 -0
- package/dist/pipeline/analyze-service.js +45 -0
- package/dist/pipeline/analyze-service.js.map +1 -0
- package/dist/pipeline/build-stored-report.js +24 -0
- package/dist/pipeline/build-stored-report.js.map +1 -0
- package/dist/pipeline/ingest-service-logs.js +39 -0
- package/dist/pipeline/ingest-service-logs.js.map +1 -0
- package/dist/pipeline/load-deploys-for-service.js +16 -0
- package/dist/pipeline/load-deploys-for-service.js.map +1 -0
- package/dist/pipeline/prepare-ticket-drafts.js +33 -0
- package/dist/pipeline/prepare-ticket-drafts.js.map +1 -0
- package/dist/pipeline/read-new-log-lines.js +39 -0
- package/dist/pipeline/read-new-log-lines.js.map +1 -0
- package/dist/pipeline/run-all-services.js +73 -0
- package/dist/pipeline/run-all-services.js.map +1 -0
- package/dist/pipeline/timestamp-range.js +20 -0
- package/dist/pipeline/timestamp-range.js.map +1 -0
- package/dist/project-setup/detect-project.js +76 -0
- package/dist/project-setup/detect-project.js.map +1 -0
- package/dist/project-setup/write-starter-files.js +68 -0
- package/dist/project-setup/write-starter-files.js.map +1 -0
- package/dist/redaction/redact-ip-addresses.js +26 -0
- package/dist/redaction/redact-ip-addresses.js.map +1 -0
- package/dist/redaction/redact-sensitive-text.js +40 -0
- package/dist/redaction/redact-sensitive-text.js.map +1 -0
- package/dist/redaction/redaction-kinds.js +5 -0
- package/dist/redaction/redaction-kinds.js.map +1 -0
- package/dist/regression-detection/count-series.js +38 -0
- package/dist/regression-detection/count-series.js.map +1 -0
- package/dist/regression-detection/detect-regressions.js +162 -0
- package/dist/regression-detection/detect-regressions.js.map +1 -0
- package/dist/regression-detection/detection-types.js +9 -0
- package/dist/regression-detection/detection-types.js.map +1 -0
- package/dist/regression-detection/noise-check.js +7 -0
- package/dist/regression-detection/noise-check.js.map +1 -0
- package/dist/regression-detection/restore-time.js +19 -0
- package/dist/regression-detection/restore-time.js.map +1 -0
- package/dist/regression-detection/severity.js +12 -0
- package/dist/regression-detection/severity.js.map +1 -0
- package/dist/regression-detection/time-units.js +12 -0
- package/dist/regression-detection/time-units.js.map +1 -0
- package/dist/reporting/build-report.js +77 -0
- package/dist/reporting/build-report.js.map +1 -0
- package/dist/reporting/compute-metrics.js +42 -0
- package/dist/reporting/compute-metrics.js.map +1 -0
- package/dist/reporting/format-terminal-report.js +104 -0
- package/dist/reporting/format-terminal-report.js.map +1 -0
- package/dist/reporting/report-schema.js +96 -0
- package/dist/reporting/report-schema.js.map +1 -0
- package/dist/reporting/report-types.js +2 -0
- package/dist/reporting/report-types.js.map +1 -0
- package/dist/suspect-attribution/apply-path-map.js +31 -0
- package/dist/suspect-attribution/apply-path-map.js.map +1 -0
- package/dist/suspect-attribution/attribute-regression.js +176 -0
- package/dist/suspect-attribution/attribute-regression.js.map +1 -0
- package/dist/suspect-attribution/bot-authors.js +8 -0
- package/dist/suspect-attribution/bot-authors.js.map +1 -0
- package/dist/suspect-attribution/callee-files.js +53 -0
- package/dist/suspect-attribution/callee-files.js.map +1 -0
- package/dist/suspect-attribution/changed-file-categories.js +39 -0
- package/dist/suspect-attribution/changed-file-categories.js.map +1 -0
- package/dist/suspect-attribution/crash-line-inputs.js +81 -0
- package/dist/suspect-attribution/crash-line-inputs.js.map +1 -0
- package/dist/suspect-attribution/function-names.js +42 -0
- package/dist/suspect-attribution/function-names.js.map +1 -0
- package/dist/suspect-attribution/git-blame-reader.js +160 -0
- package/dist/suspect-attribution/git-blame-reader.js.map +1 -0
- package/dist/suspect-attribution/score-suspects.js +100 -0
- package/dist/suspect-attribution/score-suspects.js.map +1 -0
- package/dist/suspect-attribution/suspect-types.js +2 -0
- package/dist/suspect-attribution/suspect-types.js.map +1 -0
- package/dist/ticket-trackers/create-tracker.js +21 -0
- package/dist/ticket-trackers/create-tracker.js.map +1 -0
- package/dist/ticket-trackers/jira-document.js +34 -0
- package/dist/ticket-trackers/jira-document.js.map +1 -0
- package/dist/ticket-trackers/jira-tracker.js +65 -0
- package/dist/ticket-trackers/jira-tracker.js.map +1 -0
- package/dist/ticket-trackers/linear-tracker.js +109 -0
- package/dist/ticket-trackers/linear-tracker.js.map +1 -0
- package/dist/ticket-trackers/send-tracker-request.js +84 -0
- package/dist/ticket-trackers/send-tracker-request.js.map +1 -0
- package/dist/ticket-trackers/tracker-types.js +3 -0
- package/dist/ticket-trackers/tracker-types.js.map +1 -0
- package/dist/ticketing/build-ticket-comment.js +14 -0
- package/dist/ticketing/build-ticket-comment.js.map +1 -0
- package/dist/ticketing/build-ticket-draft.js +172 -0
- package/dist/ticketing/build-ticket-draft.js.map +1 -0
- package/dist/ticketing/choose-assignee.js +30 -0
- package/dist/ticketing/choose-assignee.js.map +1 -0
- package/dist/ticketing/create-ticket.js +92 -0
- package/dist/ticketing/create-ticket.js.map +1 -0
- package/dist/ticketing/describe-ticket-outcome.js +24 -0
- package/dist/ticketing/describe-ticket-outcome.js.map +1 -0
- package/dist/ticketing/read-code-owners.js +82 -0
- package/dist/ticketing/read-code-owners.js.map +1 -0
- package/dist/ticketing/render-ticket-text.js +22 -0
- package/dist/ticketing/render-ticket-text.js.map +1 -0
- package/dist/ticketing/ticket-types.js +2 -0
- package/dist/ticketing/ticket-types.js.map +1 -0
- package/dist/webhook/build-webhook-payload.js +29 -0
- package/dist/webhook/build-webhook-payload.js.map +1 -0
- package/dist/webhook/send-regression-webhooks.js +57 -0
- package/dist/webhook/send-regression-webhooks.js.map +1 -0
- package/dist/webhook/sign-webhook-body.js +16 -0
- package/dist/webhook/sign-webhook-body.js.map +1 -0
- package/dist/webhook/webhook-payload-schema.js +32 -0
- package/dist/webhook/webhook-payload-schema.js.map +1 -0
- package/migrations/001_init.sql +66 -0
- package/migrations/002_bookmark_history_start.sql +9 -0
- package/migrations/003_tickets.sql +20 -0
- package/migrations/004_ticket_comment_count.sql +10 -0
- package/migrations/005_webhook_events.sql +18 -0
- package/migrations/README.md +1 -0
- package/package.json +37 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cluster-types.js","sourceRoot":"","sources":["../../src/error-grouping/cluster-types.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAC9B,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// Reads the exception class name ("TypeError", "java.io.IOException") from a message.
|
|
2
|
+
export function readExceptionType(message) {
|
|
3
|
+
const exceptionMatch = /^([\w$.]*(?:Error|Exception))(?::|$)/m.exec(message);
|
|
4
|
+
return exceptionMatch?.[1];
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=exception-type.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exception-type.js","sourceRoot":"","sources":["../../src/error-grouping/exception-type.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,cAAc,GAAG,uCAAuC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7E,OAAO,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7B,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// Stable ids for error groups. The normalizer version is part of every
|
|
2
|
+
// fingerprint, so changing the normalizer never mixes old and new groups.
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { NORMALIZER_VERSION } from "./normalize-message.js";
|
|
5
|
+
export const FINGERPRINT_FRAME_COUNT = 3;
|
|
6
|
+
function shortHash(parts) {
|
|
7
|
+
return createHash("sha256").update(JSON.stringify(parts)).digest("hex").slice(0, 12);
|
|
8
|
+
}
|
|
9
|
+
// Line numbers are left out on purpose: the same crash should keep its
|
|
10
|
+
// fingerprint when unrelated lines above it move.
|
|
11
|
+
export function stackFingerprint(service, exceptionName, frames) {
|
|
12
|
+
const topFrames = frames.slice(0, FINGERPRINT_FRAME_COUNT).map((frame) => [frame.file, frame.fn ?? ""]);
|
|
13
|
+
return shortHash([NORMALIZER_VERSION, service, "stack", exceptionName ?? "", topFrames]);
|
|
14
|
+
}
|
|
15
|
+
export function messageFingerprint(service, normalizedMessage) {
|
|
16
|
+
return shortHash([NORMALIZER_VERSION, service, "message", normalizedMessage]);
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=fingerprints.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fingerprints.js","sourceRoot":"","sources":["../../src/error-grouping/fingerprints.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,0EAA0E;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAEzC,SAAS,SAAS,CAAC,KAAgB;IACjC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACvF,CAAC;AAED,uEAAuE;AACvE,kDAAkD;AAClD,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,aAAiC,EAAE,MAAe;IAClG,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxG,OAAO,SAAS,CAAC,CAAC,kBAAkB,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,IAAI,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,iBAAyB;IAC3E,OAAO,SAAS,CAAC,CAAC,kBAAkB,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC;AAChF,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Replaces the parts of a message that change between occurrences (ids,
|
|
2
|
+
// numbers, IPs, query strings) with placeholders, so they group together.
|
|
3
|
+
import { redactIps } from "../redaction/redact-ip-addresses.js";
|
|
4
|
+
export const NORMALIZER_VERSION = 1;
|
|
5
|
+
const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi;
|
|
6
|
+
const HEX_NUMBER = /(?<![0-9A-Za-z])(?:0x[0-9a-f]+|(?=[0-9a-f]*\d)(?=[0-9a-f]*[a-f])[0-9a-f]{8,})(?![0-9A-Za-z])/gi;
|
|
7
|
+
const QUERY = /\?[\w.%[\]-]+=[^\s"'#)]*/g;
|
|
8
|
+
const PATH_SEGMENT = /(?<=\/)[\w.~<>-]+(?=[/?#\s"'),;:\]]|$)/g;
|
|
9
|
+
const NUMBER = /(?<![A-Za-z\d.])\d+(?:\.\d+)?/g;
|
|
10
|
+
const PLACEHOLDER = /^<(?:uuid|hex|id|num|ip|email|token|card|query)>$/;
|
|
11
|
+
function maskPathSegment(segment) {
|
|
12
|
+
if (PLACEHOLDER.test(segment))
|
|
13
|
+
return segment;
|
|
14
|
+
if (/^\d+$/.test(segment))
|
|
15
|
+
return "<id>";
|
|
16
|
+
if (segment.length >= 6 && /\d|<\w+>/.test(segment) && !/\.\w{1,4}$/.test(segment))
|
|
17
|
+
return "<id>";
|
|
18
|
+
return segment;
|
|
19
|
+
}
|
|
20
|
+
export function normalize(message) {
|
|
21
|
+
let text = message.replace(UUID, "<uuid>");
|
|
22
|
+
text = text.replace(HEX_NUMBER, "<hex>");
|
|
23
|
+
text = redactIps(text);
|
|
24
|
+
text = text.replace(QUERY, "?<query>");
|
|
25
|
+
text = text.replace(PATH_SEGMENT, maskPathSegment);
|
|
26
|
+
text = text.replace(NUMBER, "<num>");
|
|
27
|
+
return text;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=normalize-message.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"normalize-message.js","sourceRoot":"","sources":["../../src/error-grouping/normalize-message.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,0EAA0E;AAC1E,OAAO,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AAEhE,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAEpC,MAAM,IAAI,GAAG,gEAAgE,CAAC;AAE9E,MAAM,UAAU,GAAG,gGAAgG,CAAC;AAEpH,MAAM,KAAK,GAAG,2BAA2B,CAAC;AAE1C,MAAM,YAAY,GAAG,yCAAyC,CAAC;AAE/D,MAAM,MAAM,GAAG,gCAAgC,CAAC;AAEhD,MAAM,WAAW,GAAG,mDAAmD,CAAC;AAExE,SAAS,eAAe,CAAC,OAAe;IACtC,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC;IAC9C,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IACzC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAClG,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAe;IACvC,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC3C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAC,OAAO,CAAC,CAAC;IACxC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACvB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IACnD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Groups messages without a stack trace by template: messages with the same
|
|
2
|
+
// length and first token that mostly match share a group, and the tokens
|
|
3
|
+
// that differ become a wildcard.
|
|
4
|
+
import { messageFingerprint } from "./fingerprints.js";
|
|
5
|
+
import { NORMALIZER_VERSION } from "./normalize-message.js";
|
|
6
|
+
export const PREFIX_TOKEN_COUNT = 1;
|
|
7
|
+
export const SIMILARITY_THRESHOLD = 0.7;
|
|
8
|
+
export const WILDCARD = "<*>";
|
|
9
|
+
export function emptyState() {
|
|
10
|
+
return { version: NORMALIZER_VERSION, groups: [] };
|
|
11
|
+
}
|
|
12
|
+
export function tokenize(normalizedMessage) {
|
|
13
|
+
return normalizedMessage.split(/\s+/).filter((token) => token !== "");
|
|
14
|
+
}
|
|
15
|
+
function treeKey(service, tokens) {
|
|
16
|
+
return [service, String(tokens.length), ...tokens.slice(0, PREFIX_TOKEN_COUNT)].join("\0");
|
|
17
|
+
}
|
|
18
|
+
// State saved with another normalizer version is dropped: its templates
|
|
19
|
+
// were made from differently normalized messages.
|
|
20
|
+
export function importState(state) {
|
|
21
|
+
const miner = { groups: [], tree: new Map(), byFingerprint: new Map() };
|
|
22
|
+
if (state.version !== NORMALIZER_VERSION)
|
|
23
|
+
return miner;
|
|
24
|
+
for (const group of state.groups)
|
|
25
|
+
addGroup(miner, { fingerprint: group.fingerprint, service: group.service, tokens: [...group.tokens] });
|
|
26
|
+
return miner;
|
|
27
|
+
}
|
|
28
|
+
export function exportState(miner) {
|
|
29
|
+
return {
|
|
30
|
+
version: NORMALIZER_VERSION,
|
|
31
|
+
groups: miner.groups.map((group) => ({ fingerprint: group.fingerprint, service: group.service, tokens: [...group.tokens] })),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function addGroup(miner, group) {
|
|
35
|
+
miner.groups.push(group);
|
|
36
|
+
miner.byFingerprint.set(group.fingerprint, group);
|
|
37
|
+
const key = treeKey(group.service, group.tokens);
|
|
38
|
+
const groupsWithSameKey = miner.tree.get(key);
|
|
39
|
+
if (groupsWithSameKey)
|
|
40
|
+
groupsWithSameKey.push(group);
|
|
41
|
+
else
|
|
42
|
+
miner.tree.set(key, [group]);
|
|
43
|
+
}
|
|
44
|
+
// The share of positions where the template and the message have the same token.
|
|
45
|
+
function similarity(template, tokens) {
|
|
46
|
+
let matchingTokenCount = 0;
|
|
47
|
+
for (let position = 0; position < tokens.length; position++)
|
|
48
|
+
if (template[position] === tokens[position])
|
|
49
|
+
matchingTokenCount++;
|
|
50
|
+
return matchingTokenCount / tokens.length;
|
|
51
|
+
}
|
|
52
|
+
export function addMessage(miner, service, normalizedMessage) {
|
|
53
|
+
const tokens = tokenize(normalizedMessage);
|
|
54
|
+
let bestGroup;
|
|
55
|
+
let bestScore = -1;
|
|
56
|
+
for (const group of miner.tree.get(treeKey(service, tokens)) ?? []) {
|
|
57
|
+
const score = similarity(group.tokens, tokens);
|
|
58
|
+
if (score > bestScore) {
|
|
59
|
+
bestGroup = group;
|
|
60
|
+
bestScore = score;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (bestGroup && bestScore >= SIMILARITY_THRESHOLD) {
|
|
64
|
+
bestGroup.tokens = bestGroup.tokens.map((token, position) => (token === tokens[position] ? token : WILDCARD));
|
|
65
|
+
return bestGroup;
|
|
66
|
+
}
|
|
67
|
+
const fingerprint = messageFingerprint(service, tokens.join(" "));
|
|
68
|
+
const existing = miner.byFingerprint.get(fingerprint);
|
|
69
|
+
if (existing)
|
|
70
|
+
return existing;
|
|
71
|
+
const group = { fingerprint, service, tokens };
|
|
72
|
+
addGroup(miner, group);
|
|
73
|
+
return group;
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=template-miner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"template-miner.js","sourceRoot":"","sources":["../../src/error-grouping/template-miner.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,yEAAyE;AACzE,iCAAiC;AACjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AACpC,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACxC,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,CAAC;AAoB9B,MAAM,UAAU,UAAU;IACxB,OAAO,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,iBAAyB;IAChD,OAAO,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,OAAO,CAAC,OAAe,EAAE,MAAgB;IAChD,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,wEAAwE;AACxE,kDAAkD;AAClD,MAAM,UAAU,WAAW,CAAC,KAAiB;IAC3C,MAAM,KAAK,GAAU,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/E,IAAI,KAAK,CAAC,OAAO,KAAK,kBAAkB;QAAE,OAAO,KAAK,CAAC;IACvD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM;QAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACzI,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAY;IACtC,OAAO;QACL,OAAO,EAAE,kBAAkB;QAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;KAC7H,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAAY,EAAE,KAAiB;IAC/C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,iBAAiB;QAAE,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;QAChD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,iFAAiF;AACjF,SAAS,UAAU,CAAC,QAAkB,EAAE,MAAgB;IACtD,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE;QAAE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC;YAAE,kBAAkB,EAAE,CAAC;IAC/H,OAAO,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAY,EAAE,OAAe,EAAE,iBAAyB;IACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE3C,IAAI,SAAiC,CAAC;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;IACnB,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACnE,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/C,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,SAAS,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;QACnD,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9G,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACtD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,KAAK,GAAe,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3D,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACvB,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Groups lines into entries: a line a format recognizes starts an entry, and
|
|
2
|
+
// the stack trace lines after it are attached to it.
|
|
3
|
+
import { buildLogEntry } from "./build-log-entry.js";
|
|
4
|
+
import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
|
|
5
|
+
const CONTINUATION_LINE = /^(?:\s|[\w$.]*(?:Error|Exception)(?::|$)|Caused by:)/;
|
|
6
|
+
export function assembleLogEntries(lines, readLineStart, redactionKinds = ALL_REDACTION_KINDS) {
|
|
7
|
+
const entries = [];
|
|
8
|
+
let skipped = 0;
|
|
9
|
+
// "ignored" = inside an entry we don't keep (e.g. INFO), so its stack lines are dropped too.
|
|
10
|
+
let currentEntry = null;
|
|
11
|
+
for (const rawLine of lines) {
|
|
12
|
+
const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
|
|
13
|
+
if (line.trim() === "")
|
|
14
|
+
continue;
|
|
15
|
+
let lineStart;
|
|
16
|
+
try {
|
|
17
|
+
lineStart = readLineStart(line);
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
lineStart = { kind: "bad" }; // never let one line crash the run
|
|
21
|
+
}
|
|
22
|
+
if (lineStart.kind === "none") {
|
|
23
|
+
if (currentEntry === null || !CONTINUATION_LINE.test(line))
|
|
24
|
+
skipped++;
|
|
25
|
+
else if (currentEntry !== "ignored")
|
|
26
|
+
currentEntry.continuationLines.push(line);
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
if (currentEntry !== null && currentEntry !== "ignored")
|
|
30
|
+
entries.push(buildLogEntry(currentEntry, redactionKinds));
|
|
31
|
+
if (lineStart.kind === "entry") {
|
|
32
|
+
currentEntry = {
|
|
33
|
+
timestamp: lineStart.timestamp,
|
|
34
|
+
level: lineStart.level,
|
|
35
|
+
message: lineStart.message,
|
|
36
|
+
continuationLines: lineStart.stack ? lineStart.stack.split("\n") : [],
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
if (lineStart.kind === "bad")
|
|
41
|
+
skipped++;
|
|
42
|
+
currentEntry = "ignored";
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (currentEntry !== null && currentEntry !== "ignored")
|
|
46
|
+
entries.push(buildLogEntry(currentEntry, redactionKinds));
|
|
47
|
+
return { entries, skipped };
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=assemble-log-entries.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assemble-log-entries.js","sourceRoot":"","sources":["../../src/log-parsing/assemble-log-entries.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,qDAAqD;AACrD,OAAO,EAAE,aAAa,EAAmB,MAAM,sBAAsB,CAAC;AAEtE,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAE1F,MAAM,iBAAiB,GAAG,sDAAsD,CAAC;AAEjF,MAAM,UAAU,kBAAkB,CAChC,KAAe,EACf,aAA0C,EAC1C,cAAc,GAA6B,mBAAmB;IAE9D,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,6FAA6F;IAC7F,IAAI,YAAY,GAAkC,IAAI,CAAC;IAEvD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACrE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAS;QAEjC,IAAI,SAAoB,CAAC;QACzB,IAAI,CAAC;YACH,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,mCAAmC;QAClE,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC9B,IAAI,YAAY,KAAK,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,EAAE,CAAC;iBACjE,IAAI,YAAY,KAAK,SAAS;gBAAE,YAAY,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/E,SAAS;QACX,CAAC;QAED,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QACnH,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC/B,YAAY,GAAG;gBACb,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,iBAAiB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;aACtE,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,SAAS,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO,EAAE,CAAC;YACxC,YAAY,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;IACnH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Turns a first line plus its continuation lines into one redacted LogEntry.
|
|
2
|
+
import { redact } from "../redaction/redact-sensitive-text.js";
|
|
3
|
+
import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
|
|
4
|
+
import { parseFrame } from "./stack-frames.js";
|
|
5
|
+
// Redacts the configured kinds (default: all) in the message and frames.
|
|
6
|
+
export function buildLogEntry(draft, redactionKinds = ALL_REDACTION_KINDS) {
|
|
7
|
+
const frames = [];
|
|
8
|
+
const messageLines = [draft.message.trim()];
|
|
9
|
+
for (const line of draft.continuationLines) {
|
|
10
|
+
const frame = parseFrame(line);
|
|
11
|
+
if (frame)
|
|
12
|
+
frames.push(frame);
|
|
13
|
+
// "at ..." lines we can't read (e.g. "at async Promise.all (index 0)") are left out.
|
|
14
|
+
else if (!/^\s*at\s/.test(line) && line.trim() !== "")
|
|
15
|
+
messageLines.push(line.trim());
|
|
16
|
+
}
|
|
17
|
+
const safeFrames = frames.map((frame) => frame.fn === undefined
|
|
18
|
+
? { file: redact(frame.file, redactionKinds), line: frame.line }
|
|
19
|
+
: { file: redact(frame.file, redactionKinds), line: frame.line, fn: redact(frame.fn, redactionKinds) });
|
|
20
|
+
const entry = {
|
|
21
|
+
timestamp: draft.timestamp,
|
|
22
|
+
level: draft.level,
|
|
23
|
+
message: redact(messageLines.filter((messageLine) => messageLine !== "").join("\n"), redactionKinds),
|
|
24
|
+
frames: safeFrames,
|
|
25
|
+
};
|
|
26
|
+
const topFrame = safeFrames[0];
|
|
27
|
+
if (topFrame) {
|
|
28
|
+
entry.file = topFrame.file;
|
|
29
|
+
entry.lineNumber = topFrame.line;
|
|
30
|
+
}
|
|
31
|
+
return entry;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=build-log-entry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build-log-entry.js","sourceRoot":"","sources":["../../src/log-parsing/build-log-entry.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,OAAO,EAAE,MAAM,EAAE,MAAM,uCAAuC,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAE1F,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAS/C,yEAAyE;AACzE,MAAM,UAAU,aAAa,CAAC,KAAiB,EAAE,cAAc,GAA6B,mBAAmB;IAC7G,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,qFAAqF;aAChF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,EAAE,KAAK,SAAS;QACpB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;QAChE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CACzG,CAAC;IACF,MAAM,KAAK,GAAa;QACtB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC;QACpG,MAAM,EAAE,UAAU;KACnB,CAAC;IACF,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC/B,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Guesses a log's format from a sample of its lines.
|
|
2
|
+
import { LINE_START_READERS } from "./line-readers.js";
|
|
3
|
+
// How many sample lines the best format recognizes, so callers can tell
|
|
4
|
+
// "text" (recognized) from "text" (the fallback because nothing matched).
|
|
5
|
+
export function recognizeFormat(sampleLines) {
|
|
6
|
+
const format = detectFormat(sampleLines);
|
|
7
|
+
let recognizedCount = 0;
|
|
8
|
+
for (const line of sampleLines) {
|
|
9
|
+
try {
|
|
10
|
+
const kind = LINE_START_READERS[format](line).kind;
|
|
11
|
+
if (kind === "entry" || kind === "drop")
|
|
12
|
+
recognizedCount++;
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
// an unreadable line just doesn't count
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return { format, recognizedCount };
|
|
19
|
+
}
|
|
20
|
+
// Whichever format recognizes the most lines wins. Ties, or nothing
|
|
21
|
+
// recognized at all, fall back to "text".
|
|
22
|
+
export function detectFormat(sampleLines) {
|
|
23
|
+
let bestFormat = "text";
|
|
24
|
+
let bestScore = 0;
|
|
25
|
+
for (const format of ["text", "json", "nginx"]) {
|
|
26
|
+
let score = 0;
|
|
27
|
+
for (const line of sampleLines) {
|
|
28
|
+
try {
|
|
29
|
+
const kind = LINE_START_READERS[format](line).kind;
|
|
30
|
+
if (kind === "entry" || kind === "drop")
|
|
31
|
+
score++;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// an unreadable line just doesn't count
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
if (score > bestScore) {
|
|
38
|
+
bestFormat = format;
|
|
39
|
+
bestScore = score;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return bestFormat;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=detect-log-format.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"detect-log-format.js","sourceRoot":"","sources":["../../src/log-parsing/detect-log-format.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAGvD,wEAAwE;AACxE,0EAA0E;AAC1E,MAAM,UAAU,eAAe,CAAC,WAAqB;IACnD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IACzC,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YACnD,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM;gBAAE,eAAe,EAAE,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;AACrC,CAAC;AAED,oEAAoE;AACpE,0CAA0C;AAC1C,MAAM,UAAU,YAAY,CAAC,WAAqB;IAChD,IAAI,UAAU,GAAc,MAAM,CAAC;IACnC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAU,EAAE,CAAC;QACxD,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;gBACnD,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM;oBAAE,KAAK,EAAE,CAAC;YACnD,CAAC;YAAC,MAAM,CAAC;gBACP,wCAAwC;YAC1C,CAAC;QACH,CAAC;QACD,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,UAAU,GAAG,MAAM,CAAC;YACpB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// JSON log lines (pino, bunyan, winston and similar).
|
|
2
|
+
import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
|
|
3
|
+
import { assembleLogEntries } from "./assemble-log-entries.js";
|
|
4
|
+
import { normalizeLevel } from "./log-levels.js";
|
|
5
|
+
import { epochToIso, toIso } from "./timestamps.js";
|
|
6
|
+
const isJsonObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
7
|
+
function firstString(...values) {
|
|
8
|
+
return values.find((value) => typeof value === "string");
|
|
9
|
+
}
|
|
10
|
+
// The keys ("time", "ts", "lvl", ...) are what loggers write, so they stay as they are.
|
|
11
|
+
function readTime(jsonObject) {
|
|
12
|
+
const value = jsonObject.time ?? jsonObject.timestamp ?? jsonObject.ts ?? jsonObject["@timestamp"];
|
|
13
|
+
if (typeof value === "string")
|
|
14
|
+
return toIso(value);
|
|
15
|
+
if (typeof value === "number")
|
|
16
|
+
return epochToIso(value);
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
// undefined/null both mean "not a level we keep".
|
|
20
|
+
function readLevel(jsonObject) {
|
|
21
|
+
const value = jsonObject.level ?? jsonObject.severity ?? jsonObject.lvl;
|
|
22
|
+
if (typeof value === "string")
|
|
23
|
+
return normalizeLevel(value);
|
|
24
|
+
if (typeof value === "number")
|
|
25
|
+
return value >= 50 ? "ERROR" : value >= 40 ? "WARN" : null;
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
export function readJsonStart(line) {
|
|
29
|
+
if (!line.trimStart().startsWith("{"))
|
|
30
|
+
return { kind: "none" };
|
|
31
|
+
let parsedLine;
|
|
32
|
+
try {
|
|
33
|
+
parsedLine = JSON.parse(line);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return { kind: "bad" };
|
|
37
|
+
}
|
|
38
|
+
if (!isJsonObject(parsedLine))
|
|
39
|
+
return { kind: "bad" };
|
|
40
|
+
const timestamp = readTime(parsedLine);
|
|
41
|
+
if (!timestamp)
|
|
42
|
+
return { kind: "bad" };
|
|
43
|
+
const level = readLevel(parsedLine);
|
|
44
|
+
if (!level)
|
|
45
|
+
return { kind: "drop", timestamp };
|
|
46
|
+
const errorField = isJsonObject(parsedLine.err) ? parsedLine.err : isJsonObject(parsedLine.error) ? parsedLine.error : {};
|
|
47
|
+
const message = firstString(parsedLine.msg, parsedLine.message, errorField.message) ?? "";
|
|
48
|
+
const stack = firstString(parsedLine.stack, errorField.stack);
|
|
49
|
+
return stack === undefined
|
|
50
|
+
? { kind: "entry", timestamp, level, message }
|
|
51
|
+
: { kind: "entry", timestamp, level, message, stack };
|
|
52
|
+
}
|
|
53
|
+
export function parseJson(lines, redactionKinds = ALL_REDACTION_KINDS) {
|
|
54
|
+
return assembleLogEntries(lines, readJsonStart, redactionKinds);
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=json-log-format.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-log-format.js","sourceRoot":"","sources":["../../src/log-parsing/json-log-format.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAIpD,MAAM,YAAY,GAAG,CAAC,KAAc,EAAuB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEnI,SAAS,WAAW,CAAC,GAAG,MAAiB;IACvC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;AAC5E,CAAC;AAED,wFAAwF;AACxF,SAAS,QAAQ,CAAC,UAAsB;IACtC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC;IACnG,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;IACxD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,kDAAkD;AAClD,SAAS,SAAS,CAAC,UAAsB;IACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,GAAG,CAAC;IACxE,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC5D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1F,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC/D,IAAI,UAAmB,CAAC;IACxB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IACD,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACtD,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACvC,IAAI,CAAC,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAE/C,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1H,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC1F,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9D,OAAO,KAAK,KAAK,SAAS;QACxB,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE;QAC9C,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAe,EAAE,cAAc,GAA6B,mBAAmB;IACvG,OAAO,kBAAkB,CAAC,KAAK,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;AAClE,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { parseJson, readJsonStart } from "./json-log-format.js";
|
|
2
|
+
import { parseNginx, readNginxLine } from "./nginx-log-format.js";
|
|
3
|
+
import { parseText, readTextStart } from "./text-log-format.js";
|
|
4
|
+
export const LINE_START_READERS = {
|
|
5
|
+
text: readTextStart,
|
|
6
|
+
json: readJsonStart,
|
|
7
|
+
nginx: readNginxLine,
|
|
8
|
+
};
|
|
9
|
+
export const FORMAT_PARSERS = {
|
|
10
|
+
text: parseText,
|
|
11
|
+
json: parseJson,
|
|
12
|
+
nginx: parseNginx,
|
|
13
|
+
};
|
|
14
|
+
//# sourceMappingURL=line-readers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"line-readers.js","sourceRoot":"","sources":["../../src/log-parsing/line-readers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEhE,MAAM,CAAC,MAAM,kBAAkB,GAAmD;IAChF,IAAI,EAAE,aAAa;IACnB,IAAI,EAAE,aAAa;IACnB,KAAK,EAAE,aAAa;CACrB,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAmG;IAC5H,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;CAClB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-entry-types.js","sourceRoot":"","sources":["../../src/log-parsing/log-entry-types.ts"],"names":[],"mappings":"AAAA,6CAA6C"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// null = a known level we don't keep.
|
|
2
|
+
const LEVEL_WORDS = {
|
|
3
|
+
emerg: "ERROR",
|
|
4
|
+
alert: "ERROR",
|
|
5
|
+
fatal: "ERROR",
|
|
6
|
+
crit: "ERROR",
|
|
7
|
+
critical: "ERROR",
|
|
8
|
+
error: "ERROR",
|
|
9
|
+
err: "ERROR",
|
|
10
|
+
warn: "WARN",
|
|
11
|
+
warning: "WARN",
|
|
12
|
+
notice: null,
|
|
13
|
+
info: null,
|
|
14
|
+
debug: null,
|
|
15
|
+
trace: null,
|
|
16
|
+
};
|
|
17
|
+
// undefined = not a level word at all.
|
|
18
|
+
export function normalizeLevel(word) {
|
|
19
|
+
return LEVEL_WORDS[word.toLowerCase()];
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=log-levels.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-levels.js","sourceRoot":"","sources":["../../src/log-parsing/log-levels.ts"],"names":[],"mappings":"AAGA,sCAAsC;AACtC,MAAM,WAAW,GAAiC;IAChD,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,OAAO;IACb,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,OAAO;IACd,GAAG,EAAE,OAAO;IACZ,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,IAAI;IACZ,IAAI,EAAE,IAAI;IACV,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,IAAI;CACZ,CAAC;AAEF,uCAAuC;AACvC,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// nginx error log and access log lines. Access lines count only for 5xx.
|
|
2
|
+
import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
|
|
3
|
+
import { buildLogEntry } from "./build-log-entry.js";
|
|
4
|
+
import { normalizeLevel } from "./log-levels.js";
|
|
5
|
+
import { toIso } from "./timestamps.js";
|
|
6
|
+
const ERROR_LINE = /^(\d{4})\/(\d{2})\/(\d{2}) (\d{2}:\d{2}:\d{2}) \[(\w+)\] (?:\d+#\d+: )?(?:\*\d+ )?(.*)$/;
|
|
7
|
+
const ACCESS_LINE = /^\S+ \S+ \S+ \[(\d{2})\/(\w{3})\/(\d{4}):(\d{2}:\d{2}:\d{2}) ([+-]\d{4})\] "([^"]*)" (\d{3})(?:\s|$)/;
|
|
8
|
+
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
|
9
|
+
export function readNginxLine(line) {
|
|
10
|
+
const errorLineMatch = ERROR_LINE.exec(line);
|
|
11
|
+
if (errorLineMatch) {
|
|
12
|
+
const [, year, month, day, time, levelWord = "", message = ""] = errorLineMatch;
|
|
13
|
+
const timestamp = toIso(`${year}-${month}-${day}T${time}Z`);
|
|
14
|
+
const level = normalizeLevel(levelWord);
|
|
15
|
+
if (!timestamp || level === undefined)
|
|
16
|
+
return { kind: "bad" };
|
|
17
|
+
if (level === null)
|
|
18
|
+
return { kind: "drop", timestamp };
|
|
19
|
+
return { kind: "entry", timestamp, level, message };
|
|
20
|
+
}
|
|
21
|
+
const accessLineMatch = ACCESS_LINE.exec(line);
|
|
22
|
+
if (accessLineMatch) {
|
|
23
|
+
const [, day, monthName = "", year, time, zone, request, status = ""] = accessLineMatch;
|
|
24
|
+
const month = MONTHS.indexOf(monthName) + 1;
|
|
25
|
+
const timestamp = month > 0 ? toIso(`${year}-${String(month).padStart(2, "0")}-${day}T${time}${zone}`) : undefined;
|
|
26
|
+
if (!timestamp)
|
|
27
|
+
return { kind: "bad" };
|
|
28
|
+
if (!status.startsWith("5"))
|
|
29
|
+
return { kind: "drop", timestamp };
|
|
30
|
+
return { kind: "entry", timestamp, level: "ERROR", message: `"${request}" ${status}` };
|
|
31
|
+
}
|
|
32
|
+
return { kind: "none" };
|
|
33
|
+
}
|
|
34
|
+
// nginx never writes stack traces, so every line stands alone.
|
|
35
|
+
export function parseNginx(lines, redactionKinds = ALL_REDACTION_KINDS) {
|
|
36
|
+
const entries = [];
|
|
37
|
+
let skipped = 0;
|
|
38
|
+
for (const rawLine of lines) {
|
|
39
|
+
const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
|
|
40
|
+
if (line.trim() === "")
|
|
41
|
+
continue;
|
|
42
|
+
let lineStart;
|
|
43
|
+
try {
|
|
44
|
+
lineStart = readNginxLine(line);
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
lineStart = { kind: "bad" };
|
|
48
|
+
}
|
|
49
|
+
if (lineStart.kind === "entry")
|
|
50
|
+
entries.push(buildLogEntry({ ...lineStart, continuationLines: [] }, redactionKinds));
|
|
51
|
+
else if (lineStart.kind !== "drop")
|
|
52
|
+
skipped++;
|
|
53
|
+
}
|
|
54
|
+
return { entries, skipped };
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=nginx-log-format.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nginx-log-format.js","sourceRoot":"","sources":["../../src/log-parsing/nginx-log-format.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAC1F,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAExC,MAAM,UAAU,GAAG,yFAAyF,CAAC;AAC7G,MAAM,WAAW,GAAG,sGAAsG,CAAC;AAE3H,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;AAEpG,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,EAAE,OAAO,GAAG,EAAE,CAAC,GAAG,cAAc,CAAC;QAChF,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI,GAAG,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC9D,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QACvD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IACtD,CAAC;IAED,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,CAAC,EAAE,GAAG,EAAE,SAAS,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,CAAC,GAAG,eAAe,CAAC;QACxF,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACnH,IAAI,CAAC,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAChE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,OAAO,KAAK,MAAM,EAAE,EAAE,CAAC;IACzF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC1B,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,UAAU,CAAC,KAAe,EAAE,cAAc,GAA6B,mBAAmB;IACxG,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACrE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAS;QACjC,IAAI,SAAoB,CAAC;QACzB,IAAI,CAAC;YACH,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,GAAG,SAAS,EAAE,iBAAiB,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;aAChH,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,EAAE,CAAC;IAChD,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Turns log lines in any supported format into normalized entries.
|
|
2
|
+
import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
|
|
3
|
+
import { detectFormat } from "./detect-log-format.js";
|
|
4
|
+
import { FORMAT_PARSERS } from "./line-readers.js";
|
|
5
|
+
// Parses lines in the given format, or detects it from the first 200 lines.
|
|
6
|
+
// Redacts the given kinds (default: all four).
|
|
7
|
+
export function parseLines(lines, format, redactionKinds = ALL_REDACTION_KINDS) {
|
|
8
|
+
const usedFormat = format ?? detectFormat(lines.slice(0, 200));
|
|
9
|
+
return { format: usedFormat, ...FORMAT_PARSERS[usedFormat](lines, redactionKinds) };
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=parse-log-lines.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-log-lines.js","sourceRoot":"","sources":["../../src/log-parsing/parse-log-lines.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAC1F,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnD,4EAA4E;AAC5E,+CAA+C;AAC/C,MAAM,UAAU,UAAU,CACxB,KAAe,EACf,MAAkB,EAClB,cAAc,GAA6B,mBAAmB;IAE9D,MAAM,UAAU,GAAG,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/D,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,cAAc,CAAC,EAAE,CAAC;AACtF,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// The timestamp of one line in any format. Used by the command reader to
|
|
2
|
+
// drop lines it has already seen.
|
|
3
|
+
import { LINE_START_READERS } from "./line-readers.js";
|
|
4
|
+
import { toIso } from "./timestamps.js";
|
|
5
|
+
// undefined for lines without a time (e.g. stack trace lines).
|
|
6
|
+
export function lineTimestamp(line) {
|
|
7
|
+
for (const readLineStart of Object.values(LINE_START_READERS)) {
|
|
8
|
+
try {
|
|
9
|
+
const lineStart = readLineStart(line);
|
|
10
|
+
if (lineStart.kind === "entry" || lineStart.kind === "drop")
|
|
11
|
+
return lineStart.timestamp;
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
// try the next format
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
// A timestamped line with a level we don't know still has a time.
|
|
18
|
+
const timestampMatch = /^(\S+(?: \d{2}:\d{2}:\d{2}\S*)?)/.exec(line);
|
|
19
|
+
return timestampMatch?.[1] ? toIso(timestampMatch[1]) : undefined;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=read-line-timestamp.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read-line-timestamp.js","sourceRoot":"","sources":["../../src/log-parsing/read-line-timestamp.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,kCAAkC;AAClC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAExC,+DAA+D;AAC/D,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM;gBAAE,OAAO,SAAS,CAAC,SAAS,CAAC;QAC1F,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IACD,kEAAkE;IAClE,MAAM,cAAc,GAAG,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,OAAO,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACpE,CAAC"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Runs the log command from vika.yaml (docker logs, kubectl logs, ssh ...)
|
|
2
|
+
// and keeps only lines newer than the last run's timestamp bookmark.
|
|
3
|
+
import { spawn } from "node:child_process";
|
|
4
|
+
import { lineTimestamp } from "./read-line-timestamp.js";
|
|
5
|
+
export function readCommand(command, sinceTimestamp, timeoutMs) {
|
|
6
|
+
const sinceMs = sinceTimestamp === undefined ? undefined : Date.parse(sinceTimestamp);
|
|
7
|
+
const isNewerThanBookmark = (timestamp) => sinceMs === undefined || Date.parse(timestamp) > sinceMs;
|
|
8
|
+
return new Promise((resolve, reject) => {
|
|
9
|
+
const child = spawn(command, { shell: true, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] });
|
|
10
|
+
const lines = [];
|
|
11
|
+
let newestTimestamp = sinceTimestamp;
|
|
12
|
+
let timedOut = false;
|
|
13
|
+
// Lines without a timestamp (stack traces) follow the decision for the
|
|
14
|
+
// timestamped line before them on the same stream.
|
|
15
|
+
const streams = {
|
|
16
|
+
stdout: { partial: "", keep: sinceMs === undefined },
|
|
17
|
+
stderr: { partial: "", keep: sinceMs === undefined },
|
|
18
|
+
};
|
|
19
|
+
const takeLine = (streamState, rawLine) => {
|
|
20
|
+
const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
|
|
21
|
+
const timestamp = lineTimestamp(line);
|
|
22
|
+
if (timestamp !== undefined) {
|
|
23
|
+
streamState.keep = isNewerThanBookmark(timestamp);
|
|
24
|
+
if (streamState.keep && (newestTimestamp === undefined || Date.parse(timestamp) > Date.parse(newestTimestamp))) {
|
|
25
|
+
newestTimestamp = timestamp;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
if (streamState.keep)
|
|
29
|
+
lines.push(line);
|
|
30
|
+
};
|
|
31
|
+
for (const streamName of ["stdout", "stderr"]) {
|
|
32
|
+
const streamState = streams[streamName];
|
|
33
|
+
const stream = child[streamName];
|
|
34
|
+
stream.setEncoding("utf8"); // handles UTF-8 characters split across chunks
|
|
35
|
+
stream.on("data", (chunk) => {
|
|
36
|
+
const parts = (streamState.partial + chunk).split("\n");
|
|
37
|
+
streamState.partial = parts.pop() ?? "";
|
|
38
|
+
for (const part of parts)
|
|
39
|
+
takeLine(streamState, part);
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
const timer = setTimeout(() => {
|
|
43
|
+
timedOut = true;
|
|
44
|
+
try {
|
|
45
|
+
process.kill(-child.pid, "SIGKILL"); // the whole process group
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
child.kill("SIGKILL");
|
|
49
|
+
}
|
|
50
|
+
}, timeoutMs);
|
|
51
|
+
child.on("error", (error) => {
|
|
52
|
+
clearTimeout(timer);
|
|
53
|
+
reject(error);
|
|
54
|
+
});
|
|
55
|
+
child.on("close", (exitCode) => {
|
|
56
|
+
clearTimeout(timer);
|
|
57
|
+
if (!timedOut) {
|
|
58
|
+
for (const streamState of Object.values(streams)) {
|
|
59
|
+
if (streamState.partial !== "")
|
|
60
|
+
takeLine(streamState, streamState.partial);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const next = newestTimestamp === undefined ? {} : { sinceTimestamp: newestTimestamp };
|
|
64
|
+
resolve({ lines, next, timedOut, exitCode: timedOut ? null : exitCode });
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=read-log-command.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read-log-command.js","sourceRoot":"","sources":["../../src/log-parsing/read-log-command.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,qEAAqE;AACrE,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AASzD,MAAM,UAAU,WAAW,CAAC,OAAe,EAAE,cAAkC,EAAE,SAAiB;IAChG,MAAM,OAAO,GAAG,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACtF,MAAM,mBAAmB,GAAG,CAAC,SAAiB,EAAE,EAAE,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC;IAE5G,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAEzH,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,eAAe,GAAG,cAAc,CAAC;QACrC,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,uEAAuE;QACvE,mDAAmD;QACnD,MAAM,OAAO,GAAG;YACd,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,KAAK,SAAS,EAAE;YACpD,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,KAAK,SAAS,EAAE;SACrD,CAAC;QAEF,MAAM,QAAQ,GAAG,CAAC,WAA8B,EAAE,OAAe,EAAE,EAAE;YACnE,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YACrE,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;gBAClD,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;oBAC/G,eAAe,GAAG,SAAS,CAAC;gBAC9B,CAAC;YACH,CAAC;YACD,IAAI,WAAW,CAAC,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC;QAEF,KAAK,MAAM,UAAU,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAU,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;YACjC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,+CAA+C;YAC3E,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,CAAC,WAAW,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACxD,WAAW,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;gBACxC,KAAK,MAAM,IAAI,IAAI,KAAK;oBAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,CAAE,KAAK,CAAC,GAAc,EAAE,SAAS,CAAC,CAAC,CAAC,0BAA0B;YAC7E,CAAC;YAAC,MAAM,CAAC;gBACP,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE;YAC7B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;oBACjD,IAAI,WAAW,CAAC,OAAO,KAAK,EAAE;wBAAE,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YACD,MAAM,IAAI,GAAG,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;YACtF,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|