vika-cli 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +37 -2
  3. package/dist/cli.js +32 -0
  4. package/dist/cli.js.map +1 -0
  5. package/dist/commands/approve-and-create-tickets.js +51 -0
  6. package/dist/commands/approve-and-create-tickets.js.map +1 -0
  7. package/dist/commands/create-tickets-command.js +30 -0
  8. package/dist/commands/create-tickets-command.js.map +1 -0
  9. package/dist/commands/doctor-command.js +59 -0
  10. package/dist/commands/doctor-command.js.map +1 -0
  11. package/dist/commands/init-command.js +21 -0
  12. package/dist/commands/init-command.js.map +1 -0
  13. package/dist/commands/mark-deploy-command.js +59 -0
  14. package/dist/commands/mark-deploy-command.js.map +1 -0
  15. package/dist/commands/report-command.js +40 -0
  16. package/dist/commands/report-command.js.map +1 -0
  17. package/dist/commands/run-cli.js +197 -0
  18. package/dist/commands/run-cli.js.map +1 -0
  19. package/dist/commands/run-exit-code.js +11 -0
  20. package/dist/commands/run-exit-code.js.map +1 -0
  21. package/dist/commands/serve-command.js +49 -0
  22. package/dist/commands/serve-command.js.map +1 -0
  23. package/dist/configuration/configuration-errors.js +43 -0
  24. package/dist/configuration/configuration-errors.js.map +1 -0
  25. package/dist/configuration/configuration-schema.js +69 -0
  26. package/dist/configuration/configuration-schema.js.map +1 -0
  27. package/dist/configuration/database-url-checks.js +43 -0
  28. package/dist/configuration/database-url-checks.js.map +1 -0
  29. package/dist/configuration/describe-configuration-safely.js +20 -0
  30. package/dist/configuration/describe-configuration-safely.js.map +1 -0
  31. package/dist/configuration/hide-secrets.js +44 -0
  32. package/dist/configuration/hide-secrets.js.map +1 -0
  33. package/dist/configuration/load-configuration.js +97 -0
  34. package/dist/configuration/load-configuration.js.map +1 -0
  35. package/dist/configuration/read-environment-file.js +77 -0
  36. package/dist/configuration/read-environment-file.js.map +1 -0
  37. package/dist/dashboard/dashboard-page.js +267 -0
  38. package/dist/dashboard/dashboard-page.js.map +1 -0
  39. package/dist/dashboard/dashboard-server.js +144 -0
  40. package/dist/dashboard/dashboard-server.js.map +1 -0
  41. package/dist/dashboard/github-issue-link.js +23 -0
  42. package/dist/dashboard/github-issue-link.js.map +1 -0
  43. package/dist/dashboard/request-guard.js +34 -0
  44. package/dist/dashboard/request-guard.js.map +1 -0
  45. package/dist/database/database-store.js +425 -0
  46. package/dist/database/database-store.js.map +1 -0
  47. package/dist/database/store-types.js +2 -0
  48. package/dist/database/store-types.js.map +1 -0
  49. package/dist/deploys/check-sha-on-branch.js +21 -0
  50. package/dist/deploys/check-sha-on-branch.js.map +1 -0
  51. package/dist/deploys/commits-in-range.js +31 -0
  52. package/dist/deploys/commits-in-range.js.map +1 -0
  53. package/dist/deploys/marker-deploys.js +18 -0
  54. package/dist/deploys/marker-deploys.js.map +1 -0
  55. package/dist/deploys/merge-log.js +33 -0
  56. package/dist/deploys/merge-log.js.map +1 -0
  57. package/dist/deploys/pull-request-number.js +5 -0
  58. package/dist/deploys/pull-request-number.js.map +1 -0
  59. package/dist/deploys/read-merge-deploys.js +46 -0
  60. package/dist/deploys/read-merge-deploys.js.map +1 -0
  61. package/dist/deploys/record-deploys.js +12 -0
  62. package/dist/deploys/record-deploys.js.map +1 -0
  63. package/dist/deploys/run-git.js +49 -0
  64. package/dist/deploys/run-git.js.map +1 -0
  65. package/dist/deploys/starting-point.js +5 -0
  66. package/dist/deploys/starting-point.js.map +1 -0
  67. package/dist/deploys/utc-timestamp.js +6 -0
  68. package/dist/deploys/utc-timestamp.js.map +1 -0
  69. package/dist/doctor/check-configuration.js +46 -0
  70. package/dist/doctor/check-configuration.js.map +1 -0
  71. package/dist/doctor/check-database.js +30 -0
  72. package/dist/doctor/check-database.js.map +1 -0
  73. package/dist/doctor/check-deploys.js +22 -0
  74. package/dist/doctor/check-deploys.js.map +1 -0
  75. package/dist/doctor/check-service-logs.js +107 -0
  76. package/dist/doctor/check-service-logs.js.map +1 -0
  77. package/dist/doctor/check-service-repository.js +43 -0
  78. package/dist/doctor/check-service-repository.js.map +1 -0
  79. package/dist/doctor/check-tracker.js +20 -0
  80. package/dist/doctor/check-tracker.js.map +1 -0
  81. package/dist/doctor/check-webhook.js +13 -0
  82. package/dist/doctor/check-webhook.js.map +1 -0
  83. package/dist/doctor/doctor-check-types.js +4 -0
  84. package/dist/doctor/doctor-check-types.js.map +1 -0
  85. package/dist/doctor/format-doctor-report.js +15 -0
  86. package/dist/doctor/format-doctor-report.js.map +1 -0
  87. package/dist/doctor/suggest-path-map.js +35 -0
  88. package/dist/doctor/suggest-path-map.js.map +1 -0
  89. package/dist/error-grouping/build-clusters.js +77 -0
  90. package/dist/error-grouping/build-clusters.js.map +1 -0
  91. package/dist/error-grouping/cluster-types.js +3 -0
  92. package/dist/error-grouping/cluster-types.js.map +1 -0
  93. package/dist/error-grouping/exception-type.js +6 -0
  94. package/dist/error-grouping/exception-type.js.map +1 -0
  95. package/dist/error-grouping/fingerprints.js +18 -0
  96. package/dist/error-grouping/fingerprints.js.map +1 -0
  97. package/dist/error-grouping/normalize-message.js +29 -0
  98. package/dist/error-grouping/normalize-message.js.map +1 -0
  99. package/dist/error-grouping/template-miner.js +75 -0
  100. package/dist/error-grouping/template-miner.js.map +1 -0
  101. package/dist/log-parsing/assemble-log-entries.js +49 -0
  102. package/dist/log-parsing/assemble-log-entries.js.map +1 -0
  103. package/dist/log-parsing/build-log-entry.js +33 -0
  104. package/dist/log-parsing/build-log-entry.js.map +1 -0
  105. package/dist/log-parsing/detect-log-format.js +44 -0
  106. package/dist/log-parsing/detect-log-format.js.map +1 -0
  107. package/dist/log-parsing/json-log-format.js +56 -0
  108. package/dist/log-parsing/json-log-format.js.map +1 -0
  109. package/dist/log-parsing/line-readers.js +14 -0
  110. package/dist/log-parsing/line-readers.js.map +1 -0
  111. package/dist/log-parsing/log-entry-types.js +3 -0
  112. package/dist/log-parsing/log-entry-types.js.map +1 -0
  113. package/dist/log-parsing/log-levels.js +21 -0
  114. package/dist/log-parsing/log-levels.js.map +1 -0
  115. package/dist/log-parsing/nginx-log-format.js +56 -0
  116. package/dist/log-parsing/nginx-log-format.js.map +1 -0
  117. package/dist/log-parsing/parse-log-lines.js +11 -0
  118. package/dist/log-parsing/parse-log-lines.js.map +1 -0
  119. package/dist/log-parsing/read-line-timestamp.js +21 -0
  120. package/dist/log-parsing/read-line-timestamp.js.map +1 -0
  121. package/dist/log-parsing/read-log-command.js +68 -0
  122. package/dist/log-parsing/read-log-command.js.map +1 -0
  123. package/dist/log-parsing/read-log-file.js +32 -0
  124. package/dist/log-parsing/read-log-file.js.map +1 -0
  125. package/dist/log-parsing/stack-frames.js +10 -0
  126. package/dist/log-parsing/stack-frames.js.map +1 -0
  127. package/dist/log-parsing/text-log-format.js +24 -0
  128. package/dist/log-parsing/text-log-format.js.map +1 -0
  129. package/dist/log-parsing/timestamps.js +22 -0
  130. package/dist/log-parsing/timestamps.js.map +1 -0
  131. package/dist/pipeline/analyze-service.js +45 -0
  132. package/dist/pipeline/analyze-service.js.map +1 -0
  133. package/dist/pipeline/build-stored-report.js +24 -0
  134. package/dist/pipeline/build-stored-report.js.map +1 -0
  135. package/dist/pipeline/ingest-service-logs.js +39 -0
  136. package/dist/pipeline/ingest-service-logs.js.map +1 -0
  137. package/dist/pipeline/load-deploys-for-service.js +16 -0
  138. package/dist/pipeline/load-deploys-for-service.js.map +1 -0
  139. package/dist/pipeline/prepare-ticket-drafts.js +33 -0
  140. package/dist/pipeline/prepare-ticket-drafts.js.map +1 -0
  141. package/dist/pipeline/read-new-log-lines.js +39 -0
  142. package/dist/pipeline/read-new-log-lines.js.map +1 -0
  143. package/dist/pipeline/run-all-services.js +73 -0
  144. package/dist/pipeline/run-all-services.js.map +1 -0
  145. package/dist/pipeline/timestamp-range.js +20 -0
  146. package/dist/pipeline/timestamp-range.js.map +1 -0
  147. package/dist/project-setup/detect-project.js +76 -0
  148. package/dist/project-setup/detect-project.js.map +1 -0
  149. package/dist/project-setup/write-starter-files.js +68 -0
  150. package/dist/project-setup/write-starter-files.js.map +1 -0
  151. package/dist/redaction/redact-ip-addresses.js +26 -0
  152. package/dist/redaction/redact-ip-addresses.js.map +1 -0
  153. package/dist/redaction/redact-sensitive-text.js +40 -0
  154. package/dist/redaction/redact-sensitive-text.js.map +1 -0
  155. package/dist/redaction/redaction-kinds.js +5 -0
  156. package/dist/redaction/redaction-kinds.js.map +1 -0
  157. package/dist/regression-detection/count-series.js +38 -0
  158. package/dist/regression-detection/count-series.js.map +1 -0
  159. package/dist/regression-detection/detect-regressions.js +162 -0
  160. package/dist/regression-detection/detect-regressions.js.map +1 -0
  161. package/dist/regression-detection/detection-types.js +9 -0
  162. package/dist/regression-detection/detection-types.js.map +1 -0
  163. package/dist/regression-detection/noise-check.js +7 -0
  164. package/dist/regression-detection/noise-check.js.map +1 -0
  165. package/dist/regression-detection/restore-time.js +19 -0
  166. package/dist/regression-detection/restore-time.js.map +1 -0
  167. package/dist/regression-detection/severity.js +12 -0
  168. package/dist/regression-detection/severity.js.map +1 -0
  169. package/dist/regression-detection/time-units.js +12 -0
  170. package/dist/regression-detection/time-units.js.map +1 -0
  171. package/dist/reporting/build-report.js +77 -0
  172. package/dist/reporting/build-report.js.map +1 -0
  173. package/dist/reporting/compute-metrics.js +42 -0
  174. package/dist/reporting/compute-metrics.js.map +1 -0
  175. package/dist/reporting/format-terminal-report.js +104 -0
  176. package/dist/reporting/format-terminal-report.js.map +1 -0
  177. package/dist/reporting/report-schema.js +96 -0
  178. package/dist/reporting/report-schema.js.map +1 -0
  179. package/dist/reporting/report-types.js +2 -0
  180. package/dist/reporting/report-types.js.map +1 -0
  181. package/dist/suspect-attribution/apply-path-map.js +31 -0
  182. package/dist/suspect-attribution/apply-path-map.js.map +1 -0
  183. package/dist/suspect-attribution/attribute-regression.js +176 -0
  184. package/dist/suspect-attribution/attribute-regression.js.map +1 -0
  185. package/dist/suspect-attribution/bot-authors.js +8 -0
  186. package/dist/suspect-attribution/bot-authors.js.map +1 -0
  187. package/dist/suspect-attribution/callee-files.js +53 -0
  188. package/dist/suspect-attribution/callee-files.js.map +1 -0
  189. package/dist/suspect-attribution/changed-file-categories.js +39 -0
  190. package/dist/suspect-attribution/changed-file-categories.js.map +1 -0
  191. package/dist/suspect-attribution/crash-line-inputs.js +81 -0
  192. package/dist/suspect-attribution/crash-line-inputs.js.map +1 -0
  193. package/dist/suspect-attribution/function-names.js +42 -0
  194. package/dist/suspect-attribution/function-names.js.map +1 -0
  195. package/dist/suspect-attribution/git-blame-reader.js +160 -0
  196. package/dist/suspect-attribution/git-blame-reader.js.map +1 -0
  197. package/dist/suspect-attribution/score-suspects.js +100 -0
  198. package/dist/suspect-attribution/score-suspects.js.map +1 -0
  199. package/dist/suspect-attribution/suspect-types.js +2 -0
  200. package/dist/suspect-attribution/suspect-types.js.map +1 -0
  201. package/dist/ticket-trackers/create-tracker.js +21 -0
  202. package/dist/ticket-trackers/create-tracker.js.map +1 -0
  203. package/dist/ticket-trackers/jira-document.js +34 -0
  204. package/dist/ticket-trackers/jira-document.js.map +1 -0
  205. package/dist/ticket-trackers/jira-tracker.js +65 -0
  206. package/dist/ticket-trackers/jira-tracker.js.map +1 -0
  207. package/dist/ticket-trackers/linear-tracker.js +109 -0
  208. package/dist/ticket-trackers/linear-tracker.js.map +1 -0
  209. package/dist/ticket-trackers/send-tracker-request.js +84 -0
  210. package/dist/ticket-trackers/send-tracker-request.js.map +1 -0
  211. package/dist/ticket-trackers/tracker-types.js +3 -0
  212. package/dist/ticket-trackers/tracker-types.js.map +1 -0
  213. package/dist/ticketing/build-ticket-comment.js +14 -0
  214. package/dist/ticketing/build-ticket-comment.js.map +1 -0
  215. package/dist/ticketing/build-ticket-draft.js +172 -0
  216. package/dist/ticketing/build-ticket-draft.js.map +1 -0
  217. package/dist/ticketing/choose-assignee.js +30 -0
  218. package/dist/ticketing/choose-assignee.js.map +1 -0
  219. package/dist/ticketing/create-ticket.js +92 -0
  220. package/dist/ticketing/create-ticket.js.map +1 -0
  221. package/dist/ticketing/describe-ticket-outcome.js +24 -0
  222. package/dist/ticketing/describe-ticket-outcome.js.map +1 -0
  223. package/dist/ticketing/read-code-owners.js +82 -0
  224. package/dist/ticketing/read-code-owners.js.map +1 -0
  225. package/dist/ticketing/render-ticket-text.js +22 -0
  226. package/dist/ticketing/render-ticket-text.js.map +1 -0
  227. package/dist/ticketing/ticket-types.js +2 -0
  228. package/dist/ticketing/ticket-types.js.map +1 -0
  229. package/dist/webhook/build-webhook-payload.js +29 -0
  230. package/dist/webhook/build-webhook-payload.js.map +1 -0
  231. package/dist/webhook/send-regression-webhooks.js +57 -0
  232. package/dist/webhook/send-regression-webhooks.js.map +1 -0
  233. package/dist/webhook/sign-webhook-body.js +16 -0
  234. package/dist/webhook/sign-webhook-body.js.map +1 -0
  235. package/dist/webhook/webhook-payload-schema.js +32 -0
  236. package/dist/webhook/webhook-payload-schema.js.map +1 -0
  237. package/migrations/001_init.sql +66 -0
  238. package/migrations/002_bookmark_history_start.sql +9 -0
  239. package/migrations/003_tickets.sql +20 -0
  240. package/migrations/004_ticket_comment_count.sql +10 -0
  241. package/migrations/005_webhook_events.sql +18 -0
  242. package/migrations/README.md +1 -0
  243. package/package.json +37 -3
@@ -0,0 +1,3 @@
1
+ export const MAX_EXAMPLES = 3;
2
+ export const MAX_SOURCE_REFS = 3;
3
+ //# sourceMappingURL=cluster-types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cluster-types.js","sourceRoot":"","sources":["../../src/error-grouping/cluster-types.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAC9B,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC"}
@@ -0,0 +1,6 @@
1
+ // Reads the exception class name ("TypeError", "java.io.IOException") from a message.
2
+ export function readExceptionType(message) {
3
+ const exceptionMatch = /^([\w$.]*(?:Error|Exception))(?::|$)/m.exec(message);
4
+ return exceptionMatch?.[1];
5
+ }
6
+ //# sourceMappingURL=exception-type.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"exception-type.js","sourceRoot":"","sources":["../../src/error-grouping/exception-type.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,cAAc,GAAG,uCAAuC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7E,OAAO,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7B,CAAC"}
@@ -0,0 +1,18 @@
1
+ // Stable ids for error groups. The normalizer version is part of every
2
+ // fingerprint, so changing the normalizer never mixes old and new groups.
3
+ import { createHash } from "node:crypto";
4
+ import { NORMALIZER_VERSION } from "./normalize-message.js";
5
+ export const FINGERPRINT_FRAME_COUNT = 3;
6
+ function shortHash(parts) {
7
+ return createHash("sha256").update(JSON.stringify(parts)).digest("hex").slice(0, 12);
8
+ }
9
+ // Line numbers are left out on purpose: the same crash should keep its
10
+ // fingerprint when unrelated lines above it move.
11
+ export function stackFingerprint(service, exceptionName, frames) {
12
+ const topFrames = frames.slice(0, FINGERPRINT_FRAME_COUNT).map((frame) => [frame.file, frame.fn ?? ""]);
13
+ return shortHash([NORMALIZER_VERSION, service, "stack", exceptionName ?? "", topFrames]);
14
+ }
15
+ export function messageFingerprint(service, normalizedMessage) {
16
+ return shortHash([NORMALIZER_VERSION, service, "message", normalizedMessage]);
17
+ }
18
+ //# sourceMappingURL=fingerprints.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fingerprints.js","sourceRoot":"","sources":["../../src/error-grouping/fingerprints.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,0EAA0E;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAEzC,SAAS,SAAS,CAAC,KAAgB;IACjC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACvF,CAAC;AAED,uEAAuE;AACvE,kDAAkD;AAClD,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,aAAiC,EAAE,MAAe;IAClG,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxG,OAAO,SAAS,CAAC,CAAC,kBAAkB,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,IAAI,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,iBAAyB;IAC3E,OAAO,SAAS,CAAC,CAAC,kBAAkB,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC;AAChF,CAAC"}
@@ -0,0 +1,29 @@
1
+ // Replaces the parts of a message that change between occurrences (ids,
2
+ // numbers, IPs, query strings) with placeholders, so they group together.
3
+ import { redactIps } from "../redaction/redact-ip-addresses.js";
4
+ export const NORMALIZER_VERSION = 1;
5
+ const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi;
6
+ const HEX_NUMBER = /(?<![0-9A-Za-z])(?:0x[0-9a-f]+|(?=[0-9a-f]*\d)(?=[0-9a-f]*[a-f])[0-9a-f]{8,})(?![0-9A-Za-z])/gi;
7
+ const QUERY = /\?[\w.%[\]-]+=[^\s"'#)]*/g;
8
+ const PATH_SEGMENT = /(?<=\/)[\w.~<>-]+(?=[/?#\s"'),;:\]]|$)/g;
9
+ const NUMBER = /(?<![A-Za-z\d.])\d+(?:\.\d+)?/g;
10
+ const PLACEHOLDER = /^<(?:uuid|hex|id|num|ip|email|token|card|query)>$/;
11
+ function maskPathSegment(segment) {
12
+ if (PLACEHOLDER.test(segment))
13
+ return segment;
14
+ if (/^\d+$/.test(segment))
15
+ return "<id>";
16
+ if (segment.length >= 6 && /\d|<\w+>/.test(segment) && !/\.\w{1,4}$/.test(segment))
17
+ return "<id>";
18
+ return segment;
19
+ }
20
+ export function normalize(message) {
21
+ let text = message.replace(UUID, "<uuid>");
22
+ text = text.replace(HEX_NUMBER, "<hex>");
23
+ text = redactIps(text);
24
+ text = text.replace(QUERY, "?<query>");
25
+ text = text.replace(PATH_SEGMENT, maskPathSegment);
26
+ text = text.replace(NUMBER, "<num>");
27
+ return text;
28
+ }
29
+ //# sourceMappingURL=normalize-message.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"normalize-message.js","sourceRoot":"","sources":["../../src/error-grouping/normalize-message.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,0EAA0E;AAC1E,OAAO,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AAEhE,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAEpC,MAAM,IAAI,GAAG,gEAAgE,CAAC;AAE9E,MAAM,UAAU,GAAG,gGAAgG,CAAC;AAEpH,MAAM,KAAK,GAAG,2BAA2B,CAAC;AAE1C,MAAM,YAAY,GAAG,yCAAyC,CAAC;AAE/D,MAAM,MAAM,GAAG,gCAAgC,CAAC;AAEhD,MAAM,WAAW,GAAG,mDAAmD,CAAC;AAExE,SAAS,eAAe,CAAC,OAAe;IACtC,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC;IAC9C,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IACzC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAClG,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAe;IACvC,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC3C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAC,OAAO,CAAC,CAAC;IACxC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACvB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IACnD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,OAAO,IAAI,CAAC;AACd,CAAC"}
@@ -0,0 +1,75 @@
1
+ // Groups messages without a stack trace by template: messages with the same
2
+ // length and first token that mostly match share a group, and the tokens
3
+ // that differ become a wildcard.
4
+ import { messageFingerprint } from "./fingerprints.js";
5
+ import { NORMALIZER_VERSION } from "./normalize-message.js";
6
+ export const PREFIX_TOKEN_COUNT = 1;
7
+ export const SIMILARITY_THRESHOLD = 0.7;
8
+ export const WILDCARD = "<*>";
9
+ export function emptyState() {
10
+ return { version: NORMALIZER_VERSION, groups: [] };
11
+ }
12
+ export function tokenize(normalizedMessage) {
13
+ return normalizedMessage.split(/\s+/).filter((token) => token !== "");
14
+ }
15
+ function treeKey(service, tokens) {
16
+ return [service, String(tokens.length), ...tokens.slice(0, PREFIX_TOKEN_COUNT)].join("\0");
17
+ }
18
+ // State saved with another normalizer version is dropped: its templates
19
+ // were made from differently normalized messages.
20
+ export function importState(state) {
21
+ const miner = { groups: [], tree: new Map(), byFingerprint: new Map() };
22
+ if (state.version !== NORMALIZER_VERSION)
23
+ return miner;
24
+ for (const group of state.groups)
25
+ addGroup(miner, { fingerprint: group.fingerprint, service: group.service, tokens: [...group.tokens] });
26
+ return miner;
27
+ }
28
+ export function exportState(miner) {
29
+ return {
30
+ version: NORMALIZER_VERSION,
31
+ groups: miner.groups.map((group) => ({ fingerprint: group.fingerprint, service: group.service, tokens: [...group.tokens] })),
32
+ };
33
+ }
34
+ function addGroup(miner, group) {
35
+ miner.groups.push(group);
36
+ miner.byFingerprint.set(group.fingerprint, group);
37
+ const key = treeKey(group.service, group.tokens);
38
+ const groupsWithSameKey = miner.tree.get(key);
39
+ if (groupsWithSameKey)
40
+ groupsWithSameKey.push(group);
41
+ else
42
+ miner.tree.set(key, [group]);
43
+ }
44
+ // The share of positions where the template and the message have the same token.
45
+ function similarity(template, tokens) {
46
+ let matchingTokenCount = 0;
47
+ for (let position = 0; position < tokens.length; position++)
48
+ if (template[position] === tokens[position])
49
+ matchingTokenCount++;
50
+ return matchingTokenCount / tokens.length;
51
+ }
52
+ export function addMessage(miner, service, normalizedMessage) {
53
+ const tokens = tokenize(normalizedMessage);
54
+ let bestGroup;
55
+ let bestScore = -1;
56
+ for (const group of miner.tree.get(treeKey(service, tokens)) ?? []) {
57
+ const score = similarity(group.tokens, tokens);
58
+ if (score > bestScore) {
59
+ bestGroup = group;
60
+ bestScore = score;
61
+ }
62
+ }
63
+ if (bestGroup && bestScore >= SIMILARITY_THRESHOLD) {
64
+ bestGroup.tokens = bestGroup.tokens.map((token, position) => (token === tokens[position] ? token : WILDCARD));
65
+ return bestGroup;
66
+ }
67
+ const fingerprint = messageFingerprint(service, tokens.join(" "));
68
+ const existing = miner.byFingerprint.get(fingerprint);
69
+ if (existing)
70
+ return existing;
71
+ const group = { fingerprint, service, tokens };
72
+ addGroup(miner, group);
73
+ return group;
74
+ }
75
+ //# sourceMappingURL=template-miner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"template-miner.js","sourceRoot":"","sources":["../../src/error-grouping/template-miner.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,yEAAyE;AACzE,iCAAiC;AACjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AACpC,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACxC,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,CAAC;AAoB9B,MAAM,UAAU,UAAU;IACxB,OAAO,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,iBAAyB;IAChD,OAAO,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,OAAO,CAAC,OAAe,EAAE,MAAgB;IAChD,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,wEAAwE;AACxE,kDAAkD;AAClD,MAAM,UAAU,WAAW,CAAC,KAAiB;IAC3C,MAAM,KAAK,GAAU,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/E,IAAI,KAAK,CAAC,OAAO,KAAK,kBAAkB;QAAE,OAAO,KAAK,CAAC;IACvD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM;QAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACzI,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAY;IACtC,OAAO;QACL,OAAO,EAAE,kBAAkB;QAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;KAC7H,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAAY,EAAE,KAAiB;IAC/C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,iBAAiB;QAAE,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;QAChD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,iFAAiF;AACjF,SAAS,UAAU,CAAC,QAAkB,EAAE,MAAgB;IACtD,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE;QAAE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC;YAAE,kBAAkB,EAAE,CAAC;IAC/H,OAAO,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAY,EAAE,OAAe,EAAE,iBAAyB;IACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE3C,IAAI,SAAiC,CAAC;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;IACnB,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACnE,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/C,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,SAAS,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;QACnD,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9G,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACtD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,KAAK,GAAe,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3D,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACvB,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,49 @@
1
+ // Groups lines into entries: a line a format recognizes starts an entry, and
2
+ // the stack trace lines after it are attached to it.
3
+ import { buildLogEntry } from "./build-log-entry.js";
4
+ import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
5
+ const CONTINUATION_LINE = /^(?:\s|[\w$.]*(?:Error|Exception)(?::|$)|Caused by:)/;
6
+ export function assembleLogEntries(lines, readLineStart, redactionKinds = ALL_REDACTION_KINDS) {
7
+ const entries = [];
8
+ let skipped = 0;
9
+ // "ignored" = inside an entry we don't keep (e.g. INFO), so its stack lines are dropped too.
10
+ let currentEntry = null;
11
+ for (const rawLine of lines) {
12
+ const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
13
+ if (line.trim() === "")
14
+ continue;
15
+ let lineStart;
16
+ try {
17
+ lineStart = readLineStart(line);
18
+ }
19
+ catch {
20
+ lineStart = { kind: "bad" }; // never let one line crash the run
21
+ }
22
+ if (lineStart.kind === "none") {
23
+ if (currentEntry === null || !CONTINUATION_LINE.test(line))
24
+ skipped++;
25
+ else if (currentEntry !== "ignored")
26
+ currentEntry.continuationLines.push(line);
27
+ continue;
28
+ }
29
+ if (currentEntry !== null && currentEntry !== "ignored")
30
+ entries.push(buildLogEntry(currentEntry, redactionKinds));
31
+ if (lineStart.kind === "entry") {
32
+ currentEntry = {
33
+ timestamp: lineStart.timestamp,
34
+ level: lineStart.level,
35
+ message: lineStart.message,
36
+ continuationLines: lineStart.stack ? lineStart.stack.split("\n") : [],
37
+ };
38
+ }
39
+ else {
40
+ if (lineStart.kind === "bad")
41
+ skipped++;
42
+ currentEntry = "ignored";
43
+ }
44
+ }
45
+ if (currentEntry !== null && currentEntry !== "ignored")
46
+ entries.push(buildLogEntry(currentEntry, redactionKinds));
47
+ return { entries, skipped };
48
+ }
49
+ //# sourceMappingURL=assemble-log-entries.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"assemble-log-entries.js","sourceRoot":"","sources":["../../src/log-parsing/assemble-log-entries.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,qDAAqD;AACrD,OAAO,EAAE,aAAa,EAAmB,MAAM,sBAAsB,CAAC;AAEtE,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAE1F,MAAM,iBAAiB,GAAG,sDAAsD,CAAC;AAEjF,MAAM,UAAU,kBAAkB,CAChC,KAAe,EACf,aAA0C,EAC1C,cAAc,GAA6B,mBAAmB;IAE9D,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,6FAA6F;IAC7F,IAAI,YAAY,GAAkC,IAAI,CAAC;IAEvD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACrE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAS;QAEjC,IAAI,SAAoB,CAAC;QACzB,IAAI,CAAC;YACH,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,mCAAmC;QAClE,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC9B,IAAI,YAAY,KAAK,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,EAAE,CAAC;iBACjE,IAAI,YAAY,KAAK,SAAS;gBAAE,YAAY,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/E,SAAS;QACX,CAAC;QAED,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QACnH,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC/B,YAAY,GAAG;gBACb,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,iBAAiB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;aACtE,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,SAAS,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO,EAAE,CAAC;YACxC,YAAY,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;IACnH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC"}
@@ -0,0 +1,33 @@
1
+ // Turns a first line plus its continuation lines into one redacted LogEntry.
2
+ import { redact } from "../redaction/redact-sensitive-text.js";
3
+ import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
4
+ import { parseFrame } from "./stack-frames.js";
5
+ // Redacts the configured kinds (default: all) in the message and frames.
6
+ export function buildLogEntry(draft, redactionKinds = ALL_REDACTION_KINDS) {
7
+ const frames = [];
8
+ const messageLines = [draft.message.trim()];
9
+ for (const line of draft.continuationLines) {
10
+ const frame = parseFrame(line);
11
+ if (frame)
12
+ frames.push(frame);
13
+ // "at ..." lines we can't read (e.g. "at async Promise.all (index 0)") are left out.
14
+ else if (!/^\s*at\s/.test(line) && line.trim() !== "")
15
+ messageLines.push(line.trim());
16
+ }
17
+ const safeFrames = frames.map((frame) => frame.fn === undefined
18
+ ? { file: redact(frame.file, redactionKinds), line: frame.line }
19
+ : { file: redact(frame.file, redactionKinds), line: frame.line, fn: redact(frame.fn, redactionKinds) });
20
+ const entry = {
21
+ timestamp: draft.timestamp,
22
+ level: draft.level,
23
+ message: redact(messageLines.filter((messageLine) => messageLine !== "").join("\n"), redactionKinds),
24
+ frames: safeFrames,
25
+ };
26
+ const topFrame = safeFrames[0];
27
+ if (topFrame) {
28
+ entry.file = topFrame.file;
29
+ entry.lineNumber = topFrame.line;
30
+ }
31
+ return entry;
32
+ }
33
+ //# sourceMappingURL=build-log-entry.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"build-log-entry.js","sourceRoot":"","sources":["../../src/log-parsing/build-log-entry.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,OAAO,EAAE,MAAM,EAAE,MAAM,uCAAuC,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAE1F,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAS/C,yEAAyE;AACzE,MAAM,UAAU,aAAa,CAAC,KAAiB,EAAE,cAAc,GAA6B,mBAAmB;IAC7G,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,qFAAqF;aAChF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,EAAE,KAAK,SAAS;QACpB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;QAChE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CACzG,CAAC;IACF,MAAM,KAAK,GAAa;QACtB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC;QACpG,MAAM,EAAE,UAAU;KACnB,CAAC;IACF,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC/B,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,44 @@
1
+ // Guesses a log's format from a sample of its lines.
2
+ import { LINE_START_READERS } from "./line-readers.js";
3
+ // How many sample lines the best format recognizes, so callers can tell
4
+ // "text" (recognized) from "text" (the fallback because nothing matched).
5
+ export function recognizeFormat(sampleLines) {
6
+ const format = detectFormat(sampleLines);
7
+ let recognizedCount = 0;
8
+ for (const line of sampleLines) {
9
+ try {
10
+ const kind = LINE_START_READERS[format](line).kind;
11
+ if (kind === "entry" || kind === "drop")
12
+ recognizedCount++;
13
+ }
14
+ catch {
15
+ // an unreadable line just doesn't count
16
+ }
17
+ }
18
+ return { format, recognizedCount };
19
+ }
20
+ // Whichever format recognizes the most lines wins. Ties, or nothing
21
+ // recognized at all, fall back to "text".
22
+ export function detectFormat(sampleLines) {
23
+ let bestFormat = "text";
24
+ let bestScore = 0;
25
+ for (const format of ["text", "json", "nginx"]) {
26
+ let score = 0;
27
+ for (const line of sampleLines) {
28
+ try {
29
+ const kind = LINE_START_READERS[format](line).kind;
30
+ if (kind === "entry" || kind === "drop")
31
+ score++;
32
+ }
33
+ catch {
34
+ // an unreadable line just doesn't count
35
+ }
36
+ }
37
+ if (score > bestScore) {
38
+ bestFormat = format;
39
+ bestScore = score;
40
+ }
41
+ }
42
+ return bestFormat;
43
+ }
44
+ //# sourceMappingURL=detect-log-format.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"detect-log-format.js","sourceRoot":"","sources":["../../src/log-parsing/detect-log-format.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAGvD,wEAAwE;AACxE,0EAA0E;AAC1E,MAAM,UAAU,eAAe,CAAC,WAAqB;IACnD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IACzC,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YACnD,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM;gBAAE,eAAe,EAAE,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;AACrC,CAAC;AAED,oEAAoE;AACpE,0CAA0C;AAC1C,MAAM,UAAU,YAAY,CAAC,WAAqB;IAChD,IAAI,UAAU,GAAc,MAAM,CAAC;IACnC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAU,EAAE,CAAC;QACxD,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;gBACnD,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM;oBAAE,KAAK,EAAE,CAAC;YACnD,CAAC;YAAC,MAAM,CAAC;gBACP,wCAAwC;YAC1C,CAAC;QACH,CAAC;QACD,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,UAAU,GAAG,MAAM,CAAC;YACpB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}
@@ -0,0 +1,56 @@
1
+ // JSON log lines (pino, bunyan, winston and similar).
2
+ import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
3
+ import { assembleLogEntries } from "./assemble-log-entries.js";
4
+ import { normalizeLevel } from "./log-levels.js";
5
+ import { epochToIso, toIso } from "./timestamps.js";
6
+ const isJsonObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
7
+ function firstString(...values) {
8
+ return values.find((value) => typeof value === "string");
9
+ }
10
+ // The keys ("time", "ts", "lvl", ...) are what loggers write, so they stay as they are.
11
+ function readTime(jsonObject) {
12
+ const value = jsonObject.time ?? jsonObject.timestamp ?? jsonObject.ts ?? jsonObject["@timestamp"];
13
+ if (typeof value === "string")
14
+ return toIso(value);
15
+ if (typeof value === "number")
16
+ return epochToIso(value);
17
+ return undefined;
18
+ }
19
+ // undefined/null both mean "not a level we keep".
20
+ function readLevel(jsonObject) {
21
+ const value = jsonObject.level ?? jsonObject.severity ?? jsonObject.lvl;
22
+ if (typeof value === "string")
23
+ return normalizeLevel(value);
24
+ if (typeof value === "number")
25
+ return value >= 50 ? "ERROR" : value >= 40 ? "WARN" : null;
26
+ return undefined;
27
+ }
28
+ export function readJsonStart(line) {
29
+ if (!line.trimStart().startsWith("{"))
30
+ return { kind: "none" };
31
+ let parsedLine;
32
+ try {
33
+ parsedLine = JSON.parse(line);
34
+ }
35
+ catch {
36
+ return { kind: "bad" };
37
+ }
38
+ if (!isJsonObject(parsedLine))
39
+ return { kind: "bad" };
40
+ const timestamp = readTime(parsedLine);
41
+ if (!timestamp)
42
+ return { kind: "bad" };
43
+ const level = readLevel(parsedLine);
44
+ if (!level)
45
+ return { kind: "drop", timestamp };
46
+ const errorField = isJsonObject(parsedLine.err) ? parsedLine.err : isJsonObject(parsedLine.error) ? parsedLine.error : {};
47
+ const message = firstString(parsedLine.msg, parsedLine.message, errorField.message) ?? "";
48
+ const stack = firstString(parsedLine.stack, errorField.stack);
49
+ return stack === undefined
50
+ ? { kind: "entry", timestamp, level, message }
51
+ : { kind: "entry", timestamp, level, message, stack };
52
+ }
53
+ export function parseJson(lines, redactionKinds = ALL_REDACTION_KINDS) {
54
+ return assembleLogEntries(lines, readJsonStart, redactionKinds);
55
+ }
56
+ //# sourceMappingURL=json-log-format.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"json-log-format.js","sourceRoot":"","sources":["../../src/log-parsing/json-log-format.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAIpD,MAAM,YAAY,GAAG,CAAC,KAAc,EAAuB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEnI,SAAS,WAAW,CAAC,GAAG,MAAiB;IACvC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;AAC5E,CAAC;AAED,wFAAwF;AACxF,SAAS,QAAQ,CAAC,UAAsB;IACtC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC;IACnG,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;IACxD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,kDAAkD;AAClD,SAAS,SAAS,CAAC,UAAsB;IACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,GAAG,CAAC;IACxE,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC5D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1F,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC/D,IAAI,UAAmB,CAAC;IACxB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IACD,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACtD,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACvC,IAAI,CAAC,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAE/C,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1H,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC1F,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9D,OAAO,KAAK,KAAK,SAAS;QACxB,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE;QAC9C,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAe,EAAE,cAAc,GAA6B,mBAAmB;IACvG,OAAO,kBAAkB,CAAC,KAAK,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;AAClE,CAAC"}
@@ -0,0 +1,14 @@
1
+ import { parseJson, readJsonStart } from "./json-log-format.js";
2
+ import { parseNginx, readNginxLine } from "./nginx-log-format.js";
3
+ import { parseText, readTextStart } from "./text-log-format.js";
4
+ export const LINE_START_READERS = {
5
+ text: readTextStart,
6
+ json: readJsonStart,
7
+ nginx: readNginxLine,
8
+ };
9
+ export const FORMAT_PARSERS = {
10
+ text: parseText,
11
+ json: parseJson,
12
+ nginx: parseNginx,
13
+ };
14
+ //# sourceMappingURL=line-readers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"line-readers.js","sourceRoot":"","sources":["../../src/log-parsing/line-readers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEhE,MAAM,CAAC,MAAM,kBAAkB,GAAmD;IAChF,IAAI,EAAE,aAAa;IACnB,IAAI,EAAE,aAAa;IACnB,KAAK,EAAE,aAAa;CACrB,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAmG;IAC5H,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;CAClB,CAAC"}
@@ -0,0 +1,3 @@
1
+ // The shapes every log format parser shares.
2
+ export {};
3
+ //# sourceMappingURL=log-entry-types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"log-entry-types.js","sourceRoot":"","sources":["../../src/log-parsing/log-entry-types.ts"],"names":[],"mappings":"AAAA,6CAA6C"}
@@ -0,0 +1,21 @@
1
+ // null = a known level we don't keep.
2
+ const LEVEL_WORDS = {
3
+ emerg: "ERROR",
4
+ alert: "ERROR",
5
+ fatal: "ERROR",
6
+ crit: "ERROR",
7
+ critical: "ERROR",
8
+ error: "ERROR",
9
+ err: "ERROR",
10
+ warn: "WARN",
11
+ warning: "WARN",
12
+ notice: null,
13
+ info: null,
14
+ debug: null,
15
+ trace: null,
16
+ };
17
+ // undefined = not a level word at all.
18
+ export function normalizeLevel(word) {
19
+ return LEVEL_WORDS[word.toLowerCase()];
20
+ }
21
+ //# sourceMappingURL=log-levels.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"log-levels.js","sourceRoot":"","sources":["../../src/log-parsing/log-levels.ts"],"names":[],"mappings":"AAGA,sCAAsC;AACtC,MAAM,WAAW,GAAiC;IAChD,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,OAAO;IACb,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,OAAO;IACd,GAAG,EAAE,OAAO;IACZ,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,IAAI;IACZ,IAAI,EAAE,IAAI;IACV,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,IAAI;CACZ,CAAC;AAEF,uCAAuC;AACvC,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;AACzC,CAAC"}
@@ -0,0 +1,56 @@
1
+ // nginx error log and access log lines. Access lines count only for 5xx.
2
+ import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
3
+ import { buildLogEntry } from "./build-log-entry.js";
4
+ import { normalizeLevel } from "./log-levels.js";
5
+ import { toIso } from "./timestamps.js";
6
+ const ERROR_LINE = /^(\d{4})\/(\d{2})\/(\d{2}) (\d{2}:\d{2}:\d{2}) \[(\w+)\] (?:\d+#\d+: )?(?:\*\d+ )?(.*)$/;
7
+ const ACCESS_LINE = /^\S+ \S+ \S+ \[(\d{2})\/(\w{3})\/(\d{4}):(\d{2}:\d{2}:\d{2}) ([+-]\d{4})\] "([^"]*)" (\d{3})(?:\s|$)/;
8
+ const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
9
+ export function readNginxLine(line) {
10
+ const errorLineMatch = ERROR_LINE.exec(line);
11
+ if (errorLineMatch) {
12
+ const [, year, month, day, time, levelWord = "", message = ""] = errorLineMatch;
13
+ const timestamp = toIso(`${year}-${month}-${day}T${time}Z`);
14
+ const level = normalizeLevel(levelWord);
15
+ if (!timestamp || level === undefined)
16
+ return { kind: "bad" };
17
+ if (level === null)
18
+ return { kind: "drop", timestamp };
19
+ return { kind: "entry", timestamp, level, message };
20
+ }
21
+ const accessLineMatch = ACCESS_LINE.exec(line);
22
+ if (accessLineMatch) {
23
+ const [, day, monthName = "", year, time, zone, request, status = ""] = accessLineMatch;
24
+ const month = MONTHS.indexOf(monthName) + 1;
25
+ const timestamp = month > 0 ? toIso(`${year}-${String(month).padStart(2, "0")}-${day}T${time}${zone}`) : undefined;
26
+ if (!timestamp)
27
+ return { kind: "bad" };
28
+ if (!status.startsWith("5"))
29
+ return { kind: "drop", timestamp };
30
+ return { kind: "entry", timestamp, level: "ERROR", message: `"${request}" ${status}` };
31
+ }
32
+ return { kind: "none" };
33
+ }
34
+ // nginx never writes stack traces, so every line stands alone.
35
+ export function parseNginx(lines, redactionKinds = ALL_REDACTION_KINDS) {
36
+ const entries = [];
37
+ let skipped = 0;
38
+ for (const rawLine of lines) {
39
+ const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
40
+ if (line.trim() === "")
41
+ continue;
42
+ let lineStart;
43
+ try {
44
+ lineStart = readNginxLine(line);
45
+ }
46
+ catch {
47
+ lineStart = { kind: "bad" };
48
+ }
49
+ if (lineStart.kind === "entry")
50
+ entries.push(buildLogEntry({ ...lineStart, continuationLines: [] }, redactionKinds));
51
+ else if (lineStart.kind !== "drop")
52
+ skipped++;
53
+ }
54
+ return { entries, skipped };
55
+ }
56
+ //# sourceMappingURL=nginx-log-format.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"nginx-log-format.js","sourceRoot":"","sources":["../../src/log-parsing/nginx-log-format.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAC1F,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAExC,MAAM,UAAU,GAAG,yFAAyF,CAAC;AAC7G,MAAM,WAAW,GAAG,sGAAsG,CAAC;AAE3H,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;AAEpG,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,EAAE,OAAO,GAAG,EAAE,CAAC,GAAG,cAAc,CAAC;QAChF,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI,GAAG,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC9D,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QACvD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IACtD,CAAC;IAED,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,CAAC,EAAE,GAAG,EAAE,SAAS,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,CAAC,GAAG,eAAe,CAAC;QACxF,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACnH,IAAI,CAAC,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAChE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,OAAO,KAAK,MAAM,EAAE,EAAE,CAAC;IACzF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC1B,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,UAAU,CAAC,KAAe,EAAE,cAAc,GAA6B,mBAAmB;IACxG,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACrE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAS;QACjC,IAAI,SAAoB,CAAC;QACzB,IAAI,CAAC;YACH,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,GAAG,SAAS,EAAE,iBAAiB,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;aAChH,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,EAAE,CAAC;IAChD,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC"}
@@ -0,0 +1,11 @@
1
+ // Turns log lines in any supported format into normalized entries.
2
+ import { ALL_REDACTION_KINDS } from "../redaction/redaction-kinds.js";
3
+ import { detectFormat } from "./detect-log-format.js";
4
+ import { FORMAT_PARSERS } from "./line-readers.js";
5
+ // Parses lines in the given format, or detects it from the first 200 lines.
6
+ // Redacts the given kinds (default: all four).
7
+ export function parseLines(lines, format, redactionKinds = ALL_REDACTION_KINDS) {
8
+ const usedFormat = format ?? detectFormat(lines.slice(0, 200));
9
+ return { format: usedFormat, ...FORMAT_PARSERS[usedFormat](lines, redactionKinds) };
10
+ }
11
+ //# sourceMappingURL=parse-log-lines.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parse-log-lines.js","sourceRoot":"","sources":["../../src/log-parsing/parse-log-lines.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,OAAO,EAAE,mBAAmB,EAAsB,MAAM,iCAAiC,CAAC;AAC1F,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnD,4EAA4E;AAC5E,+CAA+C;AAC/C,MAAM,UAAU,UAAU,CACxB,KAAe,EACf,MAAkB,EAClB,cAAc,GAA6B,mBAAmB;IAE9D,MAAM,UAAU,GAAG,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/D,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,cAAc,CAAC,EAAE,CAAC;AACtF,CAAC"}
@@ -0,0 +1,21 @@
1
+ // The timestamp of one line in any format. Used by the command reader to
2
+ // drop lines it has already seen.
3
+ import { LINE_START_READERS } from "./line-readers.js";
4
+ import { toIso } from "./timestamps.js";
5
+ // undefined for lines without a time (e.g. stack trace lines).
6
+ export function lineTimestamp(line) {
7
+ for (const readLineStart of Object.values(LINE_START_READERS)) {
8
+ try {
9
+ const lineStart = readLineStart(line);
10
+ if (lineStart.kind === "entry" || lineStart.kind === "drop")
11
+ return lineStart.timestamp;
12
+ }
13
+ catch {
14
+ // try the next format
15
+ }
16
+ }
17
+ // A timestamped line with a level we don't know still has a time.
18
+ const timestampMatch = /^(\S+(?: \d{2}:\d{2}:\d{2}\S*)?)/.exec(line);
19
+ return timestampMatch?.[1] ? toIso(timestampMatch[1]) : undefined;
20
+ }
21
+ //# sourceMappingURL=read-line-timestamp.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"read-line-timestamp.js","sourceRoot":"","sources":["../../src/log-parsing/read-line-timestamp.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,kCAAkC;AAClC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAExC,+DAA+D;AAC/D,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM;gBAAE,OAAO,SAAS,CAAC,SAAS,CAAC;QAC1F,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IACD,kEAAkE;IAClE,MAAM,cAAc,GAAG,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,OAAO,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACpE,CAAC"}
@@ -0,0 +1,68 @@
1
+ // Runs the log command from vika.yaml (docker logs, kubectl logs, ssh ...)
2
+ // and keeps only lines newer than the last run's timestamp bookmark.
3
+ import { spawn } from "node:child_process";
4
+ import { lineTimestamp } from "./read-line-timestamp.js";
5
+ export function readCommand(command, sinceTimestamp, timeoutMs) {
6
+ const sinceMs = sinceTimestamp === undefined ? undefined : Date.parse(sinceTimestamp);
7
+ const isNewerThanBookmark = (timestamp) => sinceMs === undefined || Date.parse(timestamp) > sinceMs;
8
+ return new Promise((resolve, reject) => {
9
+ const child = spawn(command, { shell: true, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] });
10
+ const lines = [];
11
+ let newestTimestamp = sinceTimestamp;
12
+ let timedOut = false;
13
+ // Lines without a timestamp (stack traces) follow the decision for the
14
+ // timestamped line before them on the same stream.
15
+ const streams = {
16
+ stdout: { partial: "", keep: sinceMs === undefined },
17
+ stderr: { partial: "", keep: sinceMs === undefined },
18
+ };
19
+ const takeLine = (streamState, rawLine) => {
20
+ const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
21
+ const timestamp = lineTimestamp(line);
22
+ if (timestamp !== undefined) {
23
+ streamState.keep = isNewerThanBookmark(timestamp);
24
+ if (streamState.keep && (newestTimestamp === undefined || Date.parse(timestamp) > Date.parse(newestTimestamp))) {
25
+ newestTimestamp = timestamp;
26
+ }
27
+ }
28
+ if (streamState.keep)
29
+ lines.push(line);
30
+ };
31
+ for (const streamName of ["stdout", "stderr"]) {
32
+ const streamState = streams[streamName];
33
+ const stream = child[streamName];
34
+ stream.setEncoding("utf8"); // handles UTF-8 characters split across chunks
35
+ stream.on("data", (chunk) => {
36
+ const parts = (streamState.partial + chunk).split("\n");
37
+ streamState.partial = parts.pop() ?? "";
38
+ for (const part of parts)
39
+ takeLine(streamState, part);
40
+ });
41
+ }
42
+ const timer = setTimeout(() => {
43
+ timedOut = true;
44
+ try {
45
+ process.kill(-child.pid, "SIGKILL"); // the whole process group
46
+ }
47
+ catch {
48
+ child.kill("SIGKILL");
49
+ }
50
+ }, timeoutMs);
51
+ child.on("error", (error) => {
52
+ clearTimeout(timer);
53
+ reject(error);
54
+ });
55
+ child.on("close", (exitCode) => {
56
+ clearTimeout(timer);
57
+ if (!timedOut) {
58
+ for (const streamState of Object.values(streams)) {
59
+ if (streamState.partial !== "")
60
+ takeLine(streamState, streamState.partial);
61
+ }
62
+ }
63
+ const next = newestTimestamp === undefined ? {} : { sinceTimestamp: newestTimestamp };
64
+ resolve({ lines, next, timedOut, exitCode: timedOut ? null : exitCode });
65
+ });
66
+ });
67
+ }
68
+ //# sourceMappingURL=read-log-command.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"read-log-command.js","sourceRoot":"","sources":["../../src/log-parsing/read-log-command.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,qEAAqE;AACrE,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AASzD,MAAM,UAAU,WAAW,CAAC,OAAe,EAAE,cAAkC,EAAE,SAAiB;IAChG,MAAM,OAAO,GAAG,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACtF,MAAM,mBAAmB,GAAG,CAAC,SAAiB,EAAE,EAAE,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC;IAE5G,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAEzH,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,eAAe,GAAG,cAAc,CAAC;QACrC,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,uEAAuE;QACvE,mDAAmD;QACnD,MAAM,OAAO,GAAG;YACd,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,KAAK,SAAS,EAAE;YACpD,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,KAAK,SAAS,EAAE;SACrD,CAAC;QAEF,MAAM,QAAQ,GAAG,CAAC,WAA8B,EAAE,OAAe,EAAE,EAAE;YACnE,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YACrE,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,WAAW,CAAC,IAAI,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;gBAClD,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;oBAC/G,eAAe,GAAG,SAAS,CAAC;gBAC9B,CAAC;YACH,CAAC;YACD,IAAI,WAAW,CAAC,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC;QAEF,KAAK,MAAM,UAAU,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAU,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;YACjC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,+CAA+C;YAC3E,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,CAAC,WAAW,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACxD,WAAW,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;gBACxC,KAAK,MAAM,IAAI,IAAI,KAAK;oBAAE,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,CAAE,KAAK,CAAC,GAAc,EAAE,SAAS,CAAC,CAAC,CAAC,0BAA0B;YAC7E,CAAC;YAAC,MAAM,CAAC;gBACP,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE;YAC7B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;oBACjD,IAAI,WAAW,CAAC,OAAO,KAAK,EAAE;wBAAE,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YACD,MAAM,IAAI,GAAG,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;YACtF,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}