redlinegate 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +188 -0
  3. package/commands/redline-init.md +54 -0
  4. package/commands/redline-verify.md +69 -0
  5. package/dist/bin/redline.js +605 -0
  6. package/dist/bin/redline.js.map +1 -0
  7. package/dist/commands/exempt.js +45 -0
  8. package/dist/commands/exempt.js.map +1 -0
  9. package/dist/commands/init.js +615 -0
  10. package/dist/commands/init.js.map +1 -0
  11. package/dist/commands/policy.js +25 -0
  12. package/dist/commands/policy.js.map +1 -0
  13. package/dist/commands/remove.js +427 -0
  14. package/dist/commands/remove.js.map +1 -0
  15. package/dist/commands/review.js +72 -0
  16. package/dist/commands/review.js.map +1 -0
  17. package/dist/commands/sync.js +42 -0
  18. package/dist/commands/sync.js.map +1 -0
  19. package/dist/commands/verify.js +478 -0
  20. package/dist/commands/verify.js.map +1 -0
  21. package/dist/config/redline-json.js +187 -0
  22. package/dist/config/redline-json.js.map +1 -0
  23. package/dist/core/errors.js +25 -0
  24. package/dist/core/errors.js.map +1 -0
  25. package/dist/core/git.js +152 -0
  26. package/dist/core/git.js.map +1 -0
  27. package/dist/core/log.js +22 -0
  28. package/dist/core/log.js.map +1 -0
  29. package/dist/core/severity.js +16 -0
  30. package/dist/core/severity.js.map +1 -0
  31. package/dist/core/version.js +15 -0
  32. package/dist/core/version.js.map +1 -0
  33. package/dist/detect/scan.js +79 -0
  34. package/dist/detect/scan.js.map +1 -0
  35. package/dist/detect/stack.js +71 -0
  36. package/dist/detect/stack.js.map +1 -0
  37. package/dist/enforce/ladder.js +148 -0
  38. package/dist/enforce/ladder.js.map +1 -0
  39. package/dist/exempt/parse.js +86 -0
  40. package/dist/exempt/parse.js.map +1 -0
  41. package/dist/metrics/options.js +257 -0
  42. package/dist/metrics/options.js.map +1 -0
  43. package/dist/metrics/run.js +72 -0
  44. package/dist/metrics/run.js.map +1 -0
  45. package/dist/platforms/azure/client.js +53 -0
  46. package/dist/platforms/azure/client.js.map +1 -0
  47. package/dist/platforms/azure/index.js +69 -0
  48. package/dist/platforms/azure/index.js.map +1 -0
  49. package/dist/platforms/azure/install.js +917 -0
  50. package/dist/platforms/azure/install.js.map +1 -0
  51. package/dist/platforms/azure/policy-types.js +70 -0
  52. package/dist/platforms/azure/policy-types.js.map +1 -0
  53. package/dist/platforms/azure/verify.js +309 -0
  54. package/dist/platforms/azure/verify.js.map +1 -0
  55. package/dist/platforms/detect.js +40 -0
  56. package/dist/platforms/detect.js.map +1 -0
  57. package/dist/platforms/github/client.js +49 -0
  58. package/dist/platforms/github/client.js.map +1 -0
  59. package/dist/platforms/github/index.js +54 -0
  60. package/dist/platforms/github/index.js.map +1 -0
  61. package/dist/platforms/github/install.js +558 -0
  62. package/dist/platforms/github/install.js.map +1 -0
  63. package/dist/platforms/github/push.js +102 -0
  64. package/dist/platforms/github/push.js.map +1 -0
  65. package/dist/platforms/github/remote.js +36 -0
  66. package/dist/platforms/github/remote.js.map +1 -0
  67. package/dist/platforms/github/verify.js +366 -0
  68. package/dist/platforms/github/verify.js.map +1 -0
  69. package/dist/platforms/http.js +58 -0
  70. package/dist/platforms/http.js.map +1 -0
  71. package/dist/platforms/pull-request-templates.js +157 -0
  72. package/dist/platforms/pull-request-templates.js.map +1 -0
  73. package/dist/platforms/remote.js +21 -0
  74. package/dist/platforms/remote.js.map +1 -0
  75. package/dist/platforms/resolve.js +69 -0
  76. package/dist/platforms/resolve.js.map +1 -0
  77. package/dist/platforms/shape.js +19 -0
  78. package/dist/platforms/shape.js.map +1 -0
  79. package/dist/platforms/types.js +24 -0
  80. package/dist/platforms/types.js.map +1 -0
  81. package/dist/policy/checks.js +91 -0
  82. package/dist/policy/checks.js.map +1 -0
  83. package/dist/policy/diff.js +50 -0
  84. package/dist/policy/diff.js.map +1 -0
  85. package/dist/registry/discover.js +74 -0
  86. package/dist/registry/discover.js.map +1 -0
  87. package/dist/registry/serialize.js +41 -0
  88. package/dist/registry/serialize.js.map +1 -0
  89. package/dist/registry/types.js +2 -0
  90. package/dist/registry/types.js.map +1 -0
  91. package/dist/remove/host.js +294 -0
  92. package/dist/remove/host.js.map +1 -0
  93. package/dist/render/commands.js +139 -0
  94. package/dist/render/commands.js.map +1 -0
  95. package/dist/render/manifest.js +82 -0
  96. package/dist/render/manifest.js.map +1 -0
  97. package/dist/render/markers.js +166 -0
  98. package/dist/render/markers.js.map +1 -0
  99. package/dist/render/profile.js +23 -0
  100. package/dist/render/profile.js.map +1 -0
  101. package/dist/render/standards.js +169 -0
  102. package/dist/render/standards.js.map +1 -0
  103. package/dist/render/vendors.js +199 -0
  104. package/dist/render/vendors.js.map +1 -0
  105. package/dist/review/engines/api.js +95 -0
  106. package/dist/review/engines/api.js.map +1 -0
  107. package/dist/review/engines/embedded.js +20 -0
  108. package/dist/review/engines/embedded.js.map +1 -0
  109. package/dist/review/engines/types.js +2 -0
  110. package/dist/review/engines/types.js.map +1 -0
  111. package/dist/review/glob.js +29 -0
  112. package/dist/review/glob.js.map +1 -0
  113. package/dist/review/prompt.js +45 -0
  114. package/dist/review/prompt.js.map +1 -0
  115. package/dist/review/rules.js +25 -0
  116. package/dist/review/rules.js.map +1 -0
  117. package/dist/review/schema.js +84 -0
  118. package/dist/review/schema.js.map +1 -0
  119. package/dist/review/scope.js +27 -0
  120. package/dist/review/scope.js.map +1 -0
  121. package/dist/sarif/map.js +58 -0
  122. package/dist/sarif/map.js.map +1 -0
  123. package/dist/sarif/parse.js +88 -0
  124. package/dist/sarif/parse.js.map +1 -0
  125. package/dist/sarif/types.js +2 -0
  126. package/dist/sarif/types.js.map +1 -0
  127. package/dist/sync/host.js +21 -0
  128. package/dist/sync/host.js.map +1 -0
  129. package/dist/sync/plan.js +47 -0
  130. package/dist/sync/plan.js.map +1 -0
  131. package/dist/sync/render.js +57 -0
  132. package/dist/sync/render.js.map +1 -0
  133. package/dist/sync/run.js +118 -0
  134. package/dist/sync/run.js.map +1 -0
  135. package/dist/verify/host.js +52 -0
  136. package/dist/verify/host.js.map +1 -0
  137. package/dist/verify/remote.js +146 -0
  138. package/dist/verify/remote.js.map +1 -0
  139. package/package.json +52 -0
  140. package/platforms/azure/gate-template.yml +105 -0
  141. package/rulesets/redline-org-ruleset.json +46 -0
  142. package/rulesets/redline-ruleset.json +49 -0
  143. package/scripts/assign-rule-ids.mjs +129 -0
  144. package/scripts/build-baseline.mjs +124 -0
  145. package/scripts/build-correlation.mjs +74 -0
  146. package/scripts/build-dashboard.mjs +584 -0
  147. package/scripts/build-digest.mjs +164 -0
  148. package/scripts/build-inbox.mjs +133 -0
  149. package/scripts/build-registry.mjs +42 -0
  150. package/scripts/build-roi.mjs +181 -0
  151. package/scripts/check-pins.mjs +88 -0
  152. package/scripts/collect-telemetry.mjs +300 -0
  153. package/scripts/lib/__tests__/baseline.test.mjs +132 -0
  154. package/scripts/lib/__tests__/correlate.test.mjs +131 -0
  155. package/scripts/lib/__tests__/dora.test.mjs +110 -0
  156. package/scripts/lib/__tests__/exemptions.test.mjs +85 -0
  157. package/scripts/lib/__tests__/metrics-scanner.test.mjs +65 -0
  158. package/scripts/lib/__tests__/sarif.test.mjs +125 -0
  159. package/scripts/lib/__tests__/spend.test.mjs +64 -0
  160. package/scripts/lib/baseline.mjs +156 -0
  161. package/scripts/lib/correlate.mjs +125 -0
  162. package/scripts/lib/dora.mjs +133 -0
  163. package/scripts/lib/exemptions.mjs +79 -0
  164. package/scripts/lib/metrics.mjs +172 -0
  165. package/scripts/lib/rules.mjs +107 -0
  166. package/scripts/lib/sarif.mjs +108 -0
  167. package/scripts/lib/spend.mjs +69 -0
  168. package/scripts/measure-context.mjs +101 -0
  169. package/scripts/render-self.mjs +18 -0
  170. package/scripts/score-seeds.mjs +294 -0
  171. package/scripts/validate.mjs +394 -0
  172. package/standards/core.md +124 -0
  173. package/standards/manifest.json +214 -0
  174. package/standards/stacks/csharp.md +29 -0
  175. package/standards/stacks/go.md +30 -0
  176. package/standards/stacks/java.md +30 -0
  177. package/standards/stacks/javascript.md +42 -0
  178. package/standards/stacks/kotlin.md +27 -0
  179. package/standards/stacks/microservices.md +37 -0
  180. package/standards/stacks/nodejs.md +34 -0
  181. package/standards/stacks/python.md +29 -0
  182. package/standards/stacks/react-native.md +29 -0
  183. package/standards/stacks/react.md +46 -0
  184. package/standards/stacks/swift.md +26 -0
  185. package/standards/stacks/terraform.md +27 -0
  186. package/templates/CODEOWNERS +29 -0
  187. package/templates/azure/pull_request_template.md +72 -0
  188. package/templates/github/pull_request_template.md +72 -0
  189. package/templates/redline.yml +35 -0
  190. package/templates/repo-context.md +64 -0
  191. package/workflows/dashboard.yml +119 -0
  192. package/workflows/inbox.yml +97 -0
  193. package/workflows/redline-collect.yml +61 -0
  194. package/workflows/redline-gate.yml +301 -0
  195. package/workflows/redline-sync.yml +79 -0
  196. package/workflows/seed-canary.yml +231 -0
  197. package/workflows/verify-onboarding.yml +115 -0
  198. package/workflows/weekly-digest.yml +78 -0
@@ -0,0 +1,133 @@
1
+ // DORA metrics from data the collector already pulls, and nothing else.
2
+ //
3
+ // Two of the four are derivable from merged pull requests alone. The other two
4
+ // are not, and this file says so rather than approximating them:
5
+ //
6
+ // lead time for changes ....... first commit to merge, per pull request. Derived.
7
+ // change failure rate ......... share of merges that attracted a revert or a
8
+ // hotfix within a window. Derived, with its own
9
+ // caveat below.
10
+ // deployment frequency ........ needs the deployments API. Absent where a
11
+ // repository does not use it — NOT assumed to be
12
+ // "once per merge", which would silently report
13
+ // trunk-based teams and quarterly-release teams
14
+ // as identical.
15
+ // MTTR ........................ needs an incident feed. Explicitly out of scope
16
+ // in the roadmap, and not approximated here: a
17
+ // wrong MTTR is the number most likely to be
18
+ // quoted at a stakeholder who will act on it.
19
+ //
20
+ // The change-failure caveat, stated because the number will be quoted: a revert
21
+ // or a hotfix is evidence of a failed change, not proof, and a team that fixes
22
+ // forward without the word "hotfix" scores better than one that labels honestly.
23
+ // It is a floor on the true rate, and every consumer is told so.
24
+
25
+ const DAY = 86400000;
26
+
27
+ /** Lead time in hours, per pull request, from first commit to merge. */
28
+ export function leadTimes(records) {
29
+ const hours = [];
30
+ for (const record of records ?? []) {
31
+ const merged = Date.parse(record.merged_at ?? '');
32
+ const first = Date.parse(record.first_commit_at ?? '');
33
+ if (Number.isNaN(merged) || Number.isNaN(first)) continue;
34
+ if (merged < first) continue; // a rebase can move the commit date past the merge
35
+ hours.push((merged - first) / 3600000);
36
+ }
37
+ return hours.sort((a, b) => a - b);
38
+ }
39
+
40
+ /** The median, which is what DORA reports — a mean is dragged by one stale branch. */
41
+ export function median(values) {
42
+ if (!values.length) return null;
43
+ const mid = Math.floor(values.length / 2);
44
+ return values.length % 2 ? values[mid] : (values[mid - 1] + values[mid]) / 2;
45
+ }
46
+
47
+ const REVERT = /^revert[\s:"']|^revert\b/i;
48
+ const HOTFIX = /\b(hotfix|hot-fix)\b/i;
49
+
50
+ /**
51
+ * Change failure rate: the share of merged pull requests followed by a revert or
52
+ * a hotfix touching the same repository within `windowDays`.
53
+ *
54
+ * Returns null rather than 0 when there is nothing to divide. Zero here would
55
+ * read as "this estate never breaks anything", which is a claim no data supports.
56
+ */
57
+ export function changeFailureRate(records, { windowDays = 7 } = {}) {
58
+ const merged = (records ?? []).filter((r) => r.merged_at);
59
+ if (merged.length === 0) return { rate: null, failures: 0, total: 0, reason: 'no merged pull requests in the window' };
60
+
61
+ const byRepo = new Map();
62
+ for (const record of merged) {
63
+ if (!byRepo.has(record.repo)) byRepo.set(record.repo, []);
64
+ byRepo.get(record.repo).push(record);
65
+ }
66
+
67
+ let failures = 0;
68
+ for (const [, list] of byRepo) {
69
+ const sorted = [...list].sort((a, b) => Date.parse(a.merged_at) - Date.parse(b.merged_at));
70
+ for (let i = 0; i < sorted.length; i += 1) {
71
+ const at = Date.parse(sorted[i].merged_at);
72
+ // A pull request "failed" when a later one in the same repository, inside
73
+ // the window, is a revert or a hotfix. The remediation itself is not
74
+ // counted as a failure of its own, or every incident would score twice.
75
+ const remedied = sorted.slice(i + 1).some((later) => {
76
+ const gap = Date.parse(later.merged_at) - at;
77
+ if (gap < 0 || gap > windowDays * DAY) return false;
78
+ const title = later.title ?? '';
79
+ return REVERT.test(title) || HOTFIX.test(title);
80
+ });
81
+ if (remedied && !REVERT.test(sorted[i].title ?? '') && !HOTFIX.test(sorted[i].title ?? '')) {
82
+ failures += 1;
83
+ }
84
+ }
85
+ }
86
+
87
+ return {
88
+ rate: failures / merged.length,
89
+ failures,
90
+ total: merged.length,
91
+ // Carried with the number, not buried in a doc. It will be quoted.
92
+ caveat:
93
+ 'A floor, not the true rate: a revert or hotfix is evidence of a failed change rather than proof, ' +
94
+ 'and a team that fixes forward without saying "hotfix" scores better than one that labels honestly.',
95
+ };
96
+ }
97
+
98
+ /**
99
+ * Deployment frequency, from deployment timestamps the caller supplies.
100
+ *
101
+ * Absent — not zero, and not "once per merge" — where a repository does not use
102
+ * the deployments API. Assuming a merge is a deploy reports a trunk-based team
103
+ * and a quarterly-release team as identical, which is the exact distinction the
104
+ * metric exists to draw.
105
+ */
106
+ export function deploymentFrequency(deployments, windowDays) {
107
+ if (deployments === null || deployments === undefined) {
108
+ return { perDay: null, reason: 'this repository does not use the deployments API — frequency is unknown, not zero' };
109
+ }
110
+ if (windowDays <= 0) return { perDay: null, reason: 'no window to divide by' };
111
+ return { perDay: deployments.length / windowDays, count: deployments.length };
112
+ }
113
+
114
+ /** The DORA block, with every unavailable figure carrying its reason. */
115
+ export function dora(records, { windowDays = 90, deployments = null } = {}) {
116
+ const times = leadTimes(records);
117
+ const cfr = changeFailureRate(records, { windowDays: 7 });
118
+ const freq = deploymentFrequency(deployments, windowDays);
119
+
120
+ return {
121
+ leadTimeHours: times.length
122
+ ? { median: median(times), p90: times[Math.floor(times.length * 0.9)] ?? null, samples: times.length }
123
+ : { median: null, p90: null, samples: 0, reason: 'no pull request in the window carries a first-commit timestamp' },
124
+ changeFailureRate: cfr,
125
+ deploymentFrequency: freq,
126
+ // Named so a reader can see it was considered and refused, rather than
127
+ // wondering whether it was forgotten.
128
+ meanTimeToRestore: {
129
+ value: null,
130
+ reason: 'needs an incident feed Redline does not have and should not acquire — out of scope in the roadmap',
131
+ },
132
+ };
133
+ }
@@ -0,0 +1,79 @@
1
+ // Reads the structured exemption block out of a pull request body, for telemetry.
2
+ //
3
+ // Deliberately a mirror of cli/exempt/parse.ts rather than an import: the
4
+ // collector is plain .mjs running in the metrics repo with no build step, and it
5
+ // must not depend on dist/. The two are kept honest by scripts/validate.mjs,
6
+ // which fails the build if the heading or the field names diverge — the failure
7
+ // mode this guards against is the gate accepting a block the audit cannot read,
8
+ // so an exemption is enforced and then never reported.
9
+
10
+ export const EXEMPTION_HEADING = '## Redline exemption';
11
+
12
+ // See cli/exempt/parse.ts: the template's own guidance comment contains the field
13
+ // names, so a parser that does not strip comments reads the instructions.
14
+ const withoutComments = (text) => text.replace(/<!--[\s\S]*?-->/g, '');
15
+
16
+ const field = (block, name) => {
17
+ const match = new RegExp(`^\\s*[-*]?\\s*${name}\\s*:\\s*(.+)$`, 'im').exec(block);
18
+ return match?.[1]?.trim() ?? null;
19
+ };
20
+
21
+ /**
22
+ * @returns {{reason: string, until: string, scope: string[]} | null}
23
+ */
24
+ export function readExemption(body) {
25
+ if (!body) return null;
26
+ const searchable = withoutComments(body);
27
+ const start = searchable.toLowerCase().indexOf(EXEMPTION_HEADING.toLowerCase());
28
+ if (start === -1) return null;
29
+
30
+ const rest = searchable.slice(start + EXEMPTION_HEADING.length);
31
+ const next = /^#{1,2}\s/m.exec(rest);
32
+ const block = next ? rest.slice(0, next.index) : rest;
33
+
34
+ const reason = field(block, 'reason');
35
+ const until = field(block, 'until');
36
+ if (!reason || !until) return null;
37
+ // A string compare against today treats an unparseable date as standing
38
+ // forever, which is the direction that quietly hides a problem.
39
+ if (!/^\d{4}-\d{2}-\d{2}$/.test(until) || Number.isNaN(Date.parse(until))) return null;
40
+
41
+ const scopeField = field(block, 'scope');
42
+ return {
43
+ reason,
44
+ until,
45
+ scope: scopeField
46
+ ? scopeField.split(',').map((s) => s.trim()).filter(Boolean)
47
+ : ['*'],
48
+ };
49
+ }
50
+
51
+ /**
52
+ * Standing exemptions: those still in force, grouped by scope, most-used first.
53
+ * A team routing around the gate shows up here as the same scope recurring
54
+ * across pull requests — which is the signal the roadmap's guardrail wants, and
55
+ * one a per-pull-request view can never show.
56
+ */
57
+ export function standingExemptions(records, now = new Date()) {
58
+ const today = now.toISOString().slice(0, 10);
59
+ const byScope = new Map();
60
+
61
+ for (const record of records) {
62
+ const exemption = record.exemption;
63
+ if (!exemption) continue;
64
+ // An expired exemption is history, not a standing one. Counting it would
65
+ // make a resolved problem look permanent.
66
+ if (exemption.until < today) continue;
67
+ for (const scope of exemption.scope) {
68
+ const entry = byScope.get(scope) ?? { scope, count: 0, repos: new Set(), soonest: exemption.until };
69
+ entry.count += 1;
70
+ entry.repos.add(record.repo);
71
+ if (exemption.until < entry.soonest) entry.soonest = exemption.until;
72
+ byScope.set(scope, entry);
73
+ }
74
+ }
75
+
76
+ return [...byScope.values()]
77
+ .map((e) => ({ scope: e.scope, count: e.count, repos: e.repos.size, soonest: e.soonest }))
78
+ .sort((a, b) => b.count - a.count || a.scope.localeCompare(b.scope));
79
+ }
@@ -0,0 +1,172 @@
1
+ // Shared telemetry aggregation. The digest and the dashboard must not disagree about
2
+ // what "acted on" means, so both read their numbers from here.
3
+
4
+ import { readdirSync, readFileSync, existsSync } from 'node:fs';
5
+ import { join } from 'node:path';
6
+
7
+ /**
8
+ * Loads telemetry records, optionally filtered to those merged since an ISO timestamp.
9
+ */
10
+ export function loadRecords(dataDir = 'data', sinceIso = null) {
11
+ const records = [];
12
+ const problems = [];
13
+ if (!existsSync(dataDir)) return { records, problems: ['data directory is missing'] };
14
+
15
+ for (const file of readdirSync(dataDir).filter((f) => /^\d{4}-\d{2}\.jsonl$/.test(f))) {
16
+ const lines = readFileSync(join(dataDir, file), 'utf8').split('\n');
17
+ lines.forEach((line, i) => {
18
+ if (!line.trim()) return;
19
+ try {
20
+ const record = JSON.parse(line);
21
+ if (sinceIso && (record.merged_at ?? '') < sinceIso) return;
22
+ records.push(record);
23
+ } catch {
24
+ // One malformed line must not silence the whole report.
25
+ problems.push(`${file}:${i + 1} is not valid JSON`);
26
+ }
27
+ });
28
+ }
29
+ records.sort((a, b) => (a.merged_at ?? '').localeCompare(b.merged_at ?? ''));
30
+ return { records, problems };
31
+ }
32
+
33
+ const staleOf = (record) =>
34
+ Object.values(record.outcomes?.stale_by_severity ?? {}).reduce((a, b) => a + b, 0);
35
+
36
+ export function aggregate(records) {
37
+ const sum = (fn) => records.reduce((n, r) => n + (fn(r) ?? 0), 0);
38
+
39
+ const byRule = new Map();
40
+ const byRepo = new Map();
41
+ const reviewers = new Set();
42
+
43
+ for (const record of records) {
44
+ for (const [ruleId, stats] of Object.entries(record.rules ?? {})) {
45
+ const entry = byRule.get(ruleId) ?? { fired: 0, resolved: 0, stale: 0, repos: new Set(), severity: stats.severity };
46
+ entry.fired += stats.fired ?? 0;
47
+ entry.resolved += stats.resolved ?? 0;
48
+ entry.stale += stats.stale ?? 0;
49
+ entry.repos.add(record.repo);
50
+ byRule.set(ruleId, entry);
51
+ }
52
+ const repo = byRepo.get(record.repo) ?? { prs: 0, findings: 0, blocker: 0, stale: 0, exempted: 0 };
53
+ repo.prs += 1;
54
+ repo.findings += record.findings?.total ?? 0;
55
+ repo.blocker += record.findings?.blocker ?? 0;
56
+ repo.stale += staleOf(record);
57
+ repo.exempted += record.exempted ? 1 : 0;
58
+ byRepo.set(record.repo, repo);
59
+ for (const reviewer of record.reviewers ?? []) reviewers.add(reviewer);
60
+ }
61
+
62
+ const findings = sum((r) => r.findings?.total);
63
+
64
+ // Ingested scanner findings, aggregated ALONGSIDE Redline's own and never into
65
+ // them. Every number above this line describes Redline's own catalogue and is
66
+ // what rule tuning reads; folding another tool's rule ids in would tune
67
+ // Redline's rules on that tool's noise. Reported separately, they answer a
68
+ // question neither tool can answer alone: what the estate's whole finding
69
+ // surface looks like under one severity contract.
70
+ // Once per repository, not once per pull request. A scanner alert count is a
71
+ // fact about the repository; every merged PR in that repo carries the same
72
+ // snapshot, so summing them multiplied the estate's ingested findings by the
73
+ // number of merges. The most recent record per repo wins — records are sorted
74
+ // by merge time, so that is the freshest snapshot.
75
+ const scannerByRepo = new Map();
76
+ for (const record of records) {
77
+ if (!record.scanner) continue;
78
+ scannerByRepo.set(record.repo, record.scanner);
79
+ }
80
+
81
+ const scannerByTool = new Map();
82
+ const scannerBySeverity = { blocker: 0, high: 0, suggestion: 0 };
83
+ let scannerTotal = 0;
84
+ for (const scanner of scannerByRepo.values()) {
85
+ scannerTotal += scanner.findings ?? 0;
86
+ for (const [tool, n] of Object.entries(scanner.by_tool ?? {})) {
87
+ scannerByTool.set(tool, (scannerByTool.get(tool) ?? 0) + n);
88
+ }
89
+ for (const [severity, n] of Object.entries(scanner.by_severity ?? {})) {
90
+ if (severity in scannerBySeverity) scannerBySeverity[severity] += n;
91
+ }
92
+ }
93
+
94
+ return {
95
+ scanner: {
96
+ findings: scannerTotal,
97
+ bySeverity: scannerBySeverity,
98
+ byTool: [...scannerByTool.entries()]
99
+ .map(([tool, findings]) => ({ tool, findings }))
100
+ .sort((a, b) => b.findings - a.findings),
101
+ // How many repositories in this window emit anything at all. This is the
102
+ // number that decides whether Phase 1 was worth doing, and the roadmap's
103
+ // open question 1 asks for exactly it.
104
+ repos: [...scannerByRepo.values()].filter((s) => (s.findings ?? 0) > 0).length,
105
+ },
106
+ prs: records.length,
107
+ prsWithFindings: records.filter((r) => (r.findings?.total ?? 0) > 0).length,
108
+ findings,
109
+ blocker: sum((r) => r.findings?.blocker),
110
+ high: sum((r) => r.findings?.high),
111
+ suggestion: sum((r) => r.findings?.suggestion),
112
+ resolved: sum((r) => r.outcomes?.resolved),
113
+ stale: sum(staleOf),
114
+ untagged: sum((r) => r.untagged_findings),
115
+ withoutRuleId: sum((r) => r.findings_without_rule_id),
116
+ unknownRuleIds: sum((r) => r.unknown_rule_ids),
117
+ exempted: records.filter((r) => r.exempted).length,
118
+ humanThreads: sum((r) => r.human_review_threads),
119
+ reviewers: [...reviewers],
120
+ repos: byRepo.size,
121
+ byRepo: [...byRepo.entries()]
122
+ .map(([repo, s]) => ({ repo, ...s }))
123
+ .sort((a, b) => b.blocker - a.blocker || b.findings - a.findings),
124
+ byRule: [...byRule.entries()]
125
+ .map(([id, s]) => ({
126
+ id,
127
+ severity: s.severity,
128
+ fired: s.fired,
129
+ resolved: s.resolved,
130
+ stale: s.stale,
131
+ repos: s.repos.size,
132
+ // Ignored rate is the noise metric: fired a lot, acted on rarely.
133
+ ignoredRate: s.fired ? s.stale / s.fired : 0,
134
+ actedRate: s.fired ? s.resolved / s.fired : 0,
135
+ }))
136
+ .sort((a, b) => b.fired - a.fired),
137
+ };
138
+ }
139
+
140
+ /** Rules that fire often and are acted on rarely — the tuning queue. */
141
+ export function noisiestRules(agg, { minFired = 5, minIgnoredRate = 0.3, limit = 5 } = {}) {
142
+ return agg.byRule
143
+ .filter((r) => r.id !== '(untagged)' && r.fired >= minFired && r.ignoredRate >= minIgnoredRate)
144
+ .sort((a, b) => b.ignoredRate - a.ignoredRate || b.fired - a.fired)
145
+ .slice(0, limit);
146
+ }
147
+
148
+ /** Rules that fire often and are almost always acted on — evidence they earn their place. */
149
+ export function mostValuableRules(agg, { minFired = 5, limit = 5 } = {}) {
150
+ return agg.byRule
151
+ .filter((r) => r.id !== '(untagged)' && r.fired >= minFired)
152
+ .sort((a, b) => b.actedRate - a.actedRate || b.fired - a.fired)
153
+ .slice(0, limit);
154
+ }
155
+
156
+ /** Buckets records into calendar weeks for trend lines. */
157
+ export function weekly(records) {
158
+ const weeks = new Map();
159
+ for (const record of records) {
160
+ const date = new Date(record.merged_at ?? Date.now());
161
+ const monday = new Date(date);
162
+ monday.setUTCDate(date.getUTCDate() - ((date.getUTCDay() + 6) % 7));
163
+ const key = monday.toISOString().slice(0, 10);
164
+ if (!weeks.has(key)) weeks.set(key, []);
165
+ weeks.get(key).push(record);
166
+ }
167
+ return [...weeks.entries()]
168
+ .sort((a, b) => a[0].localeCompare(b[0]))
169
+ .map(([week, rows]) => ({ week, ...aggregate(rows) }));
170
+ }
171
+
172
+ export const pct = (n, d) => (d === 0 ? '—' : `${Math.round((n / d) * 100)}%`);
@@ -0,0 +1,107 @@
1
+ // Shared rule catalogue and finding parser.
2
+ //
3
+ // Every consumer of review output — telemetry, seed scoring, the digest, the dashboard —
4
+ // must agree on what a rule id is and how a finding is parsed. Keeping that in one place
5
+ // is the difference between "our numbers disagree" and a working feedback loop.
6
+
7
+ import { readFileSync } from 'node:fs';
8
+ import { join, dirname, resolve } from 'node:path';
9
+ import { fileURLToPath } from 'node:url';
10
+
11
+ export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
12
+
13
+ export const SEVERITIES = ['BLOCKER', 'HIGH', 'SUGGESTION'];
14
+ export const RANK = { SUGGESTION: 1, HIGH: 2, BLOCKER: 3 };
15
+
16
+ // Reserved ids that are not bullets in the standards.
17
+ export const RESERVED_RULE_IDS = new Set(['core/uncatalogued']);
18
+
19
+ export const RULE_ID = /^[a-z0-9][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/;
20
+ const RULE_LINE = /^-\s+`([a-z0-9][a-z0-9-]*\/[a-z0-9][a-z0-9-]*)`\s+—\s+(.*)$/;
21
+ const RULE_HEADING = /^#{2,3}\s+(BLOCKER|HIGH|SUGGESTION|Security|Type safety|Error handling|General correctness|Scope discipline)\b/i;
22
+ const ANY_HEADING = /^#{1,6}\s/;
23
+
24
+ // A finding's first line, e.g. "Redline/BLOCKER [react/key-is-index]: ...".
25
+ // The id group is optional so pre-2.1 repos still parse, reported as untagged.
26
+ const FINDING = /(?:^|\n)\s*(?:[*_`>-]*\s*)?Redline\/(BLOCKER|HIGH|SUGGESTION)\b\s*(?:\[\s*`?([a-z0-9][a-z0-9-]*\/[a-z0-9][a-z0-9-]*)`?\s*\])?/i;
27
+ const BARE_SEVERITY = /\b(BLOCKER|HIGH|SUGGESTION)\b/;
28
+
29
+ // Logins that count as automated review. Substring match, case-insensitive.
30
+ export const REVIEW_BOTS = ['copilot', 'claude', 'codex', 'cursor', 'devin', 'jules', 'redline'];
31
+ export const isReviewBot = (login = '') =>
32
+ REVIEW_BOTS.some((bot) => login.toLowerCase().includes(bot));
33
+
34
+ /**
35
+ * Reads every rule out of the standards sources.
36
+ * @returns {Map<string, {id, stack, severity, text, source, line}>}
37
+ */
38
+ export function loadRules(root = ROOT) {
39
+ const manifest = JSON.parse(readFileSync(join(root, 'standards/manifest.json'), 'utf8'));
40
+ const sources = [
41
+ ['core', manifest.core.source],
42
+ ...Object.entries(manifest.stacks).map(([id, s]) => [id, s.source]),
43
+ ];
44
+
45
+ const rules = new Map();
46
+ for (const [stack, relPath] of sources) {
47
+ const lines = readFileSync(join(root, relPath), 'utf8').split('\n');
48
+ let severity = null;
49
+ let inFence = false;
50
+
51
+ lines.forEach((line, i) => {
52
+ if (/^```/.test(line.trim())) inFence = !inFence;
53
+ if (inFence) return;
54
+
55
+ if (ANY_HEADING.test(line)) {
56
+ const heading = RULE_HEADING.exec(line);
57
+ if (!heading) {
58
+ severity = null;
59
+ return;
60
+ }
61
+ const word = heading[1].toUpperCase();
62
+ // Core groups rules by topic, not severity; each section states its own default.
63
+ severity = SEVERITIES.includes(word) ? word : /BLOCKER/i.test(line) ? 'BLOCKER' : 'HIGH';
64
+ return;
65
+ }
66
+ if (!severity) return;
67
+
68
+ const rule = RULE_LINE.exec(line);
69
+ if (!rule) return;
70
+ rules.set(rule[1], {
71
+ id: rule[1],
72
+ stack,
73
+ severity,
74
+ text: rule[2].replace(/\*\*/g, '').trim(),
75
+ source: relPath,
76
+ line: i + 1,
77
+ });
78
+ });
79
+ }
80
+ return rules;
81
+ }
82
+
83
+ /**
84
+ * Parses a review comment body into a finding.
85
+ * @returns {{severity, ruleId: string|null, tagged: boolean, hasRuleId: boolean}}
86
+ */
87
+ export function parseFinding(body = '') {
88
+ const match = FINDING.exec(body);
89
+ if (match) {
90
+ return {
91
+ severity: match[1].toUpperCase(),
92
+ ruleId: match[2] ? match[2].toLowerCase() : null,
93
+ tagged: true,
94
+ hasRuleId: Boolean(match[2]),
95
+ };
96
+ }
97
+ const bare = BARE_SEVERITY.exec(body);
98
+ return {
99
+ severity: (bare?.[1] ?? 'SUGGESTION').toUpperCase(),
100
+ ruleId: null,
101
+ tagged: false,
102
+ hasRuleId: false,
103
+ };
104
+ }
105
+
106
+ export const emptyBySeverity = () =>
107
+ Object.fromEntries(SEVERITIES.map((s) => [s.toLowerCase(), 0]));
@@ -0,0 +1,108 @@
1
+ // Ingests code-scanning alerts as findings, alongside Redline's own.
2
+ //
3
+ // Mirrors cli/sarif/ for the collector, which runs in the metrics repo with no
4
+ // build step to import dist/ from — the same arrangement as the exemption
5
+ // parser, and guarded the same way by scripts/validate.mjs.
6
+ //
7
+ // The rule this file exists to hold: an ingested finding is ALWAYS distinguishable
8
+ // from a Redline one. Redline's claim in Phase 1 is that it can measure LLM review
9
+ // and static analysis against one severity contract without either distorting the
10
+ // other, and rule tuning reads this stream. A view that cannot tell a CodeQL
11
+ // finding from a Redline one tunes Redline's rules on another tool's noise, which
12
+ // is the single way this piece can make things worse than not doing it.
13
+
14
+ export const FALLBACK_SEVERITY = 'SUGGESTION';
15
+
16
+ export const DEFAULT_SEVERITY_MAP = {
17
+ error: 'BLOCKER',
18
+ warning: 'HIGH',
19
+ note: 'SUGGESTION',
20
+ none: 'SUGGESTION',
21
+ critical: 'BLOCKER',
22
+ high: 'BLOCKER',
23
+ medium: 'HIGH',
24
+ moderate: 'HIGH',
25
+ low: 'SUGGESTION',
26
+ info: 'SUGGESTION',
27
+ informational: 'SUGGESTION',
28
+ };
29
+
30
+ export function mapSeverity(native, map = DEFAULT_SEVERITY_MAP) {
31
+ const word = String(native ?? '').trim();
32
+ const mapped = map[word.toLowerCase()];
33
+ return mapped
34
+ ? { severity: mapped, native: word, matched: true }
35
+ : { severity: FALLBACK_SEVERITY, native: word, matched: false };
36
+ }
37
+
38
+ /**
39
+ * GitHub code-scanning alerts into normalised findings.
40
+ *
41
+ * The alerts API is the practical ingestion point: it is what every SARIF upload
42
+ * on GitHub becomes, it carries the tool name and the rule id already separated,
43
+ * and it does not require the collector to fetch and parse raw SARIF blobs per
44
+ * pull request.
45
+ */
46
+ export function ingestAlerts(alerts, { map = DEFAULT_SEVERITY_MAP, where = 'alerts' } = {}) {
47
+ const findings = [];
48
+ const problems = [];
49
+
50
+ for (const alert of alerts ?? []) {
51
+ const ruleId = alert?.rule?.id?.trim?.();
52
+ if (!ruleId) {
53
+ problems.push(`${where}: an alert has no rule id — skipped`);
54
+ continue;
55
+ }
56
+ // security_severity_level is present on security rules; severity is the
57
+ // SARIF level for everything else. Security first: it is the more specific
58
+ // statement when both exist.
59
+ const native = alert.rule.security_severity_level ?? alert.rule.severity;
60
+ const mapped = mapSeverity(native, map);
61
+ if (!mapped.matched && native) {
62
+ problems.push(`${where}: severity "${native}" is not in the severity map — ${ruleId} ingested as ${mapped.severity}`);
63
+ }
64
+
65
+ findings.push({
66
+ source: 'sarif',
67
+ tool: alert.tool?.name ?? 'unknown',
68
+ ruleId,
69
+ severity: mapped.severity,
70
+ nativeSeverity: mapped.native,
71
+ state: alert.state ?? 'unknown',
72
+ file: alert.most_recent_instance?.location?.path ?? null,
73
+ line: alert.most_recent_instance?.location?.start_line ?? null,
74
+ });
75
+ }
76
+
77
+ return { findings, problems };
78
+ }
79
+
80
+ /**
81
+ * Per-source aggregate. Two catalogues, never merged into one.
82
+ *
83
+ * Acted-on is computed within each source, never across them: Redline's is
84
+ * "review thread resolved", a scanner's is "alert closed", and averaging two
85
+ * different definitions produces a number that describes neither.
86
+ */
87
+ export function aggregateBySource(findings) {
88
+ const bucket = () => ({ total: 0, blocker: 0, high: 0, suggestion: 0, acted: 0, byTool: {}, byRule: {} });
89
+ const out = { redline: bucket(), sarif: bucket() };
90
+
91
+ for (const finding of findings ?? []) {
92
+ const side = out[finding.source];
93
+ if (!side) continue;
94
+ side.total += 1;
95
+ side[finding.severity.toLowerCase()] += 1;
96
+ if (finding.acted) side.acted += 1;
97
+ side.byTool[finding.tool] = (side.byTool[finding.tool] ?? 0) + 1;
98
+ const rule = side.byRule[finding.ruleId] ?? { fired: 0, acted: 0, severity: finding.severity };
99
+ rule.fired += 1;
100
+ if (finding.acted) rule.acted += 1;
101
+ side.byRule[finding.ruleId] = rule;
102
+ }
103
+
104
+ for (const side of Object.values(out)) {
105
+ side.actedRate = side.total > 0 ? side.acted / side.total : null;
106
+ }
107
+ return out;
108
+ }
@@ -0,0 +1,69 @@
1
+ // AI spend, at whatever grain the vendor's own reporting actually offers.
2
+ //
3
+ // The roadmap's open question 2 asks whether spend is attributable per repository
4
+ // or only org-wide. The honest answer depends on the vendor and cannot be settled
5
+ // in code, so this module carries the grain WITH the number and every consumer
6
+ // reads it. The failure it prevents: an org-level figure divided by repository
7
+ // count, presented as per-repository cost. That number looks precise, is entirely
8
+ // invented, and is the one a stakeholder will act on.
9
+ //
10
+ // Redline cannot read a vendor's billing API for you. It takes the figure you
11
+ // have, records where it came from, and refuses to make it more precise than it is.
12
+
13
+ export const GRAINS = ['repo', 'org', 'unknown'];
14
+
15
+ /**
16
+ * @param {{total: number, currency?: string, grain?: string, source?: string, period?: string}} input
17
+ */
18
+ export function readSpend(input) {
19
+ if (!input || typeof input.total !== 'number' || !Number.isFinite(input.total)) {
20
+ return { available: false, reason: 'no AI spend figure was supplied — read it from the assistant vendor\'s usage reporting' };
21
+ }
22
+ if (input.total < 0) {
23
+ return { available: false, reason: `spend cannot be negative (${input.total})` };
24
+ }
25
+ const grain = GRAINS.includes(input.grain) ? input.grain : 'unknown';
26
+ return {
27
+ available: true,
28
+ total: input.total,
29
+ currency: input.currency ?? 'USD',
30
+ grain,
31
+ source: input.source ?? 'supplied by the operator',
32
+ period: input.period ?? null,
33
+ };
34
+ }
35
+
36
+ /**
37
+ * Cost per BLOCKER caught — the figure nobody else in the toolchain can compute.
38
+ *
39
+ * A cost-management tool knows spend and has no findings, so it cannot compute
40
+ * value. A DORA tool has neither. Redline knows which findings were acted on, and
41
+ * joining the two is the whole argument.
42
+ *
43
+ * It refuses in three cases rather than producing a misleading number:
44
+ * - no spend figure at all
45
+ * - no BLOCKERs in the window, which would divide by zero
46
+ * - a per-repo question asked of an org-level figure
47
+ */
48
+ export function costPerBlocker(spend, blockersCaught, { scope = 'org' } = {}) {
49
+ if (!spend?.available) {
50
+ return { value: null, reason: spend?.reason ?? 'no spend figure' };
51
+ }
52
+ if (!Number.isFinite(blockersCaught) || blockersCaught <= 0) {
53
+ return { value: null, reason: 'no BLOCKER findings were acted on in this window — nothing to divide by' };
54
+ }
55
+ if (scope === 'repo' && spend.grain !== 'repo') {
56
+ return {
57
+ value: null,
58
+ reason:
59
+ `spend is reported at ${spend.grain} grain, so it cannot be attributed per repository. ` +
60
+ 'Publish the org-level figure against org-level value instead of inventing a per-repo number.',
61
+ };
62
+ }
63
+ return {
64
+ value: spend.total / blockersCaught,
65
+ currency: spend.currency,
66
+ grain: spend.grain,
67
+ blockersCaught,
68
+ };
69
+ }