proof-pr 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +466 -7
- package/dist/index.js.map +1 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -21,6 +21,7 @@ npx proof-pr@latest init --preset security-strict
|
|
|
21
21
|
npx proof-pr@latest scan --base origin/main --head HEAD
|
|
22
22
|
npx proof-pr@latest scan --base origin/main --head HEAD --locale zh-CN
|
|
23
23
|
npx proof-pr@latest scan --base origin/main --pr-body-file pr-body.md --format json
|
|
24
|
+
npx proof-pr@latest benchmark --cases benchmarks/cases
|
|
24
25
|
```
|
|
25
26
|
|
|
26
27
|
可用预设:`balanced`、`open-source-maintainer`、`security-strict`、`ai-generated-pr`、`mcp-security`、`dependency-careful`。
|
package/dist/index.d.ts
ADDED
package/dist/index.js
CHANGED
|
@@ -23111,7 +23111,15 @@ function preprocess(fn, schema) {
|
|
|
23111
23111
|
|
|
23112
23112
|
|
|
23113
23113
|
const riskLevelSchema = schemas_enum(["low", "medium", "high"]);
|
|
23114
|
+
const findingSeveritySchema = schemas_enum(["info", "low", "medium", "high"]);
|
|
23114
23115
|
const localeSchema = schemas_enum(["en", "zh-CN"]);
|
|
23116
|
+
const evidenceRequirementSchema = schemas_enum([
|
|
23117
|
+
"verification",
|
|
23118
|
+
"reproduction",
|
|
23119
|
+
"screenshot",
|
|
23120
|
+
"changelog",
|
|
23121
|
+
"permission-rationale"
|
|
23122
|
+
]);
|
|
23115
23123
|
const configPresetSchema = schemas_enum([
|
|
23116
23124
|
"balanced",
|
|
23117
23125
|
"open-source-maintainer",
|
|
@@ -23149,6 +23157,38 @@ const DEFAULT_SENSITIVE_PATHS = [
|
|
|
23149
23157
|
"go.sum"
|
|
23150
23158
|
];
|
|
23151
23159
|
const DEFAULT_TEST_PATHS = ["src/**", "packages/**/src/**", "app/**", "lib/**"];
|
|
23160
|
+
const WORKFLOW_EVIDENCE_CONTRACTS = [
|
|
23161
|
+
{
|
|
23162
|
+
id: "workflow-permission-rationale",
|
|
23163
|
+
title: "Workflow changes need a permission rationale",
|
|
23164
|
+
paths: [".github/workflows/**", ".github/actions/**"],
|
|
23165
|
+
requires: ["verification", "permission-rationale"],
|
|
23166
|
+
severity: "high",
|
|
23167
|
+
recommendation: "Explain why the workflow needs this trigger or permission, and include verification that untrusted PR code cannot reach privileged tokens."
|
|
23168
|
+
}
|
|
23169
|
+
];
|
|
23170
|
+
const DEPENDENCY_EVIDENCE_CONTRACTS = [
|
|
23171
|
+
{
|
|
23172
|
+
id: "dependency-upgrade-evidence",
|
|
23173
|
+
title: "Dependency changes need upgrade evidence",
|
|
23174
|
+
paths: [
|
|
23175
|
+
"package.json",
|
|
23176
|
+
"**/package.json",
|
|
23177
|
+
"pnpm-lock.yaml",
|
|
23178
|
+
"package-lock.json",
|
|
23179
|
+
"yarn.lock",
|
|
23180
|
+
"requirements.txt",
|
|
23181
|
+
"**/requirements.txt",
|
|
23182
|
+
"pyproject.toml",
|
|
23183
|
+
"**/pyproject.toml",
|
|
23184
|
+
"go.mod",
|
|
23185
|
+
"**/go.mod"
|
|
23186
|
+
],
|
|
23187
|
+
requires: ["verification", "changelog"],
|
|
23188
|
+
severity: "medium",
|
|
23189
|
+
recommendation: "Link changelog or migration notes and include the test command or CI evidence used to validate the dependency change."
|
|
23190
|
+
}
|
|
23191
|
+
];
|
|
23152
23192
|
const PRESET_DEFAULTS = {
|
|
23153
23193
|
balanced: {},
|
|
23154
23194
|
"open-source-maintainer": {
|
|
@@ -23179,6 +23219,9 @@ const PRESET_DEFAULTS = {
|
|
|
23179
23219
|
requireTests: {
|
|
23180
23220
|
enabled: true,
|
|
23181
23221
|
paths: ["src/**", "packages/**/src/**", "app/**", "lib/**", "server/**", "api/**"]
|
|
23222
|
+
},
|
|
23223
|
+
evidence: {
|
|
23224
|
+
contracts: WORKFLOW_EVIDENCE_CONTRACTS
|
|
23182
23225
|
}
|
|
23183
23226
|
},
|
|
23184
23227
|
"ai-generated-pr": {
|
|
@@ -23217,9 +23260,20 @@ const PRESET_DEFAULTS = {
|
|
|
23217
23260
|
requireTests: {
|
|
23218
23261
|
enabled: true,
|
|
23219
23262
|
paths: DEFAULT_TEST_PATHS
|
|
23263
|
+
},
|
|
23264
|
+
evidence: {
|
|
23265
|
+
contracts: DEPENDENCY_EVIDENCE_CONTRACTS
|
|
23220
23266
|
}
|
|
23221
23267
|
}
|
|
23222
23268
|
};
|
|
23269
|
+
const evidenceContractSchema = object({
|
|
23270
|
+
id: schemas_string().min(1),
|
|
23271
|
+
title: schemas_string().min(1).optional(),
|
|
23272
|
+
paths: array(schemas_string().min(1)).min(1),
|
|
23273
|
+
requires: array(evidenceRequirementSchema).min(1),
|
|
23274
|
+
severity: findingSeveritySchema.default("medium"),
|
|
23275
|
+
recommendation: schemas_string().min(1).optional()
|
|
23276
|
+
});
|
|
23223
23277
|
const configSchema = object({
|
|
23224
23278
|
preset: configPresetSchema.default("balanced"),
|
|
23225
23279
|
locale: localeSchema.default("en"),
|
|
@@ -23238,6 +23292,10 @@ const configSchema = object({
|
|
|
23238
23292
|
flagLifecycleScripts: schemas_boolean().default(true)
|
|
23239
23293
|
})
|
|
23240
23294
|
.default({ flagNewPackages: true, flagMajorUpgrades: true, flagLifecycleScripts: true }),
|
|
23295
|
+
evidence: object({
|
|
23296
|
+
contracts: array(evidenceContractSchema).default([])
|
|
23297
|
+
})
|
|
23298
|
+
.default({ contracts: [] }),
|
|
23241
23299
|
comment: object({ enabled: schemas_boolean().default(true) }).default({ enabled: true })
|
|
23242
23300
|
});
|
|
23243
23301
|
function parseConfig(input) {
|
|
@@ -23375,6 +23433,9 @@ function renderEnglishMarkdownReport(result) {
|
|
|
23375
23433
|
`- PR description: ${result.summary.pullRequestDescription}`,
|
|
23376
23434
|
`- Verification evidence: ${formatBoolean(result.summary.verificationEvidence)}`,
|
|
23377
23435
|
`- Reproduction context: ${formatBoolean(result.summary.reproductionEvidence)}`,
|
|
23436
|
+
`- Screenshot evidence: ${formatBoolean(result.summary.screenshotEvidence)}`,
|
|
23437
|
+
`- Changelog evidence: ${formatBoolean(result.summary.changelogEvidence)}`,
|
|
23438
|
+
`- Permission rationale: ${formatBoolean(result.summary.permissionRationaleEvidence)}`,
|
|
23378
23439
|
""
|
|
23379
23440
|
];
|
|
23380
23441
|
appendEvidenceScoreSection(lines, result, "en");
|
|
@@ -23409,6 +23470,9 @@ function renderChineseMarkdownReport(result) {
|
|
|
23409
23470
|
`- PR 描述质量:${translateDescriptionState(result.summary.pullRequestDescription)}`,
|
|
23410
23471
|
`- 验证证据:${formatChineseBoolean(result.summary.verificationEvidence)}`,
|
|
23411
23472
|
`- 复现上下文:${formatChineseBoolean(result.summary.reproductionEvidence)}`,
|
|
23473
|
+
`- 截图或视觉证据:${formatChineseBoolean(result.summary.screenshotEvidence)}`,
|
|
23474
|
+
`- Changelog 或迁移证据:${formatChineseBoolean(result.summary.changelogEvidence)}`,
|
|
23475
|
+
`- 权限理由证据:${formatChineseBoolean(result.summary.permissionRationaleEvidence)}`,
|
|
23412
23476
|
""
|
|
23413
23477
|
];
|
|
23414
23478
|
appendEvidenceScoreSection(lines, result, "zh-CN");
|
|
@@ -23519,6 +23583,11 @@ function maintainerFocus(findings, locale) {
|
|
|
23519
23583
|
? "轮换任何可能暴露的凭证,并在移除 secret 前阻止合并。"
|
|
23520
23584
|
: "Rotate any exposed credential and block the PR until secrets are removed.");
|
|
23521
23585
|
}
|
|
23586
|
+
else if (finding.ruleId.startsWith("evidence-contract:")) {
|
|
23587
|
+
focus.add(locale === "zh-CN"
|
|
23588
|
+
? "先要求贡献者补齐仓库定义的证据契约,再投入深度 review。"
|
|
23589
|
+
: "Ask the contributor to satisfy the repository-defined evidence contract before deep review.");
|
|
23590
|
+
}
|
|
23522
23591
|
else if (finding.ruleId === "workflow-permission-change") {
|
|
23523
23592
|
focus.add(locale === "zh-CN"
|
|
23524
23593
|
? "合并前重点审查 GitHub Actions 权限。"
|
|
@@ -23559,6 +23628,11 @@ function maintainerFocus(findings, locale) {
|
|
|
23559
23628
|
? "重点审查 pull_request_target 是否会用高权限 token 执行不可信 PR 代码。"
|
|
23560
23629
|
: "Review whether pull_request_target can execute untrusted PR code with privileged tokens.");
|
|
23561
23630
|
}
|
|
23631
|
+
else if (finding.ruleId === "workflow-untrusted-checkout") {
|
|
23632
|
+
focus.add(locale === "zh-CN"
|
|
23633
|
+
? "重点审查 workflow 是否 checkout 并执行了不可信 PR head 代码。"
|
|
23634
|
+
: "Review whether the workflow checks out and executes untrusted PR head code.");
|
|
23635
|
+
}
|
|
23562
23636
|
else if (finding.ruleId === "mcp-credential-risk") {
|
|
23563
23637
|
focus.add(locale === "zh-CN"
|
|
23564
23638
|
? "重点审查 MCP command、args 和凭证处理方式。"
|
|
@@ -23573,6 +23647,13 @@ function maintainerFocus(findings, locale) {
|
|
|
23573
23647
|
return [...focus];
|
|
23574
23648
|
}
|
|
23575
23649
|
function translateFinding(finding) {
|
|
23650
|
+
if (finding.ruleId.startsWith("evidence-contract:")) {
|
|
23651
|
+
return {
|
|
23652
|
+
title: "证据契约未满足",
|
|
23653
|
+
message: "该 PR 命中了仓库自定义证据契约,但 PR 描述中缺少必需证据。",
|
|
23654
|
+
recommendation: "建议要求贡献者补齐缺失证据后再深入 review。"
|
|
23655
|
+
};
|
|
23656
|
+
}
|
|
23576
23657
|
if (finding.ruleId === "change-size") {
|
|
23577
23658
|
const files = finding.evidence?.find((item) => item.startsWith("files: "))?.replace("files: ", "");
|
|
23578
23659
|
const lines = finding.evidence?.find((item) => item.startsWith("changed lines: "))?.replace("changed lines: ", "");
|
|
@@ -23648,6 +23729,15 @@ function translateFinding(finding) {
|
|
|
23648
23729
|
recommendation: "请确认该 workflow 不会用高权限 token、secret 或写权限执行不可信 PR 代码。"
|
|
23649
23730
|
};
|
|
23650
23731
|
}
|
|
23732
|
+
if (finding.ruleId === "workflow-untrusted-checkout") {
|
|
23733
|
+
return {
|
|
23734
|
+
title: "Workflow checkout 了 PR head",
|
|
23735
|
+
message: finding.path
|
|
23736
|
+
? `${finding.path} 引用了 PR head 代码来源,需要审查它是否会在高权限上下文中执行。`
|
|
23737
|
+
: finding.message,
|
|
23738
|
+
recommendation: "避免在 pull_request_target、写权限 token 或可读取 secret 的上下文中运行不可信 PR 代码。"
|
|
23739
|
+
};
|
|
23740
|
+
}
|
|
23651
23741
|
if (finding.ruleId === "mcp-credential-risk") {
|
|
23652
23742
|
return {
|
|
23653
23743
|
title: "MCP 配置需要重点审查",
|
|
@@ -23666,8 +23756,15 @@ function translateFinding(finding) {
|
|
|
23666
23756
|
}
|
|
23667
23757
|
function translateEvidence(item) {
|
|
23668
23758
|
return item
|
|
23759
|
+
.replace("matched files: ", "命中文件:")
|
|
23760
|
+
.replace("missing evidence: ", "缺失证据:")
|
|
23669
23761
|
.replace("files: ", "文件数:")
|
|
23670
23762
|
.replace("changed lines: ", "变更行数:")
|
|
23763
|
+
.replace(/\bverification\b/g, "验证")
|
|
23764
|
+
.replace(/\breproduction\b/g, "复现")
|
|
23765
|
+
.replace(/\bscreenshot\b/g, "截图")
|
|
23766
|
+
.replace(/\bchangelog\b/g, "变更日志")
|
|
23767
|
+
.replace(/\bpermission-rationale\b/g, "权限理由")
|
|
23671
23768
|
.replace("line ", "第 ")
|
|
23672
23769
|
.replace(": ", " 行:");
|
|
23673
23770
|
}
|
|
@@ -23719,12 +23816,14 @@ function translateReviewActionTitle(actionId, fallback) {
|
|
|
23719
23816
|
"ask-for-evidence-before-review": "深入 review 前先要求补充证据",
|
|
23720
23817
|
"review-with-focus": "带着重点清单进行 review",
|
|
23721
23818
|
"normal-review": "进入常规 review",
|
|
23819
|
+
"satisfy-evidence-contract": "要求补齐证据契约",
|
|
23722
23820
|
"improve-pr-description": "要求补充更清楚的 PR 描述",
|
|
23723
23821
|
"add-verification-evidence": "要求补充测试或手动验证证据",
|
|
23724
23822
|
"add-reproduction-context": "要求补充复现或 before/after 上下文",
|
|
23725
23823
|
"rotate-secret": "轮换并移除暴露的凭证",
|
|
23726
23824
|
"justify-workflow-permissions": "要求说明 workflow 权限最小化理由",
|
|
23727
23825
|
"review-privileged-pr-trigger": "审查 pull_request_target 高权限触发器",
|
|
23826
|
+
"review-untrusted-checkout": "审查 PR head checkout 的权限边界",
|
|
23728
23827
|
"review-package-lifecycle-script": "审查包生命周期脚本",
|
|
23729
23828
|
"review-mcp-execution-surface": "审查 MCP 命令、参数和凭证处理",
|
|
23730
23829
|
"request-review-map-or-split": "要求拆分 PR 或提供逐文件 review map",
|
|
@@ -23739,12 +23838,14 @@ function translateReviewActionDetail(actionId, fallback) {
|
|
|
23739
23838
|
"ask-for-evidence-before-review": "要求测试、截图、复现步骤或更清楚的 PR 描述,再投入详细 review。",
|
|
23740
23839
|
"review-with-focus": "优先使用下面的风险发现和重点文件作为第一轮 review map。",
|
|
23741
23840
|
"normal-review": "当前证据足够支撑维护者进行常规 review。",
|
|
23841
|
+
"satisfy-evidence-contract": "该 PR 命中了仓库自定义证据契约,但 PR 描述里缺少必需证据。",
|
|
23742
23842
|
"improve-pr-description": "贡献者应说明为什么改、改了什么、如何验证,以及是否有发布或兼容性风险。",
|
|
23743
23843
|
"add-verification-evidence": "要求测试输出、CI 链接、截图,或简短的手动验证说明。",
|
|
23744
23844
|
"add-reproduction-context": "PR 应包含复现步骤、预期/实际行为,或相关 before/after 截图。",
|
|
23745
23845
|
"rotate-secret": "在 secret 从 PR 中移除并完成轮换前,不要合并。",
|
|
23746
23846
|
"justify-workflow-permissions": "确认写权限或 OIDC 是否必要,并检查不可信 PR 是否能触发该 workflow。",
|
|
23747
23847
|
"review-privileged-pr-trigger": "确认 workflow 不会用写权限 token、secret 或仓库权限执行不可信 PR 代码。",
|
|
23848
|
+
"review-untrusted-checkout": "确认 job 不会在写权限 token、仓库 secret 或 pull_request_target 高权限上下文中运行不可信 PR 代码。",
|
|
23748
23849
|
"review-package-lifecycle-script": "检查 install、postinstall、prepare 或 publish 脚本是否会执行非预期代码。",
|
|
23749
23850
|
"review-mcp-execution-surface": "检查 MCP 配置是否提交凭证,或意外扩大本地执行面。",
|
|
23750
23851
|
"request-review-map-or-split": "要求贡献者拆分无关改动,或标出最需要重点 review 的文件。",
|
|
@@ -23754,6 +23855,9 @@ function translateReviewActionDetail(actionId, fallback) {
|
|
|
23754
23855
|
}[actionId] ?? fallback;
|
|
23755
23856
|
}
|
|
23756
23857
|
function translateFocusReason(reasonId, fallback) {
|
|
23858
|
+
if (reasonId.startsWith("evidence-contract:")) {
|
|
23859
|
+
return "仓库自定义证据契约未满足";
|
|
23860
|
+
}
|
|
23757
23861
|
return {
|
|
23758
23862
|
"change-size": "review 面积相关 finding",
|
|
23759
23863
|
"sensitive-path": "敏感路径发生变更",
|
|
@@ -23762,6 +23866,7 @@ function translateFocusReason(reasonId, fallback) {
|
|
|
23762
23866
|
"dependency-lifecycle-script": "包生命周期脚本发生变更",
|
|
23763
23867
|
"workflow-permission-change": "workflow 权限发生变更",
|
|
23764
23868
|
"workflow-dangerous-trigger": "workflow 使用了高风险触发器",
|
|
23869
|
+
"workflow-untrusted-checkout": "workflow checkout 了不可信 PR head",
|
|
23765
23870
|
"mcp-credential-risk": "MCP 配置存在执行面或凭证风险",
|
|
23766
23871
|
"missing-tests": "代码改动缺少测试或验证证据"
|
|
23767
23872
|
}[reasonId] ?? fallback;
|
|
@@ -23771,6 +23876,9 @@ function translateScoreMessage(message) {
|
|
|
23771
23876
|
"PR description provides review context.": "PR 描述提供了 review 上下文。",
|
|
23772
23877
|
"Verification evidence was found.": "检测到测试或手动验证证据。",
|
|
23773
23878
|
"Reproduction or before/after context was found.": "检测到复现步骤或 before/after 上下文。",
|
|
23879
|
+
"Screenshot or visual evidence was found.": "检测到截图或视觉证据。",
|
|
23880
|
+
"Changelog or migration evidence was found.": "检测到 changelog 或迁移证据。",
|
|
23881
|
+
"Permission rationale evidence was found.": "检测到权限理由证据。",
|
|
23774
23882
|
"Test files changed with the PR.": "PR 同时修改了测试文件。",
|
|
23775
23883
|
"No configured sensitive files changed.": "没有改动已配置的敏感文件。"
|
|
23776
23884
|
}[message] ?? message;
|
|
@@ -23793,6 +23901,8 @@ function translateDeduction(reasonId, fallback) {
|
|
|
23793
23901
|
"dependency-major-upgrade": "依赖发生大版本升级。",
|
|
23794
23902
|
"dependency-lifecycle-script": "包生命周期脚本可能在安装或发布阶段执行代码。",
|
|
23795
23903
|
"workflow-dangerous-trigger": "pull_request_target workflow 需要重点审查高权限触发路径。",
|
|
23904
|
+
"workflow-untrusted-checkout": "Workflow checkout PR head 代码,需要审查权限边界。",
|
|
23905
|
+
"evidence-contract-missing": "仓库自定义证据契约未满足。",
|
|
23796
23906
|
"missing-tests": "代码发生变更,但缺少测试变更或验证说明。"
|
|
23797
23907
|
}[reasonId] ?? fallback;
|
|
23798
23908
|
}
|
|
@@ -23908,12 +24018,27 @@ const REPRODUCTION_PATTERNS = [
|
|
|
23908
24018
|
/\b(?:before|after|expected|actual)\b/i,
|
|
23909
24019
|
/复现|重现|复现步骤|期望|实际/
|
|
23910
24020
|
];
|
|
24021
|
+
const SCREENSHOT_PATTERNS = [
|
|
24022
|
+
/\b(?:screenshot|screen shot|screen recording|recording|gif|image|before\/after)\b/i,
|
|
24023
|
+
/截图|录屏|效果图|前后对比|对比图/
|
|
24024
|
+
];
|
|
24025
|
+
const CHANGELOG_PATTERNS = [
|
|
24026
|
+
/\b(?:changelog|release notes?|migration guide|breaking changes?|upgrade guide)\b/i,
|
|
24027
|
+
/变更日志|发布说明|迁移指南|升级说明|破坏性变更|兼容性/
|
|
24028
|
+
];
|
|
24029
|
+
const PERMISSION_RATIONALE_PATTERNS = [
|
|
24030
|
+
/\b(?:least privilege|permission rationale|write permission|oidc|id-token|trusted workflow|untrusted pr|token scope)\b/i,
|
|
24031
|
+
/权限理由|最小权限|写权限|OIDC|id-token|不可信 PR|高权限|token 权限|凭证权限/
|
|
24032
|
+
];
|
|
23911
24033
|
function analyzeEvidence(context) {
|
|
23912
24034
|
if (!context) {
|
|
23913
24035
|
return {
|
|
23914
24036
|
descriptionState: "unavailable",
|
|
23915
24037
|
verificationEvidence: false,
|
|
23916
|
-
reproductionEvidence: false
|
|
24038
|
+
reproductionEvidence: false,
|
|
24039
|
+
screenshotEvidence: false,
|
|
24040
|
+
changelogEvidence: false,
|
|
24041
|
+
permissionRationaleEvidence: false
|
|
23917
24042
|
};
|
|
23918
24043
|
}
|
|
23919
24044
|
const text = [context.title ?? "", context.body ?? ""].join("\n").trim();
|
|
@@ -23921,7 +24046,10 @@ function analyzeEvidence(context) {
|
|
|
23921
24046
|
return {
|
|
23922
24047
|
descriptionState: descriptionState(body),
|
|
23923
24048
|
verificationEvidence: matchesAnyPattern(text, VERIFICATION_PATTERNS),
|
|
23924
|
-
reproductionEvidence: matchesAnyPattern(text, REPRODUCTION_PATTERNS)
|
|
24049
|
+
reproductionEvidence: matchesAnyPattern(text, REPRODUCTION_PATTERNS),
|
|
24050
|
+
screenshotEvidence: matchesAnyPattern(text, SCREENSHOT_PATTERNS),
|
|
24051
|
+
changelogEvidence: matchesAnyPattern(text, CHANGELOG_PATTERNS),
|
|
24052
|
+
permissionRationaleEvidence: matchesAnyPattern(text, PERMISSION_RATIONALE_PATTERNS)
|
|
23925
24053
|
};
|
|
23926
24054
|
}
|
|
23927
24055
|
function descriptionState(body) {
|
|
@@ -24102,9 +24230,11 @@ function analyzeDiffFiles(files, config, pullRequest) {
|
|
|
24102
24230
|
findings.push(...analyzeSensitivePaths(activeFiles, config));
|
|
24103
24231
|
findings.push(...analyzeMissingTests(activeFiles, config, pullRequest));
|
|
24104
24232
|
findings.push(...analyzePullRequestEvidence(activeFiles, pullRequest));
|
|
24233
|
+
findings.push(...analyzeEvidenceContracts(activeFiles, config, pullRequest));
|
|
24105
24234
|
findings.push(...analyzeDependencyChanges(activeFiles, config));
|
|
24106
24235
|
findings.push(...analyzeWorkflowPermissions(activeFiles));
|
|
24107
24236
|
findings.push(...analyzeWorkflowDangerousTriggers(activeFiles));
|
|
24237
|
+
findings.push(...analyzeWorkflowUntrustedCheckout(activeFiles));
|
|
24108
24238
|
findings.push(...analyzeMcpConfigs(activeFiles));
|
|
24109
24239
|
if (config.secrets.enabled) {
|
|
24110
24240
|
for (const file of activeFiles) {
|
|
@@ -24124,7 +24254,10 @@ function summarizeDiffFiles(files, config, pullRequest) {
|
|
|
24124
24254
|
sensitiveFilesChanged: activeFiles.filter((file) => matchesAny(file.path, config.sensitivePaths)).length,
|
|
24125
24255
|
pullRequestDescription: evidence.descriptionState,
|
|
24126
24256
|
verificationEvidence: evidence.verificationEvidence,
|
|
24127
|
-
reproductionEvidence: evidence.reproductionEvidence
|
|
24257
|
+
reproductionEvidence: evidence.reproductionEvidence,
|
|
24258
|
+
screenshotEvidence: evidence.screenshotEvidence,
|
|
24259
|
+
changelogEvidence: evidence.changelogEvidence,
|
|
24260
|
+
permissionRationaleEvidence: evidence.permissionRationaleEvidence
|
|
24128
24261
|
};
|
|
24129
24262
|
}
|
|
24130
24263
|
function analyzeChangeSize(files) {
|
|
@@ -24229,6 +24362,55 @@ function analyzePullRequestEvidence(files, pullRequest) {
|
|
|
24229
24362
|
}
|
|
24230
24363
|
return findings;
|
|
24231
24364
|
}
|
|
24365
|
+
function analyzeEvidenceContracts(files, config, pullRequest) {
|
|
24366
|
+
if (config.evidence.contracts.length === 0) {
|
|
24367
|
+
return [];
|
|
24368
|
+
}
|
|
24369
|
+
const evidence = analyzeEvidence(pullRequest);
|
|
24370
|
+
const hasTestChanges = files.some((file) => isTestPath(file.path));
|
|
24371
|
+
const findings = [];
|
|
24372
|
+
for (const contract of config.evidence.contracts) {
|
|
24373
|
+
const matchedFiles = files.filter((file) => matchesAny(file.path, contract.paths));
|
|
24374
|
+
if (matchedFiles.length === 0) {
|
|
24375
|
+
continue;
|
|
24376
|
+
}
|
|
24377
|
+
const missingRequirements = contract.requires.filter((requirement) => !hasEvidenceRequirement(requirement, evidence, hasTestChanges));
|
|
24378
|
+
if (missingRequirements.length === 0) {
|
|
24379
|
+
continue;
|
|
24380
|
+
}
|
|
24381
|
+
findings.push({
|
|
24382
|
+
ruleId: `evidence-contract:${contract.id}`,
|
|
24383
|
+
title: contract.title ?? "Evidence contract missing",
|
|
24384
|
+
message: `Changed files match evidence contract "${contract.id}", but missing required evidence: ${missingRequirements
|
|
24385
|
+
.map(formatEvidenceRequirement)
|
|
24386
|
+
.join(", ")}.`,
|
|
24387
|
+
severity: contract.severity,
|
|
24388
|
+
path: matchedFiles[0]?.path,
|
|
24389
|
+
evidence: [
|
|
24390
|
+
`matched files: ${matchedFiles.slice(0, 5).map((file) => file.path).join(", ")}`,
|
|
24391
|
+
`missing evidence: ${missingRequirements.map(formatEvidenceRequirement).join(", ")}`
|
|
24392
|
+
],
|
|
24393
|
+
recommendation: contract.recommendation ??
|
|
24394
|
+
"Ask the contributor to add the missing evidence before spending deep review time."
|
|
24395
|
+
});
|
|
24396
|
+
}
|
|
24397
|
+
return findings;
|
|
24398
|
+
}
|
|
24399
|
+
function hasEvidenceRequirement(requirement, evidence, hasTestChanges) {
|
|
24400
|
+
if (requirement === "verification") {
|
|
24401
|
+
return evidence.verificationEvidence || hasTestChanges;
|
|
24402
|
+
}
|
|
24403
|
+
if (requirement === "reproduction") {
|
|
24404
|
+
return evidence.reproductionEvidence;
|
|
24405
|
+
}
|
|
24406
|
+
if (requirement === "screenshot") {
|
|
24407
|
+
return evidence.screenshotEvidence;
|
|
24408
|
+
}
|
|
24409
|
+
if (requirement === "changelog") {
|
|
24410
|
+
return evidence.changelogEvidence;
|
|
24411
|
+
}
|
|
24412
|
+
return evidence.permissionRationaleEvidence;
|
|
24413
|
+
}
|
|
24232
24414
|
function analyzeDependencyChanges(files, config) {
|
|
24233
24415
|
const findings = [];
|
|
24234
24416
|
for (const file of files.filter((candidate) => isDependencyManifest(candidate.path))) {
|
|
@@ -24360,7 +24542,7 @@ function extractMajorVersion(version) {
|
|
|
24360
24542
|
function analyzeWorkflowPermissions(files) {
|
|
24361
24543
|
const findings = [];
|
|
24362
24544
|
for (const file of files.filter((candidate) => isWorkflowPath(candidate.path))) {
|
|
24363
|
-
const permissionLines = file.addedLines.filter((line) =>
|
|
24545
|
+
const permissionLines = file.addedLines.filter((line) => isRiskyWorkflowPermissionLine(line.value));
|
|
24364
24546
|
if (permissionLines.length === 0) {
|
|
24365
24547
|
continue;
|
|
24366
24548
|
}
|
|
@@ -24376,6 +24558,13 @@ function analyzeWorkflowPermissions(files) {
|
|
|
24376
24558
|
}
|
|
24377
24559
|
return findings;
|
|
24378
24560
|
}
|
|
24561
|
+
function isRiskyWorkflowPermissionLine(value) {
|
|
24562
|
+
const line = value.trim();
|
|
24563
|
+
if (/^permissions:\s*write-all\b/i.test(line)) {
|
|
24564
|
+
return true;
|
|
24565
|
+
}
|
|
24566
|
+
return /^(?:actions|attestations|checks|contents|deployments|discussions|id-token|issues|models|packages|pages|pull-requests|repository-projects|security-events|statuses):\s*write\b/i.test(line);
|
|
24567
|
+
}
|
|
24379
24568
|
function analyzeWorkflowDangerousTriggers(files) {
|
|
24380
24569
|
const findings = [];
|
|
24381
24570
|
for (const file of files.filter((candidate) => isWorkflowPath(candidate.path))) {
|
|
@@ -24395,6 +24584,33 @@ function analyzeWorkflowDangerousTriggers(files) {
|
|
|
24395
24584
|
}
|
|
24396
24585
|
return findings;
|
|
24397
24586
|
}
|
|
24587
|
+
function analyzeWorkflowUntrustedCheckout(files) {
|
|
24588
|
+
const findings = [];
|
|
24589
|
+
for (const file of files.filter((candidate) => isWorkflowPath(candidate.path))) {
|
|
24590
|
+
const headCheckoutLines = file.addedLines.filter((line) => isPullRequestHeadCheckoutLine(line.value));
|
|
24591
|
+
if (headCheckoutLines.length === 0) {
|
|
24592
|
+
continue;
|
|
24593
|
+
}
|
|
24594
|
+
const hasPullRequestTarget = file.addedLines.some((line) => /\bpull_request_target\b/.test(line.value.trim()));
|
|
24595
|
+
findings.push({
|
|
24596
|
+
ruleId: "workflow-untrusted-checkout",
|
|
24597
|
+
title: "Workflow checks out pull request head",
|
|
24598
|
+
message: hasPullRequestTarget
|
|
24599
|
+
? `${file.path} combines pull_request_target with pull request head checkout references.`
|
|
24600
|
+
: `${file.path} checks out pull request head references; review the job privilege boundary before merging.`,
|
|
24601
|
+
severity: hasPullRequestTarget ? "high" : "medium",
|
|
24602
|
+
path: file.path,
|
|
24603
|
+
evidence: headCheckoutLines.slice(0, 5).map(formatEvidenceLine),
|
|
24604
|
+
recommendation: "Avoid running untrusted PR code with write tokens, repository secrets, or privileged pull_request_target context."
|
|
24605
|
+
});
|
|
24606
|
+
}
|
|
24607
|
+
return findings;
|
|
24608
|
+
}
|
|
24609
|
+
function isPullRequestHeadCheckoutLine(value) {
|
|
24610
|
+
const line = value.trim();
|
|
24611
|
+
return (/\bgithub\.head_ref\b/.test(line) ||
|
|
24612
|
+
/\bgithub\.event\.pull_request\.head(?:\.sha|\.ref|\.repo\.full_name)?\b/.test(line));
|
|
24613
|
+
}
|
|
24398
24614
|
function analyzeMcpConfigs(files) {
|
|
24399
24615
|
const findings = [];
|
|
24400
24616
|
for (const file of files.filter((candidate) => isMcpConfigPath(candidate.path))) {
|
|
@@ -24418,6 +24634,9 @@ function formatEvidenceLine(line) {
|
|
|
24418
24634
|
const value = line.value.trim();
|
|
24419
24635
|
return line.lineNumber ? `line ${line.lineNumber}: ${value}` : value;
|
|
24420
24636
|
}
|
|
24637
|
+
function formatEvidenceRequirement(requirement) {
|
|
24638
|
+
return requirement;
|
|
24639
|
+
}
|
|
24421
24640
|
function sensitivePathSeverity(path) {
|
|
24422
24641
|
if (matchesAny(path, [
|
|
24423
24642
|
"**/.env*",
|
|
@@ -24492,8 +24711,9 @@ function calculateEvidenceScore(summary, findings) {
|
|
|
24492
24711
|
"dependency-lifecycle-script",
|
|
24493
24712
|
"workflow-permission-change",
|
|
24494
24713
|
"workflow-dangerous-trigger",
|
|
24714
|
+
"workflow-untrusted-checkout",
|
|
24495
24715
|
"mcp-credential-risk"
|
|
24496
|
-
].includes(finding.ruleId));
|
|
24716
|
+
].includes(finding.ruleId) || finding.ruleId.startsWith("evidence-contract:"));
|
|
24497
24717
|
if (needsVerificationEvidence && !summary.verificationEvidence) {
|
|
24498
24718
|
addDeduction("missing-verification", 20, "No test or manual verification evidence was found.");
|
|
24499
24719
|
}
|
|
@@ -24510,6 +24730,9 @@ function calculateEvidenceScore(summary, findings) {
|
|
|
24510
24730
|
else if (finding.ruleId === "workflow-dangerous-trigger") {
|
|
24511
24731
|
addDeduction("workflow-dangerous-trigger", 30, "pull_request_target workflows need privileged trigger review.");
|
|
24512
24732
|
}
|
|
24733
|
+
else if (finding.ruleId === "workflow-untrusted-checkout") {
|
|
24734
|
+
addDeduction("workflow-untrusted-checkout", finding.severity === "high" ? 30 : 18, "Workflow checkout of pull request head needs privilege-boundary review.");
|
|
24735
|
+
}
|
|
24513
24736
|
else if (finding.ruleId === "mcp-credential-risk") {
|
|
24514
24737
|
addDeduction("mcp-credential-risk", 25, "MCP configuration expands local execution or credential risk.");
|
|
24515
24738
|
}
|
|
@@ -24530,6 +24753,9 @@ function calculateEvidenceScore(summary, findings) {
|
|
|
24530
24753
|
else if (finding.ruleId === "dependency-lifecycle-script") {
|
|
24531
24754
|
addDeduction("dependency-lifecycle-script", 25, "Package lifecycle scripts can run during install or publish.");
|
|
24532
24755
|
}
|
|
24756
|
+
else if (finding.ruleId.startsWith("evidence-contract:")) {
|
|
24757
|
+
addDeduction("evidence-contract-missing", finding.severity === "high" ? 25 : 15, "Configured evidence contract was not satisfied.");
|
|
24758
|
+
}
|
|
24533
24759
|
else if (finding.ruleId === "missing-tests") {
|
|
24534
24760
|
addDeduction("missing-tests", finding.severity === "medium" ? 20 : 12, "Code changed without test changes or verification notes.");
|
|
24535
24761
|
}
|
|
@@ -24558,6 +24784,15 @@ function collectEvidenceStrengths(summary) {
|
|
|
24558
24784
|
if (summary.reproductionEvidence) {
|
|
24559
24785
|
strengths.push("Reproduction or before/after context was found.");
|
|
24560
24786
|
}
|
|
24787
|
+
if (summary.screenshotEvidence) {
|
|
24788
|
+
strengths.push("Screenshot or visual evidence was found.");
|
|
24789
|
+
}
|
|
24790
|
+
if (summary.changelogEvidence) {
|
|
24791
|
+
strengths.push("Changelog or migration evidence was found.");
|
|
24792
|
+
}
|
|
24793
|
+
if (summary.permissionRationaleEvidence) {
|
|
24794
|
+
strengths.push("Permission rationale evidence was found.");
|
|
24795
|
+
}
|
|
24561
24796
|
if (summary.testFilesChanged > 0) {
|
|
24562
24797
|
strengths.push("Test files changed with the PR.");
|
|
24563
24798
|
}
|
|
@@ -24582,6 +24817,7 @@ function calculateReviewDecision(risk, evidenceScore, findings) {
|
|
|
24582
24817
|
const hasBlockingSecurityFinding = findings.some((finding) => finding.ruleId.startsWith("secret-detected") ||
|
|
24583
24818
|
finding.ruleId === "workflow-permission-change" ||
|
|
24584
24819
|
finding.ruleId === "workflow-dangerous-trigger" ||
|
|
24820
|
+
(finding.ruleId === "workflow-untrusted-checkout" && finding.severity === "high") ||
|
|
24585
24821
|
finding.ruleId === "dependency-lifecycle-script" ||
|
|
24586
24822
|
finding.ruleId === "mcp-credential-risk");
|
|
24587
24823
|
if (hasBlockingSecurityFinding || evidenceScore.value < 50 || risk === "high") {
|
|
@@ -24714,6 +24950,17 @@ function reviewActionsForFinding(finding) {
|
|
|
24714
24950
|
}
|
|
24715
24951
|
];
|
|
24716
24952
|
}
|
|
24953
|
+
if (finding.ruleId.startsWith("evidence-contract:")) {
|
|
24954
|
+
return [
|
|
24955
|
+
{
|
|
24956
|
+
actionId: "satisfy-evidence-contract",
|
|
24957
|
+
title: "Ask for the configured evidence contract to be satisfied.",
|
|
24958
|
+
detail: "The PR matches a repository-defined evidence contract but is missing required proof in the PR description.",
|
|
24959
|
+
priority: finding.severity === "high" ? "high" : "medium",
|
|
24960
|
+
relatedRuleIds: [finding.ruleId]
|
|
24961
|
+
}
|
|
24962
|
+
];
|
|
24963
|
+
}
|
|
24717
24964
|
if (finding.ruleId === "workflow-permission-change") {
|
|
24718
24965
|
return [
|
|
24719
24966
|
{
|
|
@@ -24736,6 +24983,17 @@ function reviewActionsForFinding(finding) {
|
|
|
24736
24983
|
}
|
|
24737
24984
|
];
|
|
24738
24985
|
}
|
|
24986
|
+
if (finding.ruleId === "workflow-untrusted-checkout") {
|
|
24987
|
+
return [
|
|
24988
|
+
{
|
|
24989
|
+
actionId: "review-untrusted-checkout",
|
|
24990
|
+
title: "Review pull request head checkout privileges.",
|
|
24991
|
+
detail: "Confirm the job does not run untrusted PR code with write tokens, repository secrets, or pull_request_target privileges.",
|
|
24992
|
+
priority: finding.severity === "high" ? "high" : "medium",
|
|
24993
|
+
relatedRuleIds: [finding.ruleId]
|
|
24994
|
+
}
|
|
24995
|
+
];
|
|
24996
|
+
}
|
|
24739
24997
|
if (finding.ruleId === "dependency-lifecycle-script") {
|
|
24740
24998
|
return [
|
|
24741
24999
|
{
|
|
@@ -24864,7 +25122,7 @@ const build_program = new Command();
|
|
|
24864
25122
|
build_program
|
|
24865
25123
|
.name("proof-pr")
|
|
24866
25124
|
.description("Review pull request evidence, scope, and safety before maintainers spend time on it.")
|
|
24867
|
-
.version("0.1.
|
|
25125
|
+
.version("0.1.7");
|
|
24868
25126
|
build_program
|
|
24869
25127
|
.command("scan", { isDefault: true })
|
|
24870
25128
|
.description("Scan a git diff and print a ProofPR report.")
|
|
@@ -24908,6 +25166,35 @@ build_program
|
|
|
24908
25166
|
await writeIfMissing(options.workflowPath, renderWorkflowTemplate(options.failOn), options.force);
|
|
24909
25167
|
process.stdout.write(`ProofPR initialized:\n- ${options.configPath}\n- ${options.workflowPath}\n`);
|
|
24910
25168
|
});
|
|
25169
|
+
build_program
|
|
25170
|
+
.command("benchmark")
|
|
25171
|
+
.description("Run ProofPR benchmark cases and compare expected risk/finding output.")
|
|
25172
|
+
.option("--cases <dir>", "Directory containing benchmark case JSON files.", "benchmarks/cases")
|
|
25173
|
+
.option("--format <format>", "Output format: text, markdown, or json.", parseBenchmarkFormat, "text")
|
|
25174
|
+
.option("--output <path>", "Write benchmark output to a file instead of stdout.")
|
|
25175
|
+
.action(async (options) => {
|
|
25176
|
+
const report = await runBenchmarks(options.cases);
|
|
25177
|
+
let output;
|
|
25178
|
+
if (options.format === "json") {
|
|
25179
|
+
output = `${JSON.stringify(report, null, 2)}\n`;
|
|
25180
|
+
}
|
|
25181
|
+
else if (options.format === "markdown") {
|
|
25182
|
+
output = renderBenchmarkMarkdown(report);
|
|
25183
|
+
}
|
|
25184
|
+
else {
|
|
25185
|
+
output = renderBenchmarkText(report);
|
|
25186
|
+
}
|
|
25187
|
+
if (options.output) {
|
|
25188
|
+
await writeOutput(options.output, output);
|
|
25189
|
+
process.stdout.write(`ProofPR benchmark report written to ${options.output}\n`);
|
|
25190
|
+
}
|
|
25191
|
+
else {
|
|
25192
|
+
process.stdout.write(output);
|
|
25193
|
+
}
|
|
25194
|
+
if (report.results.some((result) => !result.passed)) {
|
|
25195
|
+
process.exitCode = 1;
|
|
25196
|
+
}
|
|
25197
|
+
});
|
|
24911
25198
|
build_program.parseAsync(process.argv).catch((error) => {
|
|
24912
25199
|
const message = error instanceof Error ? error.message : String(error);
|
|
24913
25200
|
process.stderr.write(`ProofPR failed: ${message}\n`);
|
|
@@ -24934,6 +25221,10 @@ async function writeIfMissing(path, contents, force) {
|
|
|
24934
25221
|
await (0,promises_namespaceObject.mkdir)((0,external_node_path_.dirname)(path), { recursive: true });
|
|
24935
25222
|
await (0,promises_namespaceObject.writeFile)(path, contents, "utf8");
|
|
24936
25223
|
}
|
|
25224
|
+
async function writeOutput(path, contents) {
|
|
25225
|
+
await (0,promises_namespaceObject.mkdir)((0,external_node_path_.dirname)(path), { recursive: true });
|
|
25226
|
+
await (0,promises_namespaceObject.writeFile)(path, contents, "utf8");
|
|
25227
|
+
}
|
|
24937
25228
|
async function pathExists(path) {
|
|
24938
25229
|
try {
|
|
24939
25230
|
await (0,promises_namespaceObject.access)(path);
|
|
@@ -24993,6 +25284,18 @@ comment:
|
|
|
24993
25284
|
# flagNewPackages: true
|
|
24994
25285
|
# flagMajorUpgrades: true
|
|
24995
25286
|
# flagLifecycleScripts: true
|
|
25287
|
+
#
|
|
25288
|
+
# evidence:
|
|
25289
|
+
# contracts:
|
|
25290
|
+
# - id: ui-screenshot
|
|
25291
|
+
# title: UI changes need screenshots
|
|
25292
|
+
# paths:
|
|
25293
|
+
# - "src/components/**"
|
|
25294
|
+
# - "app/**"
|
|
25295
|
+
# requires:
|
|
25296
|
+
# - screenshot
|
|
25297
|
+
# - verification
|
|
25298
|
+
# severity: medium
|
|
24996
25299
|
`;
|
|
24997
25300
|
}
|
|
24998
25301
|
function renderWorkflowTemplate(failOn) {
|
|
@@ -25011,7 +25314,7 @@ jobs:
|
|
|
25011
25314
|
runs-on: ubuntu-latest
|
|
25012
25315
|
steps:
|
|
25013
25316
|
- uses: actions/checkout@v4
|
|
25014
|
-
- uses: linsk27/proof-pr@v0.1.
|
|
25317
|
+
- uses: linsk27/proof-pr@v0.1.7
|
|
25015
25318
|
with:
|
|
25016
25319
|
fail-on: ${failOn}
|
|
25017
25320
|
comment: "true"
|
|
@@ -25027,12 +25330,168 @@ function renderOutput(result, format, locale) {
|
|
|
25027
25330
|
}
|
|
25028
25331
|
return renderMarkdownReport(result, locale);
|
|
25029
25332
|
}
|
|
25333
|
+
async function runBenchmarks(casesDir) {
|
|
25334
|
+
const root = (0,external_node_path_.resolve)(casesDir);
|
|
25335
|
+
const entries = await (0,promises_namespaceObject.readdir)(root, { withFileTypes: true });
|
|
25336
|
+
const caseFiles = entries
|
|
25337
|
+
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
|
|
25338
|
+
.map((entry) => (0,external_node_path_.resolve)(root, entry.name))
|
|
25339
|
+
.sort();
|
|
25340
|
+
const results = [];
|
|
25341
|
+
for (const caseFile of caseFiles) {
|
|
25342
|
+
const testCase = JSON.parse(await (0,promises_namespaceObject.readFile)(caseFile, "utf8"));
|
|
25343
|
+
const diffText = await (0,promises_namespaceObject.readFile)((0,external_node_path_.resolve)((0,external_node_path_.dirname)(caseFile), testCase.diffFile), "utf8");
|
|
25344
|
+
const result = scanDiff(diffText, {
|
|
25345
|
+
config: testCase.config,
|
|
25346
|
+
pullRequest: testCase.pullRequest
|
|
25347
|
+
});
|
|
25348
|
+
const actualFindings = result.findings.map((finding) => finding.ruleId);
|
|
25349
|
+
const failures = [];
|
|
25350
|
+
if (testCase.expect.risk && result.risk !== testCase.expect.risk) {
|
|
25351
|
+
failures.push(`expected risk ${testCase.expect.risk}, got ${result.risk}`);
|
|
25352
|
+
}
|
|
25353
|
+
if (testCase.expect.reviewDecision && result.reviewDecision !== testCase.expect.reviewDecision) {
|
|
25354
|
+
failures.push(`expected review decision ${testCase.expect.reviewDecision}, got ${result.reviewDecision}`);
|
|
25355
|
+
}
|
|
25356
|
+
for (const expectedFinding of testCase.expect.findings ?? []) {
|
|
25357
|
+
if (!matchesFindingExpectation(actualFindings, expectedFinding)) {
|
|
25358
|
+
failures.push(`expected finding ${expectedFinding}`);
|
|
25359
|
+
}
|
|
25360
|
+
}
|
|
25361
|
+
for (const absentFinding of testCase.expect.absentFindings ?? []) {
|
|
25362
|
+
if (matchesFindingExpectation(actualFindings, absentFinding)) {
|
|
25363
|
+
failures.push(`unexpected finding ${absentFinding}`);
|
|
25364
|
+
}
|
|
25365
|
+
}
|
|
25366
|
+
results.push({
|
|
25367
|
+
id: testCase.id,
|
|
25368
|
+
title: testCase.title,
|
|
25369
|
+
category: testCase.category ?? "uncategorized",
|
|
25370
|
+
passed: failures.length === 0,
|
|
25371
|
+
failures,
|
|
25372
|
+
actual: {
|
|
25373
|
+
risk: result.risk,
|
|
25374
|
+
reviewDecision: result.reviewDecision,
|
|
25375
|
+
findings: actualFindings
|
|
25376
|
+
}
|
|
25377
|
+
});
|
|
25378
|
+
}
|
|
25379
|
+
return {
|
|
25380
|
+
summary: summarizeBenchmarkResults(results),
|
|
25381
|
+
results
|
|
25382
|
+
};
|
|
25383
|
+
}
|
|
25384
|
+
function summarizeBenchmarkResults(results) {
|
|
25385
|
+
const passed = results.filter((result) => result.passed).length;
|
|
25386
|
+
const categories = new Map();
|
|
25387
|
+
const findingCounts = new Map();
|
|
25388
|
+
for (const result of results) {
|
|
25389
|
+
const categoryResults = categories.get(result.category) ?? [];
|
|
25390
|
+
categoryResults.push(result);
|
|
25391
|
+
categories.set(result.category, categoryResults);
|
|
25392
|
+
for (const finding of new Set(result.actual.findings)) {
|
|
25393
|
+
findingCounts.set(finding, (findingCounts.get(finding) ?? 0) + 1);
|
|
25394
|
+
}
|
|
25395
|
+
}
|
|
25396
|
+
return {
|
|
25397
|
+
total: results.length,
|
|
25398
|
+
passed,
|
|
25399
|
+
failed: results.length - passed,
|
|
25400
|
+
passRate: ratio(passed, results.length),
|
|
25401
|
+
categories: [...categories.entries()]
|
|
25402
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
25403
|
+
.map(([category, items]) => {
|
|
25404
|
+
const categoryPassed = items.filter((item) => item.passed).length;
|
|
25405
|
+
return {
|
|
25406
|
+
category,
|
|
25407
|
+
total: items.length,
|
|
25408
|
+
passed: categoryPassed,
|
|
25409
|
+
failed: items.length - categoryPassed,
|
|
25410
|
+
passRate: ratio(categoryPassed, items.length)
|
|
25411
|
+
};
|
|
25412
|
+
}),
|
|
25413
|
+
findingCounts: [...findingCounts.entries()]
|
|
25414
|
+
.sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
|
|
25415
|
+
.map(([ruleId, count]) => ({ ruleId, count }))
|
|
25416
|
+
};
|
|
25417
|
+
}
|
|
25418
|
+
function renderBenchmarkText(report) {
|
|
25419
|
+
const lines = [
|
|
25420
|
+
"ProofPR benchmark",
|
|
25421
|
+
"",
|
|
25422
|
+
`Summary: ${report.summary.passed}/${report.summary.total} passed (${formatPercent(report.summary.passRate)})`,
|
|
25423
|
+
""
|
|
25424
|
+
];
|
|
25425
|
+
lines.push("Categories:");
|
|
25426
|
+
for (const category of report.summary.categories) {
|
|
25427
|
+
lines.push(`- ${category.category}: ${category.passed}/${category.total} passed (${formatPercent(category.passRate)})`);
|
|
25428
|
+
}
|
|
25429
|
+
if (report.summary.findingCounts.length > 0) {
|
|
25430
|
+
lines.push("", "Finding coverage:");
|
|
25431
|
+
for (const item of report.summary.findingCounts) {
|
|
25432
|
+
lines.push(`- ${item.ruleId}: ${item.count}`);
|
|
25433
|
+
}
|
|
25434
|
+
}
|
|
25435
|
+
lines.push("");
|
|
25436
|
+
for (const result of report.results) {
|
|
25437
|
+
lines.push(`${result.passed ? "PASS" : "FAIL"} ${result.id}${result.title ? ` - ${result.title}` : ""}`);
|
|
25438
|
+
for (const failure of result.failures) {
|
|
25439
|
+
lines.push(` - ${failure}`);
|
|
25440
|
+
}
|
|
25441
|
+
}
|
|
25442
|
+
lines.push("");
|
|
25443
|
+
return lines.join("\n");
|
|
25444
|
+
}
|
|
25445
|
+
function renderBenchmarkMarkdown(report) {
|
|
25446
|
+
const lines = [
|
|
25447
|
+
"# ProofPR Benchmark",
|
|
25448
|
+
"",
|
|
25449
|
+
`**Summary:** ${report.summary.passed}/${report.summary.total} passed (${formatPercent(report.summary.passRate)})`,
|
|
25450
|
+
"",
|
|
25451
|
+
"## Categories",
|
|
25452
|
+
"",
|
|
25453
|
+
"| Category | Passed | Total | Pass rate |",
|
|
25454
|
+
"| --- | ---: | ---: | ---: |"
|
|
25455
|
+
];
|
|
25456
|
+
for (const category of report.summary.categories) {
|
|
25457
|
+
lines.push(`| ${category.category} | ${category.passed} | ${category.total} | ${formatPercent(category.passRate)} |`);
|
|
25458
|
+
}
|
|
25459
|
+
lines.push("", "## Finding Coverage", "", "| Rule | Cases |", "| --- | ---: |");
|
|
25460
|
+
for (const item of report.summary.findingCounts) {
|
|
25461
|
+
lines.push(`| \`${item.ruleId}\` | ${item.count} |`);
|
|
25462
|
+
}
|
|
25463
|
+
lines.push("", "## Cases", "", "| Result | Case | Category | Actual risk | Gate |", "| --- | --- | --- | --- | --- |");
|
|
25464
|
+
for (const result of report.results) {
|
|
25465
|
+
lines.push(`| ${result.passed ? "PASS" : "FAIL"} | \`${result.id}\` | ${result.category} | ${result.actual.risk} | ${result.actual.reviewDecision} |`);
|
|
25466
|
+
}
|
|
25467
|
+
lines.push("");
|
|
25468
|
+
return lines.join("\n");
|
|
25469
|
+
}
|
|
25470
|
+
function ratio(value, total) {
|
|
25471
|
+
return total === 0 ? 0 : value / total;
|
|
25472
|
+
}
|
|
25473
|
+
function formatPercent(value) {
|
|
25474
|
+
return `${Math.round(value * 100)}%`;
|
|
25475
|
+
}
|
|
25476
|
+
function matchesFindingExpectation(actualFindings, expected) {
|
|
25477
|
+
if (expected.endsWith("*")) {
|
|
25478
|
+
const prefix = expected.slice(0, -1);
|
|
25479
|
+
return actualFindings.some((finding) => finding.startsWith(prefix));
|
|
25480
|
+
}
|
|
25481
|
+
return actualFindings.includes(expected);
|
|
25482
|
+
}
|
|
25030
25483
|
function parseFormat(value) {
|
|
25031
25484
|
if (value === "json" || value === "markdown" || value === "sarif") {
|
|
25032
25485
|
return value;
|
|
25033
25486
|
}
|
|
25034
25487
|
throw new InvalidArgumentError("format must be one of: markdown, json, sarif");
|
|
25035
25488
|
}
|
|
25489
|
+
function parseBenchmarkFormat(value) {
|
|
25490
|
+
if (value === "text" || value === "json" || value === "markdown") {
|
|
25491
|
+
return value;
|
|
25492
|
+
}
|
|
25493
|
+
throw new InvalidArgumentError("benchmark format must be one of: text, markdown, json");
|
|
25494
|
+
}
|
|
25036
25495
|
function parseFailLevel(value) {
|
|
25037
25496
|
if (value === "low" || value === "medium" || value === "high" || value === "never") {
|
|
25038
25497
|
return value;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EACL,UAAU,EACV,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EAClB,QAAQ,EAET,MAAM,gBAAgB,CAAC;AAExB,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAwB1C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,sFAAsF,CAAC;KACnG,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACpC,WAAW,CAAC,6CAA6C,CAAC;KAC1D,MAAM,CAAC,cAAc,EAAE,yDAAyD,CAAC;KACjF,MAAM,CAAC,cAAc,EAAE,gCAAgC,EAAE,MAAM,CAAC;KAChE,MAAM,CAAC,oBAAoB,EAAE,8DAA8D,CAAC;KAC5F,MAAM,CAAC,oBAAoB,EAAE,8CAA8C,CAAC;KAC5E,MAAM,CAAC,kBAAkB,EAAE,6CAA6C,CAAC;KACzE,MAAM,CAAC,uBAAuB,EAAE,gDAAgD,CAAC;KACjF,MAAM,CAAC,iBAAiB,EAAE,uBAAuB,EAAE,cAAc,CAAC;KAClE,MAAM,CAAC,mBAAmB,EAAE,0CAA0C,EAAE,WAAW,EAAE,UAAU,CAAC;KAChG,MAAM,CAAC,mBAAmB,EAAE,8DAA8D,EAAE,cAAc,EAAE,OAAO,CAAC;KACpH,MAAM,CAAC,KAAK,EAAE,OAA2B,EAAE,EAAE;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ;QAC/B,CAAC,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1C,CAAC,CAAC,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAElD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,WAAW,GACf,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;QACnD,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE;QAC1C,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC;IAEpC,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,sBAAsB,EAAE,+CAA+C,EAAE,cAAc,CAAC;KAC/F,MAAM,CACL,wBAAwB,EACxB,4CAA4C,EAC5C,+BAA+B,CAChC;KACA,MAAM,CAAC,mBAAmB,EAAE,0DAA0D,EAAE,cAAc,EAAE,MAAM,CAAC;KAC/G,MAAM,CAAC,SAAS,EAAE,2BAA2B,EAAE,KAAK,CAAC;KACrD,MAAM,CAAC,KAAK,EAAE,OAA2B,EAAE,EAAE;IAC5C,MAAM,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,oBAAoB,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAChF,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,EAAE,sBAAsB,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAClG,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,2BAA2B,OAAO,CAAC,UAAU,OAAO,OAAO,CAAC,YAAY,IAAI,CAC7E,CAAC;AACJ,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACxD,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,OAAO,IAAI,CAAC,CAAC;IACrD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC;AAEH,KAAK,UAAU,WAAW,CAAC,IAAwB,EAAE,IAAY;IAC/D,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,eAAe,EAAE,aAAa,CAAC,CAAC;IAEtD,IAAI,IAAI,EAAE,CAAC;QACT,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,IAAI,EAAE,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC;IACrF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,OAA2B;IAC5D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC;AACxB,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,QAAgB,EAAE,KAAc;IAC1E,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,gDAAgD,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB;IAC3B,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiCR,CAAC;AACF,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAiB;IAC/C,OAAO;;;;;;;;;;;;;;;;;qBAiBY,MAAM;;CAE1B,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,MAAmC,EAAE,MAAoB;IAC7E,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACvB,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,oBAAoB,CAAC,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QAClE,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,IAAI,oBAAoB,CAAC,8CAA8C,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACnF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,IAAI,oBAAoB,CAAC,kDAAkD,CAAC,CAAC;AACrF,CAAC"}
|