testguard-cli 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. testguard_cli-0.1.0/.gitattributes +1 -0
  2. testguard_cli-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md +32 -0
  3. testguard_cli-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md +19 -0
  4. testguard_cli-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +18 -0
  5. testguard_cli-0.1.0/.github/dependabot.yml +25 -0
  6. testguard_cli-0.1.0/.github/scripts/sync-release-version.mjs +37 -0
  7. testguard_cli-0.1.0/.github/workflows/auto-merge.yml +68 -0
  8. testguard_cli-0.1.0/.github/workflows/ci.yml +122 -0
  9. testguard_cli-0.1.0/.github/workflows/release.yml +177 -0
  10. testguard_cli-0.1.0/.github/workflows/scheduled-release.yml +121 -0
  11. testguard_cli-0.1.0/.github/workflows/supply-chain.yml +24 -0
  12. testguard_cli-0.1.0/.gitignore +18 -0
  13. testguard_cli-0.1.0/.npmignore +22 -0
  14. testguard_cli-0.1.0/.npmrc +7 -0
  15. testguard_cli-0.1.0/.pre-commit-hooks.yaml +34 -0
  16. testguard_cli-0.1.0/AGENTS.md +53 -0
  17. testguard_cli-0.1.0/CHANGELOG.md +39 -0
  18. testguard_cli-0.1.0/CLAUDE.md +6 -0
  19. testguard_cli-0.1.0/CODE_OF_CONDUCT.md +14 -0
  20. testguard_cli-0.1.0/CONTRIBUTING.md +84 -0
  21. testguard_cli-0.1.0/LICENSE +21 -0
  22. testguard_cli-0.1.0/PKG-INFO +158 -0
  23. testguard_cli-0.1.0/PRIVACY.md +19 -0
  24. testguard_cli-0.1.0/README.md +134 -0
  25. testguard_cli-0.1.0/SECURITY.md +52 -0
  26. testguard_cli-0.1.0/SUPPORT.md +33 -0
  27. testguard_cli-0.1.0/action.yml +72 -0
  28. testguard_cli-0.1.0/bench/README.md +46 -0
  29. testguard_cli-0.1.0/cli/testguard.mjs +6 -0
  30. testguard_cli-0.1.0/fixtures/known-answer/.gitignore +3 -0
  31. testguard_cli-0.1.0/fixtures/known-answer/README.md +48 -0
  32. testguard_cli-0.1.0/fixtures/known-answer/expected.json +15 -0
  33. testguard_cli-0.1.0/fixtures/known-answer/package.json +10 -0
  34. testguard_cli-0.1.0/fixtures/known-answer/src/export.mjs +7 -0
  35. testguard_cli-0.1.0/fixtures/known-answer/src/redact.mjs +63 -0
  36. testguard_cli-0.1.0/fixtures/known-answer/test/flaky.test.mjs +17 -0
  37. testguard_cli-0.1.0/fixtures/known-answer/test/redact.test.mjs +39 -0
  38. testguard_cli-0.1.0/fixtures/known-answer/testguard.claims.json +176 -0
  39. testguard_cli-0.1.0/fixtures/known-answer/vitest.config.mjs +9 -0
  40. testguard_cli-0.1.0/package-lock.json +1231 -0
  41. testguard_cli-0.1.0/package.json +66 -0
  42. testguard_cli-0.1.0/packaging/homebrew/testguard.rb +30 -0
  43. testguard_cli-0.1.0/pyproject.toml +41 -0
  44. testguard_cli-0.1.0/spec/GATE-SEMANTICS.md +89 -0
  45. testguard_cli-0.1.0/spec/README.md +54 -0
  46. testguard_cli-0.1.0/spec/conformance/examples/baseline.json +13 -0
  47. testguard_cli-0.1.0/spec/conformance/examples/brief.json +48 -0
  48. testguard_cli-0.1.0/spec/conformance/examples/calibration.json +44 -0
  49. testguard_cli-0.1.0/spec/conformance/examples/claims.json +85 -0
  50. testguard_cli-0.1.0/spec/conformance/examples/evidence.json +528 -0
  51. testguard_cli-0.1.0/spec/conformance/examples/ignore.json +20 -0
  52. testguard_cli-0.1.0/spec/conformance/invalid/baseline.bad-fingerprint-key.json +11 -0
  53. testguard_cli-0.1.0/spec/conformance/invalid/baseline.zero-count.json +11 -0
  54. testguard_cli-0.1.0/spec/conformance/invalid/brief.text-without-heading.json +48 -0
  55. testguard_cli-0.1.0/spec/conformance/invalid/brief.unknown-verdict-key.json +49 -0
  56. testguard_cli-0.1.0/spec/conformance/invalid/calibration.p-outside-ci.json +44 -0
  57. testguard_cli-0.1.0/spec/conformance/invalid/calibration.positives-exceed-n.json +44 -0
  58. testguard_cli-0.1.0/spec/conformance/invalid/claims.absolute-path.json +85 -0
  59. testguard_cli-0.1.0/spec/conformance/invalid/claims.duplicate-fault-id.json +85 -0
  60. testguard_cli-0.1.0/spec/conformance/invalid/claims.missing-provenance.json +80 -0
  61. testguard_cli-0.1.0/spec/conformance/invalid/claims.noop-fault.json +85 -0
  62. testguard_cli-0.1.0/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +86 -0
  63. testguard_cli-0.1.0/spec/conformance/invalid/claims.parent-traversal.json +85 -0
  64. testguard_cli-0.1.0/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +452 -0
  65. testguard_cli-0.1.0/spec/conformance/invalid/evidence.killed-by-non-assertion.json +421 -0
  66. testguard_cli-0.1.0/spec/conformance/invalid/evidence.killed-on-red-baseline.json +421 -0
  67. testguard_cli-0.1.0/spec/conformance/invalid/evidence.killed-without-n-runs.json +411 -0
  68. testguard_cli-0.1.0/spec/conformance/invalid/evidence.rank-extra-property.json +422 -0
  69. testguard_cli-0.1.0/spec/conformance/invalid/evidence.unknown-verdict.json +421 -0
  70. testguard_cli-0.1.0/spec/conformance/invalid/evidence.unverifiable-without-reason.json +420 -0
  71. testguard_cli-0.1.0/spec/conformance/invalid/evidence.wrong-fingerprint.json +421 -0
  72. testguard_cli-0.1.0/spec/conformance/invalid/ignore.missing-reason.json +9 -0
  73. testguard_cli-0.1.0/spec/conformance/invalid/ignore.short-reason.json +10 -0
  74. testguard_cli-0.1.0/spec/conformance/schemas.test.mjs +58 -0
  75. testguard_cli-0.1.0/spec/lib/fingerprint.mjs +19 -0
  76. testguard_cli-0.1.0/spec/lib/validate.mjs +119 -0
  77. testguard_cli-0.1.0/spec/schemas/baseline.schema.json +21 -0
  78. testguard_cli-0.1.0/spec/schemas/brief.schema.json +65 -0
  79. testguard_cli-0.1.0/spec/schemas/calibration.schema.json +54 -0
  80. testguard_cli-0.1.0/spec/schemas/claims.schema.json +103 -0
  81. testguard_cli-0.1.0/spec/schemas/common.schema.json +94 -0
  82. testguard_cli-0.1.0/spec/schemas/evidence.schema.json +206 -0
  83. testguard_cli-0.1.0/spec/schemas/ignore.schema.json +35 -0
  84. testguard_cli-0.1.0/src/baseline/baseline.mjs +34 -0
  85. testguard_cli-0.1.0/src/brief/brief.mjs +87 -0
  86. testguard_cli-0.1.0/src/claims/annotations.mjs +59 -0
  87. testguard_cli-0.1.0/src/claims/load.mjs +23 -0
  88. testguard_cli-0.1.0/src/cli.mjs +105 -0
  89. testguard_cli-0.1.0/src/commands/baseline.mjs +21 -0
  90. testguard_cli-0.1.0/src/commands/brief.mjs +27 -0
  91. testguard_cli-0.1.0/src/commands/claims.mjs +27 -0
  92. testguard_cli-0.1.0/src/commands/probe.mjs +53 -0
  93. testguard_cli-0.1.0/src/evidence/writer.mjs +36 -0
  94. testguard_cli-0.1.0/src/git.mjs +32 -0
  95. testguard_cli-0.1.0/src/probe/classify.mjs +37 -0
  96. testguard_cli-0.1.0/src/probe/inject.mjs +86 -0
  97. testguard_cli-0.1.0/src/probe/probe.mjs +182 -0
  98. testguard_cli-0.1.0/src/probe/rank.mjs +51 -0
  99. testguard_cli-0.1.0/src/probe/runner-vitest.mjs +85 -0
  100. testguard_cli-0.1.0/src/probe/worktree.mjs +64 -0
  101. testguard_cli-0.1.0/src/render.mjs +28 -0
  102. testguard_cli-0.1.0/src/util/glob.mjs +56 -0
  103. testguard_cli-0.1.0/src/util/hash.mjs +5 -0
  104. testguard_cli-0.1.0/test/annotations.test.mjs +45 -0
  105. testguard_cli-0.1.0/test/baseline.test.mjs +47 -0
  106. testguard_cli-0.1.0/test/brief.test.mjs +45 -0
  107. testguard_cli-0.1.0/test/classify.test.mjs +58 -0
  108. testguard_cli-0.1.0/test/glob.test.mjs +31 -0
  109. testguard_cli-0.1.0/test/inject.test.mjs +46 -0
  110. testguard_cli-0.1.0/test/probe.fixture.test.mjs +144 -0
  111. testguard_cli-0.1.0/test/runner-vitest.test.mjs +49 -0
  112. testguard_cli-0.1.0/test/worktree.test.mjs +48 -0
  113. testguard_cli-0.1.0/test/writer.test.mjs +44 -0
  114. testguard_cli-0.1.0/testguard.claims.json +120 -0
  115. testguard_cli-0.1.0/testguard_cli/__init__.py +1 -0
  116. testguard_cli-0.1.0/testguard_cli/wrapper.py +91 -0
  117. testguard_cli-0.1.0/vitest.config.mjs +9 -0
@@ -0,0 +1 @@
1
+ graphify-out/graph.json merge=graphify
@@ -0,0 +1,32 @@
1
+ ---
2
+ name: 🐛 Bug report
3
+ about: Something TestGuard did that it should not have
4
+ title: '[BUG] '
5
+ labels: bug
6
+ assignees: ''
7
+ ---
8
+
9
+ ## What happened
10
+
11
+ <!-- The command you ran and its full output. -->
12
+
13
+ ## What you expected
14
+
15
+ ## The claim involved
16
+
17
+ ```json
18
+ <!-- The relevant entry from testguard.claims.json — redact anything private. -->
19
+ ```
20
+
21
+ ## Environment
22
+
23
+ - **TestGuard**: <!-- testguard --version -->
24
+ - **Node.js**: <!-- node --version -->
25
+ - **Test runner**: <!-- vitest x.y -->
26
+ - **OS**:
27
+
28
+ ## Evidence record (if relevant)
29
+
30
+ ```json
31
+ <!-- The record for that claim from .testguard/evidence.json — paths and hashes only, no source. -->
32
+ ```
@@ -0,0 +1,19 @@
1
+ ---
2
+ name: 💡 Feature request
3
+ about: Something TestGuard should do
4
+ title: '[FEAT] '
5
+ labels: enhancement
6
+ assignees: ''
7
+ ---
8
+
9
+ ## The claim you cannot verify today
10
+
11
+ <!-- What are you trying to prove about your tests, and what stops you? -->
12
+
13
+ ## Proposed behaviour
14
+
15
+ ## Does it change what the tool emits?
16
+
17
+ <!-- If yes, it is a spec change: which schema, which GATE-SEMANTICS rule? -->
18
+
19
+ ## Alternatives considered
@@ -0,0 +1,18 @@
1
+ # Pull request
2
+
3
+ ## What & why
4
+
5
+ <!-- One or two sentences. -->
6
+
7
+ ## Checklist
8
+
9
+ - [ ] `npm test` passes
10
+ - [ ] `npm run self:probe` exits 0 (every claim about TestGuard still killed)
11
+ - [ ] `CHANGELOG.md` has an entry under `[Unreleased]`
12
+ - [ ] If output changed: schema + `GATE-SEMANTICS.md` + validator + conformance case updated together
13
+ - [ ] If the fixture changed: each new expectation verified independently before `expected.json`
14
+ - [ ] No client-identifying material; neutral domains only
15
+
16
+ ## Output
17
+
18
+ <!-- For CLI-visible changes, paste before/after terminal output. -->
@@ -0,0 +1,25 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "npm"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "monthly"
7
+ groups:
8
+ all-npm:
9
+ patterns: ["*"]
10
+ # devDependencies only (ajv for the conformance suite, vitest as the test
11
+ # runner). The published package has zero runtime dependencies.
12
+ - package-ecosystem: "pip"
13
+ directory: "/"
14
+ schedule:
15
+ interval: "monthly"
16
+ groups:
17
+ all-pip:
18
+ patterns: ["*"]
19
+ - package-ecosystem: "github-actions"
20
+ directory: "/"
21
+ schedule:
22
+ interval: "monthly"
23
+ groups:
24
+ all-actions:
25
+ patterns: ["*"]
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * package.json is the single source of truth for the version. Every other
4
+ * surface that carries it is synced from there:
5
+ * - pyproject.toml `version = "x.y.z"`
6
+ * - action.yml default of the `version` input
7
+ * - packaging/homebrew/*.rb the tarball URL (sha256 is set after publish)
8
+ *
9
+ * `--check` verifies instead of writing (used by release.yml).
10
+ */
11
+ import { readFileSync, writeFileSync } from 'node:fs';
12
+ import { resolve } from 'node:path';
13
+
14
+ const root = resolve(process.argv[1], '..', '..', '..');
15
+ const check = process.argv.includes('--check');
16
+ const version = JSON.parse(readFileSync(resolve(root, 'package.json'), 'utf8')).version;
17
+ if (!/^\d+\.\d+\.\d+$/.test(version)) throw new Error(`package.json version is not a stable semver: ${version}`);
18
+
19
+ const surfaces = [
20
+ ['pyproject.toml', /^version = "[^"]+"/m, `version = "${version}"`],
21
+ ['action.yml', /(\n version:\n description: [^\n]*\n required: false\n default: ')[^']*(')/, `$1${version}$2`],
22
+ ['packaging/homebrew/testguard.rb', /testguard-cli-\d+\.\d+\.\d+\.tgz/g, `testguard-cli-${version}.tgz`],
23
+ ];
24
+
25
+ let drift = 0;
26
+ for (const [rel, pattern, replacement] of surfaces) {
27
+ const path = resolve(root, rel);
28
+ const before = readFileSync(path, 'utf8');
29
+ if (!pattern.test(before)) throw new Error(`${rel}: version pattern not found`);
30
+ const after = before.replace(pattern, replacement);
31
+ if (after === before) continue;
32
+ drift++;
33
+ if (check) console.error(`${rel}: out of sync with package.json ${version}`);
34
+ else { writeFileSync(path, after); console.log(`${rel}: synced to ${version}`); }
35
+ }
36
+ if (check && drift) process.exit(1);
37
+ if (check) console.log(`all version surfaces at ${version}`);
@@ -0,0 +1,68 @@
1
+ name: Auto-merge (green bot PRs)
2
+
3
+ # After a successful CI run on a PR, merge it if — and only if — it is a
4
+ # non-major dependabot bump or an automated release PR touching release
5
+ # surfaces only. Everything else is left for a human. The CI run that
6
+ # triggered this is re-verified job by job; a green badge is not trusted.
7
+
8
+ on:
9
+ workflow_run:
10
+ workflows: ["TestGuard CI"]
11
+ types: [completed]
12
+
13
+ permissions:
14
+ contents: write
15
+ pull-requests: write
16
+ checks: read
17
+
18
+ jobs:
19
+ auto-merge:
20
+ if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
21
+ runs-on: ubuntu-latest
22
+ steps:
23
+ - name: Evaluate and merge if safe
24
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
25
+ with:
26
+ script: |
27
+ const { owner, repo } = context.repo;
28
+ const run = context.payload.workflow_run;
29
+ const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ owner, repo, commit_sha: run.head_sha });
30
+ const pr = prs.find((p) => p.state === 'open');
31
+ if (!pr) { core.info('No open PR for this SHA.'); return; }
32
+ const num = pr.number, author = pr.user.login;
33
+ const hold = async (why) => {
34
+ try { await github.rest.issues.addLabels({ owner, repo, issue_number: num, labels: ['needs-review'] }); } catch {}
35
+ await github.rest.issues.createComment({ owner, repo, issue_number: num, body: `⏸️ **Held for human review** — ${why}\n\nCI is green, but this falls outside the policy in \`.github/workflows/auto-merge.yml\`.` });
36
+ core.notice(`#${num} held: ${why}`);
37
+ };
38
+ if (run.head_sha !== pr.head.sha || run.head_branch !== pr.head.ref) { core.info('Run does not match current PR head.'); return; }
39
+
40
+ // Re-verify the exact run: every job succeeded and all three Node legs ran.
41
+ const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({ owner, repo, run_id: run.id, filter: 'latest', per_page: 100 });
42
+ const legs = jobs.jobs.filter((j) => /^test \((20|22|24)\)$/.test(j.name));
43
+ if (legs.length !== 3 || jobs.jobs.some((j) => j.conclusion !== 'success' && j.conclusion !== 'skipped')) {
44
+ core.info(`Not merging — jobs: ${jobs.jobs.map((j) => `${j.name}=${j.conclusion}`).join(', ')}`); return;
45
+ }
46
+
47
+ const { data: files } = await github.rest.pulls.listFiles({ owner, repo, pull_number: num, per_page: 300 });
48
+ const paths = files.map((f) => f.filename);
49
+ const RELEASE_SURFACES = new Set(['package.json', 'package-lock.json', 'pyproject.toml', 'CHANGELOG.md', 'action.yml', 'packaging/homebrew/testguard.rb']);
50
+
51
+ if (author === 'dependabot[bot]') {
52
+ const m = /from (\d+)\.(\d+)\.\S+ to (\d+)\.(\d+)\./.exec(pr.title || '');
53
+ if (!m) { await hold('could not parse the version delta from the title.'); return; }
54
+ if (m[1] !== m[3]) { await hold(`this is a **major** bump; majors are never auto-merged.`); return; }
55
+ } else if (author === 'github-actions[bot]' && /^release: v\d+\.\d+\.\d+/.test(pr.title || '')) {
56
+ const stray = paths.filter((p) => !RELEASE_SURFACES.has(p));
57
+ if (stray.length) { await hold(`release PR touches non-release files:\n${stray.map((p) => `- \`${p}\``).join('\n')}`); return; }
58
+ } else {
59
+ core.info(`#${num} by ${author} — not a handled bot; leaving alone.`); return;
60
+ }
61
+
62
+ try {
63
+ const { data: merge } = await github.rest.pulls.merge({ owner, repo, pull_number: num, merge_method: 'squash' });
64
+ if (!merge.merged) throw new Error(merge.message || 'GitHub declined the merge');
65
+ await github.rest.issues.createComment({ owner, repo, issue_number: num, body: `🤖 **Auto-merged** — the exact CI run passed all three Node legs (20/22/24) within the runtime budget and satisfied \`.github/workflows/auto-merge.yml\`.` });
66
+ } catch (e) {
67
+ await hold(`auto-merge failed: \`${e.message}\``);
68
+ }
@@ -0,0 +1,122 @@
1
+ name: TestGuard CI
2
+
3
+ on:
4
+ workflow_dispatch: # manual recovery for pushes that don't auto-trigger (e.g. release PRs opened by GITHUB_TOKEN)
5
+ push:
6
+ branches: [main]
7
+ tags: ['v*'] # publish-check below runs on tags only
8
+ pull_request:
9
+ branches: [main]
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ jobs:
15
+ test:
16
+ runs-on: ubuntu-latest
17
+ strategy:
18
+ matrix:
19
+ # 20 is the floor (node:util.parseArgs). 24 ships npm 11 — keep it so
20
+ # `npm ci` exercises the npm that treats some config as fatal where
21
+ # npm 10 is silent. Do not drop 24.
22
+ node-version: [20, 22, 24]
23
+
24
+ steps:
25
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
26
+ with:
27
+ fetch-depth: 0 # probe builds a git worktree from HEAD; the self-probe below needs history
28
+
29
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
30
+ with:
31
+ node-version: ${{ matrix.node-version }}
32
+
33
+ - name: Install (devDependencies only; zero runtime deps)
34
+ run: npm ci
35
+
36
+ # ── Syntax-check EVERY source, not just what the entry point eagerly
37
+ # imports; a lazily loaded module's parse error would otherwise slip. ──
38
+ - name: Syntax-check all sources
39
+ run: find cli src spec -name '*.mjs' -print0 | xargs -0 -n1 node --check
40
+
41
+ - name: Python wrapper imports (PyPI surface)
42
+ run: python3 -c "import testguard_cli.wrapper; print('testguard_cli.wrapper OK')"
43
+
44
+ # ── Test suite with a RUNTIME BUDGET. A test that walks the wrong tree
45
+ # or probes far more than it should still passes — it just takes
46
+ # forever. Only a duration gate catches that class. Normal: ~15s
47
+ # locally, ~30–60s here. Budget is loose on purpose. ──
48
+ - name: Run tests (with runtime budget)
49
+ shell: bash
50
+ env:
51
+ TEST_BUDGET_MS: 240000
52
+ run: |
53
+ START=$(date +%s%3N)
54
+ npm test
55
+ DURATION=$(( $(date +%s%3N) - START ))
56
+ echo "Test suite duration: ${DURATION}ms (budget: ${TEST_BUDGET_MS}ms)"
57
+ if [ "$DURATION" -gt "$TEST_BUDGET_MS" ]; then
58
+ echo "::error::Test suite took ${DURATION}ms, over the ${TEST_BUDGET_MS}ms budget — something is probing far more than it should."
59
+ exit 1
60
+ fi
61
+
62
+ # ── Dogfood: the real binary against the known-answer fixture, as a
63
+ # user would run it (the suite drives main() in-process). ──
64
+ - name: CLI version
65
+ run: node cli/testguard.mjs --version
66
+
67
+ - name: claims — fixture lists cleanly, no drift
68
+ run: node cli/testguard.mjs claims fixtures/known-answer
69
+
70
+ - name: probe → baseline → probe → brief on the fixture (through the bin)
71
+ shell: bash
72
+ run: |
73
+ set -o pipefail
74
+ S=$(mktemp -d)
75
+ cp -R fixtures/known-answer/. "$S/"
76
+ rm -rf "$S/node_modules" "$S/.flake-counter" "$S/.testguard"
77
+ ln -s "$GITHUB_WORKSPACE/node_modules" "$S/node_modules"
78
+ git -C "$S" init -q
79
+ git -C "$S" -c user.email=ci@example.invalid -c user.name=ci add -A
80
+ git -C "$S" -c user.email=ci@example.invalid -c user.name=ci commit -qm fixture
81
+ B="$GITHUB_WORKSPACE/cli/testguard.mjs"
82
+ status=0; node "$B" probe "$S" --budget 60000 --quiet || status=$?
83
+ [ "$status" -eq 1 ] || { echo "::error::expected exit 1 (unproven claims), got $status"; exit 1; }
84
+ node "$B" baseline "$S"
85
+ status=0; node "$B" probe "$S" --budget 60000 --quiet || status=$?
86
+ [ "$status" -eq 0 ] || { echo "::error::expected exit 0 after baseline, got $status"; exit 1; }
87
+ node "$B" brief "$S" --text | head -12
88
+ # every verdict must appear exactly once per expected.json
89
+ node --input-type=module -e "
90
+ import { readFileSync } from 'node:fs';
91
+ const ev = JSON.parse(readFileSync('$S/.testguard/evidence.json','utf8'));
92
+ const exp = JSON.parse(readFileSync('fixtures/known-answer/expected.json','utf8')).expected;
93
+ const bad = ev.records.filter(r => exp[r.claim.id+'/'+r.subject.id].verdict !== r.verdict);
94
+ if (bad.length) { console.error('MISMATCH', bad.map(r => r.claim.id+'/'+r.subject.id+':'+r.verdict)); process.exit(1); }
95
+ console.log('fixture verdicts match expected.json:', ev.records.length);
96
+ "
97
+
98
+ # ── Self-verification: TestGuard is governed by the standard it defines.
99
+ # testguard.claims.json states invariants of the tool itself; every
100
+ # fault must be killed by the tool's own tests. Exit 0 = all defended. ──
101
+ - name: probe TestGuard's own claims
102
+ run: node cli/testguard.mjs probe . --budget 180000 --out "$RUNNER_TEMP/self-evidence.json"
103
+
104
+ - name: Upload self-probe evidence
105
+ if: always()
106
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
107
+ with:
108
+ name: testguard-self-evidence-node-${{ matrix.node-version }}
109
+ path: ${{ runner.temp }}/self-evidence.json
110
+ if-no-files-found: warn
111
+
112
+ # ── npm publish dry-run on tags ──
113
+ publish-check:
114
+ runs-on: ubuntu-latest
115
+ if: startsWith(github.ref, 'refs/tags/v')
116
+ needs: test
117
+ steps:
118
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
119
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
120
+ with:
121
+ node-version: 24
122
+ - run: npm publish --dry-run
@@ -0,0 +1,177 @@
1
+ name: Auto Release — Tag, GitHub Release, npm, PyPI
2
+
3
+ # TAG-DRIVEN and idempotent: a version needs releasing iff no tag v<version>
4
+ # exists. Re-runs never double-publish. No long-lived secrets anywhere —
5
+ # npm and PyPI both authenticate with OIDC Trusted Publishing.
6
+ #
7
+ # ONE-TIME SETUP (maintainer, cannot be automated):
8
+ # npm: the first `npm publish` of testguard-cli is manual (Trusted Publishing
9
+ # is configured on an existing package). Then on npmjs.com →
10
+ # testguard-cli → Settings → Trusted Publisher: raccioly/testguard,
11
+ # workflow release.yml. Every later release is OIDC.
12
+ # PyPI: pypi.org → Your account → Publishing → add a PENDING publisher for
13
+ # project testguard-cli: owner raccioly, repo testguard, workflow
14
+ # release.yml, environment pypi. The first release creates the project.
15
+
16
+ on:
17
+ workflow_dispatch: # manual recovery
18
+ schedule:
19
+ - cron: '17 * * * *' # hourly sweep: publishes any merged version bump the push event missed
20
+ push:
21
+ branches: [main]
22
+
23
+ permissions:
24
+ contents: write # tags + releases
25
+
26
+ concurrency:
27
+ group: testguard-release
28
+ cancel-in-progress: false
29
+
30
+ jobs:
31
+ detect-version:
32
+ runs-on: ubuntu-latest
33
+ outputs:
34
+ version: ${{ steps.check.outputs.version }}
35
+ changed: ${{ steps.check.outputs.changed }}
36
+ tag_missing: ${{ steps.check.outputs.tag_missing }}
37
+ npm_missing: ${{ steps.check.outputs.npm_missing }}
38
+ pypi_missing: ${{ steps.check.outputs.pypi_missing }}
39
+ steps:
40
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
41
+ with:
42
+ fetch-depth: 1
43
+ - name: What does this version still need?
44
+ id: check
45
+ # Each surface is gated on ITS OWN registry, not on the tag. A tag
46
+ # whose npm or PyPI publish failed (e.g. Trusted Publishing not yet
47
+ # configured) is retried by simply re-running this workflow.
48
+ run: |
49
+ CURRENT=$(node -p "require('./package.json').version")
50
+ TAG="v${CURRENT}"
51
+ echo "version=$CURRENT" >> "$GITHUB_OUTPUT"
52
+ if git ls-remote --tags origin "refs/tags/$TAG" | grep -q "refs/tags/$TAG$"; then TAGM=false; else TAGM=true; fi
53
+ if npm view "testguard-cli@${CURRENT}" version >/dev/null 2>&1; then NPMM=false; else NPMM=true; fi
54
+ CODE=$(curl -s -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/testguard-cli/${CURRENT}/json")
55
+ if [ "$CODE" = "200" ]; then PYPIM=false; else PYPIM=true; fi
56
+ echo "tag_missing=$TAGM" >> "$GITHUB_OUTPUT"
57
+ echo "npm_missing=$NPMM" >> "$GITHUB_OUTPUT"
58
+ echo "pypi_missing=$PYPIM" >> "$GITHUB_OUTPUT"
59
+ if [ "$TAGM" = true ] || [ "$NPMM" = true ] || [ "$PYPIM" = true ]; then
60
+ echo "changed=true" >> "$GITHUB_OUTPUT"
61
+ echo "✅ $TAG — tag missing: $TAGM · npm missing: $NPMM · PyPI missing: $PYPIM"
62
+ else
63
+ echo "changed=false" >> "$GITHUB_OUTPUT"
64
+ echo "⏭️ $TAG is fully released everywhere — nothing to do"
65
+ fi
66
+
67
+ test:
68
+ needs: detect-version
69
+ if: needs.detect-version.outputs.changed == 'true'
70
+ runs-on: ubuntu-latest
71
+ steps:
72
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
73
+ with:
74
+ fetch-depth: 0
75
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
76
+ with:
77
+ node-version: 24
78
+ - run: npm ci
79
+ - run: npm test
80
+ - name: Self-probe must be fully defended before anything ships
81
+ run: node cli/testguard.mjs probe . --budget 180000 --out "$RUNNER_TEMP/self-evidence.json" --quiet
82
+ - name: Version surfaces are in sync
83
+ run: node .github/scripts/sync-release-version.mjs --check
84
+
85
+ create-release:
86
+ needs: [detect-version, test]
87
+ if: needs.detect-version.outputs.changed == 'true'
88
+ # Runs even when the tag exists: every step inside is idempotent.
89
+ runs-on: ubuntu-latest
90
+ outputs:
91
+ tag: v${{ needs.detect-version.outputs.version }}
92
+ steps:
93
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
94
+ with:
95
+ fetch-depth: 0
96
+ - name: Create and push tag
97
+ env:
98
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
99
+ run: |
100
+ TAG="v${{ needs.detect-version.outputs.version }}"
101
+ if git ls-remote --tags origin | grep -q "refs/tags/$TAG$"; then
102
+ echo "✅ Tag $TAG already exists on remote"
103
+ else
104
+ git config user.name "github-actions[bot]"
105
+ git config user.email "github-actions[bot]@users.noreply.github.com"
106
+ git tag -a "$TAG" -m "Release $TAG"
107
+ git push origin "$TAG"
108
+ fi
109
+ - name: Release notes from CHANGELOG
110
+ run: |
111
+ VERSION="${{ needs.detect-version.outputs.version }}"
112
+ NOTES=$(awk "/^## \\[${VERSION}\\]/{found=1; next} /^## \\[/{if(found) exit} found{print}" CHANGELOG.md)
113
+ [ -n "$NOTES" ] || NOTES="Release v${VERSION} — see CHANGELOG.md."
114
+ printf '%s\n' "$NOTES" > /tmp/release-notes.md
115
+ - name: Create GitHub Release
116
+ env:
117
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
118
+ run: |
119
+ TAG="v${{ needs.detect-version.outputs.version }}"
120
+ gh release view "$TAG" >/dev/null 2>&1 && { echo "✅ Release $TAG exists"; exit 0; }
121
+ gh release create "$TAG" --title "TestGuard $TAG" --notes-file /tmp/release-notes.md --latest
122
+
123
+ publish-npm:
124
+ needs: [detect-version, test, create-release]
125
+ if: needs.detect-version.outputs.npm_missing == 'true'
126
+ runs-on: ubuntu-latest
127
+ permissions:
128
+ contents: read
129
+ id-token: write # mints the OIDC token for npm Trusted Publishing
130
+ steps:
131
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
132
+ # NO registry-url on purpose: with it, setup-node writes an _authToken
133
+ # line that expands to an EMPTY token when NODE_AUTH_TOKEN is unset, and
134
+ # npm then sends that instead of attempting OIDC.
135
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
136
+ with:
137
+ node-version: 24
138
+ - run: npm install -g npm@11
139
+ - name: Assert Trusted Publishing preconditions
140
+ run: |
141
+ NPM_V=$(npm --version)
142
+ echo "node=$(node -p 'process.versions.node') npm=$NPM_V"
143
+ node -e '
144
+ const [maj, min] = process.versions.node.split(".").map(Number);
145
+ if (!(maj > 22 || (maj === 22 && min >= 14))) { console.error("::error::Node below the 22.14.0 floor npm Trusted Publishing requires."); process.exit(1); }
146
+ '
147
+ node -e '
148
+ const v = process.argv[1].split(".").map(Number);
149
+ if (!(v[0] > 11 || (v[0] === 11 && (v[1] > 5 || (v[1] === 5 && v[2] >= 1))))) { console.error("::error::npm below the 11.5.1 floor npm Trusted Publishing requires."); process.exit(1); }
150
+ ' "$NPM_V"
151
+ [ -n "$ACTIONS_ID_TOKEN_REQUEST_URL" ] || { echo "::error::no OIDC token endpoint — job lacks id-token: write"; exit 1; }
152
+ - name: Publish (provenance is automatic under Trusted Publishing)
153
+ run: npm publish --loglevel verbose
154
+
155
+ publish-pypi:
156
+ needs: [detect-version, test, create-release]
157
+ if: needs.detect-version.outputs.pypi_missing == 'true'
158
+ runs-on: ubuntu-latest
159
+ environment: pypi
160
+ permissions:
161
+ contents: read
162
+ id-token: write
163
+ steps:
164
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
165
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
166
+ with:
167
+ python-version: '3.12'
168
+ - name: Sync version into pyproject.toml
169
+ run: |
170
+ VERSION="${{ needs.detect-version.outputs.version }}"
171
+ sed -i "s/^version = \".*\"/version = \"$VERSION\"/" pyproject.toml
172
+ grep '^version' pyproject.toml
173
+ - run: pip install build
174
+ - run: python -m build
175
+ - name: Assert OIDC token is available
176
+ run: '[ -n "$ACTIONS_ID_TOKEN_REQUEST_URL" ] || { echo "::error::no OIDC token endpoint"; exit 1; }'
177
+ - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
@@ -0,0 +1,121 @@
1
+ name: Scheduled Release
2
+
3
+ # Weekly: if anything releasable merged since the last tag, bump the patch
4
+ # version, write the CHANGELOG entry, sync every version surface, verify the
5
+ # tree is green, and open a release PR with auto-merge armed. Once merged, the
6
+ # hourly sweep in release.yml tags and publishes.
7
+ #
8
+ # GITHUB_TOKEN-created PRs do not trigger `pull_request` workflows, so CI on
9
+ # the release PR must be started by the maintainer:
10
+ # gh workflow run ci.yml --ref release/vX.Y.Z
11
+ # After it goes green, auto-merge completes.
12
+
13
+ on:
14
+ schedule:
15
+ - cron: '0 9 * * 1' # Mondays 09:00 UTC
16
+ workflow_dispatch:
17
+ inputs:
18
+ bump:
19
+ description: 'Version bump'
20
+ type: choice
21
+ options: [patch, minor]
22
+ default: patch
23
+
24
+ permissions:
25
+ actions: write
26
+ contents: write
27
+ pull-requests: write
28
+
29
+ concurrency:
30
+ group: scheduled-release
31
+ cancel-in-progress: false
32
+
33
+ jobs:
34
+ cut-release:
35
+ runs-on: ubuntu-latest
36
+ steps:
37
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
38
+ with:
39
+ fetch-depth: 0
40
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
41
+ with:
42
+ node-version: 24
43
+
44
+ - name: Check for unreleased changes
45
+ id: check
46
+ env:
47
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
48
+ run: |
49
+ CURRENT=$(node -p "require('./package.json').version")
50
+ LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
51
+ echo "current=$CURRENT" >> "$GITHUB_OUTPUT"
52
+ if ! git ls-remote --tags origin "refs/tags/v$CURRENT" | grep -q "refs/tags/v$CURRENT$"; then
53
+ echo "v$CURRENT is not published — dispatching the idempotent release."
54
+ gh workflow run release.yml --ref main
55
+ echo "changed=false" >> "$GITHUB_OUTPUT"; exit 0
56
+ fi
57
+ if [ -z "$LAST_TAG" ]; then echo "changed=false" >> "$GITHUB_OUTPUT"; exit 0; fi
58
+ COMMITS=$(git log "${LAST_TAG}..HEAD" --oneline | grep -viE '^[0-9a-f]+ (release|chore\(release\)):' || true)
59
+ COUNT=$(echo "$COMMITS" | grep -c . || true)
60
+ echo "Last tag: $LAST_TAG — releasable commits since: $COUNT"
61
+ echo "$COMMITS"
62
+ if [ "$COUNT" -eq 0 ]; then
63
+ echo "changed=false" >> "$GITHUB_OUTPUT"
64
+ else
65
+ echo "changed=true" >> "$GITHUB_OUTPUT"
66
+ echo "last_tag=$LAST_TAG" >> "$GITHUB_OUTPUT"
67
+ echo "$COMMITS" > /tmp/commits.txt
68
+ fi
69
+
70
+ - name: Bump version and write changelog
71
+ if: steps.check.outputs.changed == 'true'
72
+ run: |
73
+ BUMP="${{ inputs.bump || 'patch' }}"
74
+ OLD="${{ steps.check.outputs.current }}"
75
+ NEW=$(node -e "const [a,b,c]='$OLD'.split('.').map(Number); console.log('$BUMP'==='minor' ? \`\${a}.\${b+1}.0\` : \`\${a}.\${b}.\${c+1}\`)")
76
+ echo "Bumping $OLD → $NEW"
77
+ node -e "const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.version='$NEW';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\n')"
78
+ npm install --package-lock-only --ignore-scripts
79
+ {
80
+ echo "## [$NEW] - $(date -u +%Y-%m-%d)"; echo
81
+ echo "Automated weekly release — everything merged since \`${{ steps.check.outputs.last_tag }}\`."; echo
82
+ echo "### Changed"; echo
83
+ sed -E 's/^[0-9a-f]+ /- /' /tmp/commits.txt; echo
84
+ } > /tmp/entry.md
85
+ awk '/^## \[Unreleased\]/ { print; print ""; while ((getline line < "/tmp/entry.md") > 0) print line; next } { print }' CHANGELOG.md > /tmp/CHANGELOG.new && mv /tmp/CHANGELOG.new CHANGELOG.md
86
+ node .github/scripts/sync-release-version.mjs
87
+ echo "NEW_VERSION=$NEW" >> "$GITHUB_ENV"
88
+
89
+ - name: Verify the tree is still green
90
+ if: steps.check.outputs.changed == 'true'
91
+ run: |
92
+ npm ci
93
+ npm test
94
+ node cli/testguard.mjs probe . --budget 180000 --quiet --out "$RUNNER_TEMP/self.json"
95
+
96
+ - name: Open release PR with auto-merge armed
97
+ if: steps.check.outputs.changed == 'true'
98
+ env:
99
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
100
+ run: |
101
+ BRANCH="release/v${NEW_VERSION}"
102
+ # Only release surfaces may change in a release PR.
103
+ CHANGED=$(git diff --name-only HEAD --)
104
+ for f in $CHANGED; do
105
+ case "$f" in package.json|package-lock.json|pyproject.toml|CHANGELOG.md|action.yml|packaging/homebrew/testguard.rb) ;;
106
+ *) echo "::error::unexpected file in release diff: $f"; exit 1;;
107
+ esac
108
+ done
109
+ if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then
110
+ echo "::error::branch $BRANCH already exists; refusing to overwrite"; exit 1
111
+ fi
112
+ git config user.name "github-actions[bot]"
113
+ git config user.email "github-actions[bot]@users.noreply.github.com"
114
+ git checkout -b "$BRANCH"
115
+ git add -A
116
+ git commit -m "release: v${NEW_VERSION} — automated weekly batch"
117
+ git push origin "$BRANCH"
118
+ printf 'Automated weekly release. Version bump + CHANGELOG entry + synced version surfaces. Merging tags and publishes to npm and PyPI via release.yml.\n\nCI does not start automatically on PRs opened by GITHUB_TOKEN. Run:\n\n gh workflow run ci.yml --ref %s\n\nAuto-merge is armed; it completes once the required checks pass.\n' "$BRANCH" > /tmp/pr-body.md
119
+ PR=$(gh pr create --base main --head "$BRANCH" --title "release: v${NEW_VERSION} — automated weekly batch" --body-file /tmp/pr-body.md)
120
+ gh pr merge --auto --squash "$PR"
121
+ echo "Release PR: $PR"
@@ -0,0 +1,24 @@
1
+ name: Supply-Chain Security
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ pull_request:
6
+ branches: [main]
7
+ schedule:
8
+ - cron: '30 12 * * 1' # Mondays 12:30 UTC
9
+ push:
10
+ branches: [main]
11
+
12
+ permissions:
13
+ actions: read
14
+ contents: read
15
+ security-events: write
16
+
17
+ jobs:
18
+ osv-scan-pr:
19
+ if: github.event_name == 'pull_request'
20
+ uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@6e4298ebc4db23e847df9b2e2de2939d6f066c67" # v2.5.1
21
+
22
+ osv-scan-scheduled:
23
+ if: github.event_name != 'pull_request'
24
+ uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@6e4298ebc4db23e847df9b2e2de2939d6f066c67" # v2.5.1