rungate 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/aditi.md +47 -0
- package/.claude/agents/marcus.md +50 -0
- package/.claude/agents/quinn.md +58 -0
- package/.claude/agents/rook.md +47 -0
- package/.claude/agents/serena.md +46 -0
- package/.claude/rungate.json +11 -0
- package/.fallowrc.json +28 -0
- package/.github/copilot-instructions.md +20 -0
- package/.rungate/conformity-findings.json +648 -0
- package/AGENTS.md +106 -0
- package/CODE-MAP.md +50 -0
- package/HARNESS.md +88 -0
- package/bun.lock +26 -0
- package/bunfig.toml +2 -0
- package/config/brief-policies.json +28 -0
- package/config/ceremony-profiles.json +212 -0
- package/config/consumer-cascade-map.json +24 -0
- package/config/doc-cascade-map.json +24 -0
- package/config/eval.json +84 -0
- package/config/profiles/account-plan.yaml +36 -0
- package/config/profiles/campaign.yaml +94 -0
- package/config/prove-evidence.schema.json +65 -0
- package/config/spec-policies.json +36 -0
- package/config/workflow-schema.json +281 -0
- package/gates/SCHEMA-GUIDE.md +121 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +274 -0
- package/gates/brief-assembler.test.ts +429 -0
- package/gates/brief-assembler.ts +334 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +249 -0
- package/gates/error-classifier.ts +89 -0
- package/gates/orchestrator.test.ts +950 -0
- package/gates/orchestrator.ts +569 -0
- package/gates/preload.ts +77 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prove.test.ts +164 -0
- package/gates/run-gate.ts +936 -0
- package/gates/schema-parity.test.ts +197 -0
- package/gates/schema.ts +362 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +121 -0
- package/gates/ship-orchestrator.test.ts +379 -0
- package/gates/ship-orchestrator.ts +363 -0
- package/gates/test-fixtures/broken-no-environments.json +60 -0
- package/gates/test-fixtures/valid-standard.json +54 -0
- package/gates/test-utils.ts +43 -0
- package/gates/witness.ts +220 -0
- package/gates/workflow.test.ts +1053 -0
- package/hooks/AgentBriefGuard.hook.ts +586 -0
- package/hooks/AgentVerdictCapture.hook.ts +134 -0
- package/hooks/AutoVerifyGate.hook.ts +76 -0
- package/hooks/GateEnforcement.hook.ts +273 -0
- package/hooks/IssueCloseGuard.hook.ts +116 -0
- package/hooks/MergeGuard.hook.ts +55 -0
- package/hooks/StaleTTLCleanup.hook.ts +133 -0
- package/hooks/VerifyPhaseLock.hook.ts +81 -0
- package/hooks/WorkflowStateGuard.hook.ts +37 -0
- package/hooks/lib/agentDetection.ts +70 -0
- package/hooks/lib/findWorkflow.ts +101 -0
- package/hooks/lib/parseStdin.ts +37 -0
- package/hooks/lib/paths.ts +111 -0
- package/hooks/lib/utils.ts +81 -0
- package/lib/conformity.ts +959 -0
- package/lib/paths.ts +23 -0
- package/lib/rungate-schema.ts +60 -0
- package/lib/signal-phrases.ts +7 -0
- package/package.json +20 -0
- package/prompts/ac-adversary.md +20 -0
- package/prompts/ac-format.md +34 -0
- package/prompts/aditi.md +29 -0
- package/prompts/blast-radius.md +29 -0
- package/prompts/coding-principles.md +34 -0
- package/prompts/container-rebuild.md +29 -0
- package/prompts/container-verify.md +28 -0
- package/prompts/discovery.md +27 -0
- package/prompts/environment.md +28 -0
- package/prompts/escalation-decision-tree.md +38 -0
- package/prompts/evidence-hierarchy.md +31 -0
- package/prompts/evidence-validator.md +22 -0
- package/prompts/marcus.md +29 -0
- package/prompts/prevention.md +26 -0
- package/prompts/prove-reproducer.md +43 -0
- package/prompts/quinn-decision-tree.md +31 -0
- package/prompts/quinn-ui-brief.md +106 -0
- package/prompts/quinn.md +29 -0
- package/prompts/rca.md +28 -0
- package/prompts/read-before-write.md +30 -0
- package/prompts/regression.md +26 -0
- package/prompts/rook.md +29 -0
- package/prompts/serena.md +28 -0
- package/prompts/testing-strategy.md +34 -0
- package/reference/migration-manifest.json +554 -0
- package/reference/prompts/agent-aditi.md +38 -0
- package/reference/prompts/agent-marcus.md +38 -0
- package/reference/prompts/agent-quinn.md +38 -0
- package/reference/prompts/agent-rook.md +37 -0
- package/reference/prompts/agent-serena.md +37 -0
- package/reference/prompts/council-skill.md +170 -0
- package/reference/prompts/goal-skill.md +235 -0
- package/reference/prompts/prove-skill.md +283 -0
- package/reference/prompts/quinn-standard.md +246 -0
- package/reference/prompts/ship-convergence.md +65 -0
- package/reference/prompts/ship-escalation.md +43 -0
- package/reference/prompts/ship-iteration.md +382 -0
- package/reference/prompts/ship-pending-patches.md +246 -0
- package/reference/prompts/ship-reference.md +250 -0
- package/reference/prompts/ship-skill.md +347 -0
- package/reference/prompts/ship-templates.md +215 -0
- package/reference/scripts/batch-status.sh +68 -0
- package/reference/scripts/chain-compat-check.sh +67 -0
- package/reference/scripts/context-audit.sh +314 -0
- package/reference/scripts/dead-export-check.sh +61 -0
- package/reference/scripts/generate-qc-tables.sh +92 -0
- package/reference/scripts/harness-init.sh +111 -0
- package/reference/scripts/pai-054-baseline-check.sh +40 -0
- package/reference/scripts/prove-smoke-test.sh +112 -0
- package/reference/scripts/setup-keychain-salt.sh +12 -0
- package/reference/scripts/skill-optimizer-friday.sh +88 -0
- package/reference/scripts/skill-qc-test.sh +351 -0
- package/reference/scripts/skill-synthesize-tonight.sh +66 -0
- package/reference/specs/HARNESS-EXTRACTION-SPEC.md +263 -0
- package/reference/specs/harness-v3-bash-deletion-plan.md +65 -0
- package/reference/specs/harness-v3-migration-plan.md +188 -0
- package/scripts/decision-reconcile.sh +74 -0
- package/scripts/decision-reconcile.ts +152 -0
- package/scripts/extract-constraints.ts +260 -0
- package/scripts/generate-code-map.ts +339 -0
- package/scripts/git-hooks/pre-commit +76 -0
- package/scripts/git-hooks/pre-push +271 -0
- package/scripts/harness-skill-check.sh +52 -0
- package/scripts/lib/gap-emit.sh +65 -0
- package/scripts/lib/jq-update.sh +30 -0
- package/scripts/lib/resolve-slug.sh +30 -0
- package/scripts/lib/verify-spec-compliance.sh +102 -0
- package/scripts/prove-backfill.sh +87 -0
- package/scripts/scaffold-project.ts +1488 -0
- package/scripts/scaffold-rungate-config.ts +91 -0
- package/scripts/self-containment-check.sh +109 -0
- package/scripts/skill-qc-validator.sh +402 -0
- package/scripts/slug-resolver.sh +14 -0
- package/scripts/sync-spec-tests.ts +236 -0
- package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +2493 -0
- package/specs/BOOTSTRAP-TEST-PLAN.md +571 -0
- package/specs/HARNESS-GATES.md +256 -0
- package/specs/HARNESS-SKILL-CHAIN.md +286 -0
- package/specs/HARNESS-SKILL-CONTRACT.md +93 -0
- package/specs/HARNESS-STANDARD.md +488 -0
- package/specs/SPEC-TEMPLATE.md +53 -0
- package/specs/bootstrap-data-flow.png +0 -0
- package/specs/harness-automation-matrix.md +314 -0
- package/specs/harness-automation-matrix.rules.json +64 -0
- package/test/anti.test.ts +80 -0
- package/test/contract-negative.test.ts +86 -0
- package/test/contract.test.ts +158 -0
- package/test/external-deps.test.ts +48 -0
- package/test/fixtures/golden-project/.env.example +3 -0
- package/test/fixtures/golden-project/Makefile +7 -0
- package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +23 -0
- package/test/fixtures/golden-project/docs/setup-guide.md +10 -0
- package/test/fixtures/golden-project/package.json +10 -0
- package/test/fixtures/golden-project/specs/api-spec.md +20 -0
- package/test/fixtures/golden-project/src/auth.ts +8 -0
- package/test/fixtures/golden-project/src/db.ts +11 -0
- package/test/fixtures/golden-project/src/index.ts +19 -0
- package/test/fixtures/golden-project/src/utils/format.ts +3 -0
- package/test/fixtures/golden-project/test/auth.test.ts +15 -0
- package/test/fixtures/golden-project/tsconfig.json +12 -0
- package/test/meta-sc-coverage.test.ts +159 -0
- package/test/phase-0.test.ts +846 -0
- package/test/phase-1.test.ts +617 -0
- package/test/phase-2.test.ts +620 -0
- package/test/phase-3.test.ts +99 -0
- package/test/scaffold-conformity.test.ts +8 -0
- package/test/schema-canary.test.ts +51 -0
- package/test/spec-compliance-auto.test.ts +64 -0
- package/test/spec-compliance.test.ts +295 -0
- package/test/spec-discovery.test.ts +68 -0
- package/test/spec-drift.test.ts +5 -0
- package/test/structure.test.ts +165 -0
- package/test/unit/agent-brief-guard-ship.test.ts +293 -0
- package/test/unit/extract-constraints.test.ts +145 -0
- package/test/unit/issue-close-guard.test.ts +204 -0
- package/test/unit/scaffold-project.test.ts +417 -0
- package/test/unit/skill-cooccurrence.test.ts +177 -0
- package/test/unit/skill-sequence-logger.test.ts +244 -0
- package/test/unit/workflow-state-guard.test.ts +95 -0
- package/workflows/batch-ship.js +295 -0
- package/workflows/council.js +481 -0
- package/workflows/prove.js +560 -0
- package/workflows/ship.js +789 -0
- package/workflows/verify.js +236 -0
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# skill-qc-test.sh — Automated QC tests for ADR-007 skill contracts
|
|
3
|
+
# Run: bash scripts/skill-qc-test.sh
|
|
4
|
+
set -euo pipefail
|
|
5
|
+
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
6
|
+
cd "$REPO_ROOT"
|
|
7
|
+
PASS=0; FAIL=0; TOTAL=0
|
|
8
|
+
|
|
9
|
+
pass() { echo " PASS $1"; PASS=$((PASS+1)); TOTAL=$((TOTAL+1)); }
|
|
10
|
+
fail() { echo " FAIL $1" >&2; FAIL=$((FAIL+1)); TOTAL=$((TOTAL+1)); }
|
|
11
|
+
|
|
12
|
+
echo "1. Schema prerequisite checks"
|
|
13
|
+
[[ $(jq '.properties.successCriteria.minItems' schemas/goal-record.schema.json) -ge 1 ]] && pass "goal-record successCriteria minItems" || fail "goal-record successCriteria minItems"
|
|
14
|
+
[[ $(jq '.properties.criteriaResults.minItems' skills/prove/prove-evidence.schema.json) -ge 1 ]] && pass "prove-evidence criteriaResults minItems" || fail "prove-evidence criteriaResults minItems"
|
|
15
|
+
[[ $(jq '.properties.gateOut.properties.evidence.minItems' schemas/ship-evidence.schema.json) -ge 1 ]] && pass "ship-evidence gateOut.evidence minItems" || fail "ship-evidence gateOut.evidence minItems"
|
|
16
|
+
[[ $(jq '.properties.assets.minItems' schemas/release-proof.schema.json) -ge 1 ]] && pass "release-proof assets minItems" || fail "release-proof assets minItems"
|
|
17
|
+
|
|
18
|
+
echo "2. contractVersion checks"
|
|
19
|
+
jq -e '.required | index("contractVersion")' skills/prove/prove-evidence.schema.json >/dev/null 2>&1 && pass "prove-evidence has contractVersion required" || fail "prove-evidence missing contractVersion"
|
|
20
|
+
|
|
21
|
+
echo "3. gaps[] schema check"
|
|
22
|
+
jq -e '.properties.gaps' skills/prove/prove-evidence.schema.json >/dev/null 2>&1 && pass "prove-evidence has gaps[] schema" || fail "prove-evidence missing gaps[]"
|
|
23
|
+
|
|
24
|
+
echo "4. Validator \$ref resolution"
|
|
25
|
+
# Create a minimal valid GoalRecord and validate it — if $ref fails, this catches it
|
|
26
|
+
TMPDIR=$(mktemp -d)
|
|
27
|
+
trap 'rm -rf "$TMPDIR"' EXIT
|
|
28
|
+
cat > "$TMPDIR/test-goal-record.json" << 'FIXTURE'
|
|
29
|
+
{
|
|
30
|
+
"contractVersion": "1.0",
|
|
31
|
+
"id": "test-1",
|
|
32
|
+
"goalStatement": "Test goal",
|
|
33
|
+
"successCriteria": [
|
|
34
|
+
{"id": "SC-1", "assertion": "test passes", "evidenceType": "test-pass", "threshold": {"op": "==", "value": "0", "unit": "failures"}}
|
|
35
|
+
],
|
|
36
|
+
"scopeBoundary": {"in": ["src/"], "out": ["docs/"]},
|
|
37
|
+
"artifactRef": {"type": "github-issue", "locator": "test/test#1"}
|
|
38
|
+
}
|
|
39
|
+
FIXTURE
|
|
40
|
+
bash scripts/validate-skill-output.sh goal "$TMPDIR/test-goal-record.json" >/dev/null 2>&1 && pass "GoalRecord with evidenceType validates (\$ref works)" || fail "GoalRecord validation failed (\$ref broken)"
|
|
41
|
+
|
|
42
|
+
echo "5. Validator hard-fail on missing jsonschema"
|
|
43
|
+
# Test that the validator script contains exit 1 for missing jsonschema, not exit 0
|
|
44
|
+
grep -q "sys.exit(1)" scripts/validate-skill-output.sh && pass "Validator hard-fails on missing jsonschema" || fail "Validator silently passes on missing jsonschema"
|
|
45
|
+
|
|
46
|
+
echo "6. #1318 prevention tests"
|
|
47
|
+
# Test: empty criteriaResults with PROVEN must be rejected
|
|
48
|
+
cat > "$TMPDIR/test-false-proven-empty.json" << 'FIXTURE'
|
|
49
|
+
{
|
|
50
|
+
"contractVersion": "1.0",
|
|
51
|
+
"issueNumber": 1,
|
|
52
|
+
"verdict": "PROVEN",
|
|
53
|
+
"commitSHA": "abc123",
|
|
54
|
+
"capturedAt": "2026-01-01T00:00:00Z",
|
|
55
|
+
"criteriaResults": []
|
|
56
|
+
}
|
|
57
|
+
FIXTURE
|
|
58
|
+
bash scripts/validate-skill-output.sh prove "$TMPDIR/test-false-proven-empty.json" >/dev/null 2>&1 && fail "PROVEN with empty criteriaResults accepted (should reject)" || pass "PROVEN with empty criteriaResults rejected"
|
|
59
|
+
|
|
60
|
+
# Test: valid prove-evidence with non-empty criteriaResults should pass
|
|
61
|
+
cat > "$TMPDIR/test-valid-prove.json" << 'FIXTURE'
|
|
62
|
+
{
|
|
63
|
+
"contractVersion": "1.0",
|
|
64
|
+
"issueNumber": 1,
|
|
65
|
+
"verdict": "PROVEN",
|
|
66
|
+
"commitSHA": "abc123",
|
|
67
|
+
"capturedAt": "2026-01-01T00:00:00Z",
|
|
68
|
+
"criteriaResults": [
|
|
69
|
+
{"scId": "SC-1", "verdict": "PASS", "evidence": "test passed"}
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
FIXTURE
|
|
73
|
+
bash scripts/validate-skill-output.sh prove "$TMPDIR/test-valid-prove.json" >/dev/null 2>&1 && pass "Valid prove-evidence accepted" || fail "Valid prove-evidence rejected"
|
|
74
|
+
|
|
75
|
+
# 6b. #1318 strict mode prevention
|
|
76
|
+
cat > "$TMPDIR/proven-with-fail.json" << 'FIXTURE'
|
|
77
|
+
{
|
|
78
|
+
"contractVersion": "1.0",
|
|
79
|
+
"issueNumber": 1,
|
|
80
|
+
"verdict": "PROVEN",
|
|
81
|
+
"commitSHA": "abc123",
|
|
82
|
+
"capturedAt": "2026-01-01T00:00:00Z",
|
|
83
|
+
"criteriaResults": [
|
|
84
|
+
{"scId": "SC-1", "verdict": "PASS", "evidence": "ok"},
|
|
85
|
+
{"scId": "SC-2", "verdict": "FAIL", "evidence": "not attempted"}
|
|
86
|
+
]
|
|
87
|
+
}
|
|
88
|
+
FIXTURE
|
|
89
|
+
bash scripts/validate-skill-output.sh prove "$TMPDIR/proven-with-fail.json" --strict 2>/dev/null && fail "PROVEN+FAIL accepted in strict mode" || pass "PROVEN+FAIL rejected in strict mode"
|
|
90
|
+
|
|
91
|
+
echo "7. Empty array rejection tests"
|
|
92
|
+
# Empty successCriteria
|
|
93
|
+
cat > "$TMPDIR/test-empty-sc.json" << 'FIXTURE'
|
|
94
|
+
{
|
|
95
|
+
"contractVersion": "1.0",
|
|
96
|
+
"id": "test-1",
|
|
97
|
+
"goalStatement": "Test",
|
|
98
|
+
"successCriteria": [],
|
|
99
|
+
"scopeBoundary": {"in": ["src/"], "out": ["docs/"]},
|
|
100
|
+
"artifactRef": {"type": "github-issue", "locator": "test#1"}
|
|
101
|
+
}
|
|
102
|
+
FIXTURE
|
|
103
|
+
bash scripts/validate-skill-output.sh goal "$TMPDIR/test-empty-sc.json" >/dev/null 2>&1 && fail "Empty successCriteria accepted (should reject)" || pass "Empty successCriteria rejected"
|
|
104
|
+
|
|
105
|
+
# Empty gateOut.evidence
|
|
106
|
+
cat > "$TMPDIR/test-empty-evidence.json" << 'FIXTURE'
|
|
107
|
+
{
|
|
108
|
+
"contractVersion": "1.0",
|
|
109
|
+
"issueNumber": 1,
|
|
110
|
+
"gateOut": {"status": "PASS", "evidence": []},
|
|
111
|
+
"mergeCommitSha": "abc123",
|
|
112
|
+
"capturedAt": "2026-01-01T00:00:00Z"
|
|
113
|
+
}
|
|
114
|
+
FIXTURE
|
|
115
|
+
bash scripts/validate-skill-output.sh ship "$TMPDIR/test-empty-evidence.json" >/dev/null 2>&1 && fail "Empty gateOut.evidence accepted (should reject)" || pass "Empty gateOut.evidence rejected"
|
|
116
|
+
|
|
117
|
+
# Empty assets
|
|
118
|
+
cat > "$TMPDIR/test-empty-assets.json" << 'FIXTURE'
|
|
119
|
+
{
|
|
120
|
+
"contractVersion": "1.0",
|
|
121
|
+
"version": "1.0.0",
|
|
122
|
+
"gate4Result": "PASS",
|
|
123
|
+
"assets": [],
|
|
124
|
+
"capturedAt": "2026-01-01T00:00:00Z"
|
|
125
|
+
}
|
|
126
|
+
FIXTURE
|
|
127
|
+
bash scripts/validate-skill-output.sh release "$TMPDIR/test-empty-assets.json" >/dev/null 2>&1 && fail "Empty assets accepted (should reject)" || pass "Empty assets rejected"
|
|
128
|
+
|
|
129
|
+
echo "8. Slug resolver tests"
|
|
130
|
+
[[ $(bash scripts/slug-resolver.sh hornjason/pai-config) == "pai" ]] && pass "pai-config -> pai" || fail "pai-config slug wrong"
|
|
131
|
+
[[ $(bash scripts/slug-resolver.sh hornjason/asaCommandCenter) == "ddb" ]] && pass "asaCommandCenter -> ddb" || fail "asaCommandCenter slug wrong"
|
|
132
|
+
bash scripts/slug-resolver.sh unknown/repo >/dev/null 2>&1 && fail "Unknown repo accepted (should fail)" || pass "Unknown repo rejected"
|
|
133
|
+
|
|
134
|
+
echo "9. GoalRecord TTL exemption"
|
|
135
|
+
# Verify StaleTTLCleanup.hook.sh checks for goal-record
|
|
136
|
+
grep -q 'goal-record' hooks/StaleTTLCleanup.hook.sh && pass "TTL exemption for goal-record.json" || fail "TTL exemption missing"
|
|
137
|
+
|
|
138
|
+
echo "10. Regression — prove smoke tests"
|
|
139
|
+
bash skills/prove/prove-smoke-test.sh >/dev/null 2>&1 && pass "prove-smoke-test.sh passes" || fail "prove-smoke-test.sh failed"
|
|
140
|
+
|
|
141
|
+
echo "11. Batch status tests"
|
|
142
|
+
SAVED_RUNGATE_WORK_DIR="${RUNGATE_WORK_DIR:-}"
|
|
143
|
+
export RUNGATE_WORK_DIR="$TMPDIR"
|
|
144
|
+
|
|
145
|
+
bash "$REPO_ROOT/scripts/batch-status.sh" init "test-batch" >/dev/null 2>&1 && \
|
|
146
|
+
[[ -f "$TMPDIR/batch-test-batch/batch-status.json" ]] && pass "batch-status init creates file" || fail "batch-status init"
|
|
147
|
+
|
|
148
|
+
bash "$REPO_ROOT/scripts/batch-status.sh" update "test-batch" 123 "SHIP" "IN_PROGRESS" >/dev/null 2>&1 || true
|
|
149
|
+
ISSUE_COUNT=$(python3 -c "import json; print(len(json.load(open('$TMPDIR/batch-test-batch/batch-status.json')).get('issues',[])))" 2>/dev/null || echo "0")
|
|
150
|
+
[[ "$ISSUE_COUNT" == "1" ]] && pass "batch-status update adds issue" || fail "batch-status update issue count ($ISSUE_COUNT)"
|
|
151
|
+
|
|
152
|
+
bash "$REPO_ROOT/scripts/batch-status.sh" update "test-batch" 123 "PROVE" "DONE" >/dev/null 2>&1 || true
|
|
153
|
+
PHASE=$(python3 -c "import json; print(json.load(open('$TMPDIR/batch-test-batch/batch-status.json'))['issues'][0]['phase'])" 2>/dev/null || echo "?")
|
|
154
|
+
[[ "$PHASE" == "PROVE" ]] && pass "batch-status atomic update" || fail "batch-status atomic update ($PHASE)"
|
|
155
|
+
|
|
156
|
+
if [[ -n "$SAVED_RUNGATE_WORK_DIR" ]]; then export RUNGATE_WORK_DIR="$SAVED_RUNGATE_WORK_DIR"; else unset RUNGATE_WORK_DIR; fi
|
|
157
|
+
|
|
158
|
+
echo "12. Chain detection documentation tests"
|
|
159
|
+
grep -q "Chain Detection" "$REPO_ROOT/skills/ship/SKILL.md" && pass "ship has Chain Detection section" || fail "ship missing Chain Detection"
|
|
160
|
+
grep -q "Chain Detection" "$REPO_ROOT/skills/prove/SKILL.md" && pass "prove has Chain Detection section" || fail "prove missing Chain Detection"
|
|
161
|
+
grep -q "Chain Detection" "$REPO_ROOT/skills/release/SKILL.md" && pass "release has Chain Detection section" || fail "release missing Chain Detection"
|
|
162
|
+
grep -q "test -f" "$REPO_ROOT/skills/ship/SKILL.md" && pass "ship uses file-presence detection" || fail "ship missing file-presence"
|
|
163
|
+
grep -q "test -f" "$REPO_ROOT/skills/prove/SKILL.md" && pass "prove uses file-presence detection" || fail "prove missing file-presence"
|
|
164
|
+
grep -q "test -f" "$REPO_ROOT/skills/release/SKILL.md" && pass "release uses file-presence detection" || fail "release missing file-presence"
|
|
165
|
+
|
|
166
|
+
echo "13. Council synthesis schema tests"
|
|
167
|
+
python3 -c "import json; json.load(open('$REPO_ROOT/schemas/council-synthesis.schema.json'))" 2>/dev/null && \
|
|
168
|
+
pass "council-synthesis.schema.json is valid JSON" || fail "council-synthesis.schema.json invalid"
|
|
169
|
+
|
|
170
|
+
cat > "$TMPDIR/test-council-synthesis.json" << 'FIXTURE'
|
|
171
|
+
{
|
|
172
|
+
"contractVersion": "1.0",
|
|
173
|
+
"topic": "Test topic",
|
|
174
|
+
"decisions": [
|
|
175
|
+
{"id": "D-001", "statement": "Use file presence", "disposition": "ACCEPTED", "target": {"type": "adr", "ref": "test.md"}}
|
|
176
|
+
],
|
|
177
|
+
"convergencePoints": ["all agree"],
|
|
178
|
+
"recommendation": "do it",
|
|
179
|
+
"capturedAt": "2026-01-01T00:00:00Z"
|
|
180
|
+
}
|
|
181
|
+
FIXTURE
|
|
182
|
+
bash "$REPO_ROOT/scripts/validate-skill-output.sh" council "$TMPDIR/test-council-synthesis.json" >/dev/null 2>&1 && \
|
|
183
|
+
pass "Council synthesis validates against schema" || fail "Council synthesis validation failed"
|
|
184
|
+
|
|
185
|
+
echo "14. Chain compatibility tests"
|
|
186
|
+
bash "$REPO_ROOT/scripts/chain-compat-check.sh" >/dev/null 2>&1 && \
|
|
187
|
+
pass "L3 chain compatibility check passes" || fail "L3 chain compatibility check failed"
|
|
188
|
+
|
|
189
|
+
echo "15. Chain state schema test"
|
|
190
|
+
python3 -c "import json; json.load(open('$REPO_ROOT/schemas/chain-state.schema.json'))" 2>/dev/null && \
|
|
191
|
+
pass "chain-state.schema.json is valid JSON" || fail "chain-state.schema.json invalid"
|
|
192
|
+
|
|
193
|
+
echo "16. Goal routing tests"
|
|
194
|
+
# Verify goal-record schema has artifactRef.type enum
|
|
195
|
+
python3 -c "
|
|
196
|
+
import json
|
|
197
|
+
schema = json.load(open('$REPO_ROOT/schemas/goal-record.schema.json'))
|
|
198
|
+
art_type = schema['properties']['artifactRef']['properties']['type']
|
|
199
|
+
assert 'enum' in art_type, 'artifactRef.type missing enum'
|
|
200
|
+
assert 'github-issue' in art_type['enum'], 'missing github-issue'
|
|
201
|
+
assert 'adr' in art_type['enum'], 'missing adr'
|
|
202
|
+
assert 'research-brief' in art_type['enum'], 'missing research-brief'
|
|
203
|
+
print('OK')
|
|
204
|
+
" 2>/dev/null && pass "artifactRef.type has 5-value enum" || fail "artifactRef.type enum missing"
|
|
205
|
+
|
|
206
|
+
# Verify goal SKILL.md has classification decision tree
|
|
207
|
+
grep -q "classifyGoalType\|Goal Type Classification" "$REPO_ROOT/skills/goal/SKILL.md" && \
|
|
208
|
+
pass "goal SKILL.md has type classification" || fail "goal SKILL.md missing classification"
|
|
209
|
+
|
|
210
|
+
# Validate GoalRecord with adr type
|
|
211
|
+
cat > "$TMPDIR/test-goal-adr.json" << 'FIXTURE'
|
|
212
|
+
{
|
|
213
|
+
"contractVersion": "1.0",
|
|
214
|
+
"id": "adr-007",
|
|
215
|
+
"goalStatement": "Design universal skill contract",
|
|
216
|
+
"successCriteria": [{"id": "SC-1", "assertion": "10-field contract defined", "evidenceType": "grep", "threshold": {"op": "==", "value": "10", "unit": "fields"}}],
|
|
217
|
+
"scopeBoundary": {"in": ["PAI/ADR/"], "out": ["src/"]},
|
|
218
|
+
"artifactRef": {"type": "adr", "locator": "PAI/ADR/ADR-007-universal-skill-contract.md"}
|
|
219
|
+
}
|
|
220
|
+
FIXTURE
|
|
221
|
+
bash "$REPO_ROOT/scripts/validate-skill-output.sh" goal "$TMPDIR/test-goal-adr.json" >/dev/null 2>&1 && \
|
|
222
|
+
pass "GoalRecord with adr type validates" || fail "GoalRecord with adr type rejected"
|
|
223
|
+
|
|
224
|
+
echo "17. Skill runner tests"
|
|
225
|
+
# Test pre-phase with goal skill
|
|
226
|
+
mkdir -p "$TMPDIR/runner-test"
|
|
227
|
+
echo '{"schemaVersion":2,"issue":999,"slug":"test","phase":"GOAL","issueGoal":"test","acs":[],"gates":{},"changelog":[]}' > "$TMPDIR/runner-test/workflow-state.json"
|
|
228
|
+
RUNGATE_WORK_DIR="$TMPDIR" bash "$REPO_ROOT/scripts/skill-runner.sh" --phase pre --skill goal --issue 999 --slug runner-test >/dev/null 2>&1 && \
|
|
229
|
+
pass "runner pre-phase goal passes" || fail "runner pre-phase goal failed"
|
|
230
|
+
|
|
231
|
+
# Test post-phase with valid artifact
|
|
232
|
+
cat > "$TMPDIR/runner-test/goal-record.json" << 'FIXTURE'
|
|
233
|
+
{"contractVersion":"1.0","id":"test","goalStatement":"test","successCriteria":[{"id":"SC-1","assertion":"test","evidenceType":"grep","threshold":{"op":"==","value":"0","unit":"matches"}}],"scopeBoundary":{"in":["test"],"out":["test"]},"artifactRef":{"type":"github-issue","locator":"hornjason/pai-config#999"}}
|
|
234
|
+
FIXTURE
|
|
235
|
+
RUNGATE_WORK_DIR="$TMPDIR" bash "$REPO_ROOT/scripts/skill-runner.sh" --phase post --skill goal --issue 999 --slug runner-test --artifact "$TMPDIR/runner-test/goal-record.json" >/dev/null 2>&1 && \
|
|
236
|
+
pass "runner post-phase goal passes" || fail "runner post-phase goal failed"
|
|
237
|
+
|
|
238
|
+
# Test chain-state.json was created
|
|
239
|
+
[[ -f "$TMPDIR/runner-test/chain-state.json" ]] && pass "runner creates chain-state.json" || fail "runner missing chain-state.json"
|
|
240
|
+
|
|
241
|
+
# Test post-phase catches missing artifact (Assert)
|
|
242
|
+
RUNGATE_WORK_DIR="$TMPDIR" bash "$REPO_ROOT/scripts/skill-runner.sh" --phase post --skill ship --issue 999 --slug runner-test 2>/dev/null && \
|
|
243
|
+
fail "runner should fail on missing ship artifact" || pass "runner Assert fails on missing artifact"
|
|
244
|
+
|
|
245
|
+
echo "18. Runner Assert/Suggest classification"
|
|
246
|
+
# Verify runner uses assert_fail for hard errors
|
|
247
|
+
grep -q "assert_fail" "$REPO_ROOT/scripts/skill-runner.sh" && pass "runner has assert_fail" || fail "runner missing assert_fail"
|
|
248
|
+
grep -q "suggest_warn" "$REPO_ROOT/scripts/skill-runner.sh" && pass "runner has suggest_warn" || fail "runner missing suggest_warn"
|
|
249
|
+
|
|
250
|
+
echo "19. Audit skill tests"
|
|
251
|
+
[[ -f "$REPO_ROOT/skills/audit/SKILL.md" ]] && pass "audit SKILL.md exists" || fail "audit SKILL.md missing"
|
|
252
|
+
grep -q "outputSchema:" "$REPO_ROOT/skills/audit/SKILL.md" && pass "audit has outputSchema" || fail "audit missing outputSchema"
|
|
253
|
+
|
|
254
|
+
python3 -c "import json; json.load(open('$REPO_ROOT/schemas/audit-report.schema.json'))" 2>/dev/null && \
|
|
255
|
+
pass "audit-report.schema.json valid JSON" || fail "audit-report.schema.json invalid"
|
|
256
|
+
|
|
257
|
+
cat > "$TMPDIR/test-audit-report.json" << 'FIXTURE'
|
|
258
|
+
{
|
|
259
|
+
"contractVersion": "1.0",
|
|
260
|
+
"scope": {"topic": "test"},
|
|
261
|
+
"findings": [{"id": "F-001", "severity": "MEDIUM", "description": "test finding", "status": "OPEN"}],
|
|
262
|
+
"summary": {"totalFindings": 1, "openCount": 1, "resolvedCount": 0},
|
|
263
|
+
"capturedAt": "2026-01-01T00:00:00Z"
|
|
264
|
+
}
|
|
265
|
+
FIXTURE
|
|
266
|
+
bash "$REPO_ROOT/scripts/validate-skill-output.sh" audit "$TMPDIR/test-audit-report.json" >/dev/null 2>&1 && \
|
|
267
|
+
pass "audit report validates against schema" || fail "audit report validation failed"
|
|
268
|
+
|
|
269
|
+
echo "20. Research output schema tests"
|
|
270
|
+
python3 -c "import json; json.load(open('$REPO_ROOT/schemas/research-output.schema.json'))" 2>/dev/null && \
|
|
271
|
+
pass "research-output.schema.json valid JSON" || fail "research-output.schema.json invalid"
|
|
272
|
+
|
|
273
|
+
cat > "$TMPDIR/test-research-output.json" << 'FIXTURE'
|
|
274
|
+
{
|
|
275
|
+
"contractVersion": "1.0",
|
|
276
|
+
"query": "What frameworks handle chain state?",
|
|
277
|
+
"findings": [{"claim": "Temporal uses event sourcing", "confidence": "HIGH", "source": "temporal.io docs"}],
|
|
278
|
+
"sources": [{"name": "Temporal Docs", "type": "documentation"}],
|
|
279
|
+
"capturedAt": "2026-01-01T00:00:00Z"
|
|
280
|
+
}
|
|
281
|
+
FIXTURE
|
|
282
|
+
python3 -c "
|
|
283
|
+
from jsonschema import validate
|
|
284
|
+
import json
|
|
285
|
+
schema = json.load(open('$REPO_ROOT/schemas/research-output.schema.json'))
|
|
286
|
+
data = json.load(open('$TMPDIR/test-research-output.json'))
|
|
287
|
+
validate(instance=data, schema=schema)
|
|
288
|
+
print('PASS')
|
|
289
|
+
" 2>/dev/null && pass "research output validates against schema" || fail "research output validation failed"
|
|
290
|
+
|
|
291
|
+
echo "21. Learn signal schema tests"
|
|
292
|
+
python3 -c "import json; json.load(open('$REPO_ROOT/schemas/learn-signal.schema.json'))" 2>/dev/null && \
|
|
293
|
+
pass "learn-signal.schema.json valid JSON" || fail "learn-signal.schema.json invalid"
|
|
294
|
+
|
|
295
|
+
cat > "$TMPDIR/test-learn-signal.json" << 'FIXTURE'
|
|
296
|
+
{
|
|
297
|
+
"signalType": "skill-learn",
|
|
298
|
+
"schemaVersion": "1.0",
|
|
299
|
+
"ts": "2026-01-01T00:00:00Z",
|
|
300
|
+
"skill": "prove",
|
|
301
|
+
"issue": 290,
|
|
302
|
+
"slug": "pai/290",
|
|
303
|
+
"mode": "chain",
|
|
304
|
+
"duration_ms": 1234,
|
|
305
|
+
"metrics": {"criteriaCount": 5, "criteriaPass": 4, "gapCount": 1}
|
|
306
|
+
}
|
|
307
|
+
FIXTURE
|
|
308
|
+
python3 -c "
|
|
309
|
+
from jsonschema import validate
|
|
310
|
+
import json
|
|
311
|
+
schema = json.load(open('$REPO_ROOT/schemas/learn-signal.schema.json'))
|
|
312
|
+
data = json.load(open('$TMPDIR/test-learn-signal.json'))
|
|
313
|
+
validate(instance=data, schema=schema)
|
|
314
|
+
print('PASS')
|
|
315
|
+
" 2>/dev/null && pass "learn signal validates against schema" || fail "learn signal validation failed"
|
|
316
|
+
|
|
317
|
+
echo "22. Performance profile schema tests"
|
|
318
|
+
python3 -c "import json; json.load(open('$REPO_ROOT/schemas/performance-profile.schema.json'))" 2>/dev/null && \
|
|
319
|
+
pass "performance-profile.schema.json valid JSON" || fail "performance-profile.schema.json invalid"
|
|
320
|
+
|
|
321
|
+
# Validate a sample profile
|
|
322
|
+
cat > "$TMPDIR/test-profile.json" << 'FIXTURE'
|
|
323
|
+
{
|
|
324
|
+
"skill": "prove",
|
|
325
|
+
"executions": 47,
|
|
326
|
+
"avgCriteriaHitRate": 0.82,
|
|
327
|
+
"avgIterations": 1.3,
|
|
328
|
+
"gapRate": 0.18,
|
|
329
|
+
"topGapTypes": ["UI AC never verified", "before-state not captured"],
|
|
330
|
+
"trend": "improving",
|
|
331
|
+
"generatedAt": "2026-01-01T00:00:00Z"
|
|
332
|
+
}
|
|
333
|
+
FIXTURE
|
|
334
|
+
python3 -c "
|
|
335
|
+
from jsonschema import validate
|
|
336
|
+
import json
|
|
337
|
+
schema = json.load(open('$REPO_ROOT/schemas/performance-profile.schema.json'))
|
|
338
|
+
data = json.load(open('$TMPDIR/test-profile.json'))
|
|
339
|
+
validate(instance=data, schema=schema)
|
|
340
|
+
print('PASS')
|
|
341
|
+
" 2>/dev/null && pass "performance profile validates" || fail "performance profile validation failed"
|
|
342
|
+
|
|
343
|
+
echo "23. Feedback loop documentation tests"
|
|
344
|
+
grep -q "Performance Profile\|performance profile" "$REPO_ROOT/skills/audit/SKILL.md" && \
|
|
345
|
+
pass "audit SKILL.md documents profile consumption" || fail "audit missing profile docs"
|
|
346
|
+
grep -q "Research Mode\|performance profile\|Performance Data" "$REPO_ROOT/skills/council/SKILL.md" && \
|
|
347
|
+
pass "council SKILL.md documents research mode" || fail "council missing research mode docs"
|
|
348
|
+
|
|
349
|
+
echo ""
|
|
350
|
+
echo "=== Results: $PASS/$TOTAL passed, $FAIL failed ==="
|
|
351
|
+
[[ $FAIL -eq 0 ]] && exit 0 || exit 1
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# One-time overnight skill synthesis + optimization run
|
|
3
|
+
# Crontab: 15 21 24 4 * (fires once tonight — April 24 at 9:15pm)
|
|
4
|
+
|
|
5
|
+
set -euo pipefail
|
|
6
|
+
|
|
7
|
+
BUN="/Users/jhorn/.bun/bin/bun"
|
|
8
|
+
BASE="/Users/jhorn/.claude"
|
|
9
|
+
REPORTS_DIR="$BASE/MEMORY/LEARNING/REPORTS"
|
|
10
|
+
LOG="$REPORTS_DIR/skill-synthesis-$(date +%Y%m%d).md"
|
|
11
|
+
|
|
12
|
+
mkdir -p "$REPORTS_DIR"
|
|
13
|
+
|
|
14
|
+
echo "## Skill Synthesis Run — $(date)" >> "$LOG"
|
|
15
|
+
echo "" >> "$LOG"
|
|
16
|
+
|
|
17
|
+
# Step 1: Synthesize + auto-promote
|
|
18
|
+
echo "### Step 1: Synthesize + Auto-Promote" >> "$LOG"
|
|
19
|
+
echo "Skills: infrastructure-and-devops, product-feature-development" >> "$LOG"
|
|
20
|
+
echo "" >> "$LOG"
|
|
21
|
+
|
|
22
|
+
"$BUN" "$BASE/PAI/Tools/SkillSynthesizer.ts" \
|
|
23
|
+
--skills "infrastructure-and-devops,product-feature-development" 2>&1 | tee -a "$LOG"
|
|
24
|
+
|
|
25
|
+
echo "" >> "$LOG"
|
|
26
|
+
|
|
27
|
+
# Step 2: Build benchmarks for newly promoted skills
|
|
28
|
+
echo "### Step 2: Build Benchmarks" >> "$LOG"
|
|
29
|
+
|
|
30
|
+
for skill_name in infrastructure-and-devops product-feature-development; do
|
|
31
|
+
bench="$BASE/MEMORY/LEARNING/STATE/skill-benchmark-${skill_name}.json"
|
|
32
|
+
if [ ! -f "$bench" ]; then
|
|
33
|
+
echo "Building benchmark: $skill_name" >> "$LOG"
|
|
34
|
+
cd "$BASE"
|
|
35
|
+
"$BUN" "$BASE/PAI/Tools/SkillBenchmark.ts" \
|
|
36
|
+
--skill "$skill_name" --output "$bench" 2>&1 | tee -a "$LOG" || \
|
|
37
|
+
echo "WARN: benchmark failed for $skill_name" >> "$LOG"
|
|
38
|
+
else
|
|
39
|
+
echo "Benchmark already exists: $bench — skipping" >> "$LOG"
|
|
40
|
+
fi
|
|
41
|
+
done
|
|
42
|
+
|
|
43
|
+
echo "" >> "$LOG"
|
|
44
|
+
|
|
45
|
+
# Step 3: Optimize (5 rounds each)
|
|
46
|
+
echo "### Step 3: Optimize Skills" >> "$LOG"
|
|
47
|
+
|
|
48
|
+
for skill_name in infrastructure-and-devops product-feature-development; do
|
|
49
|
+
skill_path="skills/$skill_name/SKILL.md"
|
|
50
|
+
bench="$BASE/MEMORY/LEARNING/STATE/skill-benchmark-${skill_name}.json"
|
|
51
|
+
|
|
52
|
+
if [ -f "$BASE/$skill_path" ] && [ -f "$bench" ]; then
|
|
53
|
+
echo "Optimizing: $skill_name" >> "$LOG"
|
|
54
|
+
cd "$BASE"
|
|
55
|
+
"$BUN" "$BASE/PAI/Tools/SkillOptimizer.ts" \
|
|
56
|
+
--skill "$skill_path" --benchmark "$bench" --rounds 5 2>&1 | tee -a "$LOG" || \
|
|
57
|
+
echo "WARN: optimizer failed for $skill_name" >> "$LOG"
|
|
58
|
+
else
|
|
59
|
+
echo "SKIP: $skill_name — skill or benchmark missing" >> "$LOG"
|
|
60
|
+
[ ! -f "$BASE/$skill_path" ] && echo " Missing: $BASE/$skill_path" >> "$LOG"
|
|
61
|
+
[ ! -f "$bench" ] && echo " Missing: $bench" >> "$LOG"
|
|
62
|
+
fi
|
|
63
|
+
done
|
|
64
|
+
|
|
65
|
+
echo "" >> "$LOG"
|
|
66
|
+
echo "## Complete: $(date)" >> "$LOG"
|