sinapse-ai 1.20.1 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/rules/coderabbit-integration.md +6 -0
- package/.claude/rules/documentation-first-reference.md +146 -0
- package/.claude/rules/documentation-first.md +27 -119
- package/.claude/rules/mandatory-delegation-reference.md +93 -0
- package/.claude/rules/mandatory-delegation.md +29 -93
- package/.claude/rules/project-intelligence-reference.md +159 -0
- package/.claude/rules/project-intelligence.md +37 -155
- package/.claude/rules/safe-collaboration-reference.md +163 -0
- package/.claude/rules/safe-collaboration.md +32 -173
- package/.sinapse-ai/core/doctor/checks/index.js +1 -1
- package/.sinapse-ai/core/execution/build-orchestrator.js +4 -3
- package/.sinapse-ai/core/execution/subagent-dispatcher.js +2 -2
- package/.sinapse-ai/core/ideation/ideation-engine.js +2 -1
- package/.sinapse-ai/core/orchestration/brownfield-progress.js +219 -0
- package/.sinapse-ai/core/orchestration/cli-commands.js +13 -0
- package/.sinapse-ai/core/orchestration/doc-first-resolver.js +96 -2
- package/.sinapse-ai/core/orchestration/executors/epic-3-executor.js +15 -2
- package/.sinapse-ai/core/orchestration/executors/epic-4-executor.js +3 -1
- package/.sinapse-ai/core/orchestration/greenfield-handler.js +97 -0
- package/.sinapse-ai/core/orchestration/index.js +5 -0
- package/.sinapse-ai/core/orchestration/spec-complexity.js +141 -0
- package/.sinapse-ai/core/orchestration/workflow-executor.js +17 -1
- package/.sinapse-ai/core-config.yaml +30 -0
- package/.sinapse-ai/data/entity-registry.yaml +81 -35
- package/.sinapse-ai/development/agents/architect.md +3 -15
- package/.sinapse-ai/development/agents/data-engineer.md +3 -15
- package/.sinapse-ai/development/agents/developer.md +3 -24
- package/.sinapse-ai/development/agents/devops.md +4 -25
- package/.sinapse-ai/development/agents/quality-gate.md +3 -25
- package/.sinapse-ai/development/knowledge-base/token-economy-guide.md +1 -1
- package/.sinapse-ai/development/tasks/execute-epic-plan.md +3 -0
- package/.sinapse-ai/development/tasks/resolve-github-issue.md +1 -1
- package/.sinapse-ai/development/workflows/brownfield-discovery.yaml +16 -0
- package/.sinapse-ai/development/workflows/epic-orchestration.yaml +11 -0
- package/.sinapse-ai/docs/standards/SINAPSE-LIVRO-DE-OURO-V2.1-COMPLETE.md +6 -2
- package/.sinapse-ai/docs/standards/SINAPSE-LIVRO-DE-OURO-V2.2-SUMMARY.md +4 -1
- package/.sinapse-ai/install-manifest.yaml +58 -50
- package/.sinapse-ai/product/templates/story-tmpl.yaml +11 -6
- package/.sinapse-ai/quality/judge-calibration/README.md +75 -0
- package/.sinapse-ai/quality/judge-calibration/calibration-log.md +70 -0
- package/.sinapse-ai/quality/judge-calibration/golden-set.json +105 -0
- package/.sinapse-ai/quality/judge-calibration/scenarios.json +90 -0
- package/CHANGELOG.md +55 -10
- package/README.en.md +1 -1
- package/README.md +6 -6
- package/bin/sinapse-minimal.js +5 -4
- package/docs/community/README-community-snippet-core.md +1 -1
- package/docs/community/README-community-snippet-mcp.md +2 -2
- package/docs/framework/README.md +2 -2
- package/docs/framework/architecture-overview.md +2 -2
- package/docs/framework/core-architecture.md +1 -1
- package/docs/framework/feature-process.md +5 -5
- package/docs/framework/guiding-principles.md +1 -1
- package/docs/framework/roadmap.md +4 -4
- package/docs/framework/source-tree.md +1 -1
- package/docs/framework/versioning-and-releases.md +1 -1
- package/docs/getting-started.md +1 -1
- package/docs/guides/agent-selection-guide.md +1 -1
- package/docs/guides/agents/traces/README.md +1 -1
- package/docs/guides/config-migration-guide.md +1 -1
- package/docs/guides/development-setup.md +1 -1
- package/docs/guides/docker-mcp-setup.md +5 -5
- package/docs/guides/getting-started.md +1 -1
- package/docs/guides/git-workflow-guide.md +3 -3
- package/docs/guides/ide-integration.md +3 -3
- package/docs/guides/ide-sync-guide.md +1 -1
- package/docs/guides/mcp/desktop-commander.md +2 -2
- package/docs/guides/mcp/docker-gateway-tutorial.md +1 -1
- package/docs/guides/mcp-global-setup.md +3 -3
- package/docs/guides/memory-intelligence-system.md +10 -10
- package/docs/guides/meta-agent-commands.md +1 -1
- package/docs/guides/quality-gates.md +1 -1
- package/docs/guides/security-hardening.md +1 -1
- package/docs/guides/service-discovery.md +1 -1
- package/docs/guides/squad-examples/README.md +1 -1
- package/docs/guides/squads-guide.md +1 -1
- package/docs/guides/testing-guide.md +2 -2
- package/docs/guides/user-guide.md +22 -22
- package/docs/guides/workflows/BROWNFIELD-DISCOVERY-WORKFLOW.md +3 -3
- package/docs/guides/workflows/BROWNFIELD-SERVICE-WORKFLOW.md +1 -1
- package/docs/guides/workflows/GREENFIELD-UI-WORKFLOW.md +2 -2
- package/docs/guides/workflows/SPEC-PIPELINE-WORKFLOW.md +3 -3
- package/docs/guides/workflows/STORY-DEVELOPMENT-CYCLE-WORKFLOW.md +3 -3
- package/docs/guides/workflows/pro-developer-workflow.md +2 -2
- package/docs/guides/workflows-guide.md +1 -1
- package/docs/guides/workflows-overview.md +1 -1
- package/docs/installation/npx-install.md +1 -1
- package/docs/installation/uninstallation.md +1 -1
- package/docs/installation/v4-quick-start.md +1 -1
- package/docs/pt/guides/user-guide.md +18 -18
- package/docs/pt/security.md +2 -2
- package/docs/security/overview.md +2 -2
- package/docs/security/security-best-practices.md +1 -1
- package/docs/sinapse-agent-flows/architect-system.md +1 -1
- package/docs/sinapse-agent-flows/data-engineer-system.md +1 -1
- package/docs/sinapse-agent-flows/dev-system.md +1 -1
- package/docs/sinapse-agent-flows/devops-system.md +2 -2
- package/docs/sinapse-agent-flows/qa-system.md +1 -1
- package/docs/sinapse-agent-flows/sm-system.md +2 -2
- package/docs/sinapse-agent-flows/snps-orqx-system.md +4 -4
- package/docs/sinapse-agent-flows/squad-creator-system.md +1 -1
- package/docs/sinapse-workflows/README.md +2 -2
- package/docs/sinapse-workflows/brownfield-discovery-workflow.md +3 -3
- package/docs/sinapse-workflows/brownfield-service-workflow.md +3 -3
- package/docs/sinapse-workflows/greenfield-ui-workflow.md +2 -2
- package/docs/sinapse-workflows/spec-pipeline-workflow.md +3 -3
- package/docs/sinapse-workflows/story-development-cycle-workflow.md +3 -3
- package/docs/troubleshooting.md +1 -1
- package/package.json +7 -1
- package/scripts/calibrate-judge.js +134 -0
- package/scripts/eval-e2e.js +149 -0
- package/scripts/validate-all.js +8 -0
- package/scripts/validate-article-iv.js +289 -0
- package/scripts/validate-constitution.js +58 -0
- package/scripts/validate-evals.js +9 -2
- package/scripts/validate-story-acs.js +174 -0
- package/scripts/validate-tool-descriptions.js +128 -0
- package/scripts/wave-gate.js +207 -0
- package/squads/claude-code-mastery/agents/project-integrator.md +3 -10
- package/squads/claude-code-mastery/knowledge-base/context-window-optimization.md +2 -2
- package/squads/claude-code-mastery/knowledge-base/memory-systems-reference.md +1 -1
- package/squads/squad-copy/knowledge-base/ai-copy-human-loop-canon.md +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sinapse-ai",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.22.0",
|
|
4
4
|
"description": "SINAPSE AI: Framework de orquestracao de IA — 17 squads, 172 agentes especializados",
|
|
5
5
|
"bin": {
|
|
6
6
|
"sinapse": "bin/sinapse.js",
|
|
@@ -86,6 +86,7 @@
|
|
|
86
86
|
"test:coverage": "jest --coverage",
|
|
87
87
|
"test:health-check": "jest tests/health-check",
|
|
88
88
|
"eval": "node scripts/eval-runner.js",
|
|
89
|
+
"eval:e2e": "node scripts/eval-e2e.js",
|
|
89
90
|
"validate:evals": "node scripts/validate-evals.js",
|
|
90
91
|
"validate:schemas": "node scripts/validate-schemas.js",
|
|
91
92
|
"lint": "eslint . --cache --cache-location .eslintcache",
|
|
@@ -97,6 +98,7 @@
|
|
|
97
98
|
"validate:article-vii": "node scripts/validate-article-vii.js",
|
|
98
99
|
"validate:article-viii": "node scripts/validate-article-viii.js",
|
|
99
100
|
"validate:article-xi": "node scripts/validate-article-xi.js",
|
|
101
|
+
"validate:article-iv": "node scripts/validate-article-iv.js",
|
|
100
102
|
"reconcile:squads": "node scripts/reconcile-squad-manifests.js",
|
|
101
103
|
"apply:persona-disclaimer": "node scripts/apply-persona-disclaimer.js",
|
|
102
104
|
"validate:manifest": "node scripts/validate-manifest.js",
|
|
@@ -132,6 +134,10 @@
|
|
|
132
134
|
"validate:cross-refs": "node scripts/validate-cross-refs.js",
|
|
133
135
|
"validate:agents-md": "node scripts/validate-agents-md.js",
|
|
134
136
|
"validate:agents-md:fix": "node scripts/validate-agents-md.js --fix",
|
|
137
|
+
"validate:tool-descriptions": "node scripts/validate-tool-descriptions.js",
|
|
138
|
+
"validate:story-acs": "node scripts/validate-story-acs.js",
|
|
139
|
+
"calibrate:judge": "node scripts/calibrate-judge.js",
|
|
140
|
+
"validate:constitution": "node scripts/validate-constitution.js",
|
|
135
141
|
"validate:all": "node scripts/validate-all.js",
|
|
136
142
|
"sync:squad-yaml": "node scripts/sync-squad-yaml-components.js",
|
|
137
143
|
"validate:squad-yaml": "node scripts/sync-squad-yaml-components.js --check",
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* LLM-judge calibration harness (Story mesa2-llm-judge-calibration, AF-20260704 Mesa #2).
|
|
4
|
+
*
|
|
5
|
+
* The judge that decides "Done" is the @quality-gate agent: it emits free-form
|
|
6
|
+
* prose ending in `VERDICT: X`, and Epic6Executor._parseVerdict turns that prose
|
|
7
|
+
* into the binary verdict (APPROVED → Done). This harness calibrates that
|
|
8
|
+
* DETERMINISTIC seam against a human-labeled golden set:
|
|
9
|
+
*
|
|
10
|
+
* - It runs the REAL _parseVerdict (imported, never re-implemented) over each
|
|
11
|
+
* golden output and checks it agrees with the human-assigned verdict.
|
|
12
|
+
* - _parseVerdict is deterministic, so the gate is 100%: any disagreement means
|
|
13
|
+
* the parser regressed OR a golden label is wrong — both demand action.
|
|
14
|
+
*
|
|
15
|
+
* This does NOT calibrate the LLM's semantic judgment (non-deterministic, costly)
|
|
16
|
+
* — that is the documented LIVE procedure (feed the golden scenarios to the real
|
|
17
|
+
* @quality-gate and compare). See .sinapse-ai/quality/judge-calibration/README.md.
|
|
18
|
+
*
|
|
19
|
+
* `scoreCalibration` is pure so tests can drive it with any parse function.
|
|
20
|
+
*
|
|
21
|
+
* Exit 0 = full agreement · Exit 1 = one or more disagreements.
|
|
22
|
+
*
|
|
23
|
+
* @module scripts/calibrate-judge
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
'use strict';
|
|
27
|
+
|
|
28
|
+
const fs = require('fs');
|
|
29
|
+
const path = require('path');
|
|
30
|
+
|
|
31
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
32
|
+
const GOLDEN_SET_PATH = path.join(
|
|
33
|
+
ROOT,
|
|
34
|
+
'.sinapse-ai/quality/judge-calibration/golden-set.json',
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The REAL verdict-decision logic under calibration. `_parseVerdict` uses only
|
|
39
|
+
* its `output` argument and the module-level QAVerdict constants (no `this`), so
|
|
40
|
+
* it can be invoked detached via the prototype — using the exact production
|
|
41
|
+
* function, never a copy.
|
|
42
|
+
*
|
|
43
|
+
* @returns {(output: string) => string}
|
|
44
|
+
*/
|
|
45
|
+
function loadRealParseVerdict() {
|
|
46
|
+
const Epic6Executor = require('../.sinapse-ai/core/orchestration/executors/epic-6-executor.js');
|
|
47
|
+
const fn = Epic6Executor.prototype._parseVerdict;
|
|
48
|
+
return (output) => fn.call(null, output);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Score a parse function against golden cases.
|
|
53
|
+
*
|
|
54
|
+
* @param {Array<{ id: string, output: string, expected: string }>} cases
|
|
55
|
+
* @param {(output: string) => string} parseFn
|
|
56
|
+
* @returns {{
|
|
57
|
+
* total: number,
|
|
58
|
+
* agreed: number,
|
|
59
|
+
* agreementRate: number,
|
|
60
|
+
* disagreements: Array<{ id: string, expected: string, predicted: string }>,
|
|
61
|
+
* confusion: Record<string, Record<string, number>>
|
|
62
|
+
* }}
|
|
63
|
+
*/
|
|
64
|
+
function scoreCalibration(cases, parseFn) {
|
|
65
|
+
const disagreements = [];
|
|
66
|
+
const confusion = {};
|
|
67
|
+
let agreed = 0;
|
|
68
|
+
|
|
69
|
+
for (const c of cases) {
|
|
70
|
+
const predicted = parseFn(c.output);
|
|
71
|
+
confusion[c.expected] = confusion[c.expected] || {};
|
|
72
|
+
confusion[c.expected][predicted] = (confusion[c.expected][predicted] || 0) + 1;
|
|
73
|
+
if (predicted === c.expected) {
|
|
74
|
+
agreed += 1;
|
|
75
|
+
} else {
|
|
76
|
+
disagreements.push({ id: c.id, expected: c.expected, predicted });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const total = cases.length;
|
|
81
|
+
return {
|
|
82
|
+
total,
|
|
83
|
+
agreed,
|
|
84
|
+
agreementRate: total === 0 ? 1 : agreed / total,
|
|
85
|
+
disagreements,
|
|
86
|
+
confusion,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** CLI entry point. */
|
|
91
|
+
function main() {
|
|
92
|
+
let golden;
|
|
93
|
+
try {
|
|
94
|
+
golden = JSON.parse(fs.readFileSync(GOLDEN_SET_PATH, 'utf8'));
|
|
95
|
+
} catch (err) {
|
|
96
|
+
console.error(`calibrate-judge: cannot read golden set at ${GOLDEN_SET_PATH}: ${err.message}`);
|
|
97
|
+
process.exit(1);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const cases = Array.isArray(golden.cases) ? golden.cases : [];
|
|
101
|
+
if (cases.length === 0) {
|
|
102
|
+
console.error('calibrate-judge: golden set has no cases.');
|
|
103
|
+
process.exit(1);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const parseFn = loadRealParseVerdict();
|
|
107
|
+
const result = scoreCalibration(cases, parseFn);
|
|
108
|
+
|
|
109
|
+
const pct = (result.agreementRate * 100).toFixed(1);
|
|
110
|
+
console.log(`judge calibration: ${result.agreed}/${result.total} agree with the human golden set (${pct}%).`);
|
|
111
|
+
console.log('confusion (expected → predicted):');
|
|
112
|
+
for (const expected of Object.keys(result.confusion)) {
|
|
113
|
+
const preds = result.confusion[expected];
|
|
114
|
+
const parts = Object.keys(preds).map((p) => `${p}:${preds[p]}`).join(', ');
|
|
115
|
+
console.log(` ${expected} → ${parts}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if (result.disagreements.length === 0) {
|
|
119
|
+
console.log('PASS — deterministic verdict extraction is calibrated to the human baseline.');
|
|
120
|
+
process.exit(0);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
console.log('');
|
|
124
|
+
console.log(`FAIL — ${result.disagreements.length} case(s) disagree with the human label:`);
|
|
125
|
+
for (const d of result.disagreements) {
|
|
126
|
+
console.log(` ${d.id}: expected ${d.expected}, parser produced ${d.predicted}`);
|
|
127
|
+
}
|
|
128
|
+
console.log('\nEither _parseVerdict regressed or the golden label is wrong. Fix the code or the label — do not loosen the gate.');
|
|
129
|
+
process.exit(1);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (require.main === module) main();
|
|
133
|
+
|
|
134
|
+
module.exports = { scoreCalibration, loadRealParseVerdict, GOLDEN_SET_PATH };
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* eval-e2e.js — behavioral regression eval for the epic GateEvaluator.
|
|
5
|
+
*
|
|
6
|
+
* Story onda3-s4-behavioral-eval-regression (AF-20260702 item 3.1).
|
|
7
|
+
*
|
|
8
|
+
* Two layers:
|
|
9
|
+
*
|
|
10
|
+
* DETERMINISTIC (always runs, merge gate): replays the versioned golden set
|
|
11
|
+
* (tests/evals/epic-gates/cases.json) against the REAL orchestration
|
|
12
|
+
* GateEvaluator with the DEFAULT gate config pinned (repo-local overrides in
|
|
13
|
+
* core-config.yaml never bend the baseline). Every case is a measured bug
|
|
14
|
+
* from the 2026-06-30 checkpoints — or a control case. Any mismatch = exit 1.
|
|
15
|
+
*
|
|
16
|
+
* LLM ARM PREFLIGHT (opt-in: --llm or SINAPSE_EVAL_LLM=1): verifies that a
|
|
17
|
+
* `claude` binary is spawnable and prints the exact 2-arm measurement
|
|
18
|
+
* protocol commands (tests/evals/epic-gates/PROTOCOL.md). When the spawn is
|
|
19
|
+
* unavailable (known Windows nested-spawn limitation, KNOWN-LIMITATIONS of
|
|
20
|
+
* epic-orchestration-consolidation) it reports EVAL_LLM_SKIPPED with the
|
|
21
|
+
* reason — an explicit skip, never a fake pass.
|
|
22
|
+
*
|
|
23
|
+
* Usage:
|
|
24
|
+
* npm run eval:e2e
|
|
25
|
+
* node scripts/eval-e2e.js [--llm]
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
'use strict';
|
|
29
|
+
|
|
30
|
+
const fs = require('fs');
|
|
31
|
+
const path = require('path');
|
|
32
|
+
const { execFileSync } = require('child_process');
|
|
33
|
+
|
|
34
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
35
|
+
const CASES_PATH = path.join(ROOT, 'tests', 'evals', 'epic-gates', 'cases.json');
|
|
36
|
+
const PROTOCOL_PATH = path.join('tests', 'evals', 'epic-gates', 'PROTOCOL.md');
|
|
37
|
+
|
|
38
|
+
const {
|
|
39
|
+
GateEvaluator,
|
|
40
|
+
DEFAULT_GATE_CONFIG,
|
|
41
|
+
} = require(path.join(ROOT, '.sinapse-ai', 'core', 'orchestration', 'gate-evaluator'));
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Assert one golden case against a live gate result.
|
|
45
|
+
* @returns {string[]} List of mismatch descriptions (empty = pass)
|
|
46
|
+
*/
|
|
47
|
+
function assertCase(caseObj, result) {
|
|
48
|
+
const failures = [];
|
|
49
|
+
const expect = caseObj.expect || {};
|
|
50
|
+
|
|
51
|
+
if (expect.verdict && result.verdict !== expect.verdict) {
|
|
52
|
+
failures.push(`verdict: expected "${expect.verdict}", got "${result.verdict}"`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (typeof expect.scoreEquals === 'number' && Math.abs(result.score - expect.scoreEquals) > 0.01) {
|
|
56
|
+
failures.push(`score: expected ${expect.scoreEquals}, got ${result.score}`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
for (const name of expect.failedChecks || []) {
|
|
60
|
+
const check = result.checks.find((c) => c.name === name);
|
|
61
|
+
if (!check) failures.push(`failed check "${name}": not evaluated at all`);
|
|
62
|
+
else if (check.passed) failures.push(`failed check "${name}": expected FAIL, got PASS`);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
for (const name of expect.passedChecks || []) {
|
|
66
|
+
const check = result.checks.find((c) => c.name === name);
|
|
67
|
+
if (!check) failures.push(`passed check "${name}": not evaluated at all`);
|
|
68
|
+
else if (!check.passed) failures.push(`passed check "${name}": expected PASS, got FAIL (${check.message})`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
for (const name of expect.issues || []) {
|
|
72
|
+
if (!result.issues.some((i) => i.check === name)) {
|
|
73
|
+
failures.push(`issue "${name}": expected present, not found`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return failures;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function runDeterministicLayer() {
|
|
81
|
+
const suite = JSON.parse(fs.readFileSync(CASES_PATH, 'utf8'));
|
|
82
|
+
console.log(`=== eval:e2e — ${suite.suite} (${suite.cases.length} cases) ===`);
|
|
83
|
+
|
|
84
|
+
let failedCases = 0;
|
|
85
|
+
for (const caseObj of suite.cases) {
|
|
86
|
+
// Fresh evaluator per case; DEFAULT config pinned so repo-local
|
|
87
|
+
// core-config.yaml overrides can never bend the behavioral baseline.
|
|
88
|
+
const evaluator = new GateEvaluator({
|
|
89
|
+
projectRoot: ROOT,
|
|
90
|
+
strictMode: Boolean(caseObj.strictMode),
|
|
91
|
+
gateConfig: DEFAULT_GATE_CONFIG,
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const result = await evaluator.evaluate(caseObj.gate.from, caseObj.gate.to, caseObj.epicResult);
|
|
95
|
+
const failures = assertCase(caseObj, result);
|
|
96
|
+
|
|
97
|
+
if (failures.length === 0) {
|
|
98
|
+
console.log(` PASS ${caseObj.id}`);
|
|
99
|
+
} else {
|
|
100
|
+
failedCases += 1;
|
|
101
|
+
console.log(` FAIL ${caseObj.id}`);
|
|
102
|
+
for (const f of failures) console.log(` - ${f}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
console.log('');
|
|
107
|
+
if (failedCases > 0) {
|
|
108
|
+
console.log(`eval:e2e: FAIL — ${failedCases}/${suite.cases.length} behavioral case(s) regressed.`);
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
console.log(`eval:e2e: OK — all ${suite.cases.length} behavioral cases passed.`);
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function runLlmArmPreflight() {
|
|
116
|
+
console.log('');
|
|
117
|
+
console.log('=== eval:e2e — LLM arm preflight ===');
|
|
118
|
+
try {
|
|
119
|
+
const version = execFileSync('claude', ['--version'], {
|
|
120
|
+
encoding: 'utf8',
|
|
121
|
+
timeout: 30000,
|
|
122
|
+
shell: process.platform === 'win32',
|
|
123
|
+
}).trim();
|
|
124
|
+
console.log(`claude spawnable: ${version}`);
|
|
125
|
+
console.log(`Protocolo de medição 2 braços: ${PROTOCOL_PATH}`);
|
|
126
|
+
console.log('Execute os braços conforme o protocolo e registre o scoreboard no checkpoint.');
|
|
127
|
+
return true;
|
|
128
|
+
} catch (error) {
|
|
129
|
+
console.log(`EVAL_LLM_SKIPPED: claude não spawnável daqui (${error.message.split('\n')[0]}).`);
|
|
130
|
+
console.log('Motivo conhecido no Windows: spawn aninhado falha (KNOWN-LIMITATIONS,');
|
|
131
|
+
console.log('epic-orchestration-consolidation). Rode o braço LLM via harness ou em');
|
|
132
|
+
console.log(`ambiente com spawn direto, seguindo ${PROTOCOL_PATH}.`);
|
|
133
|
+
return true; // skip explícito não é falha — mas nunca é reportado como PASS do braço
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function main() {
|
|
138
|
+
const wantLlm = process.argv.includes('--llm') || process.env.SINAPSE_EVAL_LLM === '1';
|
|
139
|
+
|
|
140
|
+
const deterministicOk = await runDeterministicLayer();
|
|
141
|
+
if (wantLlm) runLlmArmPreflight();
|
|
142
|
+
|
|
143
|
+
process.exit(deterministicOk ? 0 : 1);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
main().catch((error) => {
|
|
147
|
+
console.error(`eval:e2e crashed: ${error.message}`);
|
|
148
|
+
process.exit(1);
|
|
149
|
+
});
|
package/scripts/validate-all.js
CHANGED
|
@@ -55,6 +55,14 @@ const GUARDS = [
|
|
|
55
55
|
{ name: 'squad-yaml', script: 'validate:squad-yaml' },
|
|
56
56
|
{ name: 'squad-orqx', script: 'validate:squad-orqx' },
|
|
57
57
|
{ name: 'agent-codenames', script: 'validate:agent-codenames' },
|
|
58
|
+
{ name: 'tool-descriptions', script: 'validate:tool-descriptions' },
|
|
59
|
+
{ name: 'constitution', script: 'validate:constitution' },
|
|
60
|
+
// ACs in Given/When/Then — WARN-only (exits 0 unless STORY_ACS_STRICT=1) and
|
|
61
|
+
// a no-op PASS in CI where docs/stories/ (gitignored) is absent.
|
|
62
|
+
{ name: 'story-acs', script: 'validate:story-acs' },
|
|
63
|
+
// Article IV traceability — WARN-only during calibration: the script itself
|
|
64
|
+
// exits 0 on orphans unless ARTICLE_IV_STRICT=1 (see validate-article-iv.js).
|
|
65
|
+
{ name: 'article-iv', script: 'validate:article-iv' },
|
|
58
66
|
];
|
|
59
67
|
|
|
60
68
|
/**
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* validate-article-iv.js — Constitution Article IV (No Invention) traceability check.
|
|
5
|
+
*
|
|
6
|
+
* Story onda3-s1-article-iv-traceability (AF-20260702 item 3.2).
|
|
7
|
+
*
|
|
8
|
+
* What it does (deterministic, no LLM):
|
|
9
|
+
* 1. Resolves the push range: merge-base(HEAD, origin/main)..HEAD.
|
|
10
|
+
* 2. Extracts story refs ("[Story xyz]") from commit messages in the range.
|
|
11
|
+
* 3. Locates each referenced story under docs/stories/ and extracts its
|
|
12
|
+
* acceptance criteria ("- [ ] ACn:") and its "## File List" section.
|
|
13
|
+
* 4. Compares changed PRODUCT files against the union of the stories'
|
|
14
|
+
* File Lists. A changed product file mapped by no story is an ORPHAN.
|
|
15
|
+
*
|
|
16
|
+
* Calibration mode (default): orphans are WARNINGS, exit 0. Pass --strict
|
|
17
|
+
* (or ARTICLE_IV_STRICT=1) to turn orphans into a failure (exit 1). The
|
|
18
|
+
* warning phase exists to calibrate the matcher before it gets teeth —
|
|
19
|
+
* mirrors the rollout the audit prescribed ("órfão começa como warning").
|
|
20
|
+
*
|
|
21
|
+
* Why this runs at pre-push and NOT in GitHub CI: docs/stories/ is local-only
|
|
22
|
+
* (gitignored by design — stories are working artifacts). CI has no stories
|
|
23
|
+
* to trace against, so the only honest home for this check is the developer
|
|
24
|
+
* machine, where the stories exist. When stories are unavailable the check
|
|
25
|
+
* reports ARTICLE_IV_SKIPPED and exits 0 — an explicit skip, never a fake pass.
|
|
26
|
+
*
|
|
27
|
+
* Usage:
|
|
28
|
+
* npm run validate:article-iv
|
|
29
|
+
* node scripts/validate-article-iv.js [--strict] [--base <ref>]
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
'use strict';
|
|
33
|
+
|
|
34
|
+
const { execFileSync } = require('child_process');
|
|
35
|
+
const fs = require('fs');
|
|
36
|
+
const path = require('path');
|
|
37
|
+
|
|
38
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
39
|
+
|
|
40
|
+
// Prefixes that count as product surface for traceability purposes.
|
|
41
|
+
// Aligned with enforce-story-gate.cjs CODE_PATHS plus the framework tree
|
|
42
|
+
// (.sinapse-ai) and scripts/, which are product in this repo.
|
|
43
|
+
const PRODUCT_PREFIXES = [
|
|
44
|
+
'packages/',
|
|
45
|
+
'src/',
|
|
46
|
+
'app/',
|
|
47
|
+
'lib/',
|
|
48
|
+
'bin/',
|
|
49
|
+
'components/',
|
|
50
|
+
'pages/',
|
|
51
|
+
'api/',
|
|
52
|
+
'services/',
|
|
53
|
+
'scripts/',
|
|
54
|
+
'.sinapse-ai/',
|
|
55
|
+
];
|
|
56
|
+
|
|
57
|
+
// Never traced: docs, tests, mirrors, CI plumbing and generated artifacts —
|
|
58
|
+
// they either have their own governance or are regenerated mechanically.
|
|
59
|
+
const EXCLUDED_PREFIXES = [
|
|
60
|
+
'docs/',
|
|
61
|
+
'tests/',
|
|
62
|
+
'.claude/',
|
|
63
|
+
'.codex/',
|
|
64
|
+
'.github/',
|
|
65
|
+
'audits/',
|
|
66
|
+
'governance/',
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
const GENERATED_FILES = [
|
|
70
|
+
'.sinapse-ai/install-manifest.yaml',
|
|
71
|
+
'.sinapse-ai/data/entity-registry.yaml',
|
|
72
|
+
'.sinapse-ai/manifests/agents.csv',
|
|
73
|
+
'.sinapse-ai/manifests/workers.csv',
|
|
74
|
+
'.sinapse-ai/manifests/tasks.csv',
|
|
75
|
+
];
|
|
76
|
+
|
|
77
|
+
const STORY_REF_RE = /\[story\s+([^\]]+)\]/gi;
|
|
78
|
+
const AC_RE = /^- \[( |x)\] (AC\d+):\s*(.*)$/gim;
|
|
79
|
+
|
|
80
|
+
/** Extract unique story refs ("[Story 2.1]" → "2.1") from free text. */
|
|
81
|
+
function extractStoryRefs(text) {
|
|
82
|
+
const refs = new Set();
|
|
83
|
+
let m;
|
|
84
|
+
STORY_REF_RE.lastIndex = 0;
|
|
85
|
+
while ((m = STORY_REF_RE.exec(text)) !== null) {
|
|
86
|
+
const ref = m[1].trim();
|
|
87
|
+
if (ref) refs.add(ref);
|
|
88
|
+
}
|
|
89
|
+
return [...refs];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Extract acceptance criteria from story markdown. */
|
|
93
|
+
function extractAcceptanceCriteria(storyContent) {
|
|
94
|
+
const acs = [];
|
|
95
|
+
let m;
|
|
96
|
+
AC_RE.lastIndex = 0;
|
|
97
|
+
while ((m = AC_RE.exec(storyContent)) !== null) {
|
|
98
|
+
acs.push({ id: m[2].toUpperCase(), checked: m[1] === 'x', text: m[3].trim() });
|
|
99
|
+
}
|
|
100
|
+
return acs;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Extract path-ish entries from the story's "## File List" section.
|
|
105
|
+
* Accepts list items like "- path/to/file.js (novo)" — the first
|
|
106
|
+
* whitespace-delimited token of each bullet is taken as the path.
|
|
107
|
+
*/
|
|
108
|
+
function extractFileList(storyContent) {
|
|
109
|
+
const lines = storyContent.split(/\r?\n/);
|
|
110
|
+
const entries = [];
|
|
111
|
+
let inSection = false;
|
|
112
|
+
for (const line of lines) {
|
|
113
|
+
if (/^##\s+file list\s*$/i.test(line.trim())) {
|
|
114
|
+
inSection = true;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (inSection && /^##\s+/.test(line.trim())) break;
|
|
118
|
+
if (!inSection) continue;
|
|
119
|
+
const bullet = line.trim().match(/^[-*]\s+(.*)$/);
|
|
120
|
+
if (!bullet) continue;
|
|
121
|
+
const token = bullet[1].trim().split(/\s+/)[0];
|
|
122
|
+
if (token) entries.push(token.replace(/\\/g, '/').replace(/[,;]$/, ''));
|
|
123
|
+
}
|
|
124
|
+
return entries;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** True when a changed path counts as traceable product surface. */
|
|
128
|
+
function isProductFile(rel) {
|
|
129
|
+
const norm = rel.replace(/\\/g, '/');
|
|
130
|
+
if (GENERATED_FILES.includes(norm)) return false;
|
|
131
|
+
if (EXCLUDED_PREFIXES.some((p) => norm.startsWith(p))) return false;
|
|
132
|
+
return PRODUCT_PREFIXES.some((p) => norm.startsWith(p));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Match changed product files against File List entries.
|
|
137
|
+
* A file is mapped when some entry equals it, is a prefix directory of it,
|
|
138
|
+
* or the file path ends with the entry (tolerates entries written relative).
|
|
139
|
+
*/
|
|
140
|
+
function matchFilesToStory(changedProductFiles, fileListEntries) {
|
|
141
|
+
const norm = (s) => s.replace(/\\/g, '/').toLowerCase();
|
|
142
|
+
const entries = fileListEntries.map(norm);
|
|
143
|
+
const mapped = [];
|
|
144
|
+
const orphans = [];
|
|
145
|
+
for (const file of changedProductFiles) {
|
|
146
|
+
const f = norm(file);
|
|
147
|
+
const hit = entries.some(
|
|
148
|
+
(e) => e === f || f.startsWith(e.endsWith('/') ? e : `${e}/`) || f.endsWith(e),
|
|
149
|
+
);
|
|
150
|
+
(hit ? mapped : orphans).push(file);
|
|
151
|
+
}
|
|
152
|
+
return { mapped, orphans };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Recursively collect .md files under a directory. */
|
|
156
|
+
function walkMarkdown(dir) {
|
|
157
|
+
const out = [];
|
|
158
|
+
let items = [];
|
|
159
|
+
try {
|
|
160
|
+
items = fs.readdirSync(dir, { withFileTypes: true });
|
|
161
|
+
} catch {
|
|
162
|
+
return out;
|
|
163
|
+
}
|
|
164
|
+
for (const item of items) {
|
|
165
|
+
const full = path.join(dir, item.name);
|
|
166
|
+
if (item.isDirectory()) out.push(...walkMarkdown(full));
|
|
167
|
+
else if (item.name.endsWith('.md')) out.push(full);
|
|
168
|
+
}
|
|
169
|
+
return out;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Find the story file for a ref: filename containing the slugged ref, or
|
|
174
|
+
* story content containing the literal "[Story <ref>]".
|
|
175
|
+
*/
|
|
176
|
+
function findStoryFile(storiesDir, ref) {
|
|
177
|
+
const slug = ref.toLowerCase().replace(/\s+/g, '-');
|
|
178
|
+
const files = walkMarkdown(storiesDir);
|
|
179
|
+
for (const file of files) {
|
|
180
|
+
const base = path.basename(file, '.md').toLowerCase();
|
|
181
|
+
if (base.includes(slug)) return file;
|
|
182
|
+
}
|
|
183
|
+
const needle = `[story ${ref.toLowerCase()}]`;
|
|
184
|
+
for (const file of files) {
|
|
185
|
+
try {
|
|
186
|
+
if (fs.readFileSync(file, 'utf8').toLowerCase().includes(needle)) return file;
|
|
187
|
+
} catch {
|
|
188
|
+
/* unreadable file — keep scanning */
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function git(args) {
|
|
195
|
+
return execFileSync('git', args, { cwd: ROOT, encoding: 'utf8' });
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function main() {
|
|
199
|
+
const argv = process.argv.slice(2);
|
|
200
|
+
const strict = argv.includes('--strict') || process.env.ARTICLE_IV_STRICT === '1';
|
|
201
|
+
const baseIdx = argv.indexOf('--base');
|
|
202
|
+
const baseRef = baseIdx !== -1 ? argv[baseIdx + 1] : process.env.ARTICLE_IV_BASE || 'origin/main';
|
|
203
|
+
|
|
204
|
+
let mergeBase;
|
|
205
|
+
try {
|
|
206
|
+
mergeBase = git(['merge-base', 'HEAD', baseRef]).trim();
|
|
207
|
+
} catch {
|
|
208
|
+
console.log(`ARTICLE_IV_SKIPPED: cannot resolve merge-base with ${baseRef} (no remote?).`);
|
|
209
|
+
process.exit(0);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const changed = git(['diff', '--name-only', `${mergeBase}..HEAD`])
|
|
213
|
+
.split(/\r?\n/)
|
|
214
|
+
.filter(Boolean);
|
|
215
|
+
const commitText = git(['log', '--format=%B', `${mergeBase}..HEAD`]);
|
|
216
|
+
|
|
217
|
+
const refs = extractStoryRefs(commitText);
|
|
218
|
+
if (refs.length === 0) {
|
|
219
|
+
console.log('article-iv: no story refs in push range — nothing to trace.');
|
|
220
|
+
process.exit(0);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const storiesDir = path.join(ROOT, 'docs', 'stories');
|
|
224
|
+
if (!fs.existsSync(storiesDir)) {
|
|
225
|
+
console.log('ARTICLE_IV_SKIPPED: docs/stories unavailable (local-only artifact).');
|
|
226
|
+
process.exit(0);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const fileListUnion = [];
|
|
230
|
+
const foundStories = [];
|
|
231
|
+
const missingRefs = [];
|
|
232
|
+
for (const ref of refs) {
|
|
233
|
+
const storyFile = findStoryFile(storiesDir, ref);
|
|
234
|
+
if (!storyFile) {
|
|
235
|
+
missingRefs.push(ref);
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
const content = fs.readFileSync(storyFile, 'utf8');
|
|
239
|
+
foundStories.push({ ref, file: path.relative(ROOT, storyFile), acs: extractAcceptanceCriteria(content).length });
|
|
240
|
+
fileListUnion.push(...extractFileList(content));
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
if (foundStories.length === 0) {
|
|
244
|
+
console.log(
|
|
245
|
+
`ARTICLE_IV_SKIPPED: no referenced story found locally (refs: ${refs.join(', ')}).`,
|
|
246
|
+
);
|
|
247
|
+
process.exit(0);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const productChanged = changed.filter(isProductFile);
|
|
251
|
+
const { mapped, orphans } = matchFilesToStory(productChanged, fileListUnion);
|
|
252
|
+
|
|
253
|
+
console.log('=== article-iv traceability ===');
|
|
254
|
+
for (const s of foundStories) {
|
|
255
|
+
console.log(`story [${s.ref}] -> ${s.file} (${s.acs} ACs)`);
|
|
256
|
+
}
|
|
257
|
+
if (missingRefs.length) {
|
|
258
|
+
console.log(`refs without local story (skipped): ${missingRefs.join(', ')}`);
|
|
259
|
+
}
|
|
260
|
+
console.log(`product files changed: ${productChanged.length} | mapped: ${mapped.length} | orphans: ${orphans.length}`);
|
|
261
|
+
|
|
262
|
+
if (orphans.length > 0) {
|
|
263
|
+
console.log('');
|
|
264
|
+
console.log('ORPHAN product files (changed but absent from every referenced File List):');
|
|
265
|
+
for (const f of orphans) console.log(` WARN ${f}`);
|
|
266
|
+
console.log('');
|
|
267
|
+
console.log('Fix: add the file to the story\'s "## File List", or move the change to its own story.');
|
|
268
|
+
if (strict) {
|
|
269
|
+
console.log('article-iv: FAIL (strict mode).');
|
|
270
|
+
process.exit(1);
|
|
271
|
+
}
|
|
272
|
+
console.log('article-iv: WARN-only (calibration mode) — not blocking.');
|
|
273
|
+
process.exit(0);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
console.log('article-iv: OK — full traceability.');
|
|
277
|
+
process.exit(0);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
module.exports = {
|
|
281
|
+
extractStoryRefs,
|
|
282
|
+
extractAcceptanceCriteria,
|
|
283
|
+
extractFileList,
|
|
284
|
+
isProductFile,
|
|
285
|
+
matchFilesToStory,
|
|
286
|
+
findStoryFile,
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
if (require.main === module) main();
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Lint guard: Constitution consistency across CLAUDE.md ⇄ AGENTS.md (+ source,
|
|
4
|
+
* install template, rules). Story rodada2-m5 (AF-20260704 Mesa #7).
|
|
5
|
+
*
|
|
6
|
+
* The Mesa item framed CLAUDE.md and AGENTS.md as "twins that drift" and asked
|
|
7
|
+
* to generate both from one source. Verification REFUTED that premise: the two
|
|
8
|
+
* files serve different surfaces (Claude Code vs Codex), share almost no verbatim
|
|
9
|
+
* body, and their only shared canon — the ecosystem counts and the Constitution
|
|
10
|
+
* article set — is already single-sourced. Counts are guarded by
|
|
11
|
+
* `validate:agents-md`; the article agreement is validated by the
|
|
12
|
+
* `constitution-consistency` doctor check.
|
|
13
|
+
*
|
|
14
|
+
* The real, narrow gap this guard closes: that doctor check ran only under
|
|
15
|
+
* `sinapse doctor` and the install matrix — NOT on the PR gate. So article drift
|
|
16
|
+
* between CLAUDE.md and AGENTS.md could merge unnoticed. This wraps the SAME
|
|
17
|
+
* existing check (no new logic, no twin-generator) so it runs on every push/PR.
|
|
18
|
+
*
|
|
19
|
+
* Exit 0 = consistent (PASS/WARN) · Exit 1 = FAIL (a consumer is missing a
|
|
20
|
+
* canonical article, or the source/rules are missing).
|
|
21
|
+
*
|
|
22
|
+
* @module scripts/validate-constitution
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
'use strict';
|
|
26
|
+
|
|
27
|
+
const path = require('path');
|
|
28
|
+
|
|
29
|
+
const check = require('../.sinapse-ai/core/doctor/checks/constitution-consistency');
|
|
30
|
+
|
|
31
|
+
async function main() {
|
|
32
|
+
const projectRoot = path.resolve(__dirname, '..');
|
|
33
|
+
let result;
|
|
34
|
+
try {
|
|
35
|
+
result = await check.run({ projectRoot });
|
|
36
|
+
} catch (err) {
|
|
37
|
+
// The check declares onError:'fail' — a thrown error is a real problem.
|
|
38
|
+
console.error(`FAIL — constitution-consistency threw: ${err.message}`);
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const status = result.status || 'FAIL';
|
|
43
|
+
if (status === 'PASS') {
|
|
44
|
+
console.log(`OK — ${result.message}`);
|
|
45
|
+
process.exit(0);
|
|
46
|
+
}
|
|
47
|
+
if (status === 'WARN') {
|
|
48
|
+
// Minor divergence (≤2 issues) — surface it but do not block (matches the
|
|
49
|
+
// doctor severity gradation; Art. XI, avoid over-tightening the gate).
|
|
50
|
+
console.warn(`WARN — ${result.message}`);
|
|
51
|
+
process.exit(0);
|
|
52
|
+
}
|
|
53
|
+
console.error(`FAIL — ${result.message}`);
|
|
54
|
+
if (result.fixCommand) console.error(` fix: ${result.fixCommand}`);
|
|
55
|
+
process.exit(1);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
main();
|