pi-revit 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +483 -465
- package/README.md +802 -768
- package/docs/architecture.md +180 -260
- package/package.json +21 -17
- package/AGENTS.md +0 -167
- package/docs/evaluation.md +0 -434
- package/docs/invariants.json +0 -147
- package/scripts/benchmark-search-docs.py +0 -322
- package/scripts/check-tool-documentation.mjs +0 -287
- package/scripts/generate-contracts.mjs +0 -80
- package/scripts/lib/platform.mjs +0 -226
- package/scripts/test-extension.mjs +0 -15
|
@@ -1,287 +0,0 @@
|
|
|
1
|
-
import assert from 'node:assert/strict';
|
|
2
|
-
import { existsSync } from 'node:fs';
|
|
3
|
-
import { access, readFile, readdir } from 'node:fs/promises';
|
|
4
|
-
import path from 'node:path';
|
|
5
|
-
import { pathToFileURL } from 'node:url';
|
|
6
|
-
import {
|
|
7
|
-
CONTRACT_END, CONTRACT_START, createLoader, evaluateCorpus, extractBridgeDescriptors, manifestPath, manualRoot, registerExtension,
|
|
8
|
-
resolvePiDependencies, revitApiXmlPaths, root, skillRoot, snapshotPath,
|
|
9
|
-
} from './lib/platform.mjs';
|
|
10
|
-
import { buildArtifacts } from './generate-contracts.mjs';
|
|
11
|
-
|
|
12
|
-
// Offline platform gates: no model execution, real bridge connection or package
|
|
13
|
-
// installation. Source metadata, generated artifacts, schemas, guidance structure and
|
|
14
|
-
// prompt size are checked; Revit bodies and agent behavior need their own tests.
|
|
15
|
-
// Budgets change deliberately, never by accident: raise them only with a reviewed reason
|
|
16
|
-
// in the change log. The startup total covers the platform section plus the snippets and
|
|
17
|
-
// guidelines of tools active at session start; it must not grow with the tool count.
|
|
18
|
-
const PROMPT_BUDGET = { startupTotal: 7000, perToolGuidelines: 1200 };
|
|
19
|
-
const problems = [];
|
|
20
|
-
const warnings = [];
|
|
21
|
-
const fail = message => problems.push(message);
|
|
22
|
-
|
|
23
|
-
const piRequire = resolvePiDependencies();
|
|
24
|
-
const loader = createLoader(piRequire);
|
|
25
|
-
const { Check, Errors } = await import(pathToFileURL(piRequire.resolve('typebox/value')).href);
|
|
26
|
-
assert.equal(Check({ type: 'integer', minimum: 1 }, 0), false, 'schema validator must enforce numeric constraints');
|
|
27
|
-
assert.equal(Check({ type: 'object', required: ['id'], properties: { id: { type: 'integer' } } }, {}), false,
|
|
28
|
-
'schema validator must enforce required properties');
|
|
29
|
-
|
|
30
|
-
// 1. Generated artifacts are current: code is the single owner of contracts.
|
|
31
|
-
const { files: generated } = await buildArtifacts();
|
|
32
|
-
for (const [file, content] of generated)
|
|
33
|
-
if (!existsSync(file) || await readFile(file, 'utf8') !== content) fail(`${path.relative(root, file)}: generated artifact is stale; run npm run generate:contracts`);
|
|
34
|
-
|
|
35
|
-
const packageInfo = JSON.parse(await readFile(path.join(root, 'package.json'), 'utf8'));
|
|
36
|
-
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
|
37
|
-
const snapshot = JSON.parse(await readFile(snapshotPath, 'utf8'));
|
|
38
|
-
const descriptors = await extractBridgeDescriptors();
|
|
39
|
-
const bridgeNames = new Set(descriptors.map(tool => tool.name));
|
|
40
|
-
assert.ok(descriptors.length > 0, 'real registry must expose bridge descriptors');
|
|
41
|
-
assert.equal(bridgeNames.size, descriptors.length, 'bridge tool names must be unique');
|
|
42
|
-
|
|
43
|
-
// 2. Documentation manifest v2 structure.
|
|
44
|
-
assert.equal(manifest.schema_version, 2, 'unrecognized documentation manifest schema');
|
|
45
|
-
assert.ok(typeof manifest.documentation_revision === 'string' && manifest.documentation_revision.trim(), 'documentation revision is required');
|
|
46
|
-
const groupIds = new Set(manifest.groups.map(group => group.id));
|
|
47
|
-
const manifestNames = new Set(manifest.tools.map(tool => tool.name));
|
|
48
|
-
assert.equal(manifestNames.size, manifest.tools.length, 'manifest tool names must be unique');
|
|
49
|
-
for (const tool of manifest.tools) {
|
|
50
|
-
assert.match(tool.name, /^[a-z][a-z0-9_]*$/, 'tool names must be safe filenames');
|
|
51
|
-
assert.ok(['bridge', 'native'].includes(tool.source), `${tool.name}: source must be bridge or native`);
|
|
52
|
-
assert.ok(typeof tool.summary === 'string' && tool.summary.trim(), `${tool.name}: summary is required`);
|
|
53
|
-
if (!groupIds.has(tool.group)) fail(`${tool.name}: unknown manifest group ${tool.group}`);
|
|
54
|
-
}
|
|
55
|
-
assert.deepEqual(manifest.tools.filter(tool => tool.source === 'bridge').map(tool => tool.name).sort(), [...bridgeNames].sort(),
|
|
56
|
-
'manifest bridge inventory must equal the real C# registry');
|
|
57
|
-
const guidanceNames = new Set();
|
|
58
|
-
for (const guide of manifest.guidance) {
|
|
59
|
-
if (guidanceNames.has(guide.name)) fail(`guidance ${guide.name}: duplicate name`);
|
|
60
|
-
guidanceNames.add(guide.name);
|
|
61
|
-
if (!['workflow', 'reference', 'skill'].includes(guide.kind)) fail(`guidance ${guide.name}: kind must be workflow, reference or skill`);
|
|
62
|
-
if (!existsSync(path.join(skillRoot, guide.path))) fail(`guidance ${guide.name}: missing ${guide.path}`);
|
|
63
|
-
if (!guide.summary?.trim() || !(guide.keywords?.length >= 3)) fail(`guidance ${guide.name}: needs a summary and at least 3 keywords`);
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
// 3. Real registration against the extracted bridge catalogue.
|
|
67
|
-
const fixture = await registerExtension(loader, descriptors, packageInfo.version);
|
|
68
|
-
const { publicBridgeSchema } = await loader.import(path.join(root, 'extensions', 'pi-revit', 'tool-schema.ts'));
|
|
69
|
-
const { ALTERNATIVE_KINDS, VERIFICATION_KINDS, apiLookupQuery } = await loader.import(path.join(root, 'extensions', 'pi-revit', 'contracts.ts'));
|
|
70
|
-
const registered = fixture.registered;
|
|
71
|
-
let promptReport;
|
|
72
|
-
try {
|
|
73
|
-
assert.ok(fixture.routes.includes('/tools'), 'connector must register real extracted bridge schemas');
|
|
74
|
-
assert.deepEqual([...registered.keys()].sort(), [...manifestNames].sort(), 'manifest must cover every registered public tool');
|
|
75
|
-
assert.deepEqual(manifest.tools.filter(tool => tool.source === 'native').map(tool => tool.name).sort(),
|
|
76
|
-
[...registered.keys()].filter(name => !bridgeNames.has(name)).sort(), 'native inventory must match actual extension registrations');
|
|
77
|
-
for (const descriptor of descriptors)
|
|
78
|
-
assert.deepEqual(JSON.parse(JSON.stringify(registered.get(descriptor.name).parameters)), publicBridgeSchema(descriptor.parameters),
|
|
79
|
-
`${descriptor.name}: registered parameters must include the actual extension schema overlay`);
|
|
80
|
-
|
|
81
|
-
// 8 (measured here while registered). Prompt budget: the startup contribution must not grow with tool count.
|
|
82
|
-
await fixture.events.get('session_start')?.({}, { ui: { notify() {} } });
|
|
83
|
-
const options = { sections: {}, selectedTools: fixture.activeTools() };
|
|
84
|
-
await fixture.events.get('before_agent_start')?.({ systemPromptOptions: options }, {});
|
|
85
|
-
const section = options.sections.pi_revit ?? '';
|
|
86
|
-
const perTool = fixture.activeTools().map(name => ({ name, snippet: registered.get(name)?.promptSnippet ?? '', guidelines: (registered.get(name)?.promptGuidelines ?? []).join('\n') }));
|
|
87
|
-
const toolChars = perTool.reduce((sum, tool) => sum + tool.snippet.length + tool.guidelines.length, 0);
|
|
88
|
-
promptReport = { sectionChars: section.length, toolChars, activeTools: perTool.length, total: section.length + toolChars,
|
|
89
|
-
largestGuidelines: perTool.map(t => [t.name, t.guidelines.length]).sort((a, b) => b[1] - a[1])[0] };
|
|
90
|
-
if (!section.includes('PI-Revit protocol')) fail('platform section was not injected by before_agent_start');
|
|
91
|
-
if (/For \w+, read .*\.md when detailed usage/.test(perTool.map(t => t.guidelines).join('\n'))) fail('per-tool manual pointer lines must not return; the platform section holds one pointer');
|
|
92
|
-
for (const tool of perTool) if (tool.guidelines.length > PROMPT_BUDGET.perToolGuidelines) fail(`${tool.name}: ${tool.guidelines.length} guideline characters exceed the per-tool budget of ${PROMPT_BUDGET.perToolGuidelines}`);
|
|
93
|
-
if (promptReport.total > PROMPT_BUDGET.startupTotal) fail(`PI-Revit startup prompt contribution ${promptReport.total} characters exceeds the budget of ${PROMPT_BUDGET.startupTotal}`);
|
|
94
|
-
} finally { await fixture.restore(); }
|
|
95
|
-
|
|
96
|
-
// 4. Contract v2 validity for every tool, including limit references.
|
|
97
|
-
const publicNames = new Set([...registered.keys(), 'read']);
|
|
98
|
-
const apiXml = await Promise.all(revitApiXmlPaths().map(file => readFile(file, 'utf8')));
|
|
99
|
-
if (!apiXml.length) warnings.push('RevitAPI.xml not found; API references in declared limits were not verified (set REVIT_API_PATH).');
|
|
100
|
-
const apiMemberExists = member => apiXml.some(xml => new RegExp(`name="[TMPFE]:Autodesk\\.Revit\\.(?:[A-Za-z0-9]+\\.)*${member.replace(/\./g, '\\.')}(?:["(])`).test(xml));
|
|
101
|
-
for (const tool of snapshot.tools) {
|
|
102
|
-
if (!(tool.keywords?.length >= 3)) fail(`${tool.name}: declare at least 3 discovery keywords`);
|
|
103
|
-
if ((tool.write || tool.effects?.length) && !VERIFICATION_KINDS.includes(tool.verification))
|
|
104
|
-
fail(`${tool.name}: a tool that can write or has effects must declare verification (${VERIFICATION_KINDS.join(', ')})`);
|
|
105
|
-
if (tool.verification !== null && !VERIFICATION_KINDS.includes(tool.verification)) fail(`${tool.name}: unknown verification ${tool.verification}`);
|
|
106
|
-
if (!tool.limits.length) fail(`${tool.name}: declare at least one limit with its alternative`);
|
|
107
|
-
for (const limit of tool.limits) {
|
|
108
|
-
if (!limit.what?.trim() || !ALTERNATIVE_KINDS.includes(limit.alternative?.kind) || !limit.alternative.ref?.trim())
|
|
109
|
-
{ fail(`${tool.name}: every limit needs what, a known alternative kind and a reference`); continue; }
|
|
110
|
-
const ref = limit.alternative.ref;
|
|
111
|
-
if (limit.alternative.kind === 'tool') {
|
|
112
|
-
const named = [...ref.matchAll(/\b[a-z][a-z0-9]*(?:_[a-z0-9]+)+\b|\bread\b/g)].map(m => m[0]).filter(name => !['host_bounds', 'include_instance_count', 'count_only', 'next_offset'].includes(name));
|
|
113
|
-
if (!named.length) fail(`${tool.name}: tool alternative "${ref}" names no public tool`);
|
|
114
|
-
for (const name of named) if (!publicNames.has(name)) fail(`${tool.name}: tool alternative names unknown tool ${name}`);
|
|
115
|
-
}
|
|
116
|
-
// Every name in the generated one-call lookup must resolve, so the lookup never sends the agent searching.
|
|
117
|
-
if (limit.alternative.kind === 'api' && apiXml.length)
|
|
118
|
-
for (const member of (apiLookupQuery(ref) ?? '').split('; ').filter(Boolean))
|
|
119
|
-
if (!apiMemberExists(member)) fail(`${tool.name}: API alternative ${member} was not found in the installed RevitAPI.xml`);
|
|
120
|
-
}
|
|
121
|
-
// Required inputs must not be described as optional anywhere in their schema text.
|
|
122
|
-
for (const name of tool.parameters?.required ?? [])
|
|
123
|
-
if (/\boptional\b/i.test(tool.parameters.properties?.[name]?.description ?? '')) fail(`${tool.name}.${name}: a required input's description says "optional"`);
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// 5. Manuals: one per tool, generated block present, examples valid, limitation language declared.
|
|
127
|
-
const manualFiles = (await readdir(manualRoot)).filter(file => file.endsWith('.md')).sort();
|
|
128
|
-
assert.deepEqual(manualFiles, [...manifestNames].map(name => `${name}.md`).sort(), 'one manual per public tool; no orphan manuals');
|
|
129
|
-
const snapshotByName = new Map(snapshot.tools.map(tool => [tool.name, tool]));
|
|
130
|
-
let exampleCount = 0;
|
|
131
|
-
for (const tool of manifest.tools) {
|
|
132
|
-
const file = path.join(manualRoot, `${tool.name}.md`);
|
|
133
|
-
const markdown = await readFile(file, 'utf8');
|
|
134
|
-
if (!markdown.includes(CONTRACT_START) || !markdown.includes(CONTRACT_END)) fail(`${tool.name}: manual lacks its generated contract block`);
|
|
135
|
-
const handWritten = markdown.slice(0, markdown.indexOf(CONTRACT_START)) + markdown.slice(markdown.indexOf(CONTRACT_END) + CONTRACT_END.length);
|
|
136
|
-
if (/outside this tool|\bunsupported\b|not supported/i.test(handWritten) && !snapshotByName.get(tool.name)?.limits.length)
|
|
137
|
-
fail(`${tool.name}: the manual states a limitation but the tool declares no limits with alternatives`);
|
|
138
|
-
const examples = [...markdown.matchAll(/^```json[ \t]*\r?\n([\s\S]*?)^```[ \t]*\r?$/gm)];
|
|
139
|
-
if (examples.length === 0) fail(`${tool.name}: missing JSON input example`);
|
|
140
|
-
const schema = registered.get(tool.name).parameters;
|
|
141
|
-
for (const match of examples) {
|
|
142
|
-
const label = `${path.relative(root, file)}:${markdown.slice(0, match.index).split('\n').length + 1}`;
|
|
143
|
-
try {
|
|
144
|
-
const input = JSON.parse(match[1]);
|
|
145
|
-
assert.ok(input !== null && typeof input === 'object' && !Array.isArray(input), 'example must be an input argument object');
|
|
146
|
-
if (!Check(schema, input)) throw new Error(JSON.stringify([...Errors(schema, input)]));
|
|
147
|
-
assertDocumentedKeys(schema, input, label);
|
|
148
|
-
// Negative controls exercise the actual composed schemas rather than a copied test schema.
|
|
149
|
-
for (const key of schema.required ?? []) {
|
|
150
|
-
const missingRequired = { ...input };
|
|
151
|
-
delete missingRequired[key];
|
|
152
|
-
assert.equal(Check(schema, missingRequired), false, `${tool.name}: required ${key} was not enforced`);
|
|
153
|
-
}
|
|
154
|
-
if (schema.properties?._operation_id) assert.equal(Check(schema, { ...input, _operation_id: 42 }), false, `${tool.name}: retry ID must remain a string`);
|
|
155
|
-
exampleCount++;
|
|
156
|
-
} catch (error) { fail(`${label}: ${error.message}`); }
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
// 6. Invariant register: every rule has an owner and an enforcement, and guidance tags resolve.
|
|
161
|
-
const register = JSON.parse(await readFile(path.join(root, 'docs', 'invariants.json'), 'utf8'));
|
|
162
|
-
const invariants = new Map(register.invariants.map(item => [item.id, item]));
|
|
163
|
-
const guidanceFiles = [...await markdownUnder(path.join(root, 'skills')), ...await markdownUnder(path.join(root, 'docs')),
|
|
164
|
-
...['AGENTS.md', 'README.md', 'workspace/AGENTS.md'].map(file => path.join(root, file))];
|
|
165
|
-
const tagged = new Set();
|
|
166
|
-
const testSources = (await Promise.all((await filesUnder(path.join(root, 'tests'))).filter(file => /\.(mjs|cjs|cs)$/.test(file)).map(file => readFile(file, 'utf8')))).join('\n')
|
|
167
|
-
+ await readFile(new URL(import.meta.url), 'utf8');
|
|
168
|
-
for (const file of guidanceFiles) {
|
|
169
|
-
const text = await readFile(file, 'utf8');
|
|
170
|
-
for (const match of text.matchAll(/<!-- inv:([a-z0-9-]+) -->/g)) {
|
|
171
|
-
tagged.add(match[1]);
|
|
172
|
-
if (!invariants.has(match[1])) fail(`${path.relative(root, file)}: tag inv:${match[1]} is not in docs/invariants.json`);
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
const coreRules = ['skills/pi-revit/SKILL.md', 'skills/pi-revit/references/execution-rules.md', 'skills/pi-revit/references/operation-recovery.md',
|
|
176
|
-
'skills/pi-revit/references/visual-verification.md', 'workspace/AGENTS.md'];
|
|
177
|
-
for (const relative of coreRules)
|
|
178
|
-
for (const [index, line] of (await readFile(path.join(root, relative), 'utf8')).split(/\r?\n/).entries())
|
|
179
|
-
if (/\b(never|must not)\b/i.test(line) && !/<!-- inv:[a-z0-9-]+ -->/.test(line)) fail(`${relative}:${index + 1}: a never/must-not rule needs an <!-- inv:<id> --> tag and a register entry`);
|
|
180
|
-
for (const item of register.invariants) {
|
|
181
|
-
if (!['safety', 'correctness', 'guidance'].includes(item.severity)) fail(`inv:${item.id}: unknown severity`);
|
|
182
|
-
if (!tagged.has(item.id)) fail(`inv:${item.id}: no guidance sentence carries this tag`);
|
|
183
|
-
if (item.enforcement === 'code') {
|
|
184
|
-
if (!item.test || !testSources.includes(item.test)) fail(`inv:${item.id}: enforcement=code needs an existing test titled "${item.test}"`);
|
|
185
|
-
for (const location of item.location ?? []) if (!existsSync(path.join(root, location))) fail(`inv:${item.id}: missing location ${location}`);
|
|
186
|
-
} else if (item.enforcement === 'advisory') {
|
|
187
|
-
if (!item.reason) fail(`inv:${item.id}: advisory rules must state why code cannot enforce them`);
|
|
188
|
-
if (item.severity === 'safety' && !item.eval_scenario) fail(`inv:${item.id}: an advisory safety rule must name the agent-eval scenario that measures it`);
|
|
189
|
-
} else fail(`inv:${item.id}: enforcement must be code or advisory`);
|
|
190
|
-
}
|
|
191
|
-
const scenarioFile = path.join(root, 'tests', 'agent-eval', 'scenarios.json');
|
|
192
|
-
const scenarioIds = existsSync(scenarioFile) ? new Set(JSON.parse(await readFile(scenarioFile, 'utf8')).scenarios.map(s => s.id)) : new Set();
|
|
193
|
-
for (const item of register.invariants) if (item.eval_scenario && !scenarioIds.has(item.eval_scenario)) fail(`inv:${item.id}: eval scenario ${item.eval_scenario} is not defined in tests/agent-eval/scenarios.json`);
|
|
194
|
-
|
|
195
|
-
// Architecture gates over every bridge source file, present and future. Shared primitives
|
|
196
|
-
// own cross-cutting policy; a new tool cannot reintroduce a private variant.
|
|
197
|
-
for (const file of (await readdir(path.join(root, 'src', 'Revit', 'Tools'))).filter(name => name.endsWith('.cs'))) {
|
|
198
|
-
const source = (await readFile(path.join(root, 'src', 'Revit', 'Tools', file), 'utf8')).replace(/\/\/.*$/gm, '').replace(/"(?:[^"\\]|\\.)*"/g, '""');
|
|
199
|
-
// inv:no-tool-saves-model
|
|
200
|
-
if (/\.\s*(Save|SaveAs|Close|SynchronizeWithCentral)\s*\(/.test(source)) fail(`architecture: no bridge tool saves, closes or synchronizes a document (${file})`);
|
|
201
|
-
// inv:parameter-ambiguity: LookupParameter returns an arbitrary one of several same-named parameters.
|
|
202
|
-
if (file !== 'ParameterResolver.cs' && /\b(LookupParameter|GetParameters)\s*\(/.test(source))
|
|
203
|
-
fail(`architecture: ${file} resolves parameters by name directly; use ParameterResolver (one ambiguity policy for every tool)`);
|
|
204
|
-
// inv:special-objects-flagged: sheet-owned schedule instances include the titleblock's revision schedule.
|
|
205
|
-
if (file !== 'ElementTraits.cs' && /\bScheduleSheetInstance\b/.test(source) && !/\bElementTraits\./.test(source))
|
|
206
|
-
fail(`architecture: ${file} handles ScheduleSheetInstance without ElementTraits classification`);
|
|
207
|
-
// inv:derived-state-reported: an object made from an existing one carries its hidden content, overrides and values.
|
|
208
|
-
if (file !== 'InheritedState.cs' && /\.\s*Duplicate\s*\(|\bCopyElements?\s*\(|\bMirrorElements\s*\(|\.\s*ChangeTypeId\s*\(/.test(source) && !/\bInheritedState\./.test(source))
|
|
209
|
-
fail(`architecture: a tool that duplicates, copies or retypes reports inherited state; ${file} creates from an existing object without InheritedState`);
|
|
210
|
-
// inv:existing-objects-not-reused: names and sheet numbers go through the one collision check.
|
|
211
|
-
if (file !== 'ElementNames.cs' && /\.\s*(Name|SheetNumber)\s*=(?!=)|\.\s*(NewType|RenameCurrentType)\s*\(/.test(source))
|
|
212
|
-
fail(`architecture: names are assigned through ElementNames; ${file} assigns a Name, SheetNumber or family type name directly`);
|
|
213
|
-
// inv:document-kind-declared: every tool states where it works, so the dispatcher can refuse the rest.
|
|
214
|
-
if (/:\s*ITool\b/.test(source) && !/\bDocumentKinds\s*=>/.test(source))
|
|
215
|
-
fail(`architecture: every bridge tool declares the document kinds it supports; ${file} has no DocumentKinds`);
|
|
216
|
-
}
|
|
217
|
-
// inv:model-changes-reported: the dispatcher, not each tool, attaches model_changes to every model-changing call.
|
|
218
|
-
{
|
|
219
|
-
const server = await readFile(path.join(root, 'src', 'Revit', 'BridgeServer.cs'), 'utf8');
|
|
220
|
-
if (!/ModelChangeRecorder\.For\(\s*tool\b/.test(server) || !/\.Attach\(/.test(server))
|
|
221
|
-
fail('architecture: the bridge dispatcher records model changes for every tool call (ModelChangeRecorder.For(tool, ...) and Attach)');
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
// 7. Discovery corpus: English task phrasings (the model translates other languages).
|
|
225
|
-
// Every public tool needs coverage; overall top-N recall must stay above the threshold.
|
|
226
|
-
const corpus = JSON.parse(await readFile(path.join(root, 'tests', 'discovery', 'corpus.json'), 'utf8'));
|
|
227
|
-
const corpusResults = await evaluateCorpus(loader, corpus, packageInfo.version);
|
|
228
|
-
const recall = corpusResults.filter(r => r.pass).length / corpusResults.length;
|
|
229
|
-
if (recall < corpus.min_recall) fail(`discovery corpus recall ${recall.toFixed(3)} is below ${corpus.min_recall}; run node tests/discovery/run-corpus.mjs for the misses`);
|
|
230
|
-
for (const tool of manifest.tools) {
|
|
231
|
-
const count = corpus.entries.filter(entry => entry.expect === tool.name).length;
|
|
232
|
-
if (count < corpus.min_per_tool) fail(`${tool.name}: the discovery corpus needs at least ${corpus.min_per_tool} phrasings (has ${count})`);
|
|
233
|
-
}
|
|
234
|
-
for (const guide of manifest.guidance.filter(g => g.kind === 'workflow'))
|
|
235
|
-
if (!corpus.entries.some(entry => entry.expect_guidance === guide.name)) fail(`workflow ${guide.name}: the discovery corpus needs at least one phrasing`);
|
|
236
|
-
if (!corpus.entries.some(entry => entry.expect_fallback)) fail('the discovery corpus needs at least one query that must return the capability route');
|
|
237
|
-
|
|
238
|
-
// 9. Local links in all guidance and contributor documents.
|
|
239
|
-
let localLinkCount = 0;
|
|
240
|
-
for (const file of [...guidanceFiles, path.join(root, 'tests/tool-documentation/README.md')]) {
|
|
241
|
-
if (!existsSync(file)) continue;
|
|
242
|
-
const markdown = await readFile(file, 'utf8');
|
|
243
|
-
const prose = markdown.replace(/^```[^\n]*\r?\n[\s\S]*?^```[ \t]*\r?$/gm, '');
|
|
244
|
-
for (const match of prose.matchAll(/\[[^\]]*\]\((?:<([^>]+)>|([^\s)]+))(?:\s+"[^"]*")?\)/g)) {
|
|
245
|
-
const destination = match[1] ?? match[2];
|
|
246
|
-
if (/^(?:[a-z][a-z0-9+.-]*:|#)/i.test(destination)) continue;
|
|
247
|
-
const relative = decodeURIComponent(destination.split('#')[0]);
|
|
248
|
-
if (!relative) continue;
|
|
249
|
-
try { await access(path.resolve(path.dirname(file), relative)); localLinkCount++; }
|
|
250
|
-
catch { fail(`${path.relative(root, file)}: broken local link ${destination}`); }
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
for (const warning of warnings) console.warn(`WARNING: ${warning}`);
|
|
255
|
-
assert.equal(problems.length, 0, `Platform validation failed:\n${problems.join('\n')}`);
|
|
256
|
-
console.log(`PASS: ${descriptors.length} bridge + ${registered.size - descriptors.length} native contracts; ${manualFiles.length} manuals with generated contracts; ${exampleCount} valid input examples; ${register.invariants.length} invariants; discovery recall ${corpusResults.filter(r => r.pass).length}/${corpusResults.length}; ${localLinkCount} local links.`);
|
|
257
|
-
console.log(`Prompt: platform section ${promptReport.sectionChars} + ${promptReport.activeTools} active tools ${promptReport.toolChars} = ${promptReport.total} characters (budget ${PROMPT_BUDGET.startupTotal}).`);
|
|
258
|
-
console.log('Offline structure verification only: conditional runtime rules, Revit behavior, and agent effectiveness require their own tests.');
|
|
259
|
-
|
|
260
|
-
function assertDocumentedKeys(schema, value, location) {
|
|
261
|
-
if (!schema || typeof schema !== 'object' || value === null) return;
|
|
262
|
-
if (Array.isArray(value)) {
|
|
263
|
-
if (schema.items) value.forEach((item, index) => assertDocumentedKeys(schema.items, item, `${location}[${index}]`));
|
|
264
|
-
} else if (typeof value === 'object' && schema.properties) {
|
|
265
|
-
for (const [key, child] of Object.entries(value)) {
|
|
266
|
-
// Some schemas permit additional JSON properties, but manual argument examples must use advertised keys.
|
|
267
|
-
assert.ok(Object.hasOwn(schema.properties, key) || schema.additionalProperties === true || schema.patternProperties,
|
|
268
|
-
`${location}: undocumented argument ${key}`);
|
|
269
|
-
if (schema.properties[key]) assertDocumentedKeys(schema.properties[key], child, `${location}.${key}`);
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
async function filesUnder(directory) {
|
|
275
|
-
const files = [];
|
|
276
|
-
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
277
|
-
if (['bin', 'obj', 'node_modules'].includes(entry.name)) continue;
|
|
278
|
-
const file = path.join(directory, entry.name);
|
|
279
|
-
if (entry.isDirectory()) files.push(...await filesUnder(file));
|
|
280
|
-
else if (entry.isFile()) files.push(file);
|
|
281
|
-
}
|
|
282
|
-
return files;
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
async function markdownUnder(directory) {
|
|
286
|
-
return (await filesUnder(directory)).filter(file => file.endsWith('.md'));
|
|
287
|
-
}
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
// Generates every derived documentation artifact from code, the single owner of
|
|
2
|
-
// executable contracts:
|
|
3
|
-
// skills/pi-revit/contracts.generated.json offline contract snapshot for discovery
|
|
4
|
-
// "Contract (generated)" block in each tool manual
|
|
5
|
-
// generated tables in skills/pi-revit/references/tool-index.md
|
|
6
|
-
// Usage: node scripts/generate-contracts.mjs [--check]
|
|
7
|
-
// --check writes nothing and fails when any artifact is stale.
|
|
8
|
-
import { existsSync } from 'node:fs';
|
|
9
|
-
import { readFile, writeFile } from 'node:fs/promises';
|
|
10
|
-
import path from 'node:path';
|
|
11
|
-
import { fileURLToPath } from 'node:url';
|
|
12
|
-
import {
|
|
13
|
-
applyIndexBlock, applyManualBlock, createLoader, extractBridgeDescriptors, manifestPath, manualRoot,
|
|
14
|
-
registerExtension, renderManualBlock, renderToolIndex, resolvePiDependencies, root, snapshotPath, toolIndexPath,
|
|
15
|
-
} from './lib/platform.mjs';
|
|
16
|
-
|
|
17
|
-
export async function buildArtifacts() {
|
|
18
|
-
const piRequire = resolvePiDependencies();
|
|
19
|
-
const loader = createLoader(piRequire);
|
|
20
|
-
// The extension imports the snapshot; bootstrap an empty one on a first run.
|
|
21
|
-
if (!existsSync(snapshotPath)) await writeFile(snapshotPath, JSON.stringify({ schema_version: 1, tools: [] }, null, 2) + '\n');
|
|
22
|
-
const packageInfo = JSON.parse(await readFile(path.join(root, 'package.json'), 'utf8'));
|
|
23
|
-
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
|
24
|
-
const descriptors = await extractBridgeDescriptors();
|
|
25
|
-
const { contractHash, NATIVE_CONTRACTS, withApiLookups } = await loader.import(path.join(root, 'extensions', 'pi-revit', 'contracts.ts'));
|
|
26
|
-
const fixture = await registerExtension(loader, descriptors, packageInfo.version);
|
|
27
|
-
let tools;
|
|
28
|
-
try {
|
|
29
|
-
const bridge = new Map(descriptors.map(d => [d.name, d]));
|
|
30
|
-
tools = [...fixture.registered.keys()].sort().map(name => {
|
|
31
|
-
const d = bridge.get(name);
|
|
32
|
-
if (d) return {
|
|
33
|
-
name, source: 'bridge', tier: d.tier ?? 'core', write: d.write === true, effects: d.effects ?? [], requires_document: d.requiresDocument !== false,
|
|
34
|
-
document_kinds: d.documentKinds ?? null,
|
|
35
|
-
keywords: d.keywords ?? [], limits: (d.limits ?? []).map(l => ({ what: l.what, alternative: { kind: l.alternative.kind, ref: l.alternative.ref ?? null } })),
|
|
36
|
-
verification: d.verification ?? null, contract_hash: contractHash(d), parameters: d.parameters,
|
|
37
|
-
};
|
|
38
|
-
const native = NATIVE_CONTRACTS[name];
|
|
39
|
-
if (!native) throw new Error(`Native tool ${name} has no NATIVE_CONTRACTS entry in extensions/pi-revit/contracts.ts`);
|
|
40
|
-
const parameters = JSON.parse(JSON.stringify(fixture.registered.get(name).parameters));
|
|
41
|
-
return {
|
|
42
|
-
name, source: 'native', tier: 'core', write: native.write, effects: native.effects, requires_document: false, document_kinds: null,
|
|
43
|
-
keywords: native.keywords, limits: native.limits, verification: native.verification,
|
|
44
|
-
contract_hash: contractHash({ parameters, write: native.write, effects: native.effects, requiresDocument: false }), parameters,
|
|
45
|
-
};
|
|
46
|
-
});
|
|
47
|
-
} finally { await fixture.restore(); }
|
|
48
|
-
|
|
49
|
-
const snapshot = JSON.stringify({
|
|
50
|
-
schema_version: 1,
|
|
51
|
-
generated_by: 'scripts/generate-contracts.mjs',
|
|
52
|
-
note: 'Generated from src/Revit/Tools/*.cs (bridge) and extensions/pi-revit/contracts.ts (native). Do not edit; change the code and regenerate.',
|
|
53
|
-
tools,
|
|
54
|
-
}, null, 2) + '\n';
|
|
55
|
-
const files = new Map([[snapshotPath, snapshot]]);
|
|
56
|
-
for (const tool of tools) {
|
|
57
|
-
const file = path.join(manualRoot, `${tool.name}.md`);
|
|
58
|
-
if (!existsSync(file)) continue; // missing manuals are reported by the documentation checker
|
|
59
|
-
files.set(file, applyManualBlock(await readFile(file, 'utf8'), renderManualBlock({ ...tool, limits: withApiLookups(tool.limits) })));
|
|
60
|
-
}
|
|
61
|
-
files.set(toolIndexPath, applyIndexBlock(await readFile(toolIndexPath, 'utf8'), renderToolIndex(manifest, tools)));
|
|
62
|
-
return { files, tools, descriptors };
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
|
|
66
|
-
const check = process.argv.includes('--check');
|
|
67
|
-
const { files } = await buildArtifacts();
|
|
68
|
-
const stale = [];
|
|
69
|
-
for (const [file, content] of files) {
|
|
70
|
-
const current = existsSync(file) ? await readFile(file, 'utf8') : null;
|
|
71
|
-
if (current === content) continue;
|
|
72
|
-
stale.push(path.relative(root, file));
|
|
73
|
-
if (!check) await writeFile(file, content);
|
|
74
|
-
}
|
|
75
|
-
if (check && stale.length) {
|
|
76
|
-
console.error(`Generated artifacts are stale. Run npm run generate:contracts:\n${stale.join('\n')}`);
|
|
77
|
-
process.exit(1);
|
|
78
|
-
}
|
|
79
|
-
console.log(check ? `Generated artifacts are current (${files.size} files).` : `Updated ${stale.length} of ${files.size} generated artifacts.`);
|
|
80
|
-
}
|
package/scripts/lib/platform.mjs
DELETED
|
@@ -1,226 +0,0 @@
|
|
|
1
|
-
// Shared offline platform helpers for the contract generator and the documentation
|
|
2
|
-
// checker. Nothing here contacts Revit, the network or a live model: bridge metadata
|
|
3
|
-
// comes from the real C# registry compiled with stubbed tool bodies, and the Pi
|
|
4
|
-
// extension is registered against an isolated fake bridge.
|
|
5
|
-
import { createRequire } from 'node:module';
|
|
6
|
-
import { spawnSync } from 'node:child_process';
|
|
7
|
-
import { existsSync } from 'node:fs';
|
|
8
|
-
import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
9
|
-
import os from 'node:os';
|
|
10
|
-
import path from 'node:path';
|
|
11
|
-
import { fileURLToPath } from 'node:url';
|
|
12
|
-
|
|
13
|
-
export const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
14
|
-
export const skillRoot = path.join(root, 'skills', 'pi-revit');
|
|
15
|
-
export const manualRoot = path.join(skillRoot, 'references', 'tools');
|
|
16
|
-
export const snapshotPath = path.join(skillRoot, 'contracts.generated.json');
|
|
17
|
-
export const manifestPath = path.join(skillRoot, 'tool-manifest.json');
|
|
18
|
-
export const toolIndexPath = path.join(skillRoot, 'references', 'tool-index.md');
|
|
19
|
-
|
|
20
|
-
export function resolvePiDependencies() {
|
|
21
|
-
const candidates = [process.env.PI_CODING_AGENT_PATH,
|
|
22
|
-
path.join(path.dirname(process.execPath), 'node_modules', '@earendil-works', 'pi-coding-agent')].filter(Boolean);
|
|
23
|
-
for (const candidate of candidates) {
|
|
24
|
-
const require = createRequire(path.join(path.resolve(candidate), 'package.json'));
|
|
25
|
-
try { require.resolve('jiti'); require.resolve('typebox/value'); return require; } catch {}
|
|
26
|
-
}
|
|
27
|
-
throw new Error('Set PI_CODING_AGENT_PATH to an existing @earendil-works/pi-coding-agent directory containing jiti and typebox. These checks never install dependencies.');
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export function createLoader(piRequire) {
|
|
31
|
-
const { createJiti } = piRequire('jiti');
|
|
32
|
-
return createJiti(import.meta.url, { alias: { typebox: piRequire.resolve('typebox') }, moduleCache: false });
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/** Real ToolRegistry.DescribeAll() output, via the Roslyn schema extractor. */
|
|
36
|
-
export async function extractBridgeDescriptors() {
|
|
37
|
-
const scratch = await mkdtemp(path.join(os.tmpdir(), 'pi-revit-contracts-'));
|
|
38
|
-
try {
|
|
39
|
-
const output = path.join(scratch, 'bridge.json');
|
|
40
|
-
const run = spawnSync('dotnet', ['run', '--project', path.join(root, 'tests', 'tool-documentation', 'schema-extractor.csproj'), '--', root, output],
|
|
41
|
-
{ cwd: root, encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, env: { ...process.env, DOTNET_CLI_TELEMETRY_OPTOUT: '1', DOTNET_SKIP_FIRST_TIME_EXPERIENCE: '1' } });
|
|
42
|
-
if (run.error || run.status !== 0)
|
|
43
|
-
throw new Error(`Bridge schema extraction failed (installed .NET SDK 8+ required).\n${run.error?.message ?? ''}\n${run.stdout}\n${run.stderr}`);
|
|
44
|
-
return JSON.parse(await readFile(output, 'utf8'));
|
|
45
|
-
} finally { await rm(scratch, { recursive: true, force: true }); }
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* Register the real extension against an in-memory bridge that serves `descriptors`.
|
|
50
|
-
* Returns the registrations and event handlers. Every non-metadata route and all
|
|
51
|
-
* real network access are rejected, and no registered execute() is invoked.
|
|
52
|
-
*/
|
|
53
|
-
export async function registerExtension(loader, descriptors, addinVersion) {
|
|
54
|
-
const scratch = await mkdtemp(path.join(os.tmpdir(), 'pi-revit-register-'));
|
|
55
|
-
const original = { fetch: globalThis.fetch, appdata: process.env.APPDATA, setInterval: globalThis.setInterval };
|
|
56
|
-
process.env.APPDATA = scratch;
|
|
57
|
-
await mkdir(path.join(scratch, 'RevitBridge'));
|
|
58
|
-
await writeFile(path.join(scratch, 'RevitBridge', 'bridge.json'), JSON.stringify({ baseUrl: 'http://platform-fixture.invalid', token: 'offline-fixture' }));
|
|
59
|
-
const routes = [];
|
|
60
|
-
globalThis.fetch = async (input, init) => {
|
|
61
|
-
const url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url);
|
|
62
|
-
if (url.origin !== 'http://platform-fixture.invalid') throw new Error('real network access is forbidden');
|
|
63
|
-
if ((init?.method ?? 'GET') !== 'GET' || !['/ping', '/tools'].includes(url.pathname)) throw new Error(`tool execution is forbidden: ${url.pathname}`);
|
|
64
|
-
routes.push(url.pathname);
|
|
65
|
-
return new Response(JSON.stringify(url.pathname === '/tools' ? { tools: descriptors } : { ok: true, addinVersion }));
|
|
66
|
-
};
|
|
67
|
-
globalThis.setInterval = () => { throw new Error('Background polling is forbidden in platform fixtures'); };
|
|
68
|
-
const registered = new Map(), events = new Map();
|
|
69
|
-
let active = [];
|
|
70
|
-
const pi = {
|
|
71
|
-
registerTool(tool) { if (registered.has(tool.name)) throw new Error(`duplicate registered tool ${tool.name}`); registered.set(tool.name, tool); active = [...active, tool.name]; },
|
|
72
|
-
on(name, handler) { events.set(name, handler); },
|
|
73
|
-
getActiveTools() { return [...active]; },
|
|
74
|
-
setActiveTools(names) { active = [...names]; },
|
|
75
|
-
sendMessage() { throw new Error('Messages are forbidden in platform fixtures'); },
|
|
76
|
-
};
|
|
77
|
-
const restore = async () => {
|
|
78
|
-
await events.get('session_shutdown')?.({}, {});
|
|
79
|
-
globalThis.fetch = original.fetch; globalThis.setInterval = original.setInterval;
|
|
80
|
-
if (original.appdata === undefined) delete process.env.APPDATA; else process.env.APPDATA = original.appdata;
|
|
81
|
-
await rm(scratch, { recursive: true, force: true });
|
|
82
|
-
};
|
|
83
|
-
try {
|
|
84
|
-
const { default: connector } = await loader.import(path.join(root, 'extensions', 'pi-revit', 'index.ts'));
|
|
85
|
-
await connector(pi);
|
|
86
|
-
} catch (error) { await restore(); throw error; }
|
|
87
|
-
return { pi, registered, events, routes, activeTools: () => [...active], restore };
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/** Human-readable type, requiredness, default and allowed values of one JSON schema property. */
|
|
91
|
-
function describeProperty(schema) {
|
|
92
|
-
const values = [];
|
|
93
|
-
let type = schema?.type;
|
|
94
|
-
if (Array.isArray(schema?.enum)) values.push(...schema.enum);
|
|
95
|
-
for (const variant of schema?.anyOf ?? schema?.oneOf ?? []) {
|
|
96
|
-
if (variant.const !== undefined) { values.push(variant.const); type ??= typeof variant.const; }
|
|
97
|
-
else if (variant.type) type = type ? `${type} | ${variant.type}` : variant.type;
|
|
98
|
-
}
|
|
99
|
-
if (schema?.const !== undefined) values.push(schema.const);
|
|
100
|
-
if (type === 'array') {
|
|
101
|
-
const items = schema.items ?? {};
|
|
102
|
-
const itemValues = Array.isArray(items.enum) ? items.enum : (items.anyOf ?? []).filter(v => v.const !== undefined).map(v => v.const);
|
|
103
|
-
values.push(...itemValues);
|
|
104
|
-
type = `array of ${items.type ?? (itemValues.length ? typeof itemValues[0] : 'value')}`;
|
|
105
|
-
}
|
|
106
|
-
if (Array.isArray(type)) type = type.join(' | ');
|
|
107
|
-
return { type: type ?? 'value', values, default: schema?.default };
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
const VERIFICATION_TEXT = {
|
|
111
|
-
reread: 'query the changed state again with a read tool',
|
|
112
|
-
capture: 'capture the visible result and inspect the image',
|
|
113
|
-
inspect_output: 'open and inspect the produced file',
|
|
114
|
-
none: 'no durable outcome to check; report what was done',
|
|
115
|
-
};
|
|
116
|
-
const cell = text => String(text).replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
|
|
117
|
-
|
|
118
|
-
export function alternativeText(limit) {
|
|
119
|
-
const ref = limit.alternative.ref ? cell(limit.alternative.ref) : '';
|
|
120
|
-
switch (limit.alternative.kind) {
|
|
121
|
-
case 'tool': return `Tool: ${ref}`;
|
|
122
|
-
case 'api': return `Revit API: ${ref}. ${limit.lookup ? `Check all its members in one call: \`search_api_docs\` with query \`${cell(limit.lookup.query)}\`` : 'Check with `search_api_docs`'}, then use \`execute_csharp\` within the requested scope.`;
|
|
123
|
-
case 'user': return `User action: ${ref}`;
|
|
124
|
-
case 'revit_unsupported': return `Not offered by the Revit API (${ref}). Report this with the evidence checked.`;
|
|
125
|
-
default: return ref;
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
export const CONTRACT_START = '<!-- generated:contract:start (npm run generate:contracts; do not edit this block) -->';
|
|
130
|
-
export const CONTRACT_END = '<!-- generated:contract:end -->';
|
|
131
|
-
|
|
132
|
-
export function renderManualBlock(tool) {
|
|
133
|
-
const lines = [CONTRACT_START, '## Contract (generated)', ''];
|
|
134
|
-
const tier = tool.source === 'native' ? 'Pi extension utility, always active'
|
|
135
|
-
: tool.tier === 'advanced' ? 'bridge tool, advanced tier: activate it with `find_revit_tools`' : 'bridge tool, core tier: active by default';
|
|
136
|
-
lines.push(`- **Source:** ${tier}.`);
|
|
137
|
-
lines.push(`- **Writes model:** ${tool.write ? 'yes' : 'no'}. **Effects:** ${tool.effects.length ? tool.effects.join(', ') : 'none'}. **Requires an open document:** ${tool.requires_document ? 'yes' : 'no'}.`);
|
|
138
|
-
if (tool.document_kinds?.length && tool.requires_document)
|
|
139
|
-
lines.push(`- **Works in:** ${tool.document_kinds.length > 1 ? 'project and family documents' : `${tool.document_kinds[0]} documents only; the bridge refuses other documents before running, with the route to use instead`}.`);
|
|
140
|
-
if (tool.verification) lines.push(`- **Verify the outcome:** \`${tool.verification}\`: ${VERIFICATION_TEXT[tool.verification]}.`);
|
|
141
|
-
lines.push(`- **Contract hash:** \`${tool.contract_hash}\`. \`find_revit_tools\` compares it with the selected bridge's live contract.`);
|
|
142
|
-
lines.push('', '| Input | Type | Required | Default | Allowed values |', '| --- | --- | --- | --- | --- |');
|
|
143
|
-
const schema = tool.parameters ?? {};
|
|
144
|
-
const required = new Set(schema.required ?? []);
|
|
145
|
-
const properties = Object.entries(schema.properties ?? {});
|
|
146
|
-
if (!properties.length) lines.push('| _(none)_ | | | | |');
|
|
147
|
-
for (const [name, property] of properties) {
|
|
148
|
-
const d = describeProperty(property);
|
|
149
|
-
lines.push(`| \`${name}\` | ${cell(d.type)} | ${required.has(name) ? 'yes' : 'no'} | ${d.default === undefined ? '' : `\`${cell(JSON.stringify(d.default))}\``} | ${d.values.map(v => `\`${cell(v)}\``).join(', ')} |`);
|
|
150
|
-
}
|
|
151
|
-
if (tool.source === 'bridge') lines.push('', 'Bridge calls also accept `_operation_id`, only to retry an identical earlier request (see [operation recovery](../operation-recovery.md)).');
|
|
152
|
-
if (tool.limits.length) {
|
|
153
|
-
lines.push('', '| Not covered by this tool | Use instead |', '| --- | --- |');
|
|
154
|
-
for (const limit of tool.limits) lines.push(`| ${cell(limit.what)} | ${alternativeText(limit)} |`);
|
|
155
|
-
}
|
|
156
|
-
lines.push(CONTRACT_END);
|
|
157
|
-
return lines.join('\n');
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/** Replace the generated block, or insert it before the first "## Inputs" heading (else the second H2). */
|
|
161
|
-
export function applyManualBlock(markdown, block) {
|
|
162
|
-
const eol = markdown.includes('\r\n') ? '\r\n' : '\n';
|
|
163
|
-
const body = block.replace(/\n/g, eol);
|
|
164
|
-
const start = markdown.indexOf(CONTRACT_START), end = markdown.indexOf(CONTRACT_END);
|
|
165
|
-
if (start >= 0 && end > start) return markdown.slice(0, start) + body + markdown.slice(end + CONTRACT_END.length);
|
|
166
|
-
const headings = [...markdown.matchAll(/^## .*$/gm)];
|
|
167
|
-
const anchor = headings.find(h => /^## Inputs/.test(h[0])) ?? headings[1];
|
|
168
|
-
if (!anchor) return markdown.trimEnd() + eol + eol + body + eol;
|
|
169
|
-
return markdown.slice(0, anchor.index) + body + eol + eol + markdown.slice(anchor.index);
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
export const INDEX_START = '<!-- generated:tool-index:start (npm run generate:contracts; do not edit this block) -->';
|
|
173
|
-
export const INDEX_END = '<!-- generated:tool-index:end -->';
|
|
174
|
-
|
|
175
|
-
export function renderToolIndex(manifest, contracts) {
|
|
176
|
-
const byName = new Map(contracts.map(tool => [tool.name, tool]));
|
|
177
|
-
const lines = [INDEX_START];
|
|
178
|
-
for (const group of manifest.groups) {
|
|
179
|
-
lines.push('', `## ${group.title}`, '', '| Manual | Use | Tier | Verify |', '| --- | --- | --- | --- |');
|
|
180
|
-
for (const tool of manifest.tools.filter(entry => entry.group === group.id)) {
|
|
181
|
-
const contract = byName.get(tool.name);
|
|
182
|
-
const tier = tool.source === 'native' ? 'native' : contract?.tier ?? '';
|
|
183
|
-
lines.push(`| [${tool.name}](tools/${tool.name}.md) | ${cell(tool.summary)} | ${tier} | ${contract?.verification ?? ''} |`);
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
lines.push('', '## Workflows and shared guidance', '', '| Guide | Kind | Use |', '| --- | --- | --- |');
|
|
187
|
-
for (const guide of manifest.guidance) lines.push(`| [${guide.name}](${guide.path.replace(/^references\//, '')}) | ${guide.kind} | ${cell(guide.summary)} |`);
|
|
188
|
-
lines.push(INDEX_END);
|
|
189
|
-
return lines.join('\n');
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
export function applyIndexBlock(markdown, block) {
|
|
193
|
-
const eol = markdown.includes('\r\n') ? '\r\n' : '\n';
|
|
194
|
-
const body = block.replace(/\n/g, eol);
|
|
195
|
-
const start = markdown.indexOf(INDEX_START), end = markdown.indexOf(INDEX_END);
|
|
196
|
-
if (start < 0 || end < start) throw new Error(`tool-index.md needs ${INDEX_START} ... ${INDEX_END} markers`);
|
|
197
|
-
return markdown.slice(0, start) + body + markdown.slice(end + INDEX_END.length);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/**
|
|
201
|
-
* Score the discovery corpus against packaged documentation (no bridge needed).
|
|
202
|
-
* An entry passes when its expected tool or guidance appears in the top_n results,
|
|
203
|
-
* or, for expect_fallback, when the response carries the capability route.
|
|
204
|
-
*/
|
|
205
|
-
export async function evaluateCorpus(loader, corpus, addinVersion) {
|
|
206
|
-
const fixture = await registerExtension(loader, [], addinVersion);
|
|
207
|
-
try {
|
|
208
|
-
const find = fixture.registered.get('find_revit_tools');
|
|
209
|
-
const results = [];
|
|
210
|
-
for (const entry of corpus.entries) {
|
|
211
|
-
const response = (await find.execute('corpus', { scope: 'documentation', query: entry.q, limit: corpus.top_n })).details;
|
|
212
|
-
const tools = response.tools.map(tool => tool.name), guides = response.guidance.map(guide => guide.name);
|
|
213
|
-
const pass = entry.expect_fallback ? Boolean(response.fallback) && response.match === 'none'
|
|
214
|
-
: entry.expect_guidance ? guides.slice(0, corpus.top_n).includes(entry.expect_guidance)
|
|
215
|
-
: tools.slice(0, corpus.top_n).includes(entry.expect);
|
|
216
|
-
results.push({ ...entry, pass, match: response.match, got: entry.expect_guidance ? guides : tools });
|
|
217
|
-
}
|
|
218
|
-
return results;
|
|
219
|
-
} finally { await fixture.restore(); }
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
/** Candidate RevitAPI.xml files for optional API-reference verification. */
|
|
223
|
-
export function revitApiXmlPaths() {
|
|
224
|
-
const roots = [process.env.REVIT_API_PATH, 'C:\\Program Files\\Autodesk\\Revit 2025', 'C:\\Program Files\\Autodesk\\Revit 2026', 'C:\\Program Files\\Autodesk\\Revit 2027'].filter(Boolean);
|
|
225
|
-
return roots.map(directory => path.join(directory, 'RevitAPI.xml')).filter(file => existsSync(file));
|
|
226
|
-
}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { readdirSync } from 'node:fs';
|
|
2
|
-
import { spawnSync } from 'node:child_process';
|
|
3
|
-
import { fileURLToPath } from 'node:url';
|
|
4
|
-
|
|
5
|
-
// Run sequentially: fixtures replace fetch, timers and discovery paths in-process.
|
|
6
|
-
// Each module has a separate process and uses Node's test API directly.
|
|
7
|
-
const directory = new URL('../tests/result-contract/', import.meta.url);
|
|
8
|
-
for (const file of readdirSync(directory).filter(name => name.endsWith('.test.mjs')).sort()) {
|
|
9
|
-
process.stdout.write(`\n${file}\n`);
|
|
10
|
-
const result = spawnSync(process.execPath, [fileURLToPath(new URL(file, directory))], {
|
|
11
|
-
stdio: 'inherit', env: process.env,
|
|
12
|
-
});
|
|
13
|
-
if (result.error) throw result.error;
|
|
14
|
-
if (result.status !== 0) process.exit(result.status ?? 1);
|
|
15
|
-
}
|