structured-context 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -216
- package/dist/api.d.ts +17 -17
- package/dist/api.js +12 -12
- package/dist/commands/docs.js +1 -0
- package/dist/commands/providers.d.ts +4 -0
- package/dist/commands/providers.js +52 -0
- package/dist/commands/schemas.js +21 -2
- package/dist/commands/validate-file.d.ts +3 -0
- package/dist/commands/validate-file.js +17 -2
- package/dist/commands/validate.d.ts +4 -0
- package/dist/commands/validate.js +42 -3
- package/dist/config.d.ts +7 -0
- package/dist/config.js +26 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +59 -18
- package/dist/judge/context.d.ts +20 -0
- package/dist/judge/context.js +105 -0
- package/dist/judge/factory.d.ts +44 -0
- package/dist/judge/factory.js +148 -0
- package/dist/judge/index.d.ts +13 -0
- package/dist/judge/index.js +8 -0
- package/dist/judge/judge-node.d.ts +27 -0
- package/dist/judge/judge-node.js +122 -0
- package/dist/judge/prompt.d.ts +10 -0
- package/dist/judge/prompt.js +92 -0
- package/dist/judge/provider/anthropic.d.ts +17 -0
- package/dist/judge/provider/anthropic.js +43 -0
- package/dist/judge/provider/fake.d.ts +26 -0
- package/dist/judge/provider/fake.js +56 -0
- package/dist/judge/provider/jev.d.ts +37 -0
- package/dist/judge/provider/jev.js +169 -0
- package/dist/judge/provider/openai-decisions.d.ts +21 -0
- package/dist/judge/provider/openai-decisions.js +84 -0
- package/dist/judge/provider/openai.d.ts +20 -0
- package/dist/judge/provider/openai.js +53 -0
- package/dist/judge/provider/per-rule.d.ts +10 -0
- package/dist/judge/provider/per-rule.js +18 -0
- package/dist/judge/trim.d.ts +27 -0
- package/dist/judge/trim.js +62 -0
- package/dist/judge/types.d.ts +92 -0
- package/dist/judge/types.js +9 -0
- package/dist/judge/verdicts.d.ts +24 -0
- package/dist/judge/verdicts.js +82 -0
- package/dist/load-space-graph.d.ts +3 -3
- package/dist/load-space-graph.js +4 -4
- package/dist/plugins/loader.js +1 -1
- package/dist/plugins/markdown/parse-embedded.d.ts +2 -0
- package/dist/plugins/markdown/parse-embedded.js +161 -24
- package/dist/plugins/markdown/read-space.js +4 -9
- package/dist/plugins/markdown/render-directory.js +5 -5
- package/dist/plugins/markdown/template-sync.js +3 -4
- package/dist/plugins/markdown/util.d.ts +26 -5
- package/dist/plugins/markdown/util.js +56 -15
- package/dist/render/render.d.ts +1 -1
- package/dist/render/render.js +1 -1
- package/dist/schema/evaluate-rule.d.ts +33 -2
- package/dist/schema/evaluate-rule.js +58 -4
- package/dist/schema/gating.d.ts +69 -0
- package/dist/schema/gating.js +253 -0
- package/dist/schema/metadata-contract.d.ts +625 -12
- package/dist/schema/metadata-contract.js +110 -4
- package/dist/schema/rule-filter.d.ts +12 -0
- package/dist/schema/rule-filter.js +46 -0
- package/dist/schema/rule-severity.d.ts +13 -0
- package/dist/schema/rule-severity.js +33 -0
- package/dist/schema/schema.js +31 -1
- package/dist/schema/validate-rules.d.ts +17 -8
- package/dist/schema/validate-rules.js +91 -48
- package/dist/schemas/_knowledge_wiki.json +4 -4
- package/dist/schemas/_okf.json +172 -0
- package/dist/schemas/_ost_strict.json +15 -6
- package/dist/schemas/_sctx_base.json +1 -1
- package/dist/schemas/generated/0.13.0/_structured_context_schema_meta.json +400 -0
- package/dist/schemas/generated/latest/_structured_context_schema_meta.json +177 -6
- package/dist/schemas/okf.json +8 -0
- package/dist/space-context.d.ts +2 -2
- package/dist/space-context.js +3 -3
- package/dist/space-graph.d.ts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/types.d.ts +11 -4
- package/dist/validate.d.ts +59 -9
- package/dist/validate.js +225 -120
- package/docs/concepts.md +52 -6
- package/docs/config.md +70 -1
- package/docs/rules.md +140 -49
- package/docs/schemas.md +43 -18
- package/package.json +6 -5
- package/schemas/_knowledge_wiki.json +4 -4
- package/schemas/_okf.json +172 -0
- package/schemas/_ost_strict.json +15 -6
- package/schemas/_sctx_base.json +1 -1
- package/schemas/generated/0.13.0/_structured_context_schema_meta.json +400 -0
- package/schemas/generated/latest/_structured_context_schema_meta.json +177 -6
- package/schemas/okf.json +8 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
export const RECOMMENDED_OPENAI_DECISIONS_MODEL = 'gpt-6-luna';
|
|
2
|
+
export const OPENAI_DECISIONS_ENDPOINT = 'https://api.openai.com/v1/decisions';
|
|
3
|
+
/**
|
|
4
|
+
* Judge provider for OpenAI's Decisions API (https://developers.openai.com/api/docs/guides/decisions).
|
|
5
|
+
*
|
|
6
|
+
* Each node is sent once as the `input` text, with one `choice` question per semantic rule whose
|
|
7
|
+
* choices are the rule's closed verdict set. The API returns the chosen value with per-choice
|
|
8
|
+
* probabilities and a confidence, but no prose, so `evidence` is a summary of the distribution.
|
|
9
|
+
*/
|
|
10
|
+
export class OpenAIDecisionsJudgeProvider {
|
|
11
|
+
name = 'openai-decisions';
|
|
12
|
+
apiKey;
|
|
13
|
+
model;
|
|
14
|
+
endpoint;
|
|
15
|
+
constructor(options) {
|
|
16
|
+
this.apiKey = options.apiKey ?? '';
|
|
17
|
+
this.model = options.model || RECOMMENDED_OPENAI_DECISIONS_MODEL;
|
|
18
|
+
this.endpoint = options.endpoint || OPENAI_DECISIONS_ENDPOINT;
|
|
19
|
+
}
|
|
20
|
+
async evaluate(input) {
|
|
21
|
+
const { node, rules } = input;
|
|
22
|
+
const stateLines = [
|
|
23
|
+
`Title: ${node.title}`,
|
|
24
|
+
`Type: ${node.resolvedType}`,
|
|
25
|
+
...(node.resolvedParents?.length ? [`Parents: ${node.resolvedParents.map((p) => p.title).join(', ')}`] : []),
|
|
26
|
+
...fieldLines(node.schemaData ?? {}),
|
|
27
|
+
...(input.related ?? []).flatMap((r) => [
|
|
28
|
+
'',
|
|
29
|
+
`Related node (${r.relations.join(', ')}): ${r.title}`,
|
|
30
|
+
`Type: ${r.type}`,
|
|
31
|
+
...fieldLines(r.data),
|
|
32
|
+
]),
|
|
33
|
+
];
|
|
34
|
+
const res = await fetch(this.endpoint, {
|
|
35
|
+
method: 'POST',
|
|
36
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${this.apiKey}` },
|
|
37
|
+
body: JSON.stringify({
|
|
38
|
+
model: this.model,
|
|
39
|
+
input: stateLines.join('\n'),
|
|
40
|
+
questions: rules.map((rule) => ({
|
|
41
|
+
type: 'choice',
|
|
42
|
+
name: rule.id,
|
|
43
|
+
instructions: rule.rubric || rule.description,
|
|
44
|
+
choices: Object.entries(rule.verdicts).map(([value, description]) => ({ value, description })),
|
|
45
|
+
})),
|
|
46
|
+
}),
|
|
47
|
+
});
|
|
48
|
+
if (!res.ok) {
|
|
49
|
+
const errorText = await res.text().catch(() => '');
|
|
50
|
+
throw new Error(`OpenAI Decisions API error (${res.status}): ${errorText}`);
|
|
51
|
+
}
|
|
52
|
+
const data = (await res.json());
|
|
53
|
+
const answers = new Map((data.answers ?? []).filter((a) => a.name).map((a) => [a.name, a]));
|
|
54
|
+
const results = {};
|
|
55
|
+
for (const rule of rules) {
|
|
56
|
+
const answer = answers.get(rule.id);
|
|
57
|
+
if (!answer || answer.type === 'refusal' || !answer.choice || !(answer.choice in rule.verdicts)) {
|
|
58
|
+
results[rule.id] = { error: `OpenAI Decisions returned no valid verdict for rule "${rule.id}"` };
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const probabilities = Object.fromEntries((answer.probabilities ?? []).map((p) => [p.value, p.probability]));
|
|
62
|
+
results[rule.id] = {
|
|
63
|
+
verdict: answer.choice,
|
|
64
|
+
confidence: answer.confidence,
|
|
65
|
+
probabilities,
|
|
66
|
+
evidence: describeDistribution(answer.choice, answer.confidence, probabilities),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
return results;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function fieldLines(data) {
|
|
73
|
+
return Object.entries(data)
|
|
74
|
+
.filter(([field]) => field !== 'title' && field !== 'type')
|
|
75
|
+
.map(([field, value]) => `${field}: ${JSON.stringify(value) ?? String(value)}`);
|
|
76
|
+
}
|
|
77
|
+
function describeDistribution(choice, confidence, probabilities) {
|
|
78
|
+
const parts = Object.entries(probabilities)
|
|
79
|
+
.sort(([, a], [, b]) => b - a)
|
|
80
|
+
.map(([verdict, p]) => `${verdict} ${p.toFixed(2)}`);
|
|
81
|
+
const conf = confidence !== undefined ? `confidence ${confidence.toFixed(2)}` : undefined;
|
|
82
|
+
const detail = [conf, ...parts].filter(Boolean).join('; ');
|
|
83
|
+
return `OpenAI Decisions chose "${choice}"${detail ? ` (${detail})` : ''}`;
|
|
84
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types';
|
|
2
|
+
import { PerRuleJudgeProvider } from './per-rule';
|
|
3
|
+
export interface OpenAIJudgeProviderOptions extends HydratedProvider {
|
|
4
|
+
/** Optional: local OpenAI-compatible servers need no key. */
|
|
5
|
+
apiKey?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare const RECOMMENDED_OPENAI_MODEL = "gpt-6-luna";
|
|
8
|
+
export declare const OPENAI_COMPLETIONS_ENDPOINT = "https://api.openai.com/v1";
|
|
9
|
+
/**
|
|
10
|
+
* Judge provider for the OpenAI-compatible Chat Completions API (OpenAI, Ollama, vLLM, ...).
|
|
11
|
+
* Reads no environment: it takes hydrated values.
|
|
12
|
+
*/
|
|
13
|
+
export declare class OpenAIJudgeProvider extends PerRuleJudgeProvider {
|
|
14
|
+
readonly name = "openai";
|
|
15
|
+
private apiKey?;
|
|
16
|
+
private model;
|
|
17
|
+
private endpoint;
|
|
18
|
+
constructor(options: OpenAIJudgeProviderOptions);
|
|
19
|
+
protected judgeRule(input: SemanticJudgeInput, rule: SemanticRuleSpec): Promise<SemanticJudgeResult>;
|
|
20
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt';
|
|
2
|
+
import { PerRuleJudgeProvider } from './per-rule';
|
|
3
|
+
export const RECOMMENDED_OPENAI_MODEL = 'gpt-6-luna';
|
|
4
|
+
export const OPENAI_COMPLETIONS_ENDPOINT = 'https://api.openai.com/v1';
|
|
5
|
+
/**
|
|
6
|
+
* Judge provider for the OpenAI-compatible Chat Completions API (OpenAI, Ollama, vLLM, ...).
|
|
7
|
+
* Reads no environment: it takes hydrated values.
|
|
8
|
+
*/
|
|
9
|
+
export class OpenAIJudgeProvider extends PerRuleJudgeProvider {
|
|
10
|
+
name = 'openai';
|
|
11
|
+
apiKey;
|
|
12
|
+
model;
|
|
13
|
+
endpoint;
|
|
14
|
+
constructor(options) {
|
|
15
|
+
super();
|
|
16
|
+
this.apiKey = options.apiKey;
|
|
17
|
+
this.model = options.model || RECOMMENDED_OPENAI_MODEL;
|
|
18
|
+
const base = options.endpoint || OPENAI_COMPLETIONS_ENDPOINT;
|
|
19
|
+
this.endpoint = base.endsWith('/chat/completions') ? base : `${base.replace(/\/+$/, '')}/chat/completions`;
|
|
20
|
+
}
|
|
21
|
+
async judgeRule(input, rule) {
|
|
22
|
+
const systemPrompt = buildJudgeSystemPrompt();
|
|
23
|
+
const userPrompt = buildJudgeUserPrompt(input, rule);
|
|
24
|
+
const headers = {
|
|
25
|
+
'content-type': 'application/json',
|
|
26
|
+
};
|
|
27
|
+
if (this.apiKey) {
|
|
28
|
+
headers.authorization = `Bearer ${this.apiKey}`;
|
|
29
|
+
}
|
|
30
|
+
const res = await fetch(this.endpoint, {
|
|
31
|
+
method: 'POST',
|
|
32
|
+
headers,
|
|
33
|
+
body: JSON.stringify({
|
|
34
|
+
model: this.model,
|
|
35
|
+
response_format: { type: 'json_object' },
|
|
36
|
+
messages: [
|
|
37
|
+
{ role: 'system', content: systemPrompt },
|
|
38
|
+
{ role: 'user', content: userPrompt },
|
|
39
|
+
],
|
|
40
|
+
}),
|
|
41
|
+
});
|
|
42
|
+
if (!res.ok) {
|
|
43
|
+
const errorText = await res.text().catch(() => '');
|
|
44
|
+
throw new Error(`OpenAI API error (${res.status}): ${errorText}`);
|
|
45
|
+
}
|
|
46
|
+
const data = (await res.json());
|
|
47
|
+
const content = data.choices?.[0]?.message?.content;
|
|
48
|
+
if (!content) {
|
|
49
|
+
throw new Error('OpenAI API returned empty or missing message content in response.');
|
|
50
|
+
}
|
|
51
|
+
return parseJudgeResponse(content, Object.keys(rule.verdicts));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from '../types';
|
|
2
|
+
/**
|
|
3
|
+
* Base for providers that can only judge one rule per request (chat LLMs). `evaluate` makes one call
|
|
4
|
+
* per rule; a failure on one rule is reported for that rule and does not affect the others.
|
|
5
|
+
*/
|
|
6
|
+
export declare abstract class PerRuleJudgeProvider implements SemanticJudgeProvider {
|
|
7
|
+
abstract readonly name: string;
|
|
8
|
+
protected abstract judgeRule(input: SemanticJudgeInput, rule: SemanticRuleSpec): Promise<SemanticJudgeResult>;
|
|
9
|
+
evaluate(input: SemanticJudgeInput): Promise<Record<string, SemanticJudgeResult>>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Base for providers that can only judge one rule per request (chat LLMs). `evaluate` makes one call
|
|
3
|
+
* per rule; a failure on one rule is reported for that rule and does not affect the others.
|
|
4
|
+
*/
|
|
5
|
+
export class PerRuleJudgeProvider {
|
|
6
|
+
async evaluate(input) {
|
|
7
|
+
const results = {};
|
|
8
|
+
for (const rule of input.rules) {
|
|
9
|
+
try {
|
|
10
|
+
results[rule.id] = await this.judgeRule(input, rule);
|
|
11
|
+
}
|
|
12
|
+
catch (err) {
|
|
13
|
+
results[rule.id] = { error: err instanceof Error ? err.message : String(err) };
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
return results;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { SemanticJudgeInput } from '../types.js';
|
|
2
|
+
/** The per-node input cap: estimated tokens of one node's input (the node and its related nodes), the same for every provider. */
|
|
3
|
+
export declare const PER_NODE_INPUT_CAP_TOKENS = 32000;
|
|
4
|
+
/** Estimated tokens of a value: its JSON length divided by 4. Provider-independent, and roughly right is enough. */
|
|
5
|
+
export declare const estimateTokens: (value: unknown) => number;
|
|
6
|
+
/** The placeholder that ends a body cut to fit the per-node input cap. `removed` is the number of characters cut. */
|
|
7
|
+
export declare const trimMarker: (removed: number) => string;
|
|
8
|
+
/** The part of a judge input that the per-node input cap measures. */
|
|
9
|
+
export type NodeInput = Pick<SemanticJudgeInput, 'node' | 'related'>;
|
|
10
|
+
/**
|
|
11
|
+
* Cut `body` to at most `limit` characters, ending at the last whitespace before the limit, and end it with the
|
|
12
|
+
* trim marker. A body within the limit is returned whole.
|
|
13
|
+
*/
|
|
14
|
+
export declare function cutBody(body: string, limit: number): string;
|
|
15
|
+
/**
|
|
16
|
+
* Fit a node's input under the per-node input cap by cutting bodies only: related bodies to one shared ceiling
|
|
17
|
+
* first, then the evaluated node's body. Undefined when the input does not fit even with every body cut.
|
|
18
|
+
*/
|
|
19
|
+
export declare function trimToCap(input: NodeInput, cap?: number): {
|
|
20
|
+
input: NodeInput;
|
|
21
|
+
trimmed: boolean;
|
|
22
|
+
} | undefined;
|
|
23
|
+
/** What a run trimmed: the nodes (by label) and the rules whose input had a body cut. */
|
|
24
|
+
export interface TrimReport {
|
|
25
|
+
nodes: Set<string>;
|
|
26
|
+
ruleIds: Set<string>;
|
|
27
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/** The per-node input cap: estimated tokens of one node's input (the node and its related nodes), the same for every provider. */
|
|
2
|
+
export const PER_NODE_INPUT_CAP_TOKENS = 32_000;
|
|
3
|
+
/** Estimated tokens of a value: its JSON length divided by 4. Provider-independent, and roughly right is enough. */
|
|
4
|
+
export const estimateTokens = (value) => Math.ceil(JSON.stringify(value).length / 4);
|
|
5
|
+
/** The placeholder that ends a body cut to fit the per-node input cap. `removed` is the number of characters cut. */
|
|
6
|
+
export const trimMarker = (removed) => `[RULE_EVALUATION_CONTENT_TRIMMED ${removed} chars]`;
|
|
7
|
+
/**
|
|
8
|
+
* Cut `body` to at most `limit` characters, ending at the last whitespace before the limit, and end it with the
|
|
9
|
+
* trim marker. A body within the limit is returned whole.
|
|
10
|
+
*/
|
|
11
|
+
export function cutBody(body, limit) {
|
|
12
|
+
if (body.length <= limit)
|
|
13
|
+
return body;
|
|
14
|
+
let head = body.slice(0, limit);
|
|
15
|
+
if (!/\s/.test(body.charAt(limit))) {
|
|
16
|
+
const lastSpace = head.search(/\s\S*$/);
|
|
17
|
+
head = lastSpace < 0 ? '' : head.slice(0, lastSpace);
|
|
18
|
+
}
|
|
19
|
+
head = head.trimEnd();
|
|
20
|
+
const marker = trimMarker(body.length - head.length);
|
|
21
|
+
return head ? `${head} ${marker}` : marker;
|
|
22
|
+
}
|
|
23
|
+
function limitBodies(input, relatedLimit, nodeLimit) {
|
|
24
|
+
const cut = (data, limit) => typeof data.content === 'string' ? { ...data, content: cutBody(data.content, limit) } : data;
|
|
25
|
+
return {
|
|
26
|
+
node: { ...input.node, schemaData: cut(input.node.schemaData, nodeLimit) },
|
|
27
|
+
...(input.related && { related: input.related.map((r) => ({ ...r, data: cut(r.data, relatedLimit) })) }),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const bodyLength = (data) => (typeof data.content === 'string' ? data.content.length : 0);
|
|
31
|
+
/** The largest limit in `[0, max]` for which `fits` holds, or -1 when even 0 does not fit. `fits` shrinks as the limit falls. */
|
|
32
|
+
function largestFitting(max, fits) {
|
|
33
|
+
if (!fits(0))
|
|
34
|
+
return -1;
|
|
35
|
+
let low = 0;
|
|
36
|
+
let high = max;
|
|
37
|
+
while (low < high) {
|
|
38
|
+
const mid = Math.ceil((low + high) / 2);
|
|
39
|
+
if (fits(mid))
|
|
40
|
+
low = mid;
|
|
41
|
+
else
|
|
42
|
+
high = mid - 1;
|
|
43
|
+
}
|
|
44
|
+
return low;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Fit a node's input under the per-node input cap by cutting bodies only: related bodies to one shared ceiling
|
|
48
|
+
* first, then the evaluated node's body. Undefined when the input does not fit even with every body cut.
|
|
49
|
+
*/
|
|
50
|
+
export function trimToCap(input, cap = PER_NODE_INPUT_CAP_TOKENS) {
|
|
51
|
+
if (estimateTokens(input) <= cap)
|
|
52
|
+
return { input, trimmed: false };
|
|
53
|
+
const nodeLength = bodyLength(input.node.schemaData);
|
|
54
|
+
const relatedMax = Math.max(0, ...(input.related ?? []).map((r) => bodyLength(r.data)));
|
|
55
|
+
const fitsRelated = (limit) => estimateTokens(limitBodies(input, limit, nodeLength)) <= cap;
|
|
56
|
+
const relatedLimit = largestFitting(relatedMax, fitsRelated);
|
|
57
|
+
if (relatedLimit >= 0)
|
|
58
|
+
return { input: limitBodies(input, relatedLimit, nodeLength), trimmed: true };
|
|
59
|
+
const fitsNode = (limit) => estimateTokens(limitBodies(input, 0, limit)) <= cap;
|
|
60
|
+
const nodeLimit = largestFitting(nodeLength, fitsNode);
|
|
61
|
+
return nodeLimit >= 0 ? { input: limitBodies(input, 0, nodeLimit), trimmed: true } : undefined;
|
|
62
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type { RuleCategory, SpaceNode } from '../types';
|
|
2
|
+
/** A semantic rule as presented to a judge: the closed verdict set it must choose from. */
|
|
3
|
+
export interface SemanticRuleSpec {
|
|
4
|
+
id: string;
|
|
5
|
+
description: string;
|
|
6
|
+
rubric?: string;
|
|
7
|
+
category: RuleCategory | string;
|
|
8
|
+
/** Allowed verdicts, each with a description of when it applies. */
|
|
9
|
+
verdicts: Record<string, string>;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* A node related to the evaluated node by its rules' `scope`. It carries all the node's schema data; the body
|
|
13
|
+
* (`data.content`) is present only when the rules' `includeContent` asks for it.
|
|
14
|
+
*/
|
|
15
|
+
export interface RelatedNode {
|
|
16
|
+
/** How it relates to the evaluated node. A node can have several. */
|
|
17
|
+
relations: Array<'parent' | 'sibling' | 'child' | 'ancestor' | 'descendant'>;
|
|
18
|
+
title: string;
|
|
19
|
+
type: string;
|
|
20
|
+
/** Schema data without `title` and `type`. */
|
|
21
|
+
data: Record<string, unknown>;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* One node and the semantic rules to judge on it, which share one context: the same related nodes and the same
|
|
25
|
+
* bodies. Judges may answer all rules in a single call.
|
|
26
|
+
*/
|
|
27
|
+
export interface SemanticJudgeInput {
|
|
28
|
+
/** The evaluated node. Its body is in `schemaData.content` only when the rules' `includeContent` asks for it. */
|
|
29
|
+
node: Pick<SpaceNode, 'label' | 'title' | 'type' | 'resolvedType' | 'schemaData' | 'resolvedParents'>;
|
|
30
|
+
/** Nodes the rules' `scope` adds, as a flat list. Absent when the scope is empty. */
|
|
31
|
+
related?: RelatedNode[];
|
|
32
|
+
rules: SemanticRuleSpec[];
|
|
33
|
+
}
|
|
34
|
+
/** A judge's answer for one rule: a verdict from the rule's closed set. */
|
|
35
|
+
export interface SemanticJudgeVerdict {
|
|
36
|
+
verdict: string;
|
|
37
|
+
/** 0-1, when the judge reports one. */
|
|
38
|
+
confidence?: number;
|
|
39
|
+
/** Probability per verdict, when the judge reports a distribution. */
|
|
40
|
+
probabilities?: Record<string, number>;
|
|
41
|
+
/** Free-text reasoning; decision models may not provide it. */
|
|
42
|
+
evidence?: string;
|
|
43
|
+
}
|
|
44
|
+
/** A judge's answer for one rule: a verdict, or an error that applies to that rule only. */
|
|
45
|
+
export type SemanticJudgeResult = SemanticJudgeVerdict | {
|
|
46
|
+
error: string;
|
|
47
|
+
};
|
|
48
|
+
/** Judge interface. Results are keyed by rule id; a missing key is treated as an error. */
|
|
49
|
+
export interface SemanticJudgeProvider {
|
|
50
|
+
readonly name: string;
|
|
51
|
+
evaluate(input: SemanticJudgeInput): Promise<Record<string, SemanticJudgeResult>>;
|
|
52
|
+
/**
|
|
53
|
+
* Optional: judge many nodes in one call, so a provider can fold them into fewer model calls. Returns one
|
|
54
|
+
* result map per input, in order. A failure that affects only some inputs is reported as `{ error }`
|
|
55
|
+
* results for those inputs' rules; throw only when nothing could be judged.
|
|
56
|
+
*/
|
|
57
|
+
evaluateNodes?(inputs: SemanticJudgeInput[]): Promise<Array<Record<string, SemanticJudgeResult>>>;
|
|
58
|
+
}
|
|
59
|
+
/** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */
|
|
60
|
+
export declare const MODEL_PROVIDER_APIS: readonly ["openai-completions", "openai-decisions", "anthropic-messages", "openrouter-decisions", "typesafe-systemone", "fake"];
|
|
61
|
+
export type ModelProviderApi = (typeof MODEL_PROVIDER_APIS)[number];
|
|
62
|
+
/** How to reach a model: a bundled provider or config `modelProviders.<name>`, before hydration. */
|
|
63
|
+
export interface ModelProviderConfig {
|
|
64
|
+
/** The wire API to speak. */
|
|
65
|
+
api: ModelProviderApi;
|
|
66
|
+
/** Fixed model override. Beats `modelEnv` and the bundled default. */
|
|
67
|
+
model?: string;
|
|
68
|
+
/** Environment variable that holds the model override. */
|
|
69
|
+
modelEnv?: string;
|
|
70
|
+
/** Fixed endpoint override. Beats `endpointEnv` and the bundled default. */
|
|
71
|
+
endpoint?: string;
|
|
72
|
+
/** Environment variable that holds the endpoint override. */
|
|
73
|
+
endpointEnv?: string;
|
|
74
|
+
/** Prefer `apiKeyEnv`: keys do not belong in config files. */
|
|
75
|
+
apiKey?: string;
|
|
76
|
+
/** Environment variable holding the key. */
|
|
77
|
+
apiKeyEnv?: string;
|
|
78
|
+
}
|
|
79
|
+
/** A provider after hydration: every field is a concrete value; the implementation reads no environment. */
|
|
80
|
+
export interface HydratedProvider {
|
|
81
|
+
api: ModelProviderApi;
|
|
82
|
+
model?: string;
|
|
83
|
+
endpoint?: string;
|
|
84
|
+
apiKey?: string;
|
|
85
|
+
/** The environment variable the key was read from, when one was named or defaulted. */
|
|
86
|
+
keyEnv?: string;
|
|
87
|
+
}
|
|
88
|
+
/** Which model provider evaluates semantic rules: config `semantic`, at the top level or on a space. */
|
|
89
|
+
export interface JudgeConfig {
|
|
90
|
+
/** Name of an entry in `modelProviders`, or a bundled provider. */
|
|
91
|
+
provider?: string;
|
|
92
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */
|
|
2
|
+
export const MODEL_PROVIDER_APIS = [
|
|
3
|
+
'openai-completions',
|
|
4
|
+
'openai-decisions',
|
|
5
|
+
'anthropic-messages',
|
|
6
|
+
'openrouter-decisions',
|
|
7
|
+
'typesafe-systemone',
|
|
8
|
+
'fake',
|
|
9
|
+
];
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Rule, RuleSeverity } from '../types';
|
|
2
|
+
import type { SemanticJudgeResult, SemanticRuleSpec } from '../types.js';
|
|
3
|
+
/** Verdicts for a semantic rule that declares none. */
|
|
4
|
+
export declare const DEFAULT_VERDICTS: Record<string, string>;
|
|
5
|
+
/** A semantic rule resolved for evaluation: what the judge sees, and what each verdict means for the user. */
|
|
6
|
+
export interface ResolvedSemanticRule {
|
|
7
|
+
/** What the judge sees: the closed verdict set. */
|
|
8
|
+
spec: SemanticRuleSpec;
|
|
9
|
+
/** Severity of each verdict that fails the rule. A verdict absent here passes (including `ignore`d ones). */
|
|
10
|
+
severities: Record<string, RuleSeverity>;
|
|
11
|
+
/** User-facing guidance for verdicts that declare it. */
|
|
12
|
+
guidance: Record<string, string>;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Resolve a semantic rule: apply verdict defaults, split each verdict into its judge description and user guidance,
|
|
16
|
+
* and work out which verdicts fail at what severity. Throws on an inconsistent rule.
|
|
17
|
+
*/
|
|
18
|
+
export declare function resolveSemanticRule(rule: Rule): ResolvedSemanticRule;
|
|
19
|
+
/** The judge-facing spec for a semantic rule. */
|
|
20
|
+
export declare function semanticRuleSpec(rule: Rule): SemanticRuleSpec;
|
|
21
|
+
/** Whether a judge result is an error rather than a verdict. */
|
|
22
|
+
export declare function isJudgeError(result: SemanticJudgeResult): result is {
|
|
23
|
+
error: string;
|
|
24
|
+
};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { RESERVED_VERDICTS } from '../schema/metadata-contract';
|
|
2
|
+
import { ruleSemantic, ruleSeverity } from '../schema/rule-severity';
|
|
3
|
+
import { ruleContext } from './context';
|
|
4
|
+
/** Verdicts for a semantic rule that declares none. */
|
|
5
|
+
export const DEFAULT_VERDICTS = {
|
|
6
|
+
pass: 'The node meets the rubric.',
|
|
7
|
+
fail: 'The node does not meet the rubric.',
|
|
8
|
+
};
|
|
9
|
+
/** The reserved verdict a rule with related nodes or bodies offers, for input a trim marker shows was cut. */
|
|
10
|
+
const MISSING_CONTEXT_VERDICT = 'missing-context';
|
|
11
|
+
const MISSING_CONTEXT_DESCRIPTION = 'Content needed to judge the rule was cut: a [RULE_EVALUATION_CONTENT_TRIMMED n chars] marker shows where. Choose this only when such a marker is present and the cut content is needed. Never choose it when no marker appears.';
|
|
12
|
+
/**
|
|
13
|
+
* Resolve a semantic rule: apply verdict defaults, split each verdict into its judge description and user guidance,
|
|
14
|
+
* and work out which verdicts fail at what severity. Throws on an inconsistent rule.
|
|
15
|
+
*/
|
|
16
|
+
export function resolveSemanticRule(rule) {
|
|
17
|
+
const semantic = ruleSemantic(rule);
|
|
18
|
+
if (!semantic)
|
|
19
|
+
throw new Error(`Rule "${rule.id}" has no semantic part.`);
|
|
20
|
+
const declared = semantic.verdicts ?? DEFAULT_VERDICTS;
|
|
21
|
+
const verdicts = {};
|
|
22
|
+
const guidance = {};
|
|
23
|
+
for (const [name, value] of Object.entries(declared)) {
|
|
24
|
+
if (typeof value === 'string') {
|
|
25
|
+
verdicts[name] = value;
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
verdicts[name] = value.description;
|
|
29
|
+
if (value.guidance)
|
|
30
|
+
guidance[name] = value.guidance;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
let mapping;
|
|
34
|
+
if (semantic.verdicts) {
|
|
35
|
+
if (!semantic.severity) {
|
|
36
|
+
throw new Error(`Semantic rule "${rule.id}" declares verdicts, so semantic.severity must map the failing verdicts, e.g. { "${Object.keys(verdicts)[0]}": "warning" }.`);
|
|
37
|
+
}
|
|
38
|
+
mapping = semantic.severity;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
mapping = { ...semantic.severity, fail: ruleSeverity(rule) };
|
|
42
|
+
}
|
|
43
|
+
const reserved = RESERVED_VERDICTS;
|
|
44
|
+
const unknown = Object.keys(mapping).filter((v) => !(v in verdicts) && !reserved.includes(v));
|
|
45
|
+
if (unknown.length > 0) {
|
|
46
|
+
throw new Error(`Semantic rule "${rule.id}" has semantic.severity for verdicts not in its verdicts: ${unknown.join(', ')}.`);
|
|
47
|
+
}
|
|
48
|
+
const severities = {};
|
|
49
|
+
for (const [verdict, severity] of Object.entries(mapping)) {
|
|
50
|
+
if (severity !== 'ignore')
|
|
51
|
+
severities[verdict] = severity;
|
|
52
|
+
}
|
|
53
|
+
if (Object.keys(severities).filter((v) => v in verdicts).length >= Object.keys(verdicts).length) {
|
|
54
|
+
throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict has a severity.`);
|
|
55
|
+
}
|
|
56
|
+
// A warning by default.
|
|
57
|
+
const context = ruleContext(rule);
|
|
58
|
+
if (context.scope.length > 0 || context.content !== 'none') {
|
|
59
|
+
verdicts[MISSING_CONTEXT_VERDICT] = MISSING_CONTEXT_DESCRIPTION;
|
|
60
|
+
if (!(MISSING_CONTEXT_VERDICT in mapping))
|
|
61
|
+
severities[MISSING_CONTEXT_VERDICT] = 'warning';
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
spec: {
|
|
65
|
+
id: rule.id,
|
|
66
|
+
description: rule.description,
|
|
67
|
+
rubric: semantic.rubric,
|
|
68
|
+
category: rule.category,
|
|
69
|
+
verdicts,
|
|
70
|
+
},
|
|
71
|
+
severities,
|
|
72
|
+
guidance,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
/** The judge-facing spec for a semantic rule. */
|
|
76
|
+
export function semanticRuleSpec(rule) {
|
|
77
|
+
return resolveSemanticRule(rule).spec;
|
|
78
|
+
}
|
|
79
|
+
/** Whether a judge result is an error rather than a verdict. */
|
|
80
|
+
export function isJudgeError(result) {
|
|
81
|
+
return 'error' in result;
|
|
82
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { Config } from './config';
|
|
2
|
-
import { type SpaceGraph } from './space-graph';
|
|
3
|
-
import type { ReadSpaceResult, SpaceContext } from './types';
|
|
1
|
+
import type { Config } from './config.js';
|
|
2
|
+
import { type SpaceGraph } from './space-graph.js';
|
|
3
|
+
import type { ReadSpaceResult, SpaceContext } from './types.js';
|
|
4
4
|
export interface AssembleSpaceGraphOptions {
|
|
5
5
|
/** Named view (resolved via space.views) or a raw filter DSL expression. */
|
|
6
6
|
filter?: string;
|
package/dist/load-space-graph.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { filterNodes } from './filter/filter-nodes';
|
|
2
|
-
import { readSpace } from './read/read-space';
|
|
3
|
-
import { createSpaceContext } from './space-context';
|
|
4
|
-
import { buildSpaceGraph } from './space-graph';
|
|
1
|
+
import { filterNodes } from './filter/filter-nodes.js';
|
|
2
|
+
import { readSpace } from './read/read-space.js';
|
|
3
|
+
import { createSpaceContext } from './space-context.js';
|
|
4
|
+
import { buildSpaceGraph } from './space-graph.js';
|
|
5
5
|
/**
|
|
6
6
|
* Assemble a SpaceGraph from a space context: read the space, drop nodes that
|
|
7
7
|
* fail schema validation, build the graph for the selected hierarchy (the main
|
package/dist/plugins/loader.js
CHANGED
|
@@ -2,7 +2,7 @@ import { existsSync, readdirSync } from 'node:fs';
|
|
|
2
2
|
import { dirname, isAbsolute, join, resolve } from 'node:path';
|
|
3
3
|
import Ajv2020 from 'ajv/dist/2020';
|
|
4
4
|
import { getConfigSourceFiles } from '../config';
|
|
5
|
-
import { builtinPlugins } from '.';
|
|
5
|
+
import { builtinPlugins } from '../index.js';
|
|
6
6
|
import { CONFIG_PLUGINS_DIR, normalizePluginName, PLUGIN_PREFIX, shortenPluginName, } from './util';
|
|
7
7
|
/**
|
|
8
8
|
* Walk a plugin's configSchema and resolve any string fields annotated with
|
|
@@ -8,6 +8,8 @@ type StackEntry = {
|
|
|
8
8
|
nodeType: string;
|
|
9
9
|
/** Preferred wikilink key used when this heading acts as a parent. */
|
|
10
10
|
refTarget: string;
|
|
11
|
+
/** Node whose section this heading opens; absent for untyped placeholders and the page's own entry. */
|
|
12
|
+
node?: BaseNode;
|
|
11
13
|
};
|
|
12
14
|
/** Detect a bare wikilink `[[...]]` and return the inner target, or undefined. */
|
|
13
15
|
export declare function isWikilink(text: string): string | undefined;
|