structured-context 0.12.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -216
- package/dist/commands/docs.js +1 -0
- package/dist/commands/providers.d.ts +4 -0
- package/dist/commands/providers.js +52 -0
- package/dist/commands/schemas.js +21 -2
- package/dist/commands/validate-file.d.ts +3 -0
- package/dist/commands/validate-file.js +17 -2
- package/dist/commands/validate.d.ts +4 -0
- package/dist/commands/validate.js +42 -3
- package/dist/config.d.ts +7 -0
- package/dist/config.js +23 -0
- package/dist/index.js +44 -3
- package/dist/judge/context.d.ts +20 -0
- package/dist/judge/context.js +105 -0
- package/dist/judge/factory.d.ts +44 -0
- package/dist/judge/factory.js +148 -0
- package/dist/judge/index.d.ts +13 -0
- package/dist/judge/index.js +8 -0
- package/dist/judge/judge-node.d.ts +27 -0
- package/dist/judge/judge-node.js +122 -0
- package/dist/judge/prompt.d.ts +10 -0
- package/dist/judge/prompt.js +92 -0
- package/dist/judge/provider/anthropic.d.ts +17 -0
- package/dist/judge/provider/anthropic.js +43 -0
- package/dist/judge/provider/fake.d.ts +26 -0
- package/dist/judge/provider/fake.js +56 -0
- package/dist/judge/provider/jev.d.ts +37 -0
- package/dist/judge/provider/jev.js +169 -0
- package/dist/judge/provider/openai-decisions.d.ts +21 -0
- package/dist/judge/provider/openai-decisions.js +84 -0
- package/dist/judge/provider/openai.d.ts +20 -0
- package/dist/judge/provider/openai.js +53 -0
- package/dist/judge/provider/per-rule.d.ts +10 -0
- package/dist/judge/provider/per-rule.js +18 -0
- package/dist/judge/trim.d.ts +27 -0
- package/dist/judge/trim.js +62 -0
- package/dist/judge/types.d.ts +92 -0
- package/dist/judge/types.js +9 -0
- package/dist/judge/verdicts.d.ts +24 -0
- package/dist/judge/verdicts.js +82 -0
- package/dist/plugins/markdown/parse-embedded.d.ts +2 -0
- package/dist/plugins/markdown/parse-embedded.js +161 -24
- package/dist/plugins/markdown/read-space.js +4 -9
- package/dist/plugins/markdown/render-directory.js +4 -4
- package/dist/plugins/markdown/template-sync.js +3 -4
- package/dist/plugins/markdown/util.d.ts +25 -4
- package/dist/plugins/markdown/util.js +56 -15
- package/dist/render/render.js +1 -1
- package/dist/schema/evaluate-rule.d.ts +33 -2
- package/dist/schema/evaluate-rule.js +58 -4
- package/dist/schema/gating.d.ts +69 -0
- package/dist/schema/gating.js +253 -0
- package/dist/schema/metadata-contract.d.ts +625 -12
- package/dist/schema/metadata-contract.js +110 -4
- package/dist/schema/rule-filter.d.ts +12 -0
- package/dist/schema/rule-filter.js +46 -0
- package/dist/schema/rule-severity.d.ts +13 -0
- package/dist/schema/rule-severity.js +33 -0
- package/dist/schema/schema.js +31 -1
- package/dist/schema/validate-rules.d.ts +17 -8
- package/dist/schema/validate-rules.js +91 -48
- package/dist/schemas/_knowledge_wiki.json +4 -4
- package/dist/schemas/_okf.json +172 -0
- package/dist/schemas/_ost_strict.json +15 -6
- package/dist/schemas/_sctx_base.json +1 -1
- package/dist/schemas/generated/0.13.0/_structured_context_schema_meta.json +400 -0
- package/dist/schemas/generated/latest/_structured_context_schema_meta.json +177 -6
- package/dist/schemas/okf.json +8 -0
- package/dist/types.d.ts +8 -1
- package/dist/validate.d.ts +58 -8
- package/dist/validate.js +219 -114
- package/docs/concepts.md +52 -6
- package/docs/config.md +70 -1
- package/docs/rules.md +140 -49
- package/docs/schemas.md +43 -18
- package/package.json +10 -9
- package/schemas/_knowledge_wiki.json +4 -4
- package/schemas/_okf.json +172 -0
- package/schemas/_ost_strict.json +15 -6
- package/schemas/_sctx_base.json +1 -1
- package/schemas/generated/0.13.0/_structured_context_schema_meta.json +400 -0
- package/schemas/generated/latest/_structured_context_schema_meta.json +177 -6
- package/schemas/okf.json +8 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { SemanticJudgeInput } from './types';
|
|
2
|
+
/** The per-node input cap: estimated tokens of one node's input (the node and its related nodes), the same for every provider. */
|
|
3
|
+
export declare const PER_NODE_INPUT_CAP_TOKENS = 32000;
|
|
4
|
+
/** Estimated tokens of a value: its JSON length divided by 4. Provider-independent, and roughly right is enough. */
|
|
5
|
+
export declare const estimateTokens: (value: unknown) => number;
|
|
6
|
+
/** The placeholder that ends a body cut to fit the per-node input cap. `removed` is the number of characters cut. */
|
|
7
|
+
export declare const trimMarker: (removed: number) => string;
|
|
8
|
+
/** The part of a judge input that the per-node input cap measures. */
|
|
9
|
+
export type NodeInput = Pick<SemanticJudgeInput, 'node' | 'related'>;
|
|
10
|
+
/**
|
|
11
|
+
* Cut `body` to at most `limit` characters, ending at the last whitespace before the limit, and end it with the
|
|
12
|
+
* trim marker. A body within the limit is returned whole.
|
|
13
|
+
*/
|
|
14
|
+
export declare function cutBody(body: string, limit: number): string;
|
|
15
|
+
/**
|
|
16
|
+
* Fit a node's input under the per-node input cap by cutting bodies only: related bodies to one shared ceiling
|
|
17
|
+
* first, then the evaluated node's body. Undefined when the input does not fit even with every body cut.
|
|
18
|
+
*/
|
|
19
|
+
export declare function trimToCap(input: NodeInput, cap?: number): {
|
|
20
|
+
input: NodeInput;
|
|
21
|
+
trimmed: boolean;
|
|
22
|
+
} | undefined;
|
|
23
|
+
/** What a run trimmed: the nodes (by label) and the rules whose input had a body cut. */
|
|
24
|
+
export interface TrimReport {
|
|
25
|
+
nodes: Set<string>;
|
|
26
|
+
ruleIds: Set<string>;
|
|
27
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/** The per-node input cap: estimated tokens of one node's input (the node and its related nodes), the same for every provider. */
|
|
2
|
+
export const PER_NODE_INPUT_CAP_TOKENS = 32_000;
|
|
3
|
+
/** Estimated tokens of a value: its JSON length divided by 4. Provider-independent, and roughly right is enough. */
|
|
4
|
+
export const estimateTokens = (value) => Math.ceil(JSON.stringify(value).length / 4);
|
|
5
|
+
/** The placeholder that ends a body cut to fit the per-node input cap. `removed` is the number of characters cut. */
|
|
6
|
+
export const trimMarker = (removed) => `[RULE_EVALUATION_CONTENT_TRIMMED ${removed} chars]`;
|
|
7
|
+
/**
|
|
8
|
+
* Cut `body` to at most `limit` characters, ending at the last whitespace before the limit, and end it with the
|
|
9
|
+
* trim marker. A body within the limit is returned whole.
|
|
10
|
+
*/
|
|
11
|
+
export function cutBody(body, limit) {
|
|
12
|
+
if (body.length <= limit)
|
|
13
|
+
return body;
|
|
14
|
+
let head = body.slice(0, limit);
|
|
15
|
+
if (!/\s/.test(body.charAt(limit))) {
|
|
16
|
+
const lastSpace = head.search(/\s\S*$/);
|
|
17
|
+
head = lastSpace < 0 ? '' : head.slice(0, lastSpace);
|
|
18
|
+
}
|
|
19
|
+
head = head.trimEnd();
|
|
20
|
+
const marker = trimMarker(body.length - head.length);
|
|
21
|
+
return head ? `${head} ${marker}` : marker;
|
|
22
|
+
}
|
|
23
|
+
function limitBodies(input, relatedLimit, nodeLimit) {
|
|
24
|
+
const cut = (data, limit) => typeof data.content === 'string' ? { ...data, content: cutBody(data.content, limit) } : data;
|
|
25
|
+
return {
|
|
26
|
+
node: { ...input.node, schemaData: cut(input.node.schemaData, nodeLimit) },
|
|
27
|
+
...(input.related && { related: input.related.map((r) => ({ ...r, data: cut(r.data, relatedLimit) })) }),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const bodyLength = (data) => (typeof data.content === 'string' ? data.content.length : 0);
|
|
31
|
+
/** The largest limit in `[0, max]` for which `fits` holds, or -1 when even 0 does not fit. `fits` shrinks as the limit falls. */
|
|
32
|
+
function largestFitting(max, fits) {
|
|
33
|
+
if (!fits(0))
|
|
34
|
+
return -1;
|
|
35
|
+
let low = 0;
|
|
36
|
+
let high = max;
|
|
37
|
+
while (low < high) {
|
|
38
|
+
const mid = Math.ceil((low + high) / 2);
|
|
39
|
+
if (fits(mid))
|
|
40
|
+
low = mid;
|
|
41
|
+
else
|
|
42
|
+
high = mid - 1;
|
|
43
|
+
}
|
|
44
|
+
return low;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Fit a node's input under the per-node input cap by cutting bodies only: related bodies to one shared ceiling
|
|
48
|
+
* first, then the evaluated node's body. Undefined when the input does not fit even with every body cut.
|
|
49
|
+
*/
|
|
50
|
+
export function trimToCap(input, cap = PER_NODE_INPUT_CAP_TOKENS) {
|
|
51
|
+
if (estimateTokens(input) <= cap)
|
|
52
|
+
return { input, trimmed: false };
|
|
53
|
+
const nodeLength = bodyLength(input.node.schemaData);
|
|
54
|
+
const relatedMax = Math.max(0, ...(input.related ?? []).map((r) => bodyLength(r.data)));
|
|
55
|
+
const fitsRelated = (limit) => estimateTokens(limitBodies(input, limit, nodeLength)) <= cap;
|
|
56
|
+
const relatedLimit = largestFitting(relatedMax, fitsRelated);
|
|
57
|
+
if (relatedLimit >= 0)
|
|
58
|
+
return { input: limitBodies(input, relatedLimit, nodeLength), trimmed: true };
|
|
59
|
+
const fitsNode = (limit) => estimateTokens(limitBodies(input, 0, limit)) <= cap;
|
|
60
|
+
const nodeLimit = largestFitting(nodeLength, fitsNode);
|
|
61
|
+
return nodeLimit >= 0 ? { input: limitBodies(input, 0, nodeLimit), trimmed: true } : undefined;
|
|
62
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type { RuleCategory, SpaceNode } from '../types';
|
|
2
|
+
/** A semantic rule as presented to a judge: the closed verdict set it must choose from. */
|
|
3
|
+
export interface SemanticRuleSpec {
|
|
4
|
+
id: string;
|
|
5
|
+
description: string;
|
|
6
|
+
rubric?: string;
|
|
7
|
+
category: RuleCategory | string;
|
|
8
|
+
/** Allowed verdicts, each with a description of when it applies. */
|
|
9
|
+
verdicts: Record<string, string>;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* A node related to the evaluated node by its rules' `scope`. It carries all the node's schema data; the body
|
|
13
|
+
* (`data.content`) is present only when the rules' `includeContent` asks for it.
|
|
14
|
+
*/
|
|
15
|
+
export interface RelatedNode {
|
|
16
|
+
/** How it relates to the evaluated node. A node can have several. */
|
|
17
|
+
relations: Array<'parent' | 'sibling' | 'child' | 'ancestor' | 'descendant'>;
|
|
18
|
+
title: string;
|
|
19
|
+
type: string;
|
|
20
|
+
/** Schema data without `title` and `type`. */
|
|
21
|
+
data: Record<string, unknown>;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* One node and the semantic rules to judge on it, which share one context: the same related nodes and the same
|
|
25
|
+
* bodies. Judges may answer all rules in a single call.
|
|
26
|
+
*/
|
|
27
|
+
export interface SemanticJudgeInput {
|
|
28
|
+
/** The evaluated node. Its body is in `schemaData.content` only when the rules' `includeContent` asks for it. */
|
|
29
|
+
node: Pick<SpaceNode, 'label' | 'title' | 'type' | 'resolvedType' | 'schemaData' | 'resolvedParents'>;
|
|
30
|
+
/** Nodes the rules' `scope` adds, as a flat list. Absent when the scope is empty. */
|
|
31
|
+
related?: RelatedNode[];
|
|
32
|
+
rules: SemanticRuleSpec[];
|
|
33
|
+
}
|
|
34
|
+
/** A judge's answer for one rule: a verdict from the rule's closed set. */
|
|
35
|
+
export interface SemanticJudgeVerdict {
|
|
36
|
+
verdict: string;
|
|
37
|
+
/** 0-1, when the judge reports one. */
|
|
38
|
+
confidence?: number;
|
|
39
|
+
/** Probability per verdict, when the judge reports a distribution. */
|
|
40
|
+
probabilities?: Record<string, number>;
|
|
41
|
+
/** Free-text reasoning; decision models may not provide it. */
|
|
42
|
+
evidence?: string;
|
|
43
|
+
}
|
|
44
|
+
/** A judge's answer for one rule: a verdict, or an error that applies to that rule only. */
|
|
45
|
+
export type SemanticJudgeResult = SemanticJudgeVerdict | {
|
|
46
|
+
error: string;
|
|
47
|
+
};
|
|
48
|
+
/** Judge interface. Results are keyed by rule id; a missing key is treated as an error. */
|
|
49
|
+
export interface SemanticJudgeProvider {
|
|
50
|
+
readonly name: string;
|
|
51
|
+
evaluate(input: SemanticJudgeInput): Promise<Record<string, SemanticJudgeResult>>;
|
|
52
|
+
/**
|
|
53
|
+
* Optional: judge many nodes in one call, so a provider can fold them into fewer model calls. Returns one
|
|
54
|
+
* result map per input, in order. A failure that affects only some inputs is reported as `{ error }`
|
|
55
|
+
* results for those inputs' rules; throw only when nothing could be judged.
|
|
56
|
+
*/
|
|
57
|
+
evaluateNodes?(inputs: SemanticJudgeInput[]): Promise<Array<Record<string, SemanticJudgeResult>>>;
|
|
58
|
+
}
|
|
59
|
+
/** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */
|
|
60
|
+
export declare const MODEL_PROVIDER_APIS: readonly ["openai-completions", "openai-decisions", "anthropic-messages", "openrouter-decisions", "typesafe-systemone", "fake"];
|
|
61
|
+
export type ModelProviderApi = (typeof MODEL_PROVIDER_APIS)[number];
|
|
62
|
+
/** How to reach a model: a bundled provider or config `modelProviders.<name>`, before hydration. */
|
|
63
|
+
export interface ModelProviderConfig {
|
|
64
|
+
/** The wire API to speak. */
|
|
65
|
+
api: ModelProviderApi;
|
|
66
|
+
/** Fixed model override. Beats `modelEnv` and the bundled default. */
|
|
67
|
+
model?: string;
|
|
68
|
+
/** Environment variable that holds the model override. */
|
|
69
|
+
modelEnv?: string;
|
|
70
|
+
/** Fixed endpoint override. Beats `endpointEnv` and the bundled default. */
|
|
71
|
+
endpoint?: string;
|
|
72
|
+
/** Environment variable that holds the endpoint override. */
|
|
73
|
+
endpointEnv?: string;
|
|
74
|
+
/** Prefer `apiKeyEnv`: keys do not belong in config files. */
|
|
75
|
+
apiKey?: string;
|
|
76
|
+
/** Environment variable holding the key. */
|
|
77
|
+
apiKeyEnv?: string;
|
|
78
|
+
}
|
|
79
|
+
/** A provider after hydration: every field is a concrete value; the implementation reads no environment. */
|
|
80
|
+
export interface HydratedProvider {
|
|
81
|
+
api: ModelProviderApi;
|
|
82
|
+
model?: string;
|
|
83
|
+
endpoint?: string;
|
|
84
|
+
apiKey?: string;
|
|
85
|
+
/** The environment variable the key was read from, when one was named or defaulted. */
|
|
86
|
+
keyEnv?: string;
|
|
87
|
+
}
|
|
88
|
+
/** Which model provider evaluates semantic rules: config `semantic`, at the top level or on a space. */
|
|
89
|
+
export interface JudgeConfig {
|
|
90
|
+
/** Name of an entry in `modelProviders`, or a bundled provider. */
|
|
91
|
+
provider?: string;
|
|
92
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */
|
|
2
|
+
export const MODEL_PROVIDER_APIS = [
|
|
3
|
+
'openai-completions',
|
|
4
|
+
'openai-decisions',
|
|
5
|
+
'anthropic-messages',
|
|
6
|
+
'openrouter-decisions',
|
|
7
|
+
'typesafe-systemone',
|
|
8
|
+
'fake',
|
|
9
|
+
];
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Rule, RuleSeverity } from '../types';
|
|
2
|
+
import type { SemanticJudgeResult, SemanticRuleSpec } from './types';
|
|
3
|
+
/** Verdicts for a semantic rule that declares none. */
|
|
4
|
+
export declare const DEFAULT_VERDICTS: Record<string, string>;
|
|
5
|
+
/** A semantic rule resolved for evaluation: what the judge sees, and what each verdict means for the user. */
|
|
6
|
+
export interface ResolvedSemanticRule {
|
|
7
|
+
/** What the judge sees: the closed verdict set. */
|
|
8
|
+
spec: SemanticRuleSpec;
|
|
9
|
+
/** Severity of each verdict that fails the rule. A verdict absent here passes (including `ignore`d ones). */
|
|
10
|
+
severities: Record<string, RuleSeverity>;
|
|
11
|
+
/** User-facing guidance for verdicts that declare it. */
|
|
12
|
+
guidance: Record<string, string>;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Resolve a semantic rule: apply verdict defaults, split each verdict into its judge description and user guidance,
|
|
16
|
+
* and work out which verdicts fail at what severity. Throws on an inconsistent rule.
|
|
17
|
+
*/
|
|
18
|
+
export declare function resolveSemanticRule(rule: Rule): ResolvedSemanticRule;
|
|
19
|
+
/** The judge-facing spec for a semantic rule. */
|
|
20
|
+
export declare function semanticRuleSpec(rule: Rule): SemanticRuleSpec;
|
|
21
|
+
/** Whether a judge result is an error rather than a verdict. */
|
|
22
|
+
export declare function isJudgeError(result: SemanticJudgeResult): result is {
|
|
23
|
+
error: string;
|
|
24
|
+
};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { RESERVED_VERDICTS } from '../schema/metadata-contract';
|
|
2
|
+
import { ruleSemantic, ruleSeverity } from '../schema/rule-severity';
|
|
3
|
+
import { ruleContext } from './context';
|
|
4
|
+
/** Verdicts for a semantic rule that declares none. */
|
|
5
|
+
export const DEFAULT_VERDICTS = {
|
|
6
|
+
pass: 'The node meets the rubric.',
|
|
7
|
+
fail: 'The node does not meet the rubric.',
|
|
8
|
+
};
|
|
9
|
+
/** The reserved verdict a rule with related nodes or bodies offers, for input a trim marker shows was cut. */
|
|
10
|
+
const MISSING_CONTEXT_VERDICT = 'missing-context';
|
|
11
|
+
const MISSING_CONTEXT_DESCRIPTION = 'Content needed to judge the rule was cut: a [RULE_EVALUATION_CONTENT_TRIMMED n chars] marker shows where. Choose this only when such a marker is present and the cut content is needed. Never choose it when no marker appears.';
|
|
12
|
+
/**
|
|
13
|
+
* Resolve a semantic rule: apply verdict defaults, split each verdict into its judge description and user guidance,
|
|
14
|
+
* and work out which verdicts fail at what severity. Throws on an inconsistent rule.
|
|
15
|
+
*/
|
|
16
|
+
export function resolveSemanticRule(rule) {
|
|
17
|
+
const semantic = ruleSemantic(rule);
|
|
18
|
+
if (!semantic)
|
|
19
|
+
throw new Error(`Rule "${rule.id}" has no semantic part.`);
|
|
20
|
+
const declared = semantic.verdicts ?? DEFAULT_VERDICTS;
|
|
21
|
+
const verdicts = {};
|
|
22
|
+
const guidance = {};
|
|
23
|
+
for (const [name, value] of Object.entries(declared)) {
|
|
24
|
+
if (typeof value === 'string') {
|
|
25
|
+
verdicts[name] = value;
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
verdicts[name] = value.description;
|
|
29
|
+
if (value.guidance)
|
|
30
|
+
guidance[name] = value.guidance;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
let mapping;
|
|
34
|
+
if (semantic.verdicts) {
|
|
35
|
+
if (!semantic.severity) {
|
|
36
|
+
throw new Error(`Semantic rule "${rule.id}" declares verdicts, so semantic.severity must map the failing verdicts, e.g. { "${Object.keys(verdicts)[0]}": "warning" }.`);
|
|
37
|
+
}
|
|
38
|
+
mapping = semantic.severity;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
mapping = { ...semantic.severity, fail: ruleSeverity(rule) };
|
|
42
|
+
}
|
|
43
|
+
const reserved = RESERVED_VERDICTS;
|
|
44
|
+
const unknown = Object.keys(mapping).filter((v) => !(v in verdicts) && !reserved.includes(v));
|
|
45
|
+
if (unknown.length > 0) {
|
|
46
|
+
throw new Error(`Semantic rule "${rule.id}" has semantic.severity for verdicts not in its verdicts: ${unknown.join(', ')}.`);
|
|
47
|
+
}
|
|
48
|
+
const severities = {};
|
|
49
|
+
for (const [verdict, severity] of Object.entries(mapping)) {
|
|
50
|
+
if (severity !== 'ignore')
|
|
51
|
+
severities[verdict] = severity;
|
|
52
|
+
}
|
|
53
|
+
if (Object.keys(severities).filter((v) => v in verdicts).length >= Object.keys(verdicts).length) {
|
|
54
|
+
throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict has a severity.`);
|
|
55
|
+
}
|
|
56
|
+
// A warning by default.
|
|
57
|
+
const context = ruleContext(rule);
|
|
58
|
+
if (context.scope.length > 0 || context.content !== 'none') {
|
|
59
|
+
verdicts[MISSING_CONTEXT_VERDICT] = MISSING_CONTEXT_DESCRIPTION;
|
|
60
|
+
if (!(MISSING_CONTEXT_VERDICT in mapping))
|
|
61
|
+
severities[MISSING_CONTEXT_VERDICT] = 'warning';
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
spec: {
|
|
65
|
+
id: rule.id,
|
|
66
|
+
description: rule.description,
|
|
67
|
+
rubric: semantic.rubric,
|
|
68
|
+
category: rule.category,
|
|
69
|
+
verdicts,
|
|
70
|
+
},
|
|
71
|
+
severities,
|
|
72
|
+
guidance,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
/** The judge-facing spec for a semantic rule. */
|
|
76
|
+
export function semanticRuleSpec(rule) {
|
|
77
|
+
return resolveSemanticRule(rule).spec;
|
|
78
|
+
}
|
|
79
|
+
/** Whether a judge result is an error rather than a verdict. */
|
|
80
|
+
export function isJudgeError(result) {
|
|
81
|
+
return 'error' in result;
|
|
82
|
+
}
|
|
@@ -8,6 +8,8 @@ type StackEntry = {
|
|
|
8
8
|
nodeType: string;
|
|
9
9
|
/** Preferred wikilink key used when this heading acts as a parent. */
|
|
10
10
|
refTarget: string;
|
|
11
|
+
/** Node whose section this heading opens; absent for untyped placeholders and the page's own entry. */
|
|
12
|
+
node?: BaseNode;
|
|
11
13
|
};
|
|
12
14
|
/** Detect a bare wikilink `[[...]]` and return the inner target, or undefined. */
|
|
13
15
|
export declare function isWikilink(text: string): string | undefined;
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { load as yamlLoad } from 'js-yaml';
|
|
2
1
|
import { toString as mdastToString } from 'mdast-util-to-string';
|
|
3
2
|
import remarkGfm from 'remark-gfm';
|
|
4
3
|
import remarkParse from 'remark-parse';
|
|
5
4
|
import { unified } from 'unified';
|
|
6
5
|
import { extractLinksFromAstNode } from './extract-content-links';
|
|
7
|
-
import { applyFieldMap,
|
|
6
|
+
import { applyFieldMap, parseYaml } from './util';
|
|
8
7
|
/** Type values that identify a space_on_a_page container (not themselves space nodes). */
|
|
9
8
|
export const ON_A_PAGE_TYPES = ['ost_on_a_page', 'space_on_a_page'];
|
|
10
9
|
const DEFAULT_STATUS = 'identified';
|
|
@@ -141,18 +140,91 @@ export function defaultNodeType(stack, hierarchy) {
|
|
|
141
140
|
}
|
|
142
141
|
return hierarchy[idx + 1];
|
|
143
142
|
}
|
|
144
|
-
|
|
145
|
-
|
|
143
|
+
/** Start and end offsets of an mdast node. Throws when the parser gave no position. */
|
|
144
|
+
function offsetsOf(node) {
|
|
145
|
+
const start = node.position?.start.offset;
|
|
146
|
+
const end = node.position?.end.offset;
|
|
147
|
+
if (start === undefined || end === undefined) {
|
|
148
|
+
throw new Error(`Markdown ${node.type} node has no source position; cannot slice its content`);
|
|
149
|
+
}
|
|
150
|
+
return { start, end };
|
|
151
|
+
}
|
|
152
|
+
function addRange(claims, node, start, end) {
|
|
153
|
+
if (end <= start)
|
|
146
154
|
return;
|
|
147
|
-
const
|
|
148
|
-
const
|
|
149
|
-
|
|
150
|
-
|
|
155
|
+
const ranges = claims.ranges.get(node) ?? [];
|
|
156
|
+
const last = ranges[ranges.length - 1];
|
|
157
|
+
if (last && last.end === start)
|
|
158
|
+
last.end = end;
|
|
159
|
+
else
|
|
160
|
+
ranges.push({ start, end });
|
|
161
|
+
claims.ranges.set(node, ranges);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Give a node a block of source, plus the blank lines between `from` and the block's `start`.
|
|
165
|
+
* Those lines are skipped when a consumed block just before already kept its own.
|
|
166
|
+
*/
|
|
167
|
+
function claim(claims, node, from, start, end) {
|
|
168
|
+
addRange(claims, node, claims.gapKept.delete(node) ? start : from, end);
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Take a block out of a node's content (it was consumed as a node, field or other syntax).
|
|
172
|
+
* The blank lines before it stay, so the content on either side does not run together.
|
|
173
|
+
*/
|
|
174
|
+
function consume(claims, node, from, start) {
|
|
175
|
+
addRange(claims, node, claims.gapKept.delete(node) ? start : from, start);
|
|
176
|
+
claims.gapKept.add(node);
|
|
177
|
+
}
|
|
178
|
+
/** Sets each node's content from its claimed source. Leading blank lines and trailing whitespace are dropped. */
|
|
179
|
+
function applyClaims(claims, body) {
|
|
180
|
+
for (const [node, ranges] of claims.ranges) {
|
|
181
|
+
let content = ranges
|
|
182
|
+
.map((r) => body.slice(r.start, r.end))
|
|
183
|
+
.join('')
|
|
184
|
+
.replace(/^(?:[ \t]*\r?\n)+/, '')
|
|
185
|
+
.trimEnd();
|
|
186
|
+
const indent = claims.contentIndent.get(node);
|
|
187
|
+
if (indent) {
|
|
188
|
+
const strip = new RegExp(`^[ \\t]{1,${indent}}`);
|
|
189
|
+
content = content
|
|
190
|
+
.split(/(?<=\n)/)
|
|
191
|
+
.map((line) => line.replace(strip, ''))
|
|
192
|
+
.join('');
|
|
193
|
+
}
|
|
194
|
+
if (!content)
|
|
195
|
+
continue;
|
|
196
|
+
node.schemaData.content = content;
|
|
197
|
+
node.content = content;
|
|
198
|
+
}
|
|
151
199
|
}
|
|
152
|
-
|
|
200
|
+
/** Source lines (with terminators) of a range, as absolute offsets. */
|
|
201
|
+
function sourceLines(body, range) {
|
|
202
|
+
const lines = [];
|
|
203
|
+
const re = /[^\n]*\n|[^\n]+$/g;
|
|
204
|
+
const text = body.slice(range.start, range.end);
|
|
205
|
+
for (const m of text.matchAll(re)) {
|
|
206
|
+
lines.push({
|
|
207
|
+
start: range.start + m.index,
|
|
208
|
+
end: range.start + m.index + m[0].length,
|
|
209
|
+
text: m[0].replace(/\r?\n$/, ''),
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
return lines;
|
|
213
|
+
}
|
|
214
|
+
/** Whether a line holds only inline fields: bracketed `[key:: value]` pieces, or one `key:: value`. */
|
|
215
|
+
function isFieldOnlyLine(line) {
|
|
216
|
+
const { cleanText, fields } = extractBracketedFields(line);
|
|
217
|
+
const hasBracketed = Object.keys(fields).length > 0;
|
|
218
|
+
if (cleanText === '')
|
|
219
|
+
return hasBracketed;
|
|
220
|
+
return /^[a-zA-Z][a-zA-Z0-9_-]*:: *.*$/.test(cleanText);
|
|
221
|
+
}
|
|
222
|
+
function processListItem(item, parentRef, contentTarget, nodes, makeLabel, buildLinkTargets, typeAliases, fieldMap, pendingType, parentFieldAppend, activeNodeFieldAppend, file, lineOffset, walk) {
|
|
223
|
+
const itemEnd = offsetsOf(item).end;
|
|
153
224
|
const firstPara = item.children.find((c) => c.type === 'paragraph');
|
|
154
225
|
if (!firstPara) {
|
|
155
|
-
|
|
226
|
+
claim(walk.claims, contentTarget, walk.pos, offsetsOf(item).start, itemEnd);
|
|
227
|
+
walk.pos = itemEnd;
|
|
156
228
|
return;
|
|
157
229
|
}
|
|
158
230
|
const rawText = mdastToString(firstPara);
|
|
@@ -161,11 +233,15 @@ function processListItem(item, parentRef, contentTarget, nodes, makeLabel, build
|
|
|
161
233
|
if (wikiTarget) {
|
|
162
234
|
const linkRef = `[[${wikiTarget}]]`;
|
|
163
235
|
if (parentFieldAppend) {
|
|
236
|
+
consume(walk.claims, contentTarget, walk.pos, offsetsOf(item).start);
|
|
164
237
|
appendParentField(parentFieldAppend.node, parentFieldAppend.field, linkRef);
|
|
238
|
+
walk.pos = itemEnd;
|
|
165
239
|
return;
|
|
166
240
|
}
|
|
167
241
|
if (activeNodeFieldAppend) {
|
|
242
|
+
consume(walk.claims, contentTarget, walk.pos, offsetsOf(item).start);
|
|
168
243
|
appendParentField(activeNodeFieldAppend.node, activeNodeFieldAppend.field, linkRef);
|
|
244
|
+
walk.pos = itemEnd;
|
|
169
245
|
return;
|
|
170
246
|
}
|
|
171
247
|
// No field append context — fall through to node creation or content append
|
|
@@ -204,21 +280,34 @@ function processListItem(item, parentRef, contentTarget, nodes, makeLabel, build
|
|
|
204
280
|
...(source ? { source } : {}),
|
|
205
281
|
};
|
|
206
282
|
nodes.push(newNode);
|
|
283
|
+
const contentColumn = firstPara.position?.start.column;
|
|
284
|
+
if (contentColumn !== undefined)
|
|
285
|
+
walk.claims.contentIndent.set(newNode, contentColumn - 1);
|
|
207
286
|
if (parentFieldAppend) {
|
|
208
287
|
appendParentField(parentFieldAppend.node, parentFieldAppend.field, `[[${linkTargets[0] ?? title}]]`);
|
|
209
288
|
}
|
|
210
289
|
const nestedParentRef = `[[${linkTargets[0] ?? title}]]`;
|
|
290
|
+
consume(walk.claims, contentTarget, walk.pos, offsetsOf(item).start);
|
|
291
|
+
walk.pos = offsetsOf(firstPara).end;
|
|
211
292
|
for (const child of item.children) {
|
|
293
|
+
if (child === firstPara)
|
|
294
|
+
continue;
|
|
212
295
|
if (child.type === 'list') {
|
|
213
296
|
for (const subItem of child.children) {
|
|
214
|
-
processListItem(subItem, nestedParentRef, newNode, nodes, makeLabel, buildLinkTargets, typeAliases, fieldMap, pendingType, undefined, undefined, file, lineOffset);
|
|
297
|
+
processListItem(subItem, nestedParentRef, newNode, nodes, makeLabel, buildLinkTargets, typeAliases, fieldMap, pendingType, undefined, undefined, file, lineOffset, walk);
|
|
215
298
|
}
|
|
216
299
|
}
|
|
300
|
+
else {
|
|
301
|
+
const { start, end } = offsetsOf(child);
|
|
302
|
+
claim(walk.claims, newNode, walk.pos, start, end);
|
|
303
|
+
walk.pos = end;
|
|
304
|
+
}
|
|
217
305
|
}
|
|
218
306
|
}
|
|
219
307
|
else {
|
|
220
|
-
|
|
308
|
+
claim(walk.claims, contentTarget, walk.pos, offsetsOf(item).start, itemEnd);
|
|
221
309
|
}
|
|
310
|
+
walk.pos = itemEnd;
|
|
222
311
|
}
|
|
223
312
|
/**
|
|
224
313
|
* Extract space nodes from markdown body text.
|
|
@@ -418,7 +507,32 @@ export function extractEmbeddedNodes(body, options) {
|
|
|
418
507
|
}
|
|
419
508
|
let grouping = null;
|
|
420
509
|
let activeNode = rootNode;
|
|
510
|
+
const claims = { ranges: new Map(), gapKept: new Set(), contentIndent: new Map() };
|
|
511
|
+
/** End of the previous top-level block; a block's span starts here so the blank lines before it stay with it. */
|
|
512
|
+
let prevEnd = 0;
|
|
421
513
|
for (const child of tree.children) {
|
|
514
|
+
const { start: blockStart, end: blockEnd } = offsetsOf(child);
|
|
515
|
+
const spanStart = prevEnd;
|
|
516
|
+
prevEnd = blockEnd;
|
|
517
|
+
/** Give a node this block as source content, minus any `removed` ranges inside it. */
|
|
518
|
+
const claimBlock = (node, removed = []) => {
|
|
519
|
+
let from = blockStart;
|
|
520
|
+
let first = true;
|
|
521
|
+
for (const gap of removed) {
|
|
522
|
+
if (first)
|
|
523
|
+
claim(claims, node, spanStart, blockStart, gap.start);
|
|
524
|
+
else
|
|
525
|
+
addRange(claims, node, from, gap.start);
|
|
526
|
+
first = false;
|
|
527
|
+
from = gap.end;
|
|
528
|
+
}
|
|
529
|
+
if (first)
|
|
530
|
+
claim(claims, node, spanStart, blockStart, blockEnd);
|
|
531
|
+
else
|
|
532
|
+
addRange(claims, node, from, blockEnd);
|
|
533
|
+
};
|
|
534
|
+
/** Take this whole block out of a node's content, keeping the blank lines before it. */
|
|
535
|
+
const consumeBlock = (node) => consume(claims, node, spanStart, blockStart);
|
|
422
536
|
if (parseState === 'done') {
|
|
423
537
|
if (child.type === 'heading') {
|
|
424
538
|
const rawTitle = mdastToString(child);
|
|
@@ -436,8 +550,11 @@ export function extractEmbeddedNodes(body, options) {
|
|
|
436
550
|
if (child.type === 'heading') {
|
|
437
551
|
const heading = child;
|
|
438
552
|
const depth = heading.depth;
|
|
439
|
-
if (depth > 5)
|
|
553
|
+
if (depth > 5) {
|
|
554
|
+
if (parseState === 'active')
|
|
555
|
+
claimBlock(activeNode);
|
|
440
556
|
continue;
|
|
557
|
+
}
|
|
441
558
|
parseState = 'active';
|
|
442
559
|
const rawText = mdastToString(heading);
|
|
443
560
|
const { cleanText: afterBracketed, fields: rawInlineFields } = extractBracketedFields(rawText);
|
|
@@ -455,6 +572,10 @@ export function extractEmbeddedNodes(body, options) {
|
|
|
455
572
|
stack.push({ depth, title, nodeType: '', refTarget: title });
|
|
456
573
|
// Discard any pending grouping (untyped heading has no implied type)
|
|
457
574
|
grouping = null;
|
|
575
|
+
// An untyped heading is part of the body of the nearest enclosing node. Popping above removed every
|
|
576
|
+
// node at the same or a higher level, so those sections end here; deeper nodes' sections continue.
|
|
577
|
+
activeNode = stack.findLast((entry) => entry.node)?.node ?? rootNode;
|
|
578
|
+
claimBlock(activeNode);
|
|
458
579
|
continue;
|
|
459
580
|
}
|
|
460
581
|
if (isOnAPageMode && stack.length > 0) {
|
|
@@ -494,7 +615,7 @@ export function extractEmbeddedNodes(body, options) {
|
|
|
494
615
|
};
|
|
495
616
|
// Push to stack BEFORE resolving semantic parent — stack[-2] is the correct parent.
|
|
496
617
|
const refTarget = linkTargets[0] ?? title;
|
|
497
|
-
stack.push({ depth, title, nodeType: type, refTarget });
|
|
618
|
+
stack.push({ depth, title, nodeType: type, refTarget, node: headingNode });
|
|
498
619
|
// If this match came from a relationship/hierarchy and no explicit type was given,
|
|
499
620
|
// create a grouping — delay adding the node until we see the following content.
|
|
500
621
|
// Discard any previous grouping (not flushed — original agnostic-parse behaviour).
|
|
@@ -532,14 +653,16 @@ export function extractEmbeddedNodes(body, options) {
|
|
|
532
653
|
const activeNodeFieldAppendArg = isParentLevelMatch && semanticParent.node
|
|
533
654
|
? { node: semanticParent.node, field: definition.field }
|
|
534
655
|
: undefined;
|
|
656
|
+
const walk = { claims, pos: spanStart };
|
|
535
657
|
for (const item of list.children) {
|
|
536
|
-
processListItem(item, isParentSide ? undefined : semanticParent.ref, grouping.headingNode, nodes, makeLabel, buildListItemLinkTargets, typeAliases, fieldMap, definition.type, parentFieldAppendArg, activeNodeFieldAppendArg, options.file, options.lineOffset);
|
|
658
|
+
processListItem(item, isParentSide ? undefined : semanticParent.ref, grouping.headingNode, nodes, makeLabel, buildListItemLinkTargets, typeAliases, fieldMap, definition.type, parentFieldAppendArg, activeNodeFieldAppendArg, options.file, options.lineOffset, walk);
|
|
537
659
|
}
|
|
538
660
|
grouping = null;
|
|
539
661
|
}
|
|
540
662
|
else {
|
|
663
|
+
const walk = { claims, pos: spanStart };
|
|
541
664
|
for (const item of list.children) {
|
|
542
|
-
processListItem(item, parentRef, activeNode, nodes, makeLabel, buildListItemLinkTargets, typeAliases, fieldMap, undefined, undefined, undefined, options.file, options.lineOffset);
|
|
665
|
+
processListItem(item, parentRef, activeNode, nodes, makeLabel, buildListItemLinkTargets, typeAliases, fieldMap, undefined, undefined, undefined, options.file, options.lineOffset, walk);
|
|
543
666
|
}
|
|
544
667
|
}
|
|
545
668
|
}
|
|
@@ -648,7 +771,7 @@ export function extractEmbeddedNodes(body, options) {
|
|
|
648
771
|
grouping = null;
|
|
649
772
|
}
|
|
650
773
|
else {
|
|
651
|
-
|
|
774
|
+
claimBlock(activeNode);
|
|
652
775
|
}
|
|
653
776
|
}
|
|
654
777
|
}
|
|
@@ -662,38 +785,52 @@ export function extractEmbeddedNodes(body, options) {
|
|
|
662
785
|
if (child.type === 'paragraph') {
|
|
663
786
|
const rawText = mdastToString(child);
|
|
664
787
|
const { cleanText: afterBracketed, fields: bracketedFields } = extractBracketedFields(rawText);
|
|
665
|
-
const {
|
|
788
|
+
const { fields: unbracketedFields } = extractUnbracketedFields(afterBracketed);
|
|
666
789
|
const allFields = applyFieldMap({ ...unbracketedFields, ...bracketedFields }, fieldMap);
|
|
667
790
|
if ('type' in allFields) {
|
|
668
791
|
throw new Error(`Type override via paragraph field is not supported at "${activeNode.schemaData.title ?? activeNode.label}". ` +
|
|
669
792
|
`Put [type:: ${allFields.type}] directly in the heading text.`);
|
|
670
793
|
}
|
|
671
794
|
Object.assign(activeNode.schemaData, allFields);
|
|
672
|
-
|
|
673
|
-
|
|
795
|
+
// Lines holding only fields are consumed as fields; any other line, including prose that mentions a field, is content.
|
|
796
|
+
const lines = sourceLines(body, { start: blockStart, end: blockEnd });
|
|
797
|
+
const fieldLines = lines.filter((l) => isFieldOnlyLine(l.text));
|
|
798
|
+
if (fieldLines.length === lines.length) {
|
|
799
|
+
consumeBlock(activeNode);
|
|
800
|
+
}
|
|
801
|
+
else {
|
|
802
|
+
// Each field line goes with its terminator; a last line without one takes the preceding line break instead.
|
|
803
|
+
claimBlock(activeNode, fieldLines.map((l) => {
|
|
804
|
+
if (/\n$/.test(body.slice(l.start, l.end)))
|
|
805
|
+
return { start: l.start, end: l.end };
|
|
806
|
+
return { start: l.start - (body[l.start - 2] === '\r' ? 2 : 1), end: l.end };
|
|
807
|
+
}));
|
|
808
|
+
}
|
|
674
809
|
activeNode.contentLinks ??= [];
|
|
675
810
|
activeNode.contentLinks.push(...extractLinksFromAstNode(child));
|
|
676
811
|
}
|
|
677
812
|
else if (child.type === 'code' && child.lang?.trim() === 'yaml') {
|
|
678
813
|
const code = child;
|
|
679
|
-
const parsed =
|
|
814
|
+
const parsed = parseYaml(code.value);
|
|
680
815
|
if (parsed && !Array.isArray(parsed) && typeof parsed === 'object') {
|
|
681
|
-
Object.assign(activeNode.schemaData,
|
|
816
|
+
Object.assign(activeNode.schemaData, applyFieldMap(parsed, fieldMap));
|
|
817
|
+
consumeBlock(activeNode);
|
|
682
818
|
}
|
|
683
819
|
else if (Array.isArray(parsed)) {
|
|
684
820
|
throw new Error(`YAML block must be an object at "${activeNode.label}".`);
|
|
685
821
|
}
|
|
686
822
|
else {
|
|
687
|
-
|
|
823
|
+
claimBlock(activeNode);
|
|
688
824
|
}
|
|
689
825
|
}
|
|
690
826
|
else {
|
|
691
|
-
|
|
827
|
+
claimBlock(activeNode);
|
|
692
828
|
activeNode.contentLinks ??= [];
|
|
693
829
|
activeNode.contentLinks.push(...extractLinksFromAstNode(child));
|
|
694
830
|
}
|
|
695
831
|
}
|
|
696
832
|
}
|
|
833
|
+
applyClaims(claims, body);
|
|
697
834
|
const preambleContent = rootNode.schemaData.content;
|
|
698
835
|
const { type: _rootType, content: _rootContent, ...preambleFields } = rootNode.schemaData;
|
|
699
836
|
return {
|