structured-context 0.12.0 → 0.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +29 -216
  2. package/dist/commands/docs.js +1 -0
  3. package/dist/commands/providers.d.ts +4 -0
  4. package/dist/commands/providers.js +52 -0
  5. package/dist/commands/schemas.js +21 -2
  6. package/dist/commands/validate-file.d.ts +3 -0
  7. package/dist/commands/validate-file.js +17 -2
  8. package/dist/commands/validate.d.ts +4 -0
  9. package/dist/commands/validate.js +42 -3
  10. package/dist/config.d.ts +7 -0
  11. package/dist/config.js +23 -0
  12. package/dist/index.js +44 -3
  13. package/dist/judge/context.d.ts +20 -0
  14. package/dist/judge/context.js +105 -0
  15. package/dist/judge/factory.d.ts +44 -0
  16. package/dist/judge/factory.js +148 -0
  17. package/dist/judge/index.d.ts +13 -0
  18. package/dist/judge/index.js +8 -0
  19. package/dist/judge/judge-node.d.ts +27 -0
  20. package/dist/judge/judge-node.js +122 -0
  21. package/dist/judge/prompt.d.ts +10 -0
  22. package/dist/judge/prompt.js +92 -0
  23. package/dist/judge/provider/anthropic.d.ts +17 -0
  24. package/dist/judge/provider/anthropic.js +43 -0
  25. package/dist/judge/provider/fake.d.ts +26 -0
  26. package/dist/judge/provider/fake.js +56 -0
  27. package/dist/judge/provider/jev.d.ts +37 -0
  28. package/dist/judge/provider/jev.js +169 -0
  29. package/dist/judge/provider/openai-decisions.d.ts +21 -0
  30. package/dist/judge/provider/openai-decisions.js +84 -0
  31. package/dist/judge/provider/openai.d.ts +20 -0
  32. package/dist/judge/provider/openai.js +53 -0
  33. package/dist/judge/provider/per-rule.d.ts +10 -0
  34. package/dist/judge/provider/per-rule.js +18 -0
  35. package/dist/judge/trim.d.ts +27 -0
  36. package/dist/judge/trim.js +62 -0
  37. package/dist/judge/types.d.ts +92 -0
  38. package/dist/judge/types.js +9 -0
  39. package/dist/judge/verdicts.d.ts +24 -0
  40. package/dist/judge/verdicts.js +82 -0
  41. package/dist/plugins/markdown/parse-embedded.d.ts +2 -0
  42. package/dist/plugins/markdown/parse-embedded.js +161 -24
  43. package/dist/plugins/markdown/read-space.js +4 -9
  44. package/dist/plugins/markdown/render-directory.js +4 -4
  45. package/dist/plugins/markdown/template-sync.js +3 -4
  46. package/dist/plugins/markdown/util.d.ts +25 -4
  47. package/dist/plugins/markdown/util.js +56 -15
  48. package/dist/render/render.js +1 -1
  49. package/dist/schema/evaluate-rule.d.ts +33 -2
  50. package/dist/schema/evaluate-rule.js +58 -4
  51. package/dist/schema/gating.d.ts +69 -0
  52. package/dist/schema/gating.js +253 -0
  53. package/dist/schema/metadata-contract.d.ts +625 -12
  54. package/dist/schema/metadata-contract.js +110 -4
  55. package/dist/schema/rule-filter.d.ts +12 -0
  56. package/dist/schema/rule-filter.js +46 -0
  57. package/dist/schema/rule-severity.d.ts +13 -0
  58. package/dist/schema/rule-severity.js +33 -0
  59. package/dist/schema/schema.js +31 -1
  60. package/dist/schema/validate-rules.d.ts +17 -8
  61. package/dist/schema/validate-rules.js +91 -48
  62. package/dist/schemas/_knowledge_wiki.json +4 -4
  63. package/dist/schemas/_okf.json +172 -0
  64. package/dist/schemas/_ost_strict.json +15 -6
  65. package/dist/schemas/_sctx_base.json +1 -1
  66. package/dist/schemas/generated/0.13.0/_structured_context_schema_meta.json +400 -0
  67. package/dist/schemas/generated/latest/_structured_context_schema_meta.json +177 -6
  68. package/dist/schemas/okf.json +8 -0
  69. package/dist/types.d.ts +8 -1
  70. package/dist/validate.d.ts +58 -8
  71. package/dist/validate.js +219 -114
  72. package/docs/concepts.md +52 -6
  73. package/docs/config.md +70 -1
  74. package/docs/rules.md +140 -49
  75. package/docs/schemas.md +43 -18
  76. package/package.json +10 -9
  77. package/schemas/_knowledge_wiki.json +4 -4
  78. package/schemas/_okf.json +172 -0
  79. package/schemas/_ost_strict.json +15 -6
  80. package/schemas/_sctx_base.json +1 -1
  81. package/schemas/generated/0.13.0/_structured_context_schema_meta.json +400 -0
  82. package/schemas/generated/latest/_structured_context_schema_meta.json +177 -6
  83. package/schemas/okf.json +8 -0
@@ -0,0 +1,27 @@
1
+ import type { SemanticJudgeInput } from './types';
2
+ /** The per-node input cap: estimated tokens of one node's input (the node and its related nodes), the same for every provider. */
3
+ export declare const PER_NODE_INPUT_CAP_TOKENS = 32000;
4
+ /** Estimated tokens of a value: its JSON length divided by 4. Provider-independent, and roughly right is enough. */
5
+ export declare const estimateTokens: (value: unknown) => number;
6
+ /** The placeholder that ends a body cut to fit the per-node input cap. `removed` is the number of characters cut. */
7
+ export declare const trimMarker: (removed: number) => string;
8
+ /** The part of a judge input that the per-node input cap measures. */
9
+ export type NodeInput = Pick<SemanticJudgeInput, 'node' | 'related'>;
10
+ /**
11
+ * Cut `body` to at most `limit` characters, ending at the last whitespace before the limit, and end it with the
12
+ * trim marker. A body within the limit is returned whole.
13
+ */
14
+ export declare function cutBody(body: string, limit: number): string;
15
+ /**
16
+ * Fit a node's input under the per-node input cap by cutting bodies only: related bodies to one shared ceiling
17
+ * first, then the evaluated node's body. Undefined when the input does not fit even with every body cut.
18
+ */
19
+ export declare function trimToCap(input: NodeInput, cap?: number): {
20
+ input: NodeInput;
21
+ trimmed: boolean;
22
+ } | undefined;
23
+ /** What a run trimmed: the nodes (by label) and the rules whose input had a body cut. */
24
+ export interface TrimReport {
25
+ nodes: Set<string>;
26
+ ruleIds: Set<string>;
27
+ }
@@ -0,0 +1,62 @@
1
+ /** The per-node input cap: estimated tokens of one node's input (the node and its related nodes), the same for every provider. */
2
+ export const PER_NODE_INPUT_CAP_TOKENS = 32_000;
3
+ /** Estimated tokens of a value: its JSON length divided by 4. Provider-independent, and roughly right is enough. */
4
+ export const estimateTokens = (value) => Math.ceil(JSON.stringify(value).length / 4);
5
+ /** The placeholder that ends a body cut to fit the per-node input cap. `removed` is the number of characters cut. */
6
+ export const trimMarker = (removed) => `[RULE_EVALUATION_CONTENT_TRIMMED ${removed} chars]`;
7
+ /**
8
+ * Cut `body` to at most `limit` characters, ending at the last whitespace before the limit, and end it with the
9
+ * trim marker. A body within the limit is returned whole.
10
+ */
11
+ export function cutBody(body, limit) {
12
+ if (body.length <= limit)
13
+ return body;
14
+ let head = body.slice(0, limit);
15
+ if (!/\s/.test(body.charAt(limit))) {
16
+ const lastSpace = head.search(/\s\S*$/);
17
+ head = lastSpace < 0 ? '' : head.slice(0, lastSpace);
18
+ }
19
+ head = head.trimEnd();
20
+ const marker = trimMarker(body.length - head.length);
21
+ return head ? `${head} ${marker}` : marker;
22
+ }
23
+ function limitBodies(input, relatedLimit, nodeLimit) {
24
+ const cut = (data, limit) => typeof data.content === 'string' ? { ...data, content: cutBody(data.content, limit) } : data;
25
+ return {
26
+ node: { ...input.node, schemaData: cut(input.node.schemaData, nodeLimit) },
27
+ ...(input.related && { related: input.related.map((r) => ({ ...r, data: cut(r.data, relatedLimit) })) }),
28
+ };
29
+ }
30
+ const bodyLength = (data) => (typeof data.content === 'string' ? data.content.length : 0);
31
+ /** The largest limit in `[0, max]` for which `fits` holds, or -1 when even 0 does not fit. `fits` shrinks as the limit falls. */
32
+ function largestFitting(max, fits) {
33
+ if (!fits(0))
34
+ return -1;
35
+ let low = 0;
36
+ let high = max;
37
+ while (low < high) {
38
+ const mid = Math.ceil((low + high) / 2);
39
+ if (fits(mid))
40
+ low = mid;
41
+ else
42
+ high = mid - 1;
43
+ }
44
+ return low;
45
+ }
46
+ /**
47
+ * Fit a node's input under the per-node input cap by cutting bodies only: related bodies to one shared ceiling
48
+ * first, then the evaluated node's body. Undefined when the input does not fit even with every body cut.
49
+ */
50
+ export function trimToCap(input, cap = PER_NODE_INPUT_CAP_TOKENS) {
51
+ if (estimateTokens(input) <= cap)
52
+ return { input, trimmed: false };
53
+ const nodeLength = bodyLength(input.node.schemaData);
54
+ const relatedMax = Math.max(0, ...(input.related ?? []).map((r) => bodyLength(r.data)));
55
+ const fitsRelated = (limit) => estimateTokens(limitBodies(input, limit, nodeLength)) <= cap;
56
+ const relatedLimit = largestFitting(relatedMax, fitsRelated);
57
+ if (relatedLimit >= 0)
58
+ return { input: limitBodies(input, relatedLimit, nodeLength), trimmed: true };
59
+ const fitsNode = (limit) => estimateTokens(limitBodies(input, 0, limit)) <= cap;
60
+ const nodeLimit = largestFitting(nodeLength, fitsNode);
61
+ return nodeLimit >= 0 ? { input: limitBodies(input, 0, nodeLimit), trimmed: true } : undefined;
62
+ }
@@ -0,0 +1,92 @@
1
+ import type { RuleCategory, SpaceNode } from '../types';
2
+ /** A semantic rule as presented to a judge: the closed verdict set it must choose from. */
3
+ export interface SemanticRuleSpec {
4
+ id: string;
5
+ description: string;
6
+ rubric?: string;
7
+ category: RuleCategory | string;
8
+ /** Allowed verdicts, each with a description of when it applies. */
9
+ verdicts: Record<string, string>;
10
+ }
11
+ /**
12
+ * A node related to the evaluated node by its rules' `scope`. It carries all the node's schema data; the body
13
+ * (`data.content`) is present only when the rules' `includeContent` asks for it.
14
+ */
15
+ export interface RelatedNode {
16
+ /** How it relates to the evaluated node. A node can have several. */
17
+ relations: Array<'parent' | 'sibling' | 'child' | 'ancestor' | 'descendant'>;
18
+ title: string;
19
+ type: string;
20
+ /** Schema data without `title` and `type`. */
21
+ data: Record<string, unknown>;
22
+ }
23
+ /**
24
+ * One node and the semantic rules to judge on it, which share one context: the same related nodes and the same
25
+ * bodies. Judges may answer all rules in a single call.
26
+ */
27
+ export interface SemanticJudgeInput {
28
+ /** The evaluated node. Its body is in `schemaData.content` only when the rules' `includeContent` asks for it. */
29
+ node: Pick<SpaceNode, 'label' | 'title' | 'type' | 'resolvedType' | 'schemaData' | 'resolvedParents'>;
30
+ /** Nodes the rules' `scope` adds, as a flat list. Absent when the scope is empty. */
31
+ related?: RelatedNode[];
32
+ rules: SemanticRuleSpec[];
33
+ }
34
+ /** A judge's answer for one rule: a verdict from the rule's closed set. */
35
+ export interface SemanticJudgeVerdict {
36
+ verdict: string;
37
+ /** 0-1, when the judge reports one. */
38
+ confidence?: number;
39
+ /** Probability per verdict, when the judge reports a distribution. */
40
+ probabilities?: Record<string, number>;
41
+ /** Free-text reasoning; decision models may not provide it. */
42
+ evidence?: string;
43
+ }
44
+ /** A judge's answer for one rule: a verdict, or an error that applies to that rule only. */
45
+ export type SemanticJudgeResult = SemanticJudgeVerdict | {
46
+ error: string;
47
+ };
48
+ /** Judge interface. Results are keyed by rule id; a missing key is treated as an error. */
49
+ export interface SemanticJudgeProvider {
50
+ readonly name: string;
51
+ evaluate(input: SemanticJudgeInput): Promise<Record<string, SemanticJudgeResult>>;
52
+ /**
53
+ * Optional: judge many nodes in one call, so a provider can fold them into fewer model calls. Returns one
54
+ * result map per input, in order. A failure that affects only some inputs is reported as `{ error }`
55
+ * results for those inputs' rules; throw only when nothing could be judged.
56
+ */
57
+ evaluateNodes?(inputs: SemanticJudgeInput[]): Promise<Array<Record<string, SemanticJudgeResult>>>;
58
+ }
59
+ /** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */
60
+ export declare const MODEL_PROVIDER_APIS: readonly ["openai-completions", "openai-decisions", "anthropic-messages", "openrouter-decisions", "typesafe-systemone", "fake"];
61
+ export type ModelProviderApi = (typeof MODEL_PROVIDER_APIS)[number];
62
+ /** How to reach a model: a bundled provider or config `modelProviders.<name>`, before hydration. */
63
+ export interface ModelProviderConfig {
64
+ /** The wire API to speak. */
65
+ api: ModelProviderApi;
66
+ /** Fixed model override. Beats `modelEnv` and the bundled default. */
67
+ model?: string;
68
+ /** Environment variable that holds the model override. */
69
+ modelEnv?: string;
70
+ /** Fixed endpoint override. Beats `endpointEnv` and the bundled default. */
71
+ endpoint?: string;
72
+ /** Environment variable that holds the endpoint override. */
73
+ endpointEnv?: string;
74
+ /** Prefer `apiKeyEnv`: keys do not belong in config files. */
75
+ apiKey?: string;
76
+ /** Environment variable holding the key. */
77
+ apiKeyEnv?: string;
78
+ }
79
+ /** A provider after hydration: every field is a concrete value; the implementation reads no environment. */
80
+ export interface HydratedProvider {
81
+ api: ModelProviderApi;
82
+ model?: string;
83
+ endpoint?: string;
84
+ apiKey?: string;
85
+ /** The environment variable the key was read from, when one was named or defaulted. */
86
+ keyEnv?: string;
87
+ }
88
+ /** Which model provider evaluates semantic rules: config `semantic`, at the top level or on a space. */
89
+ export interface JudgeConfig {
90
+ /** Name of an entry in `modelProviders`, or a bundled provider. */
91
+ provider?: string;
92
+ }
@@ -0,0 +1,9 @@
1
+ /** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */
2
+ export const MODEL_PROVIDER_APIS = [
3
+ 'openai-completions',
4
+ 'openai-decisions',
5
+ 'anthropic-messages',
6
+ 'openrouter-decisions',
7
+ 'typesafe-systemone',
8
+ 'fake',
9
+ ];
@@ -0,0 +1,24 @@
1
+ import type { Rule, RuleSeverity } from '../types';
2
+ import type { SemanticJudgeResult, SemanticRuleSpec } from './types';
3
+ /** Verdicts for a semantic rule that declares none. */
4
+ export declare const DEFAULT_VERDICTS: Record<string, string>;
5
+ /** A semantic rule resolved for evaluation: what the judge sees, and what each verdict means for the user. */
6
+ export interface ResolvedSemanticRule {
7
+ /** What the judge sees: the closed verdict set. */
8
+ spec: SemanticRuleSpec;
9
+ /** Severity of each verdict that fails the rule. A verdict absent here passes (including `ignore`d ones). */
10
+ severities: Record<string, RuleSeverity>;
11
+ /** User-facing guidance for verdicts that declare it. */
12
+ guidance: Record<string, string>;
13
+ }
14
+ /**
15
+ * Resolve a semantic rule: apply verdict defaults, split each verdict into its judge description and user guidance,
16
+ * and work out which verdicts fail at what severity. Throws on an inconsistent rule.
17
+ */
18
+ export declare function resolveSemanticRule(rule: Rule): ResolvedSemanticRule;
19
+ /** The judge-facing spec for a semantic rule. */
20
+ export declare function semanticRuleSpec(rule: Rule): SemanticRuleSpec;
21
+ /** Whether a judge result is an error rather than a verdict. */
22
+ export declare function isJudgeError(result: SemanticJudgeResult): result is {
23
+ error: string;
24
+ };
@@ -0,0 +1,82 @@
1
+ import { RESERVED_VERDICTS } from '../schema/metadata-contract';
2
+ import { ruleSemantic, ruleSeverity } from '../schema/rule-severity';
3
+ import { ruleContext } from './context';
4
+ /** Verdicts for a semantic rule that declares none. */
5
+ export const DEFAULT_VERDICTS = {
6
+ pass: 'The node meets the rubric.',
7
+ fail: 'The node does not meet the rubric.',
8
+ };
9
+ /** The reserved verdict a rule with related nodes or bodies offers, for input a trim marker shows was cut. */
10
+ const MISSING_CONTEXT_VERDICT = 'missing-context';
11
+ const MISSING_CONTEXT_DESCRIPTION = 'Content needed to judge the rule was cut: a [RULE_EVALUATION_CONTENT_TRIMMED n chars] marker shows where. Choose this only when such a marker is present and the cut content is needed. Never choose it when no marker appears.';
12
+ /**
13
+ * Resolve a semantic rule: apply verdict defaults, split each verdict into its judge description and user guidance,
14
+ * and work out which verdicts fail at what severity. Throws on an inconsistent rule.
15
+ */
16
+ export function resolveSemanticRule(rule) {
17
+ const semantic = ruleSemantic(rule);
18
+ if (!semantic)
19
+ throw new Error(`Rule "${rule.id}" has no semantic part.`);
20
+ const declared = semantic.verdicts ?? DEFAULT_VERDICTS;
21
+ const verdicts = {};
22
+ const guidance = {};
23
+ for (const [name, value] of Object.entries(declared)) {
24
+ if (typeof value === 'string') {
25
+ verdicts[name] = value;
26
+ }
27
+ else {
28
+ verdicts[name] = value.description;
29
+ if (value.guidance)
30
+ guidance[name] = value.guidance;
31
+ }
32
+ }
33
+ let mapping;
34
+ if (semantic.verdicts) {
35
+ if (!semantic.severity) {
36
+ throw new Error(`Semantic rule "${rule.id}" declares verdicts, so semantic.severity must map the failing verdicts, e.g. { "${Object.keys(verdicts)[0]}": "warning" }.`);
37
+ }
38
+ mapping = semantic.severity;
39
+ }
40
+ else {
41
+ mapping = { ...semantic.severity, fail: ruleSeverity(rule) };
42
+ }
43
+ const reserved = RESERVED_VERDICTS;
44
+ const unknown = Object.keys(mapping).filter((v) => !(v in verdicts) && !reserved.includes(v));
45
+ if (unknown.length > 0) {
46
+ throw new Error(`Semantic rule "${rule.id}" has semantic.severity for verdicts not in its verdicts: ${unknown.join(', ')}.`);
47
+ }
48
+ const severities = {};
49
+ for (const [verdict, severity] of Object.entries(mapping)) {
50
+ if (severity !== 'ignore')
51
+ severities[verdict] = severity;
52
+ }
53
+ if (Object.keys(severities).filter((v) => v in verdicts).length >= Object.keys(verdicts).length) {
54
+ throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict has a severity.`);
55
+ }
56
+ // A warning by default.
57
+ const context = ruleContext(rule);
58
+ if (context.scope.length > 0 || context.content !== 'none') {
59
+ verdicts[MISSING_CONTEXT_VERDICT] = MISSING_CONTEXT_DESCRIPTION;
60
+ if (!(MISSING_CONTEXT_VERDICT in mapping))
61
+ severities[MISSING_CONTEXT_VERDICT] = 'warning';
62
+ }
63
+ return {
64
+ spec: {
65
+ id: rule.id,
66
+ description: rule.description,
67
+ rubric: semantic.rubric,
68
+ category: rule.category,
69
+ verdicts,
70
+ },
71
+ severities,
72
+ guidance,
73
+ };
74
+ }
75
+ /** The judge-facing spec for a semantic rule. */
76
+ export function semanticRuleSpec(rule) {
77
+ return resolveSemanticRule(rule).spec;
78
+ }
79
+ /** Whether a judge result is an error rather than a verdict. */
80
+ export function isJudgeError(result) {
81
+ return 'error' in result;
82
+ }
@@ -8,6 +8,8 @@ type StackEntry = {
8
8
  nodeType: string;
9
9
  /** Preferred wikilink key used when this heading acts as a parent. */
10
10
  refTarget: string;
11
+ /** Node whose section this heading opens; absent for untyped placeholders and the page's own entry. */
12
+ node?: BaseNode;
11
13
  };
12
14
  /** Detect a bare wikilink `[[...]]` and return the inner target, or undefined. */
13
15
  export declare function isWikilink(text: string): string | undefined;
@@ -1,10 +1,9 @@
1
- import { load as yamlLoad } from 'js-yaml';
2
1
  import { toString as mdastToString } from 'mdast-util-to-string';
3
2
  import remarkGfm from 'remark-gfm';
4
3
  import remarkParse from 'remark-parse';
5
4
  import { unified } from 'unified';
6
5
  import { extractLinksFromAstNode } from './extract-content-links';
7
- import { applyFieldMap, coerceDates } from './util';
6
+ import { applyFieldMap, parseYaml } from './util';
8
7
  /** Type values that identify a space_on_a_page container (not themselves space nodes). */
9
8
  export const ON_A_PAGE_TYPES = ['ost_on_a_page', 'space_on_a_page'];
10
9
  const DEFAULT_STATUS = 'identified';
@@ -141,18 +140,91 @@ export function defaultNodeType(stack, hierarchy) {
141
140
  }
142
141
  return hierarchy[idx + 1];
143
142
  }
144
- function appendContent(node, text) {
145
- if (!text)
143
+ /** Start and end offsets of an mdast node. Throws when the parser gave no position. */
144
+ function offsetsOf(node) {
145
+ const start = node.position?.start.offset;
146
+ const end = node.position?.end.offset;
147
+ if (start === undefined || end === undefined) {
148
+ throw new Error(`Markdown ${node.type} node has no source position; cannot slice its content`);
149
+ }
150
+ return { start, end };
151
+ }
152
+ function addRange(claims, node, start, end) {
153
+ if (end <= start)
146
154
  return;
147
- const existing = node.schemaData.content;
148
- const updated = existing ? `${existing}\n${text}` : text;
149
- node.schemaData.content = updated;
150
- node.content = updated;
155
+ const ranges = claims.ranges.get(node) ?? [];
156
+ const last = ranges[ranges.length - 1];
157
+ if (last && last.end === start)
158
+ last.end = end;
159
+ else
160
+ ranges.push({ start, end });
161
+ claims.ranges.set(node, ranges);
162
+ }
163
+ /**
164
+ * Give a node a block of source, plus the blank lines between `from` and the block's `start`.
165
+ * Those lines are skipped when a consumed block just before already kept its own.
166
+ */
167
+ function claim(claims, node, from, start, end) {
168
+ addRange(claims, node, claims.gapKept.delete(node) ? start : from, end);
169
+ }
170
+ /**
171
+ * Take a block out of a node's content (it was consumed as a node, field or other syntax).
172
+ * The blank lines before it stay, so the content on either side does not run together.
173
+ */
174
+ function consume(claims, node, from, start) {
175
+ addRange(claims, node, claims.gapKept.delete(node) ? start : from, start);
176
+ claims.gapKept.add(node);
177
+ }
178
+ /** Sets each node's content from its claimed source. Leading blank lines and trailing whitespace are dropped. */
179
+ function applyClaims(claims, body) {
180
+ for (const [node, ranges] of claims.ranges) {
181
+ let content = ranges
182
+ .map((r) => body.slice(r.start, r.end))
183
+ .join('')
184
+ .replace(/^(?:[ \t]*\r?\n)+/, '')
185
+ .trimEnd();
186
+ const indent = claims.contentIndent.get(node);
187
+ if (indent) {
188
+ const strip = new RegExp(`^[ \\t]{1,${indent}}`);
189
+ content = content
190
+ .split(/(?<=\n)/)
191
+ .map((line) => line.replace(strip, ''))
192
+ .join('');
193
+ }
194
+ if (!content)
195
+ continue;
196
+ node.schemaData.content = content;
197
+ node.content = content;
198
+ }
151
199
  }
152
- function processListItem(item, parentRef, contentTarget, nodes, makeLabel, buildLinkTargets, typeAliases, fieldMap, pendingType, parentFieldAppend, activeNodeFieldAppend, file, lineOffset) {
200
+ /** Source lines (with terminators) of a range, as absolute offsets. */
201
+ function sourceLines(body, range) {
202
+ const lines = [];
203
+ const re = /[^\n]*\n|[^\n]+$/g;
204
+ const text = body.slice(range.start, range.end);
205
+ for (const m of text.matchAll(re)) {
206
+ lines.push({
207
+ start: range.start + m.index,
208
+ end: range.start + m.index + m[0].length,
209
+ text: m[0].replace(/\r?\n$/, ''),
210
+ });
211
+ }
212
+ return lines;
213
+ }
214
+ /** Whether a line holds only inline fields: bracketed `[key:: value]` pieces, or one `key:: value`. */
215
+ function isFieldOnlyLine(line) {
216
+ const { cleanText, fields } = extractBracketedFields(line);
217
+ const hasBracketed = Object.keys(fields).length > 0;
218
+ if (cleanText === '')
219
+ return hasBracketed;
220
+ return /^[a-zA-Z][a-zA-Z0-9_-]*:: *.*$/.test(cleanText);
221
+ }
222
+ function processListItem(item, parentRef, contentTarget, nodes, makeLabel, buildLinkTargets, typeAliases, fieldMap, pendingType, parentFieldAppend, activeNodeFieldAppend, file, lineOffset, walk) {
223
+ const itemEnd = offsetsOf(item).end;
153
224
  const firstPara = item.children.find((c) => c.type === 'paragraph');
154
225
  if (!firstPara) {
155
- appendContent(contentTarget, `- ${mdastToString(item)}`);
226
+ claim(walk.claims, contentTarget, walk.pos, offsetsOf(item).start, itemEnd);
227
+ walk.pos = itemEnd;
156
228
  return;
157
229
  }
158
230
  const rawText = mdastToString(firstPara);
@@ -161,11 +233,15 @@ function processListItem(item, parentRef, contentTarget, nodes, makeLabel, build
161
233
  if (wikiTarget) {
162
234
  const linkRef = `[[${wikiTarget}]]`;
163
235
  if (parentFieldAppend) {
236
+ consume(walk.claims, contentTarget, walk.pos, offsetsOf(item).start);
164
237
  appendParentField(parentFieldAppend.node, parentFieldAppend.field, linkRef);
238
+ walk.pos = itemEnd;
165
239
  return;
166
240
  }
167
241
  if (activeNodeFieldAppend) {
242
+ consume(walk.claims, contentTarget, walk.pos, offsetsOf(item).start);
168
243
  appendParentField(activeNodeFieldAppend.node, activeNodeFieldAppend.field, linkRef);
244
+ walk.pos = itemEnd;
169
245
  return;
170
246
  }
171
247
  // No field append context — fall through to node creation or content append
@@ -204,21 +280,34 @@ function processListItem(item, parentRef, contentTarget, nodes, makeLabel, build
204
280
  ...(source ? { source } : {}),
205
281
  };
206
282
  nodes.push(newNode);
283
+ const contentColumn = firstPara.position?.start.column;
284
+ if (contentColumn !== undefined)
285
+ walk.claims.contentIndent.set(newNode, contentColumn - 1);
207
286
  if (parentFieldAppend) {
208
287
  appendParentField(parentFieldAppend.node, parentFieldAppend.field, `[[${linkTargets[0] ?? title}]]`);
209
288
  }
210
289
  const nestedParentRef = `[[${linkTargets[0] ?? title}]]`;
290
+ consume(walk.claims, contentTarget, walk.pos, offsetsOf(item).start);
291
+ walk.pos = offsetsOf(firstPara).end;
211
292
  for (const child of item.children) {
293
+ if (child === firstPara)
294
+ continue;
212
295
  if (child.type === 'list') {
213
296
  for (const subItem of child.children) {
214
- processListItem(subItem, nestedParentRef, newNode, nodes, makeLabel, buildLinkTargets, typeAliases, fieldMap, pendingType, undefined, undefined, file, lineOffset);
297
+ processListItem(subItem, nestedParentRef, newNode, nodes, makeLabel, buildLinkTargets, typeAliases, fieldMap, pendingType, undefined, undefined, file, lineOffset, walk);
215
298
  }
216
299
  }
300
+ else {
301
+ const { start, end } = offsetsOf(child);
302
+ claim(walk.claims, newNode, walk.pos, start, end);
303
+ walk.pos = end;
304
+ }
217
305
  }
218
306
  }
219
307
  else {
220
- appendContent(contentTarget, `- ${rawText}`);
308
+ claim(walk.claims, contentTarget, walk.pos, offsetsOf(item).start, itemEnd);
221
309
  }
310
+ walk.pos = itemEnd;
222
311
  }
223
312
  /**
224
313
  * Extract space nodes from markdown body text.
@@ -418,7 +507,32 @@ export function extractEmbeddedNodes(body, options) {
418
507
  }
419
508
  let grouping = null;
420
509
  let activeNode = rootNode;
510
+ const claims = { ranges: new Map(), gapKept: new Set(), contentIndent: new Map() };
511
+ /** End of the previous top-level block; a block's span starts here so the blank lines before it stay with it. */
512
+ let prevEnd = 0;
421
513
  for (const child of tree.children) {
514
+ const { start: blockStart, end: blockEnd } = offsetsOf(child);
515
+ const spanStart = prevEnd;
516
+ prevEnd = blockEnd;
517
+ /** Give a node this block as source content, minus any `removed` ranges inside it. */
518
+ const claimBlock = (node, removed = []) => {
519
+ let from = blockStart;
520
+ let first = true;
521
+ for (const gap of removed) {
522
+ if (first)
523
+ claim(claims, node, spanStart, blockStart, gap.start);
524
+ else
525
+ addRange(claims, node, from, gap.start);
526
+ first = false;
527
+ from = gap.end;
528
+ }
529
+ if (first)
530
+ claim(claims, node, spanStart, blockStart, blockEnd);
531
+ else
532
+ addRange(claims, node, from, blockEnd);
533
+ };
534
+ /** Take this whole block out of a node's content, keeping the blank lines before it. */
535
+ const consumeBlock = (node) => consume(claims, node, spanStart, blockStart);
422
536
  if (parseState === 'done') {
423
537
  if (child.type === 'heading') {
424
538
  const rawTitle = mdastToString(child);
@@ -436,8 +550,11 @@ export function extractEmbeddedNodes(body, options) {
436
550
  if (child.type === 'heading') {
437
551
  const heading = child;
438
552
  const depth = heading.depth;
439
- if (depth > 5)
553
+ if (depth > 5) {
554
+ if (parseState === 'active')
555
+ claimBlock(activeNode);
440
556
  continue;
557
+ }
441
558
  parseState = 'active';
442
559
  const rawText = mdastToString(heading);
443
560
  const { cleanText: afterBracketed, fields: rawInlineFields } = extractBracketedFields(rawText);
@@ -455,6 +572,10 @@ export function extractEmbeddedNodes(body, options) {
455
572
  stack.push({ depth, title, nodeType: '', refTarget: title });
456
573
  // Discard any pending grouping (untyped heading has no implied type)
457
574
  grouping = null;
575
+ // An untyped heading is part of the body of the nearest enclosing node. Popping above removed every
576
+ // node at the same or a higher level, so those sections end here; deeper nodes' sections continue.
577
+ activeNode = stack.findLast((entry) => entry.node)?.node ?? rootNode;
578
+ claimBlock(activeNode);
458
579
  continue;
459
580
  }
460
581
  if (isOnAPageMode && stack.length > 0) {
@@ -494,7 +615,7 @@ export function extractEmbeddedNodes(body, options) {
494
615
  };
495
616
  // Push to stack BEFORE resolving semantic parent — stack[-2] is the correct parent.
496
617
  const refTarget = linkTargets[0] ?? title;
497
- stack.push({ depth, title, nodeType: type, refTarget });
618
+ stack.push({ depth, title, nodeType: type, refTarget, node: headingNode });
498
619
  // If this match came from a relationship/hierarchy and no explicit type was given,
499
620
  // create a grouping — delay adding the node until we see the following content.
500
621
  // Discard any previous grouping (not flushed — original agnostic-parse behaviour).
@@ -532,14 +653,16 @@ export function extractEmbeddedNodes(body, options) {
532
653
  const activeNodeFieldAppendArg = isParentLevelMatch && semanticParent.node
533
654
  ? { node: semanticParent.node, field: definition.field }
534
655
  : undefined;
656
+ const walk = { claims, pos: spanStart };
535
657
  for (const item of list.children) {
536
- processListItem(item, isParentSide ? undefined : semanticParent.ref, grouping.headingNode, nodes, makeLabel, buildListItemLinkTargets, typeAliases, fieldMap, definition.type, parentFieldAppendArg, activeNodeFieldAppendArg, options.file, options.lineOffset);
658
+ processListItem(item, isParentSide ? undefined : semanticParent.ref, grouping.headingNode, nodes, makeLabel, buildListItemLinkTargets, typeAliases, fieldMap, definition.type, parentFieldAppendArg, activeNodeFieldAppendArg, options.file, options.lineOffset, walk);
537
659
  }
538
660
  grouping = null;
539
661
  }
540
662
  else {
663
+ const walk = { claims, pos: spanStart };
541
664
  for (const item of list.children) {
542
- processListItem(item, parentRef, activeNode, nodes, makeLabel, buildListItemLinkTargets, typeAliases, fieldMap, undefined, undefined, undefined, options.file, options.lineOffset);
665
+ processListItem(item, parentRef, activeNode, nodes, makeLabel, buildListItemLinkTargets, typeAliases, fieldMap, undefined, undefined, undefined, options.file, options.lineOffset, walk);
543
666
  }
544
667
  }
545
668
  }
@@ -648,7 +771,7 @@ export function extractEmbeddedNodes(body, options) {
648
771
  grouping = null;
649
772
  }
650
773
  else {
651
- appendContent(activeNode, mdastToString(child));
774
+ claimBlock(activeNode);
652
775
  }
653
776
  }
654
777
  }
@@ -662,38 +785,52 @@ export function extractEmbeddedNodes(body, options) {
662
785
  if (child.type === 'paragraph') {
663
786
  const rawText = mdastToString(child);
664
787
  const { cleanText: afterBracketed, fields: bracketedFields } = extractBracketedFields(rawText);
665
- const { remainingText, fields: unbracketedFields } = extractUnbracketedFields(afterBracketed);
788
+ const { fields: unbracketedFields } = extractUnbracketedFields(afterBracketed);
666
789
  const allFields = applyFieldMap({ ...unbracketedFields, ...bracketedFields }, fieldMap);
667
790
  if ('type' in allFields) {
668
791
  throw new Error(`Type override via paragraph field is not supported at "${activeNode.schemaData.title ?? activeNode.label}". ` +
669
792
  `Put [type:: ${allFields.type}] directly in the heading text.`);
670
793
  }
671
794
  Object.assign(activeNode.schemaData, allFields);
672
- if (remainingText)
673
- appendContent(activeNode, remainingText);
795
+ // Lines holding only fields are consumed as fields; any other line, including prose that mentions a field, is content.
796
+ const lines = sourceLines(body, { start: blockStart, end: blockEnd });
797
+ const fieldLines = lines.filter((l) => isFieldOnlyLine(l.text));
798
+ if (fieldLines.length === lines.length) {
799
+ consumeBlock(activeNode);
800
+ }
801
+ else {
802
+ // Each field line goes with its terminator; a last line without one takes the preceding line break instead.
803
+ claimBlock(activeNode, fieldLines.map((l) => {
804
+ if (/\n$/.test(body.slice(l.start, l.end)))
805
+ return { start: l.start, end: l.end };
806
+ return { start: l.start - (body[l.start - 2] === '\r' ? 2 : 1), end: l.end };
807
+ }));
808
+ }
674
809
  activeNode.contentLinks ??= [];
675
810
  activeNode.contentLinks.push(...extractLinksFromAstNode(child));
676
811
  }
677
812
  else if (child.type === 'code' && child.lang?.trim() === 'yaml') {
678
813
  const code = child;
679
- const parsed = yamlLoad(code.value);
814
+ const parsed = parseYaml(code.value);
680
815
  if (parsed && !Array.isArray(parsed) && typeof parsed === 'object') {
681
- Object.assign(activeNode.schemaData, coerceDates(applyFieldMap(parsed, fieldMap)));
816
+ Object.assign(activeNode.schemaData, applyFieldMap(parsed, fieldMap));
817
+ consumeBlock(activeNode);
682
818
  }
683
819
  else if (Array.isArray(parsed)) {
684
820
  throw new Error(`YAML block must be an object at "${activeNode.label}".`);
685
821
  }
686
822
  else {
687
- appendContent(activeNode, code.value);
823
+ claimBlock(activeNode);
688
824
  }
689
825
  }
690
826
  else {
691
- appendContent(activeNode, mdastToString(child));
827
+ claimBlock(activeNode);
692
828
  activeNode.contentLinks ??= [];
693
829
  activeNode.contentLinks.push(...extractLinksFromAstNode(child));
694
830
  }
695
831
  }
696
832
  }
833
+ applyClaims(claims, body);
697
834
  const preambleContent = rootNode.schemaData.content;
698
835
  const { type: _rootType, content: _rootContent, ...preambleFields } = rootNode.schemaData;
699
836
  return {