vocs 2.0.13 → 2.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/internal/mcp-transport.d.ts +1 -1
- package/dist/internal/mcp-transport.d.ts.map +1 -1
- package/dist/internal/mcp-transport.js +20 -24
- package/dist/internal/mcp-transport.js.map +1 -1
- package/dist/internal/mdx.d.ts +6 -0
- package/dist/internal/mdx.d.ts.map +1 -1
- package/dist/internal/mdx.js +61 -18
- package/dist/internal/mdx.js.map +1 -1
- package/dist/internal/search.d.ts +1 -1
- package/dist/internal/search.d.ts.map +1 -1
- package/dist/internal/search.js +12 -5
- package/dist/internal/search.js.map +1 -1
- package/dist/waku/internal/middleware/md-router.d.ts.map +1 -1
- package/dist/waku/internal/middleware/md-router.js +10 -1
- package/dist/waku/internal/middleware/md-router.js.map +1 -1
- package/dist/waku/internal/patches/adapters/node.d.ts.map +1 -1
- package/dist/waku/internal/patches/adapters/node.js +9 -6
- package/dist/waku/internal/patches/adapters/node.js.map +1 -1
- package/dist/waku/internal/vite-plugins.d.ts +7 -5
- package/dist/waku/internal/vite-plugins.d.ts.map +1 -1
- package/dist/waku/internal/vite-plugins.js +20 -108
- package/dist/waku/internal/vite-plugins.js.map +1 -1
- package/dist/waku/vite.js +1 -1
- package/dist/waku/vite.js.map +1 -1
- package/package.json +2 -2
- package/src/internal/mcp-transport.test.ts +132 -0
- package/src/internal/mcp-transport.ts +20 -33
- package/src/internal/mdx.test.ts +83 -0
- package/src/internal/mdx.ts +72 -23
- package/src/internal/search.test.ts +40 -0
- package/src/internal/search.ts +12 -5
- package/src/waku/internal/middleware/md-router.test.ts +85 -0
- package/src/waku/internal/middleware/md-router.ts +11 -1
- package/src/waku/internal/patches/adapters/node.ts +23 -9
- package/src/waku/internal/vite-plugins.ts +21 -118
- package/src/waku/vite.ts +1 -1
- package/src/waku/internal/vite-plugins.test.ts +0 -118
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto'
|
|
2
|
-
import type
|
|
2
|
+
import { isJSONRPCRequest, type JSONRPCMessage } from '@modelcontextprotocol/sdk/types.js'
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* SSE Server Transport for Web APIs (Request/Response).
|
|
@@ -67,31 +67,20 @@ export class WebSSEServerTransport {
|
|
|
67
67
|
* Returns the response to send back.
|
|
68
68
|
*/
|
|
69
69
|
async handlePostMessage(body: unknown): Promise<Response> {
|
|
70
|
-
|
|
71
|
-
if (!this._writer) {
|
|
72
|
-
console.log('[MCP:SSE] No writer - SSE connection not established')
|
|
73
|
-
return new Response('SSE connection not established', { status: 500 })
|
|
74
|
-
}
|
|
70
|
+
if (!this._writer) return new Response('SSE connection not established', { status: 500 })
|
|
75
71
|
|
|
76
72
|
try {
|
|
77
|
-
|
|
78
|
-
this.onmessage?.(message)
|
|
79
|
-
console.log('[MCP:SSE] Message dispatched, returning 202')
|
|
73
|
+
this.onmessage?.(body as JSONRPCMessage)
|
|
80
74
|
return new Response('Accepted', { status: 202 })
|
|
81
75
|
} catch (error) {
|
|
82
|
-
console.error('[MCP:SSE] Error handling message:', error)
|
|
83
76
|
this.onerror?.(error as Error)
|
|
84
77
|
return new Response(`Invalid message: ${String(error)}`, { status: 400 })
|
|
85
78
|
}
|
|
86
79
|
}
|
|
87
80
|
|
|
88
81
|
async send(message: JSONRPCMessage): Promise<void> {
|
|
89
|
-
|
|
90
|
-
if (!this._writer) {
|
|
91
|
-
throw new Error('Not connected')
|
|
92
|
-
}
|
|
82
|
+
if (!this._writer) throw new Error('Not connected')
|
|
93
83
|
await this._write(`event: message\ndata: ${JSON.stringify(message)}\n\n`)
|
|
94
|
-
console.log('[MCP:SSE] Message sent to SSE stream')
|
|
95
84
|
}
|
|
96
85
|
|
|
97
86
|
async close(): Promise<void> {
|
|
@@ -136,33 +125,37 @@ export class WebStreamableHTTPServerTransport {
|
|
|
136
125
|
* Handle a POST request. Returns a Response (either SSE stream or JSON).
|
|
137
126
|
*/
|
|
138
127
|
async handleRequest(body: unknown, useSSE = true): Promise<Response> {
|
|
139
|
-
|
|
140
|
-
const message = body as JSONRPCMessage
|
|
128
|
+
const messages = Array.isArray(body) ? (body as JSONRPCMessage[]) : [body as JSONRPCMessage]
|
|
141
129
|
|
|
142
|
-
|
|
130
|
+
const isInitialize = messages.some(
|
|
131
|
+
(message) => 'method' in message && message.method === 'initialize',
|
|
132
|
+
)
|
|
133
|
+
if (!this.sessionId && isInitialize) {
|
|
143
134
|
this.sessionId = this._options.sessionIdGenerator?.() ?? randomUUID()
|
|
144
|
-
console.log('[MCP:HTTP] Session initialized:', this.sessionId)
|
|
145
135
|
this._options.onsessioninitialized?.(this.sessionId)
|
|
146
136
|
}
|
|
147
137
|
|
|
138
|
+
// Notifications/responses get no JSON-RPC reply -- ack with 202 instead of hanging.
|
|
139
|
+
if (!messages.some(isJSONRPCRequest)) {
|
|
140
|
+
for (const message of messages) this.onmessage?.(message)
|
|
141
|
+
return new Response(null, {
|
|
142
|
+
status: 202,
|
|
143
|
+
headers: this.sessionId ? { 'mcp-session-id': this.sessionId } : {},
|
|
144
|
+
})
|
|
145
|
+
}
|
|
146
|
+
|
|
148
147
|
if (useSSE) {
|
|
149
|
-
console.log('[MCP:HTTP] Using SSE response mode')
|
|
150
148
|
const { readable, writable } = new TransformStream<Uint8Array, Uint8Array>()
|
|
151
149
|
this._responseWriter = writable.getWriter()
|
|
152
150
|
|
|
153
151
|
this._pendingResponse = async (response) => {
|
|
154
|
-
console.log(
|
|
155
|
-
'[MCP:HTTP] SSE pending response received',
|
|
156
|
-
JSON.stringify(response).slice(0, 200),
|
|
157
|
-
)
|
|
158
152
|
await this._write(`event: message\ndata: ${JSON.stringify(response)}\n\n`)
|
|
159
153
|
try {
|
|
160
154
|
await this._responseWriter?.close()
|
|
161
155
|
} catch {}
|
|
162
156
|
}
|
|
163
157
|
|
|
164
|
-
this.onmessage?.(message)
|
|
165
|
-
console.log('[MCP:HTTP] Message dispatched, returning SSE stream')
|
|
158
|
+
for (const message of messages) this.onmessage?.(message)
|
|
166
159
|
|
|
167
160
|
return new Response(readable, {
|
|
168
161
|
headers: {
|
|
@@ -173,21 +166,15 @@ export class WebStreamableHTTPServerTransport {
|
|
|
173
166
|
})
|
|
174
167
|
}
|
|
175
168
|
|
|
176
|
-
console.log('[MCP:HTTP] Using JSON response mode')
|
|
177
169
|
return new Promise((resolve) => {
|
|
178
170
|
this._pendingResponse = (response) => {
|
|
179
|
-
console.log(
|
|
180
|
-
'[MCP:HTTP] JSON pending response received',
|
|
181
|
-
JSON.stringify(response).slice(0, 200),
|
|
182
|
-
)
|
|
183
171
|
resolve(
|
|
184
172
|
Response.json(response, {
|
|
185
173
|
headers: this.sessionId ? { 'mcp-session-id': this.sessionId } : {},
|
|
186
174
|
}),
|
|
187
175
|
)
|
|
188
176
|
}
|
|
189
|
-
this.onmessage?.(message)
|
|
190
|
-
console.log('[MCP:HTTP] Message dispatched, waiting for response')
|
|
177
|
+
for (const message of messages) this.onmessage?.(message)
|
|
191
178
|
})
|
|
192
179
|
}
|
|
193
180
|
|
package/src/internal/mdx.test.ts
CHANGED
|
@@ -1,13 +1,19 @@
|
|
|
1
|
+
import type * as MdAst from 'mdast'
|
|
2
|
+
import remarkFrontmatter from 'remark-frontmatter'
|
|
3
|
+
import remarkParse from 'remark-parse'
|
|
1
4
|
import ruby from 'shiki/langs/ruby.mjs'
|
|
5
|
+
import { unified } from 'unified'
|
|
2
6
|
import { describe, expect, it } from 'vitest'
|
|
3
7
|
import * as Config from './config.js'
|
|
4
8
|
import {
|
|
5
9
|
getCompileOptions,
|
|
6
10
|
remarkCodeTitle,
|
|
11
|
+
remarkDefaultFrontmatter,
|
|
7
12
|
remarkFilename,
|
|
8
13
|
remarkFileTree,
|
|
9
14
|
remarkLangCommaAttrs,
|
|
10
15
|
remarkRestoreUnknownTextDirectives,
|
|
16
|
+
remarkSubheading,
|
|
11
17
|
} from './mdx.js'
|
|
12
18
|
|
|
13
19
|
type CodeNode = {
|
|
@@ -62,6 +68,72 @@ function transformCodeNode(
|
|
|
62
68
|
return tree.children[0] as CodeNode
|
|
63
69
|
}
|
|
64
70
|
|
|
71
|
+
describe('remarkDefaultFrontmatter', () => {
|
|
72
|
+
it('infers title and description from heading subtext with inline code', async () => {
|
|
73
|
+
const tree = await runRemark(
|
|
74
|
+
'# Common Package [How `packages/common` is structured, why it exists]',
|
|
75
|
+
[remarkDefaultFrontmatter],
|
|
76
|
+
)
|
|
77
|
+
const frontmatter = tree.children[0]
|
|
78
|
+
|
|
79
|
+
expect(frontmatter).toMatchObject({
|
|
80
|
+
type: 'yaml',
|
|
81
|
+
value:
|
|
82
|
+
'title: "Common Package"\ndescription: "How packages/common is structured, why it exists"',
|
|
83
|
+
})
|
|
84
|
+
})
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
describe('remarkSubheading', () => {
|
|
88
|
+
it('extracts heading subtext with inline markdown nodes', async () => {
|
|
89
|
+
const tree = await runRemark(
|
|
90
|
+
[
|
|
91
|
+
'---',
|
|
92
|
+
'title: Test',
|
|
93
|
+
'---',
|
|
94
|
+
'',
|
|
95
|
+
'# **Common** Package [How `packages/common` is *structured*]',
|
|
96
|
+
].join('\n'),
|
|
97
|
+
[remarkFrontmatter, remarkSubheading],
|
|
98
|
+
)
|
|
99
|
+
const hgroup = tree.children[1] as MdAst.Paragraph
|
|
100
|
+
const heading = hgroup.children[0] as unknown as MdAst.Heading
|
|
101
|
+
const subheading = hgroup.children[1] as unknown as MdAst.Paragraph
|
|
102
|
+
|
|
103
|
+
expect(stripPositions(heading.children)).toEqual([
|
|
104
|
+
{
|
|
105
|
+
type: 'strong',
|
|
106
|
+
children: [{ type: 'text', value: 'Common' }],
|
|
107
|
+
},
|
|
108
|
+
{ type: 'text', value: ' Package' },
|
|
109
|
+
])
|
|
110
|
+
expect(stripPositions(subheading.children)).toEqual([
|
|
111
|
+
{ type: 'text', value: 'How ' },
|
|
112
|
+
{ type: 'inlineCode', value: 'packages/common' },
|
|
113
|
+
{ type: 'text', value: ' is ' },
|
|
114
|
+
{
|
|
115
|
+
type: 'emphasis',
|
|
116
|
+
children: [{ type: 'text', value: 'structured' }],
|
|
117
|
+
},
|
|
118
|
+
])
|
|
119
|
+
})
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
async function runRemark(markdown: string, plugins: unknown[]) {
|
|
123
|
+
const processor = unified().use(remarkParse)
|
|
124
|
+
for (const plugin of plugins) processor.use(plugin as never)
|
|
125
|
+
const tree = processor.parse(markdown) as MdAst.Root
|
|
126
|
+
await processor.run(tree)
|
|
127
|
+
return tree
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function stripPositions(children: MdAst.PhrasingContent[]): unknown[] {
|
|
131
|
+
return children.map(({ position: _, ...child }) => {
|
|
132
|
+
if ('children' in child) return { ...child, children: stripPositions(child.children) }
|
|
133
|
+
return child
|
|
134
|
+
})
|
|
135
|
+
}
|
|
136
|
+
|
|
65
137
|
describe('remarkFilename', () => {
|
|
66
138
|
it('scopes duplicate code-group filenames to their group', () => {
|
|
67
139
|
const firstExample = {
|
|
@@ -223,6 +295,17 @@ describe('getCompileOptions', () => {
|
|
|
223
295
|
expect(codeNode.lang).toBe('ts')
|
|
224
296
|
expect(codeNode.meta).toBe('[example.ts]')
|
|
225
297
|
})
|
|
298
|
+
|
|
299
|
+
it('threads user-configured remark plugins into the txt profile', () => {
|
|
300
|
+
function userRemarkPlugin() {}
|
|
301
|
+
const config = Config.define({
|
|
302
|
+
markdown: { remarkPlugins: [userRemarkPlugin] },
|
|
303
|
+
rootDir: process.cwd(),
|
|
304
|
+
})
|
|
305
|
+
|
|
306
|
+
expect(getCompileOptions('txt', config).remarkPlugins).toContain(userRemarkPlugin)
|
|
307
|
+
expect(getCompileOptions('react', config).remarkPlugins).toContain(userRemarkPlugin)
|
|
308
|
+
})
|
|
226
309
|
})
|
|
227
310
|
|
|
228
311
|
describe('remarkRestoreUnknownTextDirectives', () => {
|
package/src/internal/mdx.ts
CHANGED
|
@@ -213,6 +213,9 @@ export function getCompileOptions(
|
|
|
213
213
|
remarkStripFrontmatter,
|
|
214
214
|
remarkStripJs,
|
|
215
215
|
remarkStripInlineCache,
|
|
216
|
+
// User plugins extend the parser (e.g. `remark-math`) so syntax
|
|
217
|
+
// recognized in the React build also parses for llms/search.
|
|
218
|
+
...(markdown?.remarkPlugins ?? []),
|
|
216
219
|
],
|
|
217
220
|
}
|
|
218
221
|
if (type === 'react')
|
|
@@ -809,17 +812,15 @@ export function remarkDefaultFrontmatter() {
|
|
|
809
812
|
const heading = (tree.children.find(
|
|
810
813
|
(node) => node.type === 'heading' && (node as { depth: number }).depth === 1,
|
|
811
814
|
) ?? tree.children.find((node) => node.type === 'heading')) as
|
|
812
|
-
| { type: 'heading'; children:
|
|
815
|
+
| { type: 'heading'; children: MdAst.PhrasingContent[] }
|
|
813
816
|
| undefined
|
|
814
817
|
if (!heading) return
|
|
815
818
|
|
|
816
|
-
|
|
817
|
-
const
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
const title = match?.[1]?.trim() ?? textContent.trim()
|
|
822
|
-
const description = match?.[2]?.trim()
|
|
819
|
+
const subheading = extractSubheading(heading.children)
|
|
820
|
+
const title = getPhrasingContentText(subheading?.headingChildren ?? heading.children).trim()
|
|
821
|
+
const description = subheading
|
|
822
|
+
? getPhrasingContentText(subheading.subheadingChildren).trim()
|
|
823
|
+
: undefined
|
|
823
824
|
|
|
824
825
|
// Build new frontmatter
|
|
825
826
|
const newLines: string[] = []
|
|
@@ -1035,25 +1036,15 @@ export function remarkStripFrontmatter() {
|
|
|
1035
1036
|
* Converts `# Title [Subheading text]` into a `<header>` with both title and subtitle.
|
|
1036
1037
|
*/
|
|
1037
1038
|
export function remarkSubheading() {
|
|
1038
|
-
const subheadingRegex = / \[(.*)\]$/
|
|
1039
|
-
|
|
1040
1039
|
return (tree: MdAst.Root) => {
|
|
1041
1040
|
UnistUtil.visit(tree, 'heading', (node, index, parent) => {
|
|
1042
1041
|
if (index === undefined || !parent) return
|
|
1043
1042
|
if (node.depth !== 1) return
|
|
1044
1043
|
if (node.children.length === 0) return
|
|
1045
1044
|
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
)
|
|
1050
|
-
if (!textChild) return
|
|
1051
|
-
|
|
1052
|
-
// Extract and remove subheading from text
|
|
1053
|
-
const match = textChild.value.match(subheadingRegex)
|
|
1054
|
-
if (!match) return
|
|
1055
|
-
const subheading = match[1]
|
|
1056
|
-
textChild.value = textChild.value.replace(match[0], '')
|
|
1045
|
+
const subheading = extractSubheading(node.children)
|
|
1046
|
+
if (!subheading) return
|
|
1047
|
+
node.children = subheading.headingChildren
|
|
1057
1048
|
|
|
1058
1049
|
// Build hgroup wrapper with h1 and optional subtitle (p)
|
|
1059
1050
|
const hgroup = {
|
|
@@ -1061,12 +1052,12 @@ export function remarkSubheading() {
|
|
|
1061
1052
|
data: { hName: 'hgroup' },
|
|
1062
1053
|
children: [
|
|
1063
1054
|
node as unknown as MdAst.PhrasingContent,
|
|
1064
|
-
...(subheading
|
|
1055
|
+
...(subheading.subheadingChildren.length > 0
|
|
1065
1056
|
? [
|
|
1066
1057
|
{
|
|
1067
1058
|
type: 'paragraph',
|
|
1068
1059
|
data: { hName: 'p' },
|
|
1069
|
-
children:
|
|
1060
|
+
children: subheading.subheadingChildren,
|
|
1070
1061
|
} as unknown as MdAst.PhrasingContent,
|
|
1071
1062
|
]
|
|
1072
1063
|
: []),
|
|
@@ -1080,6 +1071,64 @@ export function remarkSubheading() {
|
|
|
1080
1071
|
}
|
|
1081
1072
|
}
|
|
1082
1073
|
|
|
1074
|
+
type Subheading = {
|
|
1075
|
+
headingChildren: MdAst.PhrasingContent[]
|
|
1076
|
+
subheadingChildren: MdAst.PhrasingContent[]
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
export function extractSubheading(children: MdAst.PhrasingContent[]): Subheading | undefined {
|
|
1080
|
+
const lastChild = children[children.length - 1]
|
|
1081
|
+
if (!isText(lastChild) || !lastChild.value.endsWith(']')) return
|
|
1082
|
+
|
|
1083
|
+
for (let index = children.length - 1; index >= 0; index--) {
|
|
1084
|
+
const child = children[index]
|
|
1085
|
+
if (!isText(child)) continue
|
|
1086
|
+
|
|
1087
|
+
const openIndex = child.value.lastIndexOf(' [')
|
|
1088
|
+
if (openIndex === -1) continue
|
|
1089
|
+
|
|
1090
|
+
const headingChildren = clonePhrasingContent(children.slice(0, index))
|
|
1091
|
+
const headingText = child.value.slice(0, openIndex)
|
|
1092
|
+
if (headingText) headingChildren.push({ ...child, value: headingText })
|
|
1093
|
+
|
|
1094
|
+
const subheadingChildren = clonePhrasingContent(children.slice(index + 1))
|
|
1095
|
+
const subheadingText = child.value.slice(openIndex + 2)
|
|
1096
|
+
if (subheadingText) subheadingChildren.unshift({ ...child, value: subheadingText })
|
|
1097
|
+
|
|
1098
|
+
const finalChild = subheadingChildren[subheadingChildren.length - 1]
|
|
1099
|
+
if (!isText(finalChild)) return
|
|
1100
|
+
finalChild.value = finalChild.value.slice(0, -1)
|
|
1101
|
+
|
|
1102
|
+
return {
|
|
1103
|
+
headingChildren,
|
|
1104
|
+
subheadingChildren: subheadingChildren.filter(
|
|
1105
|
+
(child) => !isText(child) || child.value.length > 0,
|
|
1106
|
+
),
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
return undefined
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
export function getPhrasingContentText(children: MdAst.PhrasingContent[]): string {
|
|
1114
|
+
return children.map(getPhrasingNodeText).join('')
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
function clonePhrasingContent(children: MdAst.PhrasingContent[]) {
|
|
1118
|
+
return children.map((child) => structuredClone(child))
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
function getPhrasingNodeText(child: MdAst.PhrasingContent): string {
|
|
1122
|
+
if ('value' in child && typeof child.value === 'string') return child.value
|
|
1123
|
+
if ('children' in child) return getPhrasingContentText(child.children)
|
|
1124
|
+
if ('alt' in child && typeof child.alt === 'string') return child.alt
|
|
1125
|
+
return ''
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
function isText(child: MdAst.PhrasingContent | undefined): child is MdAst.Text {
|
|
1129
|
+
return child?.type === 'text'
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1083
1132
|
const filenameRegex = /filename=["']([^"']+)["']/
|
|
1084
1133
|
const titleFilenameRegex = /\[([./\w-]+\.(?:[cm]?[jt]sx?|json|sol|rs|toml|ya?ml|css|html|mdx?))\]/
|
|
1085
1134
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type * as MdAst from 'mdast'
|
|
1
2
|
import { describe, expect, it } from 'vitest'
|
|
2
3
|
import * as Config from './config.js'
|
|
3
4
|
import * as Search from './search.js'
|
|
@@ -746,6 +747,26 @@ Some content.
|
|
|
746
747
|
`)
|
|
747
748
|
})
|
|
748
749
|
|
|
750
|
+
it('handles title descriptions with inline code', () => {
|
|
751
|
+
const content = `
|
|
752
|
+
# Common Package [How \`packages/common\` is structured, why it exists]
|
|
753
|
+
|
|
754
|
+
Some content.
|
|
755
|
+
`
|
|
756
|
+
expect(Search.extract(content, config).sections).toMatchInlineSnapshot(`
|
|
757
|
+
[
|
|
758
|
+
{
|
|
759
|
+
"anchor": "common-package",
|
|
760
|
+
"isPage": true,
|
|
761
|
+
"subtitle": "How packages/common is structured, why it exists",
|
|
762
|
+
"text": " Some content.",
|
|
763
|
+
"title": "Common Package",
|
|
764
|
+
"titles": [],
|
|
765
|
+
},
|
|
766
|
+
]
|
|
767
|
+
`)
|
|
768
|
+
})
|
|
769
|
+
|
|
749
770
|
it('includes JSX text content', () => {
|
|
750
771
|
const content = `
|
|
751
772
|
# Title
|
|
@@ -817,6 +838,25 @@ Some content here.
|
|
|
817
838
|
]
|
|
818
839
|
`)
|
|
819
840
|
})
|
|
841
|
+
|
|
842
|
+
it('applies user-configured remark plugins from config', () => {
|
|
843
|
+
function remarkUppercaseHeadings() {
|
|
844
|
+
return (tree: MdAst.Root) => {
|
|
845
|
+
for (const node of tree.children) {
|
|
846
|
+
if (node.type !== 'heading') continue
|
|
847
|
+
for (const child of node.children)
|
|
848
|
+
if (child.type === 'text') child.value = child.value.toUpperCase()
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
const customConfig = Config.define({
|
|
854
|
+
markdown: { remarkPlugins: [remarkUppercaseHeadings] },
|
|
855
|
+
})
|
|
856
|
+
|
|
857
|
+
const { sections } = Search.extract('# hello world\n\nSome text.', customConfig)
|
|
858
|
+
expect(sections[0]?.title).toBe('HELLO WORLD')
|
|
859
|
+
})
|
|
820
860
|
})
|
|
821
861
|
|
|
822
862
|
describe('Search fields and storeFields', () => {
|
package/src/internal/search.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { unified } from 'unified'
|
|
|
14
14
|
import * as UnistUtil from 'unist-util-visit'
|
|
15
15
|
import * as yaml from 'yaml'
|
|
16
16
|
import type * as Config from './config.js'
|
|
17
|
+
import { extractSubheading, getPhrasingContentText } from './mdx.js'
|
|
17
18
|
import * as Path from './path.js'
|
|
18
19
|
import { searchFields, storeFields, tokenize } from './search.client.js'
|
|
19
20
|
import * as Sidebar from './sidebar.js'
|
|
@@ -310,13 +311,16 @@ type Section = {
|
|
|
310
311
|
* - `content`: raw markdown (sliced from source) for rendering in search UI
|
|
311
312
|
* - `text`: stripped plain text (directives/JSX removed) for search indexing
|
|
312
313
|
*/
|
|
313
|
-
export function extract(source: string,
|
|
314
|
+
export function extract(source: string, config: Config.Config): extract.ReturnType {
|
|
314
315
|
const remarkPlugins = [
|
|
315
316
|
remarkFrontmatter,
|
|
316
317
|
remarkDirective,
|
|
317
318
|
remarkGfm,
|
|
318
319
|
remarkStripJsx,
|
|
319
320
|
remarkStripDirectives,
|
|
321
|
+
// User plugins extend the parser (e.g. `remark-math`) so syntax
|
|
322
|
+
// recognized in the React build also parses during indexing.
|
|
323
|
+
...(config.markdown?.remarkPlugins ?? []),
|
|
320
324
|
]
|
|
321
325
|
|
|
322
326
|
// Extract frontmatter with regex (avoids extra parse)
|
|
@@ -337,14 +341,17 @@ export function extract(source: string, _config: Config.Config): extract.ReturnT
|
|
|
337
341
|
const headingPositions = tree.children
|
|
338
342
|
.filter((node): node is MdAst.Heading => node.type === 'heading')
|
|
339
343
|
.map((heading) => {
|
|
340
|
-
const
|
|
341
|
-
const
|
|
344
|
+
const subheading = extractSubheading(heading.children)
|
|
345
|
+
const title = getPhrasingContentText(subheading?.headingChildren ?? heading.children).trim()
|
|
346
|
+
const subtitle = subheading
|
|
347
|
+
? getPhrasingContentText(subheading.subheadingChildren).trim()
|
|
348
|
+
: ''
|
|
342
349
|
return {
|
|
343
350
|
depth: heading.depth,
|
|
344
351
|
endOffset: heading.position?.end.offset ?? 0,
|
|
345
352
|
startOffset: heading.position?.start.offset ?? 0,
|
|
346
|
-
subtitle
|
|
347
|
-
title
|
|
353
|
+
subtitle,
|
|
354
|
+
title,
|
|
348
355
|
}
|
|
349
356
|
})
|
|
350
357
|
|
|
@@ -137,4 +137,89 @@ describe('middleware', () => {
|
|
|
137
137
|
expect(await response.text()).toBe('# Hello from disk')
|
|
138
138
|
expect(readFile).toHaveBeenCalledOnce()
|
|
139
139
|
})
|
|
140
|
+
|
|
141
|
+
it('passes static assets through for AI agents without resolving a twin', async () => {
|
|
142
|
+
process.env['NODE_ENV'] = 'production'
|
|
143
|
+
|
|
144
|
+
const { readFile } = await import('node:fs/promises')
|
|
145
|
+
const fetchSpy = vi.fn()
|
|
146
|
+
globalThis.fetch = fetchSpy
|
|
147
|
+
|
|
148
|
+
const response = await request('http://localhost/specs/v0.1/openapi.json', {
|
|
149
|
+
'user-agent': 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)',
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
expect(response.status).toBe(200)
|
|
153
|
+
expect(response.headers.get('content-type')).toContain('text/html')
|
|
154
|
+
expect(await response.text()).toBe('<p>ok</p>')
|
|
155
|
+
expect(readFile).not.toHaveBeenCalled()
|
|
156
|
+
expect(fetchSpy).not.toHaveBeenCalled()
|
|
157
|
+
})
|
|
158
|
+
|
|
159
|
+
it('passes static assets through for terminal user-agents', async () => {
|
|
160
|
+
process.env['NODE_ENV'] = 'production'
|
|
161
|
+
|
|
162
|
+
const { readFile } = await import('node:fs/promises')
|
|
163
|
+
const fetchSpy = vi.fn()
|
|
164
|
+
globalThis.fetch = fetchSpy
|
|
165
|
+
|
|
166
|
+
const response = await request('http://localhost/specs/v0.1/openapi.json', {
|
|
167
|
+
'user-agent': 'curl/8.7.1',
|
|
168
|
+
})
|
|
169
|
+
|
|
170
|
+
expect(response.status).toBe(200)
|
|
171
|
+
expect(await response.text()).toBe('<p>ok</p>')
|
|
172
|
+
expect(readFile).not.toHaveBeenCalled()
|
|
173
|
+
expect(fetchSpy).not.toHaveBeenCalled()
|
|
174
|
+
})
|
|
175
|
+
|
|
176
|
+
it('passes static assets through even when text/markdown is accepted', async () => {
|
|
177
|
+
process.env['NODE_ENV'] = 'production'
|
|
178
|
+
|
|
179
|
+
const { readFile } = await import('node:fs/promises')
|
|
180
|
+
const fetchSpy = vi.fn()
|
|
181
|
+
globalThis.fetch = fetchSpy
|
|
182
|
+
|
|
183
|
+
const response = await request('http://localhost/specs/openapi.json', {
|
|
184
|
+
accept: 'text/markdown',
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
expect(response.status).toBe(200)
|
|
188
|
+
expect(await response.text()).toBe('<p>ok</p>')
|
|
189
|
+
expect(readFile).not.toHaveBeenCalled()
|
|
190
|
+
expect(fetchSpy).not.toHaveBeenCalled()
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
it('passes generated markdown twins through without re-resolving', async () => {
|
|
194
|
+
process.env['NODE_ENV'] = 'production'
|
|
195
|
+
|
|
196
|
+
const { readFile } = await import('node:fs/promises')
|
|
197
|
+
const fetchSpy = vi.fn()
|
|
198
|
+
globalThis.fetch = fetchSpy
|
|
199
|
+
|
|
200
|
+
const response = await request('http://localhost/assets/md/docs.md', {
|
|
201
|
+
'user-agent': 'Mozilla/5.0 (compatible; GPTBot/1.0; +https://openai.com/gptbot)',
|
|
202
|
+
})
|
|
203
|
+
|
|
204
|
+
expect(response.status).toBe(200)
|
|
205
|
+
expect(await response.text()).toBe('<p>ok</p>')
|
|
206
|
+
expect(readFile).not.toHaveBeenCalled()
|
|
207
|
+
expect(fetchSpy).not.toHaveBeenCalled()
|
|
208
|
+
})
|
|
209
|
+
|
|
210
|
+
it('serves markdown for clean routes whose parent segments contain dots', async () => {
|
|
211
|
+
process.env['NODE_ENV'] = 'production'
|
|
212
|
+
|
|
213
|
+
const { readFile } = await import('node:fs/promises')
|
|
214
|
+
vi.mocked(readFile).mockResolvedValue('# Hello from disk')
|
|
215
|
+
|
|
216
|
+
const response = await request('http://localhost/docs/v2.0/intro', {
|
|
217
|
+
'user-agent': 'Mozilla/5.0 (compatible; GPTBot/1.0; +https://openai.com/gptbot)',
|
|
218
|
+
})
|
|
219
|
+
|
|
220
|
+
expect(response.status).toBe(200)
|
|
221
|
+
expect(response.headers.get('content-type')).toBe('text/markdown; charset=utf-8')
|
|
222
|
+
expect(await response.text()).toBe('# Hello from disk')
|
|
223
|
+
expect(readFile).toHaveBeenCalledOnce()
|
|
224
|
+
})
|
|
140
225
|
})
|
|
@@ -97,6 +97,10 @@ export function middleware(): MiddlewareHandler {
|
|
|
97
97
|
return async (context, next) => {
|
|
98
98
|
const url = new URL(context.req.url)
|
|
99
99
|
|
|
100
|
+
// Generated markdown twins are static output; routing them back through
|
|
101
|
+
// twin resolution would trigger a recursive self-fetch.
|
|
102
|
+
if (url.pathname.startsWith('/assets/md/')) return next()
|
|
103
|
+
|
|
100
104
|
const userAgent = context.req.header('user-agent') ?? ''
|
|
101
105
|
const isOgBot = ogBotUserAgents.some((agent) => userAgent.includes(agent))
|
|
102
106
|
if (isOgBot) return next()
|
|
@@ -126,6 +130,12 @@ export function middleware(): MiddlewareHandler {
|
|
|
126
130
|
}
|
|
127
131
|
|
|
128
132
|
const isMarkdownRequest = url.pathname.endsWith('.md')
|
|
133
|
+
|
|
134
|
+
// Static assets (`.json`, `.svg`, `.png`, ...) have no markdown twin. Skip
|
|
135
|
+
// twin resolution so a disk miss never falls back to a slow self-fetch.
|
|
136
|
+
const filename = url.pathname.split('/').pop() ?? ''
|
|
137
|
+
if (!isMarkdownRequest && filename.includes('.')) return next()
|
|
138
|
+
|
|
129
139
|
if (!isMarkdownRequest && (isSearchEngine || (!isAiAgent && !isTerminal && !acceptsMarkdown)))
|
|
130
140
|
return next()
|
|
131
141
|
|
|
@@ -148,7 +158,7 @@ export function middleware(): MiddlewareHandler {
|
|
|
148
158
|
|
|
149
159
|
context.res = new Response(text, {
|
|
150
160
|
headers: {
|
|
151
|
-
'Content-Type':
|
|
161
|
+
'Content-Type': 'text/markdown; charset=utf-8',
|
|
152
162
|
},
|
|
153
163
|
})
|
|
154
164
|
return
|
|
@@ -2,6 +2,7 @@ import path from 'node:path'
|
|
|
2
2
|
import { serve } from '@hono/node-server'
|
|
3
3
|
import { serveStatic } from '@hono/node-server/serve-static'
|
|
4
4
|
import type { MiddlewareHandler } from 'hono'
|
|
5
|
+
import { bodyLimit } from 'hono/body-limit'
|
|
5
6
|
import { Hono } from 'hono/tiny'
|
|
6
7
|
import { unstable_createServerEntryAdapter as createServerEntryAdapter } from 'waku/adapter-builders'
|
|
7
8
|
import {
|
|
@@ -10,13 +11,22 @@ import {
|
|
|
10
11
|
} from 'waku/internals'
|
|
11
12
|
|
|
12
13
|
const { DIST_PUBLIC } = constants
|
|
13
|
-
const {
|
|
14
|
+
const { rscMiddleware, middlewareRunner } = honoMiddleware
|
|
14
15
|
|
|
15
|
-
|
|
16
|
+
const DEFAULT_BODY_LIMIT_MAX_SIZE = 100 * 1024 * 1024
|
|
17
|
+
|
|
18
|
+
type MiddlewareModules = Record<
|
|
19
|
+
string,
|
|
20
|
+
() => Promise<{ default: (opts: { app: Hono }) => MiddlewareHandler }>
|
|
21
|
+
>
|
|
16
22
|
|
|
17
23
|
const adapter: typeof import('waku/adapters/node').default = createServerEntryAdapter(
|
|
18
24
|
({ processRequest, processBuild, config, isBuild, notFoundHtml }, options) => {
|
|
19
|
-
const {
|
|
25
|
+
const {
|
|
26
|
+
bodyLimit: bodyLimitOptions,
|
|
27
|
+
middlewareFns = [],
|
|
28
|
+
middlewareModules = {},
|
|
29
|
+
} = options || {}
|
|
20
30
|
const typedMiddlewareModules = middlewareModules as MiddlewareModules
|
|
21
31
|
const app = new Hono()
|
|
22
32
|
|
|
@@ -29,9 +39,12 @@ const adapter: typeof import('waku/adapters/node').default = createServerEntryAd
|
|
|
29
39
|
// output before middleware in build mode, but Vocs needs mdRouter to run first
|
|
30
40
|
// for partial-static clean URLs that negotiate `text/markdown`.
|
|
31
41
|
if (isBuild && typedMiddlewareModules['mdRouter']) {
|
|
32
|
-
const mdRouterMiddleware = middlewareRunner(
|
|
33
|
-
|
|
34
|
-
|
|
42
|
+
const mdRouterMiddleware = middlewareRunner(
|
|
43
|
+
{
|
|
44
|
+
mdRouter: typedMiddlewareModules['mdRouter'],
|
|
45
|
+
},
|
|
46
|
+
{ app },
|
|
47
|
+
)
|
|
35
48
|
app.use(`${config.basePath}*`, (context, next) => {
|
|
36
49
|
const url = new URL(context.req.url)
|
|
37
50
|
// Generated markdown assets are already static output. Passing them through
|
|
@@ -50,9 +63,10 @@ const adapter: typeof import('waku/adapters/node').default = createServerEntryAd
|
|
|
50
63
|
}),
|
|
51
64
|
)
|
|
52
65
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
app.use(
|
|
66
|
+
if (bodyLimitOptions !== false)
|
|
67
|
+
app.use(bodyLimit(bodyLimitOptions ?? { maxSize: DEFAULT_BODY_LIMIT_MAX_SIZE }))
|
|
68
|
+
for (const middlewareFn of middlewareFns) app.use(middlewareFn({ app }))
|
|
69
|
+
app.use(middlewareRunner(typedMiddlewareModules, { app }))
|
|
56
70
|
app.use(rscMiddleware({ processRequest }))
|
|
57
71
|
|
|
58
72
|
return {
|