skrypt-ai 0.4.2 → 0.6.0

package/dist/template/scripts/build-search-index.mjs

@@ -19,13 +19,15 @@ async function dirExists(dir) {
 async function getAllMDXFiles(dir) {
   const files = []
 
-  async function walk(currentDir) {
+  async function walk(currentDir, depth = 0) {
+    if (depth > 20) return
     try {
       const entries = await readdir(currentDir, { withFileTypes: true })
       for (const entry of entries) {
         const fullPath = join(currentDir, entry.name)
+        if (entry.isSymbolicLink?.()) continue
         if (entry.isDirectory()) {
-          await walk(fullPath)
+          await walk(fullPath, depth + 1)
         } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
           files.push(fullPath)
         }
@@ -39,14 +41,30 @@ async function getAllMDXFiles(dir) {
   return files
 }
 
+/**
+ * Extract headings (h2, h3) as a separate searchable field.
+ * This lets users find pages by section names.
+ */
+function extractHeadings(content) {
+  const headings = []
+  const regex = /^#{2,3}\s+(.+)$/gm
+  let match
+  while ((match = regex.exec(content)) !== null) {
+    // Strip any inline formatting
+    const clean = match[1].replace(/[*_`[\]]/g, '').trim()
+    if (clean) headings.push(clean)
+  }
+  return headings.join(' | ')
+}
+
 function extractPlainText(content) {
   return content
     // Remove import statements
     .replace(/^import\s+.*$/gm, '')
     // Remove export statements
     .replace(/^export\s+.*$/gm, '')
-    // Remove MDX/JSX components
-    .replace(/<[^>]+>/g, '')
+    // Remove MDX/JSX components (but keep text content inside)
+    .replace(/<[^>]+>/g, ' ')
     // Remove code blocks
     .replace(/```[\s\S]*?```/g, '')
     .replace(/`[^`]+`/g, '')
@@ -68,19 +86,95 @@ function extractPlainText(content) {
     .trim()
 }
 
+/**
+ * Slugify a heading for use as an anchor hash.
+ * Matches the logic in heading components (H2, H3).
+ */
+function slugifyHeading(text) {
+  return text
+    .replace(/`/g, '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/(^-|-$)/g, '')
+}
+
+/**
+ * Split content into sections by h2 headings.
+ * Returns an array of { heading, slug, content } objects.
+ */
+function splitBySections(content) {
+  const sections = []
+  const h2Regex = /^## (.+)$/gm
+  let lastIndex = 0
+  let lastHeading = null
+  let lastSlug = null
+  let match
+
+  while ((match = h2Regex.exec(content)) !== null) {
+    if (lastHeading !== null) {
+      sections.push({
+        heading: lastHeading,
+        slug: lastSlug,
+        content: content.slice(lastIndex, match.index),
+      })
+    }
+    lastHeading = match[1].replace(/[`*_[\]]/g, '').trim()
+    lastSlug = slugifyHeading(lastHeading)
+    lastIndex = match.index
+  }
+
+  // Last section
+  if (lastHeading !== null) {
+    sections.push({
+      heading: lastHeading,
+      slug: lastSlug,
+      content: content.slice(lastIndex),
+    })
+  }
+
+  return sections
+}
+
+/**
+ * Extract keywords from content — important terms that appear frequently
+ * or in headings, used to boost search relevance.
+ */
+function extractKeywords(content, title) {
+  const words = new Map()
+
+  // Title words get high weight
+  for (const word of title.toLowerCase().split(/\s+/)) {
+    if (word.length > 2) words.set(word, (words.get(word) || 0) + 3)
+  }
+
+  // Heading words get medium weight
+  const headingRegex = /^#{1,3}\s+(.+)$/gm
+  let match
+  while ((match = headingRegex.exec(content)) !== null) {
+    for (const word of match[1].toLowerCase().replace(/[^a-z0-9\s]/g, '').split(/\s+/)) {
+      if (word.length > 2) words.set(word, (words.get(word) || 0) + 2)
+    }
+  }
+
+  // Sort by weight and return top keywords
+  return [...words.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 20)
+    .map(([word]) => word)
+    .join(' ')
+}
+
 function getSlugFromContentPath(filePath) {
   const rel = relative(CONTENT_DIR, filePath)
   const slug = rel
     .replace(/\.(mdx?|md)$/, '')
-    .replace(/\/index$/, '')
+    .replace(/(^|\/)index$/, '')
     .replace(/\\/g, '/')
 
   return slug ? `/docs/${slug}` : '/docs'
 }
 
 function getSlugFromAppPath(filePath) {
-  // For App Router: src/app/docs/quickstart/page.mdx -> /docs/quickstart
-  // src/app/docs/page.mdx -> /docs
   const rel = relative(APP_DOCS_DIR, filePath)
   const dir = dirname(rel)
   const name = basename(rel)
@@ -99,11 +193,9 @@ function getSlugFromAppPath(filePath) {
 }
 
 function extractTitle(content, filePath) {
-  // Try to get title from first heading
   const h1Match = content.match(/^#\s+(.+)$/m)
   if (h1Match) return h1Match[1].trim()
 
-  // Derive from file path
   const dir = dirname(filePath)
   const folderName = basename(dir)
   if (folderName && folderName !== 'docs' && folderName !== '.') {
@@ -122,10 +214,18 @@ async function buildSearchIndex() {
     schema: {
       id: 'string',
       title: 'string',
+      headings: 'string',
+      keywords: 'string',
       content: 'string',
       href: 'string',
       section: 'string',
     },
+    components: {
+      tokenizer: {
+        stemming: true,
+        language: 'english',
+      },
+    },
   })
 
   const documents = []
@@ -135,26 +235,57 @@ async function buildSearchIndex() {
   if (await dirExists(CONTENT_DIR)) {
     const contentFiles = await getAllMDXFiles(CONTENT_DIR)
     for (const file of contentFiles) {
+      // Skip metadata files
+      const fileName = basename(file).replace(/\.(mdx?|md)$/, '')
+      if (fileName.startsWith('_') || fileName.startsWith('llms') || fileName === 'README') continue
+
       try {
         const raw = await readFile(file, 'utf-8')
         const { data, content } = matter(raw)
 
         const title = data.title || extractTitle(content, file)
-        const plainContent = extractPlainText(content)
         const href = getSlugFromContentPath(file)
 
         if (seen.has(href)) continue
         seen.add(href)
 
+        // Index the page itself
+        const plainContent = extractPlainText(content)
+        const headings = extractHeadings(content)
+        const keywords = extractKeywords(content, title)
+        const sectionLabel = data.section || data.category || title
+
         documents.push({
           id: href,
           title,
+          headings,
+          keywords,
           content: plainContent.slice(0, 5000),
           href,
-          section: data.section || data.category || '',
+          section: sectionLabel,
         })
 
-        console.log(`  Indexed: ${href} (content/)`)
+        // Also index individual h2 sections as separate search documents
+        // This enables deep-linking search results (e.g. /docs/core#autofixbatch)
+        const sections = splitBySections(content)
+        for (const sec of sections) {
+          const sectionHref = `${href}#${sec.slug}`
+          if (seen.has(sectionHref)) continue
+          seen.add(sectionHref)
+
+          const sectionContent = extractPlainText(sec.content).slice(0, 2000)
+          documents.push({
+            id: sectionHref,
+            title: sec.heading,
+            headings: '',
+            keywords: sec.heading.toLowerCase(),
+            content: sectionContent,
+            href: sectionHref,
+            section: title, // parent page title as section label
+          })
+        }
+
+        console.log(`  Indexed: ${href} (content/) + ${sections.length} sections`)
       } catch (err) {
         console.error(`  Error indexing ${file}:`, err.message)
       }
@@ -165,8 +296,9 @@ async function buildSearchIndex() {
   if (await dirExists(APP_DOCS_DIR)) {
     const appFiles = await getAllMDXFiles(APP_DOCS_DIR)
     for (const file of appFiles) {
-      // Skip catch-all route files and layout files
       if (file.includes('[...slug]') || file.includes('layout.')) continue
+      const appFileName = basename(file).replace(/\.(mdx?|md)$/, '')
+      if (appFileName.startsWith('_') || appFileName.startsWith('llms') || appFileName === 'README') continue
 
       try {
         const raw = await readFile(file, 'utf-8')
@@ -174,6 +306,8 @@ async function buildSearchIndex() {
 
         const title = data.title || extractTitle(content, file)
         const plainContent = extractPlainText(content)
+        const headings = extractHeadings(content)
+        const keywords = extractKeywords(content, title)
         const href = getSlugFromAppPath(file)
 
         if (seen.has(href)) continue
@@ -182,6 +316,8 @@ async function buildSearchIndex() {
         documents.push({
           id: href,
           title,
+          headings,
+          keywords,
           content: plainContent.slice(0, 5000),
           href,
           section: data.section || data.category || '',

package/dist/template/src/app/api/chat/route.ts

@@ -1,10 +1,24 @@
-import { NextResponse } from 'next/server'
+import { streamText, type CoreMessage } from 'ai'
+import { createOpenAI } from '@ai-sdk/openai'
+import { createAnthropic } from '@ai-sdk/anthropic'
 import { readFileSync, readdirSync, statSync, existsSync } from 'fs'
 import { join, extname } from 'path'
 
-interface Message {
-  role: 'user' | 'assistant'
-  content: string
+// In-memory rate limiter
+const rateLimitMap = new Map<string, { count: number; resetAt: number }>()
+const RATE_LIMIT = 20
+const RATE_WINDOW = 60_000
+
+function checkRateLimit(ip: string): boolean {
+  const now = Date.now()
+  const entry = rateLimitMap.get(ip)
+  if (!entry || now > entry.resetAt) {
+    rateLimitMap.set(ip, { count: 1, resetAt: now + RATE_WINDOW })
+    return true
+  }
+  if (entry.count >= RATE_LIMIT) return false
+  entry.count++
+  return true
 }
 
 interface DocChunk {
@@ -19,13 +33,12 @@ let docChunks: DocChunk[] | null = null
 let lastIndexTime = 0
 
 function indexDocs(): DocChunk[] {
-  // Return cached if recent (5 min)
   if (docChunks && Date.now() - lastIndexTime < 5 * 60 * 1000) {
     return docChunks
   }
 
   const chunks: DocChunk[] = []
-  const docsDir = join(process.cwd(), 'src/app/docs')
+  const docsDir = join(process.cwd(), 'content/docs')
 
   if (!existsSync(docsDir)) {
     return chunks
@@ -45,7 +58,6 @@ function indexDocs(): DocChunk[] {
           const content = readFileSync(fullPath, 'utf-8')
           const docPath = `${basePath}/${entry.replace(/\.(mdx?|md)$/, '')}`.replace('/page', '')
 
-          // Extract title from frontmatter or first heading
           let title = entry.replace(/\.(mdx?|md)$/, '')
           const titleMatch = content.match(/title:\s*["']?([^"'\n]+)["']?/m)
           if (titleMatch) {
@@ -55,16 +67,14 @@ function indexDocs(): DocChunk[] {
             if (h1Match) title = h1Match[1]
           }
 
-          // Extract headings
           const headings = [...content.matchAll(/^#{1,3}\s+(.+)$/gm)].map(m => m[1])
 
-          // Clean content (remove frontmatter, code blocks for search)
           let cleanContent = content
-            .replace(/^---[\s\S]*?---/m, '') // Remove frontmatter
-            .replace(/```[\s\S]*?```/g, '') // Remove code blocks
-            .replace(/<[^>]+>/g, '') // Remove JSX/HTML
-            .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert links to text
-            .slice(0, 2000) // Limit size
+            .replace(/^---[\s\S]*?---/m, '')
+            .replace(/```[\s\S]*?```/g, '')
+            .replace(/<[^>]+>/g, '')
+            .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+            .slice(0, 2000)
 
           chunks.push({
             path: docPath || '/',
@@ -100,13 +110,11 @@ function searchDocs(query: string, chunks: DocChunk[]): DocChunk[] {
       const contentLower = chunk.content.toLowerCase()
       const titleLower = chunk.title.toLowerCase()
 
-      // Title matches (high weight)
       if (titleLower.includes(queryLower)) score += 20
       queryWords.forEach(word => {
         if (titleLower.includes(word)) score += 5
       })
 
-      // Heading matches (medium weight)
       chunk.headings.forEach(h => {
         const hLower = h.toLowerCase()
         if (hLower.includes(queryLower)) score += 10
@@ -115,10 +123,9 @@ function searchDocs(query: string, chunks: DocChunk[]): DocChunk[] {
         })
       })
 
-      // Content matches (lower weight)
       queryWords.forEach(word => {
         const matches = (contentLower.match(new RegExp(escapeRegex(word), 'g')) || []).length
-        score += Math.min(matches, 5) // Cap at 5 matches per word
+        score += Math.min(matches, 5)
       })
 
       return { chunk, score }
@@ -129,137 +136,85 @@ function searchDocs(query: string, chunks: DocChunk[]): DocChunk[] {
     .map(r => r.chunk)
 }
 
-async function generateAnswer(
-  query: string,
-  context: DocChunk[],
-  history: Message[]
-): Promise<{ content: string; citations: Array<{ title: string; path: string; snippet: string }> }> {
-  // Check for API key
-  const apiKey = process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY
-
-  if (!apiKey) {
-    // Fallback: return a simple search result without AI
-    if (context.length === 0) {
-      return {
-        content: "I couldn't find any relevant documentation for your question. Try rephrasing or browsing the docs directly.",
-        citations: [],
-      }
-    }
-
-    return {
-      content: `Based on the documentation, here's what I found:\n\n${context[0].content.slice(0, 500)}...\n\nSee the linked sources for more details.`,
-      citations: context.slice(0, 3).map(c => ({
-        title: c.title,
-        path: `/docs${c.path}`,
-        snippet: c.content.slice(0, 100),
-      })),
-    }
+function getModel() {
+  if (process.env.OPENAI_API_KEY) {
+    const openai = createOpenAI({ apiKey: process.env.OPENAI_API_KEY })
+    return openai('gpt-4o-mini')
   }
-
-  // Build context for AI
-  const contextText = context
-    .map(c => `## ${c.title}\nPath: ${c.path}\n\n${c.content}`)
-    .join('\n\n---\n\n')
-
-  const systemPrompt = `You are a helpful documentation assistant. Answer questions based ONLY on the provided documentation context. If the answer isn't in the context, say so.
-
-Always cite your sources by mentioning which document you're referencing. Be concise and helpful.
-
-Documentation context:
-${contextText}`
-
-  const messages = [
-    { role: 'system' as const, content: systemPrompt },
-    ...history.slice(-4), // Last 4 messages for context
-    { role: 'user' as const, content: query },
-  ]
-
-  try {
-    let answer: string
-
-    if (process.env.OPENAI_API_KEY) {
-      const response = await fetch('https://api.openai.com/v1/chat/completions', {
-        method: 'POST',
-        headers: {
-          'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          model: 'gpt-4o-mini',
-          messages,
-          max_tokens: 500,
-        }),
-      })
-
-      const data = await response.json()
-      answer = data.choices?.[0]?.message?.content || 'Sorry, I could not generate an answer.'
-    } else {
-      const response = await fetch('https://api.anthropic.com/v1/messages', {
-        method: 'POST',
-        headers: {
-          'x-api-key': process.env.ANTHROPIC_API_KEY!,
-          'anthropic-version': '2023-06-01',
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          model: 'claude-sonnet-4-20250514',
-          max_tokens: 500,
-          system: systemPrompt,
-          messages: messages.filter(m => m.role !== 'system'),
-        }),
-      })
-
-      const data = await response.json()
-      answer = data.content?.[0]?.text || 'Sorry, I could not generate an answer.'
-    }
-
-    return {
-      content: answer,
-      citations: context.slice(0, 3).map(c => ({
-        title: c.title,
-        path: `/docs${c.path}`,
-        snippet: c.content.slice(0, 100),
-      })),
-    }
-  } catch (err) {
-    console.error('AI generation error:', err)
-    return {
-      content: 'Sorry, I encountered an error generating an answer. Please try again.',
-      citations: [],
-    }
+  if (process.env.ANTHROPIC_API_KEY) {
+    const anthropic = createAnthropic({ apiKey: process.env.ANTHROPIC_API_KEY })
+    return anthropic('claude-sonnet-4-20250514')
   }
+  return null
 }
 
 export async function POST(request: Request) {
+  const ip = request.headers.get('x-forwarded-for') ?? request.headers.get('x-real-ip') ?? 'unknown'
+  if (!checkRateLimit(ip)) {
+    return new Response(JSON.stringify({ error: 'Rate limit exceeded' }), { status: 429 })
+  }
+
   try {
     const body = await request.json()
-    const { messages } = body as { messages: Message[] }
+    const { messages } = body as { messages: CoreMessage[] }
 
     if (!Array.isArray(messages) || messages.length === 0 || messages.length > 50) {
-      return NextResponse.json({ error: 'Invalid request' }, { status: 400 })
+      return new Response(JSON.stringify({ error: 'Invalid request' }), { status: 400 })
     }
 
     const lastMessage = messages[messages.length - 1]
     if (typeof lastMessage?.content !== 'string' || lastMessage.content.length > 10000) {
-      return NextResponse.json({ error: 'Message too long' }, { status: 400 })
+      return new Response(JSON.stringify({ error: 'Message too long' }), { status: 400 })
     }
 
-    if (lastMessage.role !== 'user') {
-      return NextResponse.json({ error: 'Last message must be from user' }, { status: 400 })
+    // Index and search docs
+    const chunks = indexDocs()
+    const relevantDocs = searchDocs(lastMessage.content as string, chunks)
+
+    const contextText = relevantDocs
+      .map(c => `## ${c.title}\nPath: ${c.path}\n\n${c.content}`)
+      .join('\n\n---\n\n')
+
+    const citations = relevantDocs.slice(0, 3).map(c => ({
+      title: c.title,
+      path: `/docs${c.path}`,
+      snippet: c.content.slice(0, 100),
+    }))
+
+    const model = getModel()
+
+    // No API key — return plain search results as a non-streaming response
+    if (!model) {
+      const fallback = relevantDocs.length > 0
+        ? `Based on the docs:\n\n${relevantDocs[0].content.slice(0, 500)}...\n\nSee the linked sources for more.`
+        : "I couldn't find relevant documentation. Try rephrasing or browse the docs directly."
+
+      return new Response(JSON.stringify({ content: fallback, citations }), {
+        headers: { 'Content-Type': 'application/json' },
+      })
     }
 
-    // Index docs
-    const chunks = indexDocs()
+    // Stream the response using Vercel AI SDK
+    const result = streamText({
+      model,
+      system: `You are a helpful documentation assistant. Answer questions based ONLY on the provided documentation context. If the answer isn't in the context, say so. Be concise. Use markdown formatting (bold, code blocks, lists) to make answers scannable.
 
-    // Search for relevant docs
-    const relevantDocs = searchDocs(lastMessage.content, chunks)
+Documentation context:
+${contextText}
 
-    // Generate answer
-    const answer = await generateAnswer(lastMessage.content, relevantDocs, messages.slice(0, -1))
+Sources available (reference these when relevant):
+${citations.map(c => `- [${c.title}](${c.path})`).join('\n')}`,
+      messages: messages.slice(-5) as CoreMessage[],
+      maxTokens: 800,
+    })
 
-    return NextResponse.json(answer)
+    return result.toDataStreamResponse({
+      headers: {
+        'X-Citations': Buffer.from(JSON.stringify(citations)).toString('base64'),
+      },
+    })
   } catch (err) {
     console.error('Chat error:', err)
-    return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
+    return new Response(JSON.stringify({ error: 'Internal server error' }), { status: 500 })
   }
 }