skrypt-ai 0.5.0 → 0.6.0

package/dist/template/scripts/build-search-index.mjs

@@ -86,6 +86,55 @@ function extractPlainText(content) {
     .trim()
 }
 
+/**
+ * Slugify a heading for use as an anchor hash.
+ * Matches the logic in heading components (H2, H3).
+ */
+function slugifyHeading(text) {
+  return text
+    .replace(/`/g, '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/(^-|-$)/g, '')
+}
+
+/**
+ * Split content into sections by h2 headings.
+ * Returns an array of { heading, slug, content } objects.
+ */
+function splitBySections(content) {
+  const sections = []
+  const h2Regex = /^## (.+)$/gm
+  let lastIndex = 0
+  let lastHeading = null
+  let lastSlug = null
+  let match
+
+  while ((match = h2Regex.exec(content)) !== null) {
+    if (lastHeading !== null) {
+      sections.push({
+        heading: lastHeading,
+        slug: lastSlug,
+        content: content.slice(lastIndex, match.index),
+      })
+    }
+    lastHeading = match[1].replace(/[`*_[\]]/g, '').trim()
+    lastSlug = slugifyHeading(lastHeading)
+    lastIndex = match.index
+  }
+
+  // Last section
+  if (lastHeading !== null) {
+    sections.push({
+      heading: lastHeading,
+      slug: lastSlug,
+      content: content.slice(lastIndex),
+    })
+  }
+
+  return sections
+}
+
 /**
  * Extract keywords from content — important terms that appear frequently
  * or in headings, used to boost search relevance.
@@ -119,7 +168,7 @@ function getSlugFromContentPath(filePath) {
   const rel = relative(CONTENT_DIR, filePath)
   const slug = rel
     .replace(/\.(mdx?|md)$/, '')
-    .replace(/\/index$/, '')
+    .replace(/(^|\/)index$/, '')
     .replace(/\\/g, '/')
 
   return slug ? `/docs/${slug}` : '/docs'
@@ -186,19 +235,26 @@ async function buildSearchIndex() {
   if (await dirExists(CONTENT_DIR)) {
     const contentFiles = await getAllMDXFiles(CONTENT_DIR)
     for (const file of contentFiles) {
+      // Skip metadata files
+      const fileName = basename(file).replace(/\.(mdx?|md)$/, '')
+      if (fileName.startsWith('_') || fileName.startsWith('llms') || fileName === 'README') continue
+
       try {
         const raw = await readFile(file, 'utf-8')
         const { data, content } = matter(raw)
 
         const title = data.title || extractTitle(content, file)
-        const plainContent = extractPlainText(content)
-        const headings = extractHeadings(content)
-        const keywords = extractKeywords(content, title)
         const href = getSlugFromContentPath(file)
 
         if (seen.has(href)) continue
         seen.add(href)
 
+        // Index the page itself
+        const plainContent = extractPlainText(content)
+        const headings = extractHeadings(content)
+        const keywords = extractKeywords(content, title)
+        const sectionLabel = data.section || data.category || title
+
         documents.push({
           id: href,
           title,
@@ -206,10 +262,30 @@ async function buildSearchIndex() {
           keywords,
           content: plainContent.slice(0, 5000),
           href,
-          section: data.section || data.category || '',
+          section: sectionLabel,
         })
 
-        console.log(`  Indexed: ${href} (content/)`)
+        // Also index individual h2 sections as separate search documents
+        // This enables deep-linking search results (e.g. /docs/core#autofixbatch)
+        const sections = splitBySections(content)
+        for (const sec of sections) {
+          const sectionHref = `${href}#${sec.slug}`
+          if (seen.has(sectionHref)) continue
+          seen.add(sectionHref)
+
+          const sectionContent = extractPlainText(sec.content).slice(0, 2000)
+          documents.push({
+            id: sectionHref,
+            title: sec.heading,
+            headings: '',
+            keywords: sec.heading.toLowerCase(),
+            content: sectionContent,
+            href: sectionHref,
+            section: title, // parent page title as section label
+          })
+        }
+
+        console.log(`  Indexed: ${href} (content/) + ${sections.length} sections`)
       } catch (err) {
         console.error(`  Error indexing ${file}:`, err.message)
       }
@@ -221,6 +297,8 @@ async function buildSearchIndex() {
     const appFiles = await getAllMDXFiles(APP_DOCS_DIR)
     for (const file of appFiles) {
       if (file.includes('[...slug]') || file.includes('layout.')) continue
+      const appFileName = basename(file).replace(/\.(mdx?|md)$/, '')
+      if (appFileName.startsWith('_') || appFileName.startsWith('llms') || appFileName === 'README') continue
 
       try {
         const raw = await readFile(file, 'utf-8')

package/dist/template/src/app/api/chat/route.ts

@@ -1,10 +1,24 @@
-import { NextResponse } from 'next/server'
+import { streamText, type CoreMessage } from 'ai'
+import { createOpenAI } from '@ai-sdk/openai'
+import { createAnthropic } from '@ai-sdk/anthropic'
 import { readFileSync, readdirSync, statSync, existsSync } from 'fs'
 import { join, extname } from 'path'
 
-interface Message {
-  role: 'user' | 'assistant'
-  content: string
+// In-memory rate limiter
+const rateLimitMap = new Map<string, { count: number; resetAt: number }>()
+const RATE_LIMIT = 20
+const RATE_WINDOW = 60_000
+
+function checkRateLimit(ip: string): boolean {
+  const now = Date.now()
+  const entry = rateLimitMap.get(ip)
+  if (!entry || now > entry.resetAt) {
+    rateLimitMap.set(ip, { count: 1, resetAt: now + RATE_WINDOW })
+    return true
+  }
+  if (entry.count >= RATE_LIMIT) return false
+  entry.count++
+  return true
 }
 
 interface DocChunk {
@@ -19,13 +33,12 @@ let docChunks: DocChunk[] | null = null
 let lastIndexTime = 0
 
 function indexDocs(): DocChunk[] {
-  // Return cached if recent (5 min)
   if (docChunks && Date.now() - lastIndexTime < 5 * 60 * 1000) {
     return docChunks
   }
 
   const chunks: DocChunk[] = []
-  const docsDir = join(process.cwd(), 'src/app/docs')
+  const docsDir = join(process.cwd(), 'content/docs')
 
   if (!existsSync(docsDir)) {
     return chunks
@@ -45,7 +58,6 @@ function indexDocs(): DocChunk[] {
           const content = readFileSync(fullPath, 'utf-8')
           const docPath = `${basePath}/${entry.replace(/\.(mdx?|md)$/, '')}`.replace('/page', '')
 
-          // Extract title from frontmatter or first heading
           let title = entry.replace(/\.(mdx?|md)$/, '')
           const titleMatch = content.match(/title:\s*["']?([^"'\n]+)["']?/m)
           if (titleMatch) {
@@ -55,16 +67,14 @@ function indexDocs(): DocChunk[] {
             if (h1Match) title = h1Match[1]
           }
 
-          // Extract headings
           const headings = [...content.matchAll(/^#{1,3}\s+(.+)$/gm)].map(m => m[1])
 
-          // Clean content (remove frontmatter, code blocks for search)
           let cleanContent = content
-            .replace(/^---[\s\S]*?---/m, '') // Remove frontmatter
-            .replace(/```[\s\S]*?```/g, '') // Remove code blocks
-            .replace(/<[^>]+>/g, '') // Remove JSX/HTML
-            .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert links to text
-            .slice(0, 2000) // Limit size
+            .replace(/^---[\s\S]*?---/m, '')
+            .replace(/```[\s\S]*?```/g, '')
+            .replace(/<[^>]+>/g, '')
+            .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+            .slice(0, 2000)
 
           chunks.push({
             path: docPath || '/',
@@ -100,13 +110,11 @@ function searchDocs(query: string, chunks: DocChunk[]): DocChunk[] {
       const contentLower = chunk.content.toLowerCase()
       const titleLower = chunk.title.toLowerCase()
 
-      // Title matches (high weight)
       if (titleLower.includes(queryLower)) score += 20
       queryWords.forEach(word => {
         if (titleLower.includes(word)) score += 5
       })
 
-      // Heading matches (medium weight)
       chunk.headings.forEach(h => {
         const hLower = h.toLowerCase()
         if (hLower.includes(queryLower)) score += 10
@@ -115,10 +123,9 @@ function searchDocs(query: string, chunks: DocChunk[]): DocChunk[] {
         })
       })
 
-      // Content matches (lower weight)
       queryWords.forEach(word => {
         const matches = (contentLower.match(new RegExp(escapeRegex(word), 'g')) || []).length
-        score += Math.min(matches, 5) // Cap at 5 matches per word
+        score += Math.min(matches, 5)
       })
 
       return { chunk, score }
@@ -129,137 +136,85 @@ function searchDocs(query: string, chunks: DocChunk[]): DocChunk[] {
     .map(r => r.chunk)
 }
 
-async function generateAnswer(
-  query: string,
-  context: DocChunk[],
-  history: Message[]
-): Promise<{ content: string; citations: Array<{ title: string; path: string; snippet: string }> }> {
-  // Check for API key
-  const apiKey = process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY
-
-  if (!apiKey) {
-    // Fallback: return a simple search result without AI
-    if (context.length === 0) {
-      return {
-        content: "I couldn't find any relevant documentation for your question. Try rephrasing or browsing the docs directly.",
-        citations: [],
-      }
-    }
-
-    return {
-      content: `Based on the documentation, here's what I found:\n\n${context[0].content.slice(0, 500)}...\n\nSee the linked sources for more details.`,
-      citations: context.slice(0, 3).map(c => ({
-        title: c.title,
-        path: `/docs${c.path}`,
-        snippet: c.content.slice(0, 100),
-      })),
-    }
+function getModel() {
+  if (process.env.OPENAI_API_KEY) {
+    const openai = createOpenAI({ apiKey: process.env.OPENAI_API_KEY })
+    return openai('gpt-4o-mini')
   }
-
-  // Build context for AI
-  const contextText = context
-    .map(c => `## ${c.title}\nPath: ${c.path}\n\n${c.content}`)
-    .join('\n\n---\n\n')
-
-  const systemPrompt = `You are a helpful documentation assistant. Answer questions based ONLY on the provided documentation context. If the answer isn't in the context, say so.
-
-Always cite your sources by mentioning which document you're referencing. Be concise and helpful.
-
-Documentation context:
-${contextText}`
-
-  const messages = [
-    { role: 'system' as const, content: systemPrompt },
-    ...history.slice(-4), // Last 4 messages for context
-    { role: 'user' as const, content: query },
-  ]
-
-  try {
-    let answer: string
-
-    if (process.env.OPENAI_API_KEY) {
-      const response = await fetch('https://api.openai.com/v1/chat/completions', {
-        method: 'POST',
-        headers: {
-          'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          model: 'gpt-4o-mini',
-          messages,
-          max_tokens: 500,
-        }),
-      })
-
-      const data = await response.json()
-      answer = data.choices?.[0]?.message?.content || 'Sorry, I could not generate an answer.'
-    } else {
-      const response = await fetch('https://api.anthropic.com/v1/messages', {
-        method: 'POST',
-        headers: {
-          'x-api-key': process.env.ANTHROPIC_API_KEY!,
-          'anthropic-version': '2023-06-01',
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          model: 'claude-sonnet-4-20250514',
-          max_tokens: 500,
-          system: systemPrompt,
-          messages: messages.filter(m => m.role !== 'system'),
-        }),
-      })
-
-      const data = await response.json()
-      answer = data.content?.[0]?.text || 'Sorry, I could not generate an answer.'
-    }
-
-    return {
-      content: answer,
-      citations: context.slice(0, 3).map(c => ({
-        title: c.title,
-        path: `/docs${c.path}`,
-        snippet: c.content.slice(0, 100),
-      })),
-    }
-  } catch (err) {
-    console.error('AI generation error:', err)
-    return {
-      content: 'Sorry, I encountered an error generating an answer. Please try again.',
-      citations: [],
-    }
+  if (process.env.ANTHROPIC_API_KEY) {
+    const anthropic = createAnthropic({ apiKey: process.env.ANTHROPIC_API_KEY })
+    return anthropic('claude-sonnet-4-20250514')
   }
+  return null
 }
 
 export async function POST(request: Request) {
+  const ip = request.headers.get('x-forwarded-for') ?? request.headers.get('x-real-ip') ?? 'unknown'
+  if (!checkRateLimit(ip)) {
+    return new Response(JSON.stringify({ error: 'Rate limit exceeded' }), { status: 429 })
+  }
+
   try {
     const body = await request.json()
-    const { messages } = body as { messages: Message[] }
+    const { messages } = body as { messages: CoreMessage[] }
 
     if (!Array.isArray(messages) || messages.length === 0 || messages.length > 50) {
-      return NextResponse.json({ error: 'Invalid request' }, { status: 400 })
+      return new Response(JSON.stringify({ error: 'Invalid request' }), { status: 400 })
     }
 
     const lastMessage = messages[messages.length - 1]
     if (typeof lastMessage?.content !== 'string' || lastMessage.content.length > 10000) {
-      return NextResponse.json({ error: 'Message too long' }, { status: 400 })
+      return new Response(JSON.stringify({ error: 'Message too long' }), { status: 400 })
     }
 
-    if (lastMessage.role !== 'user') {
-      return NextResponse.json({ error: 'Last message must be from user' }, { status: 400 })
+    // Index and search docs
+    const chunks = indexDocs()
+    const relevantDocs = searchDocs(lastMessage.content as string, chunks)
+
+    const contextText = relevantDocs
+      .map(c => `## ${c.title}\nPath: ${c.path}\n\n${c.content}`)
+      .join('\n\n---\n\n')
+
+    const citations = relevantDocs.slice(0, 3).map(c => ({
+      title: c.title,
+      path: `/docs${c.path}`,
+      snippet: c.content.slice(0, 100),
+    }))
+
+    const model = getModel()
+
+    // No API key — return plain search results as a non-streaming response
+    if (!model) {
+      const fallback = relevantDocs.length > 0
+        ? `Based on the docs:\n\n${relevantDocs[0].content.slice(0, 500)}...\n\nSee the linked sources for more.`
+        : "I couldn't find relevant documentation. Try rephrasing or browse the docs directly."
+
+      return new Response(JSON.stringify({ content: fallback, citations }), {
+        headers: { 'Content-Type': 'application/json' },
+      })
     }
 
-    // Index docs
-    const chunks = indexDocs()
+    // Stream the response using Vercel AI SDK
+    const result = streamText({
+      model,
+      system: `You are a helpful documentation assistant. Answer questions based ONLY on the provided documentation context. If the answer isn't in the context, say so. Be concise. Use markdown formatting (bold, code blocks, lists) to make answers scannable.
 
-    // Search for relevant docs
-    const relevantDocs = searchDocs(lastMessage.content, chunks)
+Documentation context:
+${contextText}
 
-    // Generate answer
-    const answer = await generateAnswer(lastMessage.content, relevantDocs, messages.slice(0, -1))
+Sources available (reference these when relevant):
+${citations.map(c => `- [${c.title}](${c.path})`).join('\n')}`,
+      messages: messages.slice(-5) as CoreMessage[],
+      maxTokens: 800,
+    })
 
-    return NextResponse.json(answer)
+    return result.toDataStreamResponse({
+      headers: {
+        'X-Citations': Buffer.from(JSON.stringify(citations)).toString('base64'),
+      },
+    })
   } catch (err) {
     console.error('Chat error:', err)
-    return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
+    return new Response(JSON.stringify({ error: 'Internal server error' }), { status: 500 })
   }
 }

package/dist/template/src/app/docs/[...slug]/page.tsx

@@ -3,7 +3,60 @@ import { readFile, readdir } from 'fs/promises'
 import { join } from 'path'
 import { compileMDX } from 'next-mdx-remote/rsc'
 import remarkGfm from 'remark-gfm'
-import * as components from '@/components/mdx'
+import * as mdxComponents from '@/components/mdx'
+import { findPageInNavigation, type Navigation } from '@/lib/navigation'
+import { Children, isValidElement, type ReactNode } from 'react'
+
+/** Wrapper for <pre> that routes mermaid code blocks to the Mermaid component */
+function PreWithMermaid(props: { children: ReactNode; className?: string }) {
+  let isMermaid = false
+  let codeText = ''
+
+  Children.forEach(props.children, (child) => {
+    if (isValidElement(child) && child.type === 'code') {
+      const childProps = child.props as { className?: string; children?: string }
+      if (childProps.className?.includes('language-mermaid')) {
+        isMermaid = true
+        codeText = childProps.children || ''
+      }
+    }
+  })
+
+  if (isMermaid) {
+    return <mdxComponents.Mermaid chart={codeText} />
+  }
+
+  return <mdxComponents.HighlightedCode {...props} />
+}
+
+/** Passthrough component for generated content that references unknown JSX tags */
+function Passthrough({ children }: { children?: ReactNode }) {
+  return <>{children}</>
+}
+
+/** Strip empty <a id="..."></a> anchor tags — headings already have IDs from the Heading component */
+function AnchorOrLink(props: { id?: string; href?: string; children?: ReactNode; className?: string }) {
+  // Empty anchor-only elements (e.g. <a id="pluginmanager"></a>) — strip them
+  if (props.id && !props.href && !props.children) {
+    return null
+  }
+  // Regular links — render normally
+  return <a {...props} />
+}
+
+const components = {
+  ...mdxComponents,
+  pre: PreWithMermaid,
+  h1: mdxComponents.H1,
+  h2: mdxComponents.H2,
+  h3: mdxComponents.H3,
+  h4: mdxComponents.H4,
+  a: AnchorOrLink,
+  // Passthrough for JSX tags that appear in generated code examples
+  Tabs: Passthrough,
+  TabItem: Passthrough,
+  Details: Passthrough,
+}
 import type { Metadata } from 'next'
 
 interface PageProps {
@@ -23,6 +76,13 @@ function getDocsConfig() {
 
 async function getContent(slug: string[]) {
   const contentDir = join(process.cwd(), 'content', 'docs')
+
+  // Skip metadata files (llms.txt, llms-full.md, _navigation.json, _sidebars.json, README)
+  const lastName = slug[slug.length - 1]
+  if (lastName.startsWith('_') || lastName.startsWith('llms') || lastName === 'README') {
+    return null
+  }
+
   const filePath = join(contentDir, ...slug)
 
   // Try .mdx first, then .md
@@ -72,17 +132,11 @@ export async function generateMetadata({ params }: PageProps): Promise<Metadata>
 
   const frontmatter = await getFrontmatter(slug)
 
-  // Fall back to nav config title
+  // Fall back to nav config title (works with both flat and tabbed navigation)
   if (!frontmatter.title) {
     const path = '/docs/' + slug.join('/')
-    for (const group of docsConfig.navigation || []) {
-      for (const page of group.pages || []) {
-        if (page.path === path) {
-          frontmatter.title = page.title
-          break
-        }
-      }
-    }
+    const pageInfo = findPageInNavigation(docsConfig.navigation as Navigation || [], path)
+    if (pageInfo.title) frontmatter.title = pageInfo.title
   }
 
   const title = frontmatter.title || slug[slug.length - 1].replace(/-/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase())
@@ -93,18 +147,12 @@ export async function generateMetadata({ params }: PageProps): Promise<Metadata>
   }
 }
 
-/** Look up page title from docs.json navigation by pathname */
+/** Look up page title from docs.json navigation by pathname (works with both formats) */
 function getNavTitle(slug: string[]): string | undefined {
   const docsConfig = getDocsConfig()
   const path = '/docs/' + slug.join('/')
-  for (const group of docsConfig.navigation || []) {
-    for (const page of group.pages || []) {
-      if (page.path === path) {
-        return page.title
-      }
-    }
-  }
-  return undefined
+  const pageInfo = findPageInNavigation(docsConfig.navigation as Navigation || [], path)
+  return pageInfo.title
 }
 
 export default async function DocPage({ params }: PageProps) {
@@ -121,8 +169,11 @@ export default async function DocPage({ params }: PageProps) {
   const pageDescription = frontmatter.description
 
   try {
+    // Strip empty <a id="..."></a> anchors from generated content — headings already have IDs
+    const cleanedSource = result.content.replace(/<a\s+id="[^"]*"\s*><\/a>\s*/g, '')
+
     const { content } = await compileMDX({
-      source: result.content,
+      source: cleanedSource,
       components: components as any,
       options: {
         parseFrontmatter: true,
@@ -160,6 +211,8 @@ export default async function DocPage({ params }: PageProps) {
         {pageDescription && (
           <div data-page-description={pageDescription} className="hidden" />
         )}
+        {/* Raw content for copy page button */}
+        <script id="raw-page-content" type="text/plain" data-content={Buffer.from(result.content).toString('base64')} />
         <div className="prose max-w-none">
           {content}
         </div>
@@ -189,6 +242,8 @@ export async function generateStaticParams() {
           await walkDir(join(dir, entry.name), [...prefix, entry.name])
         } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
           const name = entry.name.replace(/\.(mdx?|md)$/, '')
+          // Skip metadata files
+          if (name.startsWith('_') || name.startsWith('llms') || name === 'README') continue
           if (name === 'index') {
             if (prefix.length > 0) {
               slugs.push({ slug: prefix })