skrypt-ai 0.3.4 → 0.4.1

package/dist/template/scripts/build-search-index.mjs

@@ -1,11 +1,21 @@
 import { create, insertMultiple, save } from '@orama/orama'
-import { readdir, readFile, writeFile, mkdir } from 'fs/promises'
-import { join, relative } from 'path'
+import { readdir, readFile, writeFile, mkdir, access } from 'fs/promises'
+import { join, relative, dirname, basename } from 'path'
 import matter from 'gray-matter'
 
 const CONTENT_DIR = join(process.cwd(), 'content', 'docs')
+const APP_DOCS_DIR = join(process.cwd(), 'src', 'app', 'docs')
 const OUTPUT_DIR = join(process.cwd(), 'public')
 
+async function dirExists(dir) {
+  try {
+    await access(dir)
+    return true
+  } catch {
+    return false
+  }
+}
+
 async function getAllMDXFiles(dir) {
   const files = []
 
@@ -31,6 +41,10 @@ async function getAllMDXFiles(dir) {
 
 function extractPlainText(content) {
   return content
+    // Remove import statements
+    .replace(/^import\s+.*$/gm, '')
+    // Remove export statements
+    .replace(/^export\s+.*$/gm, '')
     // Remove MDX/JSX components
     .replace(/<[^>]+>/g, '')
     // Remove code blocks
@@ -46,12 +60,15 @@ function extractPlainText(content) {
     .replace(/[*_]{1,2}([^*_]+)[*_]{1,2}/g, '$1')
     // Remove horizontal rules
     .replace(/^[-*_]{3,}$/gm, '')
+    // Remove table formatting
+    .replace(/\|/g, ' ')
+    .replace(/^[\s-:]+$/gm, '')
     // Remove extra whitespace
     .replace(/\s+/g, ' ')
     .trim()
 }
 
-function getSlugFromPath(filePath) {
+function getSlugFromContentPath(filePath) {
   const rel = relative(CONTENT_DIR, filePath)
   const slug = rel
     .replace(/\.(mdx?|md)$/, '')
@@ -61,6 +78,43 @@ function getSlugFromPath(filePath) {
   return slug ? `/docs/${slug}` : '/docs'
 }
 
+function getSlugFromAppPath(filePath) {
+  // For App Router: src/app/docs/quickstart/page.mdx -> /docs/quickstart
+  // src/app/docs/page.mdx -> /docs
+  const rel = relative(APP_DOCS_DIR, filePath)
+  const dir = dirname(rel)
+  const name = basename(rel)
+
+  if (name === 'page.mdx' || name === 'page.md') {
+    if (dir === '.') return '/docs'
+    return `/docs/${dir.replace(/\\/g, '/')}`
+  }
+
+  const slug = rel
+    .replace(/\.(mdx?|md)$/, '')
+    .replace(/\/index$/, '')
+    .replace(/\\/g, '/')
+
+  return `/docs/${slug}`
+}
+
+function extractTitle(content, filePath) {
+  // Try to get title from first heading
+  const h1Match = content.match(/^#\s+(.+)$/m)
+  if (h1Match) return h1Match[1].trim()
+
+  // Derive from file path
+  const dir = dirname(filePath)
+  const folderName = basename(dir)
+  if (folderName && folderName !== 'docs' && folderName !== '.') {
+    return folderName
+      .replace(/-/g, ' ')
+      .replace(/\b\w/g, c => c.toUpperCase())
+  }
+
+  return basename(filePath).replace(/\.(mdx?|md)$/, '')
+}
+
 async function buildSearchIndex() {
   console.log('Building search index...')
 
@@ -74,36 +128,77 @@ async function buildSearchIndex() {
     },
   })
 
-  const files = await getAllMDXFiles(CONTENT_DIR)
   const documents = []
+  const seen = new Set()
+
+  // 1. Index content/docs/ (generated docs)
+  if (await dirExists(CONTENT_DIR)) {
+    const contentFiles = await getAllMDXFiles(CONTENT_DIR)
+    for (const file of contentFiles) {
+      try {
+        const raw = await readFile(file, 'utf-8')
+        const { data, content } = matter(raw)
+
+        const title = data.title || extractTitle(content, file)
+        const plainContent = extractPlainText(content)
+        const href = getSlugFromContentPath(file)
+
+        if (seen.has(href)) continue
+        seen.add(href)
+
+        documents.push({
+          id: href,
+          title,
+          content: plainContent.slice(0, 5000),
+          href,
+          section: data.section || data.category || '',
+        })
+
+        console.log(`  Indexed: ${href} (content/)`)
+      } catch (err) {
+        console.error(`  Error indexing ${file}:`, err.message)
+      }
+    }
+  }
 
-  for (const file of files) {
-    try {
-      const raw = await readFile(file, 'utf-8')
-      const { data, content } = matter(raw)
-
-      const title = data.title || file.split('/').pop()?.replace(/\.(mdx?|md)$/, '') || 'Untitled'
-      const plainContent = extractPlainText(content)
-      const href = getSlugFromPath(file)
-      const section = data.section || data.category || ''
-
-      documents.push({
-        id: href,
-        title,
-        content: plainContent.slice(0, 5000), // Limit content size
-        href,
-        section,
-      })
-
-      console.log(`  Indexed: ${href}`)
-    } catch (err) {
-      console.error(`  Error indexing ${file}:`, err.message)
+  // 2. Index src/app/docs/ (App Router MDX pages)
+  if (await dirExists(APP_DOCS_DIR)) {
+    const appFiles = await getAllMDXFiles(APP_DOCS_DIR)
+    for (const file of appFiles) {
+      // Skip catch-all route files and layout files
+      if (file.includes('[...slug]') || file.includes('layout.')) continue
+
+      try {
+        const raw = await readFile(file, 'utf-8')
+        const { data, content } = matter(raw)
+
+        const title = data.title || extractTitle(content, file)
+        const plainContent = extractPlainText(content)
+        const href = getSlugFromAppPath(file)
+
+        if (seen.has(href)) continue
+        seen.add(href)
+
+        documents.push({
+          id: href,
+          title,
+          content: plainContent.slice(0, 5000),
+          href,
+          section: data.section || data.category || '',
+        })
+
+        console.log(`  Indexed: ${href} (app/)`)
+      } catch (err) {
+        console.error(`  Error indexing ${file}:`, err.message)
+      }
     }
   }
 
   if (documents.length > 0) {
     await insertMultiple(db, documents)
     console.log(`\nIndexed ${documents.length} documents`)
+  } else {
+    console.log('\nNo documents found to index')
   }
 
   // Export the index

package/dist/template/src/app/api/chat/route.ts

@@ -86,6 +86,10 @@ function indexDocs(): DocChunk[] {
   return chunks
 }
 
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
 function searchDocs(query: string, chunks: DocChunk[]): DocChunk[] {
   const queryLower = query.toLowerCase()
   const queryWords = queryLower.split(/\s+/).filter(w => w.length > 2)
@@ -113,7 +117,7 @@ function searchDocs(query: string, chunks: DocChunk[]): DocChunk[] {
 
       // Content matches (lower weight)
       queryWords.forEach(word => {
-        const matches = (contentLower.match(new RegExp(word, 'g')) || []).length
+        const matches = (contentLower.match(new RegExp(escapeRegex(word), 'g')) || []).length
         score += Math.min(matches, 5) // Cap at 5 matches per word
       })
 
@@ -231,11 +235,15 @@ export async function POST(request: Request) {
     const body = await request.json()
     const { messages } = body as { messages: Message[] }
 
-    if (!messages || messages.length === 0) {
-      return NextResponse.json({ error: 'Messages required' }, { status: 400 })
+    if (!Array.isArray(messages) || messages.length === 0 || messages.length > 50) {
+      return NextResponse.json({ error: 'Invalid request' }, { status: 400 })
     }
 
     const lastMessage = messages[messages.length - 1]
+    if (typeof lastMessage?.content !== 'string' || lastMessage.content.length > 10000) {
+      return NextResponse.json({ error: 'Message too long' }, { status: 400 })
+    }
+
     if (lastMessage.role !== 'user') {
       return NextResponse.json({ error: 'Last message must be from user' }, { status: 400 })
     }

package/dist/template/src/app/docs/README.md

@@ -0,0 +1,28 @@
+# API Documentation
+
+Generated by [skrypt](https://github.com/debgotwired/skrypt)
+
+## Summary
+
+- **Total elements:** 81
+
+## Files
+
+- [auth/index.ts](./auth/index.md) (6 elements)
+- [autofix/index.ts](./autofix/index.md) (4 elements)
+- [config/loader.ts](./config/loader.md) (4 elements)
+- [generator/generator.ts](./generator/generator.md) (3 elements)
+- [generator/organizer.ts](./generator/organizer.md) (6 elements)
+- [generator/writer.ts](./generator/writer.md) (4 elements)
+- [github/pr-comments.ts](./github/pr-comments.md) (3 elements)
+- [llm/anthropic-client.ts](./llm/anthropic-client.md) (4 elements)
+- [llm/index.ts](./llm/index.md) (3 elements)
+- [llm/openai-client.ts](./llm/openai-client.md) (4 elements)
+- [plugins/index.ts](./plugins/index.md) (14 elements)
+- [scanner/content-type.ts](./scanner/content-type.md) (4 elements)
+- [scanner/go.ts](./scanner/go.md) (3 elements)
+- [scanner/index.ts](./scanner/index.md) (2 elements)
+- [scanner/python.ts](./scanner/python.md) (3 elements)
+- [scanner/python_parser.py](./scanner/python_parser.md) (8 elements)
+- [scanner/rust.ts](./scanner/rust.md) (3 elements)
+- [scanner/typescript.ts](./scanner/typescript.md) (3 elements)

package/dist/template/src/app/docs/[...slug]/page.tsx

@@ -1,14 +1,26 @@
 import { notFound } from 'next/navigation'
-import { readFile, readdir, stat } from 'fs/promises'
+import { readFile, readdir } from 'fs/promises'
 import { join } from 'path'
 import { compileMDX } from 'next-mdx-remote/rsc'
 import remarkGfm from 'remark-gfm'
 import * as components from '@/components/mdx'
+import type { Metadata } from 'next'
 
 interface PageProps {
   params: Promise<{ slug: string[] }>
 }
 
+interface Frontmatter {
+  title?: string
+  description?: string
+}
+
+function getDocsConfig() {
+  const fs = require('fs')
+  const configPath = join(process.cwd(), 'docs.json')
+  return JSON.parse(fs.readFileSync(configPath, 'utf-8'))
+}
+
 async function getContent(slug: string[]) {
   const contentDir = join(process.cwd(), 'content', 'docs')
   const filePath = join(contentDir, ...slug)
@@ -27,6 +39,74 @@ async function getContent(slug: string[]) {
   return null
 }
 
+async function getFrontmatter(slug: string[]): Promise<Frontmatter> {
+  const result = await getContent(slug)
+  if (!result) return {}
+
+  // Simple frontmatter extraction without full MDX compilation
+  const fmMatch = result.content.match(/^---\s*\n([\s\S]*?)\n---/)
+  if (!fmMatch) return {}
+
+  const fm: Frontmatter = {}
+  const lines = fmMatch[1].split('\n')
+  for (const line of lines) {
+    const match = line.match(/^(\w+)\s*:\s*(.+)$/)
+    if (match) {
+      const key = match[1].trim()
+      let value = match[2].trim()
+      // Strip quotes
+      if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+        value = value.slice(1, -1)
+      }
+      if (key === 'title') fm.title = value
+      if (key === 'description') fm.description = value
+    }
+  }
+  return fm
+}
+
+export async function generateMetadata({ params }: PageProps): Promise<Metadata> {
+  const { slug } = await params
+  const docsConfig = getDocsConfig()
+  const siteName = docsConfig.name || 'Documentation'
+
+  const frontmatter = await getFrontmatter(slug)
+
+  // Fall back to nav config title
+  if (!frontmatter.title) {
+    const path = '/docs/' + slug.join('/')
+    for (const group of docsConfig.navigation || []) {
+      for (const page of group.pages || []) {
+        if (page.path === path) {
+          frontmatter.title = page.title
+          break
+        }
+      }
+    }
+  }
+
+  const title = frontmatter.title || slug[slug.length - 1].replace(/-/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase())
+
+  return {
+    title,
+    description: frontmatter.description || `${title} - ${siteName} documentation`,
+  }
+}
+
+/** Look up page title from docs.json navigation by pathname */
+function getNavTitle(slug: string[]): string | undefined {
+  const docsConfig = getDocsConfig()
+  const path = '/docs/' + slug.join('/')
+  for (const group of docsConfig.navigation || []) {
+    for (const page of group.pages || []) {
+      if (page.path === path) {
+        return page.title
+      }
+    }
+  }
+  return undefined
+}
+
 export default async function DocPage({ params }: PageProps) {
   const { slug } = await params
   const result = await getContent(slug)
@@ -35,22 +115,65 @@ export default async function DocPage({ params }: PageProps) {
     notFound()
   }
 
-  const { content } = await compileMDX({
-    source: result.content,
-    components: components as any,
-    options: {
-      parseFrontmatter: true,
-      mdxOptions: {
-        remarkPlugins: [remarkGfm],
+  // Resolve page title from docs.json navigation (preferred) and description from frontmatter
+  const navTitle = getNavTitle(slug)
+  const frontmatter = await getFrontmatter(slug)
+  const pageDescription = frontmatter.description
+
+  try {
+    const { content } = await compileMDX({
+      source: result.content,
+      components: components as any,
+      options: {
+        parseFrontmatter: true,
+        mdxOptions: {
+          remarkPlugins: [remarkGfm],
+        },
       },
-    },
-  })
-
-  return (
-    <div className="prose max-w-none">
-      {content}
-    </div>
-  )
+    })
+
+    // Resolve metadata for JSON-LD
+    const docsConfig = getDocsConfig()
+    const siteUrl = docsConfig.siteUrl || ''
+    const siteName = docsConfig.name || 'Documentation'
+    const pageTitle = navTitle || frontmatter.title || slug[slug.length - 1].replace(/-/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase())
+
+    const jsonLd = {
+      '@context': 'https://schema.org',
+      '@type': 'TechArticle',
+      headline: pageTitle,
+      description: pageDescription || `${pageTitle} - ${siteName} documentation`,
+      url: `${siteUrl}/docs/${slug.join('/')}`,
+      author: { '@type': 'Organization', name: siteName },
+    }
+
+    return (
+      <>
+        <script
+          type="application/ld+json"
+          dangerouslySetInnerHTML={{ __html: JSON.stringify(jsonLd) }}
+        />
+        {/*
+          Page title is resolved by DocsLayout from docs.json navigation via usePathname().
+          Description from MDX frontmatter is passed via data attribute for DocsLayout to read.
+        */}
+        {pageDescription && (
+          <div data-page-description={pageDescription} className="hidden" />
+        )}
+        <div className="prose max-w-none">
+          {content}
+        </div>
+      </>
+    )
+  } catch (error) {
+    console.error(`MDX compilation error for ${slug.join('/')}:`, error)
+    return (
+      <article className="prose">
+        <h1>Error loading page</h1>
+        <p>This documentation page could not be rendered. The MDX content may contain syntax errors.</p>
+      </article>
+    )
+  }
 }
 
 export async function generateStaticParams() {