skrypt-ai 0.4.2 → 0.5.0

package/dist/template/scripts/build-search-index.mjs

@@ -19,13 +19,15 @@ async function dirExists(dir) {
 async function getAllMDXFiles(dir) {
   const files = []
 
-  async function walk(currentDir) {
+  async function walk(currentDir, depth = 0) {
+    if (depth > 20) return
     try {
       const entries = await readdir(currentDir, { withFileTypes: true })
       for (const entry of entries) {
         const fullPath = join(currentDir, entry.name)
+        if (entry.isSymbolicLink?.()) continue
         if (entry.isDirectory()) {
-          await walk(fullPath)
+          await walk(fullPath, depth + 1)
         } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
           files.push(fullPath)
         }
@@ -39,14 +41,30 @@ async function getAllMDXFiles(dir) {
   return files
 }
 
+/**
+ * Extract headings (h2, h3) as a separate searchable field.
+ * This lets users find pages by section names.
+ */
+function extractHeadings(content) {
+  const headings = []
+  const regex = /^#{2,3}\s+(.+)$/gm
+  let match
+  while ((match = regex.exec(content)) !== null) {
+    // Strip any inline formatting
+    const clean = match[1].replace(/[*_`[\]]/g, '').trim()
+    if (clean) headings.push(clean)
+  }
+  return headings.join(' | ')
+}
+
 function extractPlainText(content) {
   return content
     // Remove import statements
     .replace(/^import\s+.*$/gm, '')
     // Remove export statements
     .replace(/^export\s+.*$/gm, '')
-    // Remove MDX/JSX components
-    .replace(/<[^>]+>/g, '')
+    // Remove MDX/JSX components (but keep text content inside)
+    .replace(/<[^>]+>/g, ' ')
     // Remove code blocks
     .replace(/```[\s\S]*?```/g, '')
     .replace(/`[^`]+`/g, '')
@@ -68,6 +86,35 @@ function extractPlainText(content) {
     .trim()
 }
 
+/**
+ * Extract keywords from content — important terms that appear frequently
+ * or in headings, used to boost search relevance.
+ */
+function extractKeywords(content, title) {
+  const words = new Map()
+
+  // Title words get high weight
+  for (const word of title.toLowerCase().split(/\s+/)) {
+    if (word.length > 2) words.set(word, (words.get(word) || 0) + 3)
+  }
+
+  // Heading words get medium weight
+  const headingRegex = /^#{1,3}\s+(.+)$/gm
+  let match
+  while ((match = headingRegex.exec(content)) !== null) {
+    for (const word of match[1].toLowerCase().replace(/[^a-z0-9\s]/g, '').split(/\s+/)) {
+      if (word.length > 2) words.set(word, (words.get(word) || 0) + 2)
+    }
+  }
+
+  // Sort by weight and return top keywords
+  return [...words.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 20)
+    .map(([word]) => word)
+    .join(' ')
+}
+
 function getSlugFromContentPath(filePath) {
   const rel = relative(CONTENT_DIR, filePath)
   const slug = rel
@@ -79,8 +126,6 @@ function getSlugFromContentPath(filePath) {
 }
 
 function getSlugFromAppPath(filePath) {
-  // For App Router: src/app/docs/quickstart/page.mdx -> /docs/quickstart
-  // src/app/docs/page.mdx -> /docs
   const rel = relative(APP_DOCS_DIR, filePath)
   const dir = dirname(rel)
   const name = basename(rel)
@@ -99,11 +144,9 @@ function getSlugFromAppPath(filePath) {
 }
 
 function extractTitle(content, filePath) {
-  // Try to get title from first heading
   const h1Match = content.match(/^#\s+(.+)$/m)
   if (h1Match) return h1Match[1].trim()
 
-  // Derive from file path
   const dir = dirname(filePath)
   const folderName = basename(dir)
   if (folderName && folderName !== 'docs' && folderName !== '.') {
@@ -122,10 +165,18 @@ async function buildSearchIndex() {
     schema: {
       id: 'string',
       title: 'string',
+      headings: 'string',
+      keywords: 'string',
       content: 'string',
       href: 'string',
       section: 'string',
     },
+    components: {
+      tokenizer: {
+        stemming: true,
+        language: 'english',
+      },
+    },
   })
 
   const documents = []
@@ -141,6 +192,8 @@ async function buildSearchIndex() {
 
         const title = data.title || extractTitle(content, file)
         const plainContent = extractPlainText(content)
+        const headings = extractHeadings(content)
+        const keywords = extractKeywords(content, title)
         const href = getSlugFromContentPath(file)
 
         if (seen.has(href)) continue
@@ -149,6 +202,8 @@ async function buildSearchIndex() {
         documents.push({
           id: href,
           title,
+          headings,
+          keywords,
           content: plainContent.slice(0, 5000),
           href,
           section: data.section || data.category || '',
@@ -165,7 +220,6 @@ async function buildSearchIndex() {
   if (await dirExists(APP_DOCS_DIR)) {
     const appFiles = await getAllMDXFiles(APP_DOCS_DIR)
     for (const file of appFiles) {
-      // Skip catch-all route files and layout files
       if (file.includes('[...slug]') || file.includes('layout.')) continue
 
       try {
@@ -174,6 +228,8 @@ async function buildSearchIndex() {
 
         const title = data.title || extractTitle(content, file)
         const plainContent = extractPlainText(content)
+        const headings = extractHeadings(content)
+        const keywords = extractKeywords(content, title)
         const href = getSlugFromAppPath(file)
 
         if (seen.has(href)) continue
@@ -182,6 +238,8 @@ async function buildSearchIndex() {
         documents.push({
           id: href,
           title,
+          headings,
+          keywords,
           content: plainContent.slice(0, 5000),
           href,
           section: data.section || data.category || '',

package/dist/template/src/lib/search-types.ts

@@ -1,6 +1,5 @@
 /**
  * Type definitions for Orama search database
- * Fixes P0: Removes `any` types from search functionality
  */
 
 import type { Orama, Results, SearchParams } from '@orama/orama'
@@ -9,6 +8,8 @@ import type { Orama, Results, SearchParams } from '@orama/orama'
 export interface SearchDocument {
   id: string
   title: string
+  headings: string
+  keywords: string
   content: string
   href: string
   section: string
@@ -18,6 +19,8 @@ export interface SearchDocument {
 export type SearchDatabase = Orama<{
   id: 'string'
   title: 'string'
+  headings: 'string'
+  keywords: 'string'
   content: 'string'
   href: 'string'
   section: 'string'

package/dist/template/src/lib/search.ts

@@ -7,6 +7,8 @@ let loadPromise: Promise<void> | null = null
 const schema = {
   id: 'string' as const,
   title: 'string' as const,
+  headings: 'string' as const,
+  keywords: 'string' as const,
   content: 'string' as const,
   href: 'string' as const,
   section: 'string' as const,
@@ -41,7 +43,15 @@ async function loadSearchIndex(): Promise<void> {
         return
       }
 
-      const newDb = await create({ schema }) as SearchDatabase
+      const newDb = await create({
+        schema,
+        components: {
+          tokenizer: {
+            stemming: true,
+            language: 'english',
+          },
+        },
+      }) as SearchDatabase
       await load(newDb, data as RawData)
       db = newDb
     } catch (err) {
@@ -55,10 +65,21 @@ async function loadSearchIndex(): Promise<void> {
 }
 
 /**
- * Generate a snippet with highlighted search terms
+ * Generate a snippet with highlighted search terms, preferring heading matches
  */
-function generateSnippet(content: string, query: string, maxLength: number = 150): string {
+function generateSnippet(content: string, headings: string, query: string, maxLength: number = 150): string {
   const terms = query.toLowerCase().split(/\s+/).filter(Boolean)
+
+  // Check if any headings match — show that context first
+  if (headings) {
+    const headingList = headings.split(' | ')
+    for (const heading of headingList) {
+      if (terms.some(term => heading.toLowerCase().includes(term))) {
+        return heading
+      }
+    }
+  }
+
   const contentLower = content.toLowerCase()
 
   // Find the best position to start the snippet (where most terms match)
@@ -110,11 +131,13 @@ export async function search(query: string): Promise<SearchResultWithHighlight[]
   try {
     const results = await oramaSearch(db, {
       term: query,
-      properties: ['title', 'content'],
+      properties: ['title', 'headings', 'keywords', 'content'],
       limit: 10,
-      tolerance: 1,
+      tolerance: 2,
       boost: {
-        title: 2,
+        title: 5,
+        headings: 3,
+        keywords: 2,
         content: 1,
       },
     })
@@ -124,7 +147,7 @@ export async function search(query: string): Promise<SearchResultWithHighlight[]
       href: hit.document.href,
       content: hit.document.content,
       section: hit.document.section || undefined,
-      snippet: generateSnippet(hit.document.content, query),
+      snippet: generateSnippet(hit.document.content, hit.document.headings, query),
       score: hit.score,
     }))
   } catch (err) {

package/dist/utils/files.d.ts

@@ -1,9 +1,17 @@
 /**
  * Recursively find all .md and .mdx files in a directory.
- * Skips hidden directories and node_modules.
+ * Skips hidden directories, node_modules, and symlinks.
  */
 export declare function findMdxFiles(dir: string): string[];
 /**
  * Convert string to URL-safe slug
  */
 export declare function slugify(str: string): string;
+/**
+ * Simple YAML frontmatter parser — splits on --- markers.
+ * Returns parsed key-value data and remaining content body.
+ */
+export declare function parseFrontmatter(content: string): {
+    data: Record<string, unknown>;
+    content: string;
+};

package/dist/utils/files.js

@@ -1,25 +1,41 @@
-import { readdirSync, statSync } from 'fs';
+import { readdirSync, statSync, lstatSync } from 'fs';
 import { join, extname } from 'path';
 /**
  * Recursively find all .md and .mdx files in a directory.
- * Skips hidden directories and node_modules.
+ * Skips hidden directories, node_modules, and symlinks.
  */
 export function findMdxFiles(dir) {
     const files = [];
-    function walk(currentDir) {
-        const entries = readdirSync(currentDir);
+    function walk(currentDir, depth) {
+        if (depth > 30)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(currentDir);
+        }
+        catch {
+            return;
+        }
         for (const entry of entries) {
             const fullPath = join(currentDir, entry);
-            const stat = statSync(fullPath);
-            if (stat.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') {
-                walk(fullPath);
+            try {
+                // Skip symlinks to prevent infinite loops
+                if (lstatSync(fullPath).isSymbolicLink())
+                    continue;
+                const stat = statSync(fullPath);
+                if (stat.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') {
+                    walk(fullPath, depth + 1);
+                }
+                else if (stat.isFile() && (extname(entry) === '.mdx' || extname(entry) === '.md')) {
+                    files.push(fullPath);
+                }
             }
-            else if (stat.isFile() && (extname(entry) === '.mdx' || extname(entry) === '.md')) {
-                files.push(fullPath);
+            catch {
+                continue;
             }
         }
     }
-    walk(dir);
+    walk(dir, 0);
     return files;
 }
 /**
@@ -31,3 +47,36 @@ export function slugify(str) {
         .replace(/[^a-z0-9]+/g, '-')
         .replace(/^-|-$/g, '');
 }
+/**
+ * Simple YAML frontmatter parser — splits on --- markers.
+ * Returns parsed key-value data and remaining content body.
+ */
+export function parseFrontmatter(content) {
+    const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
+    if (!match)
+        return { data: {}, content };
+    const yamlStr = match[1];
+    const body = match[2];
+    const data = {};
+    for (const line of yamlStr.split('\n')) {
+        const kvMatch = line.match(/^(\w[\w-]*)\s*:\s*(.*)$/);
+        if (!kvMatch)
+            continue;
+        const key = kvMatch[1];
+        let value = kvMatch[2].trim();
+        if (typeof value === 'string') {
+            if ((value.startsWith('"') && value.endsWith('"')) ||
+                (value.startsWith("'") && value.endsWith("'"))) {
+                value = value.slice(1, -1);
+            }
+            else if (value === 'true')
+                value = true;
+            else if (value === 'false')
+                value = false;
+            else if (/^\d+$/.test(value))
+                value = parseInt(value, 10);
+        }
+        data[key] = value;
+    }
+    return { data, content: body };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skrypt-ai",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "AI-powered documentation generator with code examples",
   "type": "module",
   "main": "dist/cli.js",
@@ -54,6 +54,9 @@
     "openai": "^6.27.0",
     "typescript": "^5.9.3"
   },
+  "optionalDependencies": {
+    "@napi-rs/keyring": "^1.1.6"
+  },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
     "@types/js-yaml": "^4.0.9",