voyageai-cli 1.15.0 → 1.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "voyageai-cli",
-  "version": "1.15.0",
+  "version": "1.16.0",
   "description": "CLI for Voyage AI embeddings, reranking, and MongoDB Atlas Vector Search",
   "bin": {
     "vai": "./src/cli.js"

package/src/cli.js

@@ -21,6 +21,8 @@ const { registerCompletions } = require('./commands/completions');
 const { registerPlayground } = require('./commands/playground');
 const { registerBenchmark } = require('./commands/benchmark');
 const { registerEstimate } = require('./commands/estimate');
+const { registerInit } = require('./commands/init');
+const { registerChunk } = require('./commands/chunk');
 const { registerAbout } = require('./commands/about');
 const { showBanner, showQuickStart, getVersion } = require('./lib/banner');
 
@@ -47,6 +49,8 @@ registerCompletions(program);
 registerPlayground(program);
 registerBenchmark(program);
 registerEstimate(program);
+registerInit(program);
+registerChunk(program);
 registerAbout(program);
 
 // Append disclaimer to all help output

package/src/commands/chunk.js

@@ -0,0 +1,277 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { chunk, estimateTokens, STRATEGIES, DEFAULTS } = require('../lib/chunker');
+const { readFile, scanDirectory, isSupported, getReaderType } = require('../lib/readers');
+const { loadProject, mergeOptions } = require('../lib/project');
+const ui = require('../lib/ui');
+
+/**
+ * Format a number with commas.
+ */
+function fmtNum(n) {
+  return n.toLocaleString('en-US');
+}
+
+/**
+ * Build chunk metadata for a source file.
+ * @param {string} filePath - Source file path
+ * @param {string} basePath - Base directory for relative paths
+ * @param {number} index - Chunk index within the file
+ * @param {number} total - Total chunks from this file
+ * @returns {object}
+ */
+function buildMetadata(filePath, basePath, index, total) {
+  return {
+    source: path.relative(basePath, filePath),
+    chunk_index: index,
+    total_chunks: total,
+  };
+}
+
+/**
+ * Register the chunk command on a Commander program.
+ * @param {import('commander').Command} program
+ */
+function registerChunk(program) {
+  program
+    .command('chunk [input]')
+    .description('Chunk documents for embedding — files, directories, or stdin')
+    .option('-s, --strategy <strategy>', `Chunking strategy: ${STRATEGIES.join(', ')}`)
+    .option('-c, --chunk-size <n>', 'Target chunk size in characters', (v) => parseInt(v, 10))
+    .option('--overlap <n>', 'Overlap between chunks in characters', (v) => parseInt(v, 10))
+    .option('--min-size <n>', 'Minimum chunk size (drop smaller)', (v) => parseInt(v, 10))
+    .option('-o, --output <path>', 'Output file (JSONL). Omit for stdout')
+    .option('--text-field <name>', 'Text field name for JSON/JSONL input', 'text')
+    .option('--extensions <exts>', 'Comma-separated file extensions to include when scanning directories')
+    .option('--ignore <dirs>', 'Comma-separated directory names to skip', 'node_modules,.git,__pycache__')
+    .option('--dry-run', 'Show what would be chunked without processing')
+    .option('--stats', 'Show chunking statistics after processing')
+    .option('--json', 'Machine-readable JSON output')
+    .option('-q, --quiet', 'Suppress non-essential output')
+    .action(async (input, opts) => {
+      try {
+        // Load project config, merge with CLI opts
+        const { config: projectConfig } = loadProject();
+        const chunkConfig = projectConfig.chunk || {};
+
+        const strategy = opts.strategy || chunkConfig.strategy || DEFAULTS.strategy || 'recursive';
+        const chunkSize = opts.chunkSize || chunkConfig.size || DEFAULTS.size;
+        const overlap = opts.overlap != null ? opts.overlap : (chunkConfig.overlap != null ? chunkConfig.overlap : DEFAULTS.overlap);
+        const minSize = opts.minSize || chunkConfig.minSize || DEFAULTS.minSize;
+        const textField = opts.textField || 'text';
+
+        if (!STRATEGIES.includes(strategy)) {
+          console.error(ui.error(`Unknown strategy: "${strategy}". Available: ${STRATEGIES.join(', ')}`));
+          process.exit(1);
+        }
+
+        // Resolve input files
+        const files = resolveInput(input, opts);
+
+        if (files.length === 0) {
+          console.error(ui.error('No supported files found. Supported types: .txt, .md, .html, .json, .jsonl, .pdf'));
+          process.exit(1);
+        }
+
+        // Dry run
+        if (opts.dryRun) {
+          if (opts.json) {
+            console.log(JSON.stringify({ files: files.map(f => path.relative(process.cwd(), f)), strategy, chunkSize, overlap }, null, 2));
+          } else {
+            console.log(ui.bold(`Would chunk ${files.length} file(s) with strategy: ${strategy}`));
+            console.log(ui.dim(`  Chunk size: ${chunkSize} chars, overlap: ${overlap} chars`));
+            console.log('');
+            for (const f of files) {
+              const size = fs.statSync(f).size;
+              console.log(`  ${ui.dim(path.relative(process.cwd(), f))} (${fmtNum(size)} bytes)`);
+            }
+          }
+          return;
+        }
+
+        // Process files
+        const basePath = input && fs.existsSync(input) && fs.statSync(input).isDirectory()
+          ? path.resolve(input)
+          : process.cwd();
+
+        const allChunks = [];
+        const fileStats = [];
+
+        const showProgress = !opts.json && !opts.quiet && files.length > 1;
+        if (showProgress) {
+          console.log(ui.bold(`Chunking ${files.length} file(s) with strategy: ${strategy}`));
+          console.log(ui.dim(`  Chunk size: ${chunkSize}, overlap: ${overlap}, min: ${minSize}`));
+          console.log('');
+        }
+
+        for (let fi = 0; fi < files.length; fi++) {
+          const filePath = files[fi];
+          const relPath = path.relative(basePath, filePath);
+          const readerType = getReaderType(filePath);
+
+          try {
+            const content = await readFile(filePath, { textField });
+
+            // readFile returns string for text/html/pdf, array for json/jsonl
+            let textsToChunk = [];
+
+            if (typeof content === 'string') {
+              textsToChunk = [{ text: content, metadata: {} }];
+            } else if (Array.isArray(content)) {
+              textsToChunk = content;
+            }
+
+            let fileChunkCount = 0;
+            for (const item of textsToChunk) {
+              const effectiveStrategy = readerType === 'text' && filePath.endsWith('.md') ? 'markdown' : strategy;
+              // Auto-detect markdown for .md files when using default strategy
+              const useStrategy = (strategy === 'recursive' && filePath.endsWith('.md')) ? 'markdown' : strategy;
+
+              const chunks = chunk(item.text, {
+                strategy: useStrategy,
+                size: chunkSize,
+                overlap,
+                minSize,
+              });
+
+              for (let ci = 0; ci < chunks.length; ci++) {
+                allChunks.push({
+                  text: chunks[ci],
+                  metadata: {
+                    ...item.metadata,
+                    ...buildMetadata(filePath, basePath, ci, chunks.length),
+                  },
+                });
+              }
+              fileChunkCount += chunks.length;
+            }
+
+            fileStats.push({
+              file: relPath,
+              inputChars: textsToChunk.reduce((sum, t) => sum + t.text.length, 0),
+              chunks: fileChunkCount,
+            });
+
+            if (showProgress) {
+              console.log(`  ${ui.green('✓')} ${relPath} → ${fileChunkCount} chunks`);
+            }
+          } catch (err) {
+            fileStats.push({ file: relPath, error: err.message, chunks: 0 });
+            if (!opts.quiet) {
+              console.error(`  ${ui.red('✗')} ${relPath}: ${err.message}`);
+            }
+          }
+        }
+
+        // Output
+        if (opts.json) {
+          const output = {
+            totalChunks: allChunks.length,
+            totalTokens: allChunks.reduce((sum, c) => sum + estimateTokens(c.text), 0),
+            strategy,
+            chunkSize,
+            overlap,
+            files: fileStats,
+            chunks: allChunks,
+          };
+          const jsonStr = JSON.stringify(output, null, 2);
+          if (opts.output) {
+            fs.writeFileSync(opts.output, jsonStr + '\n');
+          } else {
+            console.log(jsonStr);
+          }
+        } else {
+          // JSONL output
+          const lines = allChunks.map(c => JSON.stringify(c));
+          const jsonlStr = lines.join('\n') + '\n';
+
+          if (opts.output) {
+            fs.writeFileSync(opts.output, jsonlStr);
+            if (!opts.quiet) {
+              console.log('');
+              console.log(ui.success(`Wrote ${fmtNum(allChunks.length)} chunks to ${opts.output}`));
+            }
+          } else if (opts.quiet || !showProgress) {
+            // Stdout — write JSONL directly
+            process.stdout.write(jsonlStr);
+          } else {
+            // Progress was shown, write to stdout with separator
+            console.log('');
+            process.stdout.write(jsonlStr);
+          }
+        }
+
+        // Stats summary
+        if ((opts.stats || showProgress) && !opts.json) {
+          const totalChars = fileStats.reduce((sum, f) => sum + (f.inputChars || 0), 0);
+          const totalTokens = allChunks.reduce((sum, c) => sum + estimateTokens(c.text), 0);
+          const avgChunkSize = allChunks.length > 0
+            ? Math.round(allChunks.reduce((sum, c) => sum + c.text.length, 0) / allChunks.length)
+            : 0;
+          const errors = fileStats.filter(f => f.error).length;
+
+          console.log('');
+          console.log(ui.bold('Summary'));
+          console.log(ui.label('Files', `${fmtNum(files.length)}${errors ? ` (${errors} failed)` : ''}`));
+          console.log(ui.label('Input', `${fmtNum(totalChars)} chars`));
+          console.log(ui.label('Chunks', fmtNum(allChunks.length)));
+          console.log(ui.label('Avg chunk', `${fmtNum(avgChunkSize)} chars (~${fmtNum(Math.round(avgChunkSize / 4))} tokens)`));
+          console.log(ui.label('Est. tokens', `~${fmtNum(totalTokens)}`));
+
+          // Cost hint
+          const pricePerMToken = 0.12; // voyage-4-large default
+          const cost = (totalTokens / 1e6) * pricePerMToken;
+          if (cost > 0) {
+            console.log(ui.label('Est. cost', ui.dim(`~$${cost < 0.01 ? cost.toFixed(4) : cost.toFixed(2)} with voyage-4-large`)));
+          }
+        }
+      } catch (err) {
+        console.error(ui.error(err.message));
+        process.exit(1);
+      }
+    });
+}
+
+/**
+ * Resolve input to a list of file paths.
+ * @param {string} input - File path, directory path, or glob
+ * @param {object} opts
+ * @returns {string[]}
+ */
+function resolveInput(input, opts) {
+  if (!input) {
+    console.error(ui.error('Please provide a file or directory path.'));
+    console.error(ui.dim('  Usage: vai chunk <file-or-directory> [options]'));
+    process.exit(1);
+  }
+
+  const resolved = path.resolve(input);
+
+  if (!fs.existsSync(resolved)) {
+    console.error(ui.error(`Not found: ${input}`));
+    process.exit(1);
+  }
+
+  const stat = fs.statSync(resolved);
+
+  if (stat.isFile()) {
+    return [resolved];
+  }
+
+  if (stat.isDirectory()) {
+    const scanOpts = {};
+    if (opts.extensions) {
+      scanOpts.extensions = opts.extensions.split(',').map(e => e.trim());
+    }
+    if (opts.ignore) {
+      scanOpts.ignore = opts.ignore.split(',').map(d => d.trim());
+    }
+    return scanDirectory(resolved, scanOpts);
+  }
+
+  return [];
+}
+
+module.exports = { registerChunk };

package/src/commands/completions.js

@@ -19,7 +19,7 @@ _vai_completions() {
   prev="\${COMP_WORDS[COMP_CWORD-1]}"
 
   # Top-level commands
-  commands="embed rerank store search index models ping config demo explain similarity ingest estimate completions help"
+  commands="embed rerank store search index models ping config demo explain similarity ingest estimate init chunk completions help"
 
   # Subcommands
   local index_subs="create list delete"
@@ -106,6 +106,14 @@ _vai_completions() {
       COMPREPLY=( \$(compgen -W "--docs --queries --doc-tokens --query-tokens --doc-model --query-model --months --json --quiet --help" -- "\$cur") )
       return 0
       ;;
+    init)
+      COMPREPLY=( \$(compgen -W "--yes --force --json --quiet --help" -- "\$cur") )
+      return 0
+      ;;
+    chunk)
+      COMPREPLY=( \$(compgen -W "--strategy --chunk-size --overlap --min-size --output --text-field --extensions --ignore --dry-run --stats --json --quiet --help" -- "\$cur") )
+      return 0
+      ;;
     completions)
       COMPREPLY=( \$(compgen -W "bash zsh --help" -- "\$cur") )
       return 0
@@ -177,6 +185,8 @@ _vai() {
     'similarity:Compute cosine similarity between texts'
     'ingest:Bulk import documents with progress'
     'estimate:Estimate embedding costs — symmetric vs asymmetric'
+    'init:Initialize project with .vai.json'
+    'chunk:Chunk documents for embedding'
     'completions:Generate shell completion scripts'
     'help:Display help for command'
   )
@@ -392,6 +402,29 @@ _vai() {
             '--json[Machine-readable JSON output]' \\
             '(-q --quiet)'{-q,--quiet}'[Suppress non-essential output]'
           ;;
+        init)
+          _arguments \\
+            '(-y --yes)'{-y,--yes}'[Accept all defaults]' \\
+            '--force[Overwrite existing .vai.json]' \\
+            '--json[Output config as JSON]' \\
+            '(-q --quiet)'{-q,--quiet}'[Suppress non-essential output]'
+          ;;
+        chunk)
+          _arguments \\
+            '1:input:_files' \\
+            '(-s --strategy)'{-s,--strategy}'[Chunking strategy]:strategy:(fixed sentence paragraph recursive markdown)' \\
+            '(-c --chunk-size)'{-c,--chunk-size}'[Target chunk size]:size:' \\
+            '--overlap[Overlap between chunks]:chars:' \\
+            '--min-size[Minimum chunk size]:chars:' \\
+            '(-o --output)'{-o,--output}'[Output file]:file:_files' \\
+            '--text-field[Text field for JSON]:field:' \\
+            '--extensions[File extensions]:exts:' \\
+            '--ignore[Dirs to skip]:dirs:' \\
+            '--dry-run[Preview without processing]' \\
+            '--stats[Show statistics]' \\
+            '--json[JSON output]' \\
+            '(-q --quiet)'{-q,--quiet}'[Suppress non-essential output]'
+          ;;
         completions)
           _arguments \\
             '1:shell:(bash zsh)'

package/src/commands/init.js

@@ -0,0 +1,153 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const readline = require('readline');
+const { MODEL_CATALOG } = require('../lib/catalog');
+const { STRATEGIES } = require('../lib/chunker');
+const { defaultProjectConfig, saveProject, findProjectFile, PROJECT_FILE } = require('../lib/project');
+const ui = require('../lib/ui');
+
+/**
+ * Prompt the user for input with a default value.
+ * @param {readline.Interface} rl
+ * @param {string} question
+ * @param {string} [defaultVal]
+ * @returns {Promise<string>}
+ */
+function ask(rl, question, defaultVal) {
+  const suffix = defaultVal ? ` ${ui.dim(`(${defaultVal})`)}` : '';
+  return new Promise((resolve) => {
+    rl.question(`  ${question}${suffix}: `, (answer) => {
+      resolve(answer.trim() || defaultVal || '');
+    });
+  });
+}
+
+/**
+ * Prompt for a choice from a list.
+ * @param {readline.Interface} rl
+ * @param {string} question
+ * @param {string[]} choices
+ * @param {string} defaultVal
+ * @returns {Promise<string>}
+ */
+async function askChoice(rl, question, choices, defaultVal) {
+  console.log('');
+  for (let i = 0; i < choices.length; i++) {
+    const marker = choices[i] === defaultVal ? ui.cyan('→') : ' ';
+    console.log(`  ${marker} ${i + 1}. ${choices[i]}`);
+  }
+  const answer = await ask(rl, question, defaultVal);
+  // Accept number or value
+  const num = parseInt(answer, 10);
+  if (num >= 1 && num <= choices.length) return choices[num - 1];
+  if (choices.includes(answer)) return answer;
+  return defaultVal;
+}
+
+/**
+ * Register the init command on a Commander program.
+ * @param {import('commander').Command} program
+ */
+function registerInit(program) {
+  program
+    .command('init')
+    .description('Initialize a project with .vai.json configuration')
+    .option('-y, --yes', 'Accept all defaults (non-interactive)')
+    .option('--force', 'Overwrite existing .vai.json')
+    .option('--json', 'Output created config as JSON (non-interactive)')
+    .option('-q, --quiet', 'Suppress non-essential output')
+    .action(async (opts) => {
+      // Check for existing config
+      const existing = findProjectFile();
+      if (existing && !opts.force) {
+        const relPath = path.relative(process.cwd(), existing);
+        console.error(ui.warn(`Project already initialized: ${relPath}`));
+        console.error(ui.dim('  Use --force to overwrite.'));
+        process.exit(1);
+      }
+
+      const defaults = defaultProjectConfig();
+
+      // Non-interactive mode
+      if (opts.yes || opts.json) {
+        const filePath = saveProject(defaults);
+        if (opts.json) {
+          console.log(JSON.stringify(defaults, null, 2));
+        } else if (!opts.quiet) {
+          console.log(ui.success(`Created ${PROJECT_FILE}`));
+        }
+        return;
+      }
+
+      // Interactive mode
+      console.log('');
+      console.log(ui.bold('  🚀 Initialize Voyage AI Project'));
+      console.log(ui.dim('  Creates .vai.json in the current directory.'));
+      console.log(ui.dim('  Press Enter to accept defaults.'));
+      console.log('');
+
+      const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+      });
+
+      try {
+        // Embedding model
+        const embeddingModels = MODEL_CATALOG
+          .filter(m => m.type === 'embedding' && !m.legacy && !m.unreleased)
+          .map(m => m.name);
+        const model = await askChoice(rl, 'Embedding model', embeddingModels, defaults.model);
+
+        // MongoDB settings
+        console.log('');
+        console.log(ui.bold('  MongoDB Atlas'));
+        const db = await ask(rl, 'Database name', defaults.db || 'myapp');
+        const collection = await ask(rl, 'Collection name', defaults.collection || 'documents');
+        const field = await ask(rl, 'Embedding field', defaults.field);
+        const index = await ask(rl, 'Vector index name', defaults.index);
+
+        // Dimensions
+        const modelInfo = MODEL_CATALOG.find(m => m.name === model);
+        const defaultDims = modelInfo && modelInfo.dimensions.includes('1024') ? '1024' : '512';
+        const dimensions = parseInt(await ask(rl, 'Dimensions', defaultDims), 10) || parseInt(defaultDims, 10);
+
+        // Chunking
+        console.log('');
+        console.log(ui.bold('  Chunking'));
+        const strategy = await askChoice(rl, 'Chunk strategy', STRATEGIES, defaults.chunk.strategy);
+        const chunkSize = parseInt(await ask(rl, 'Chunk size (chars)', String(defaults.chunk.size)), 10);
+        const chunkOverlap = parseInt(await ask(rl, 'Chunk overlap (chars)', String(defaults.chunk.overlap)), 10);
+
+        const config = {
+          model,
+          db,
+          collection,
+          field,
+          inputType: 'document',
+          dimensions,
+          index,
+          chunk: {
+            strategy,
+            size: chunkSize,
+            overlap: chunkOverlap,
+          },
+        };
+
+        const filePath = saveProject(config);
+        console.log('');
+        console.log(ui.success(`Created ${path.relative(process.cwd(), filePath)}`));
+        console.log('');
+        console.log(ui.dim('  Next steps:'));
+        console.log(ui.dim('    vai chunk ./docs/           # Chunk your documents'));
+        console.log(ui.dim('    vai pipeline ./docs/        # Chunk → embed → store (coming soon)'));
+        console.log(ui.dim('    vai search --query "..."    # Search your collection'));
+        console.log('');
+      } finally {
+        rl.close();
+      }
+    });
+}
+
+module.exports = { registerInit };

package/src/lib/chunker.js

@@ -0,0 +1,341 @@
+'use strict';
+
+/**
+ * Available chunking strategies.
+ */
+const STRATEGIES = ['fixed', 'sentence', 'paragraph', 'recursive', 'markdown'];
+
+/**
+ * Default chunk options.
+ */
+const DEFAULTS = {
+  size: 512,
+  overlap: 50,
+  minSize: 20,
+};
+
+// ── Sentence splitting ──
+
+/**
+ * Split text into sentences. Handles common abbreviations and edge cases.
+ * @param {string} text
+ * @returns {string[]}
+ */
+function splitSentences(text) {
+  // Split on sentence-ending punctuation followed by whitespace or EOL.
+  // Negative lookbehind for common abbreviations (Mr., Dr., etc.)
+  const parts = text.split(/(?<=[.!?])\s+(?=[A-Z\u00C0-\u024F"])/);
+  return parts.map(s => s.trim()).filter(s => s.length > 0);
+}
+
+// ── Strategy implementations ──
+
+/**
+ * Fixed-size chunking with character count and overlap.
+ * @param {string} text
+ * @param {object} opts
+ * @param {number} opts.size - Target chunk size in characters
+ * @param {number} opts.overlap - Overlap between chunks in characters
+ * @returns {string[]}
+ */
+function chunkFixed(text, opts) {
+  const { size, overlap } = opts;
+  const chunks = [];
+  let start = 0;
+
+  while (start < text.length) {
+    const end = start + size;
+    chunks.push(text.slice(start, end).trim());
+    start = end - overlap;
+    if (start >= text.length) break;
+    // Prevent infinite loop with tiny overlap
+    if (end >= text.length) break;
+  }
+
+  return chunks.filter(c => c.length >= (opts.minSize || DEFAULTS.minSize));
+}
+
+/**
+ * Sentence-boundary chunking. Groups sentences until size limit.
+ * @param {string} text
+ * @param {object} opts
+ * @returns {string[]}
+ */
+function chunkSentence(text, opts) {
+  const { size, overlap } = opts;
+  const sentences = splitSentences(text);
+  return groupUnits(sentences, size, overlap, opts.minSize || DEFAULTS.minSize);
+}
+
+/**
+ * Paragraph chunking. Splits on double newlines, groups if needed.
+ * @param {string} text
+ * @param {object} opts
+ * @returns {string[]}
+ */
+function chunkParagraph(text, opts) {
+  const { size, overlap } = opts;
+  const paragraphs = text.split(/\n\s*\n/).map(p => p.trim()).filter(p => p.length > 0);
+  return groupUnits(paragraphs, size, overlap, opts.minSize || DEFAULTS.minSize);
+}
+
+/**
+ * Recursive chunking. Tries largest delimiters first, falls back to smaller.
+ * This is the most commonly used strategy for RAG pipelines.
+ * @param {string} text
+ * @param {object} opts
+ * @returns {string[]}
+ */
+function chunkRecursive(text, opts) {
+  const { size, minSize } = opts;
+  const separators = ['\n\n', '\n', '. ', '! ', '? ', '; ', ', ', ' '];
+
+  return recursiveSplit(text, separators, size, minSize || DEFAULTS.minSize);
+}
+
+/**
+ * Internal recursive split implementation.
+ * @param {string} text
+ * @param {string[]} separators
+ * @param {number} maxSize
+ * @param {number} minSize
+ * @returns {string[]}
+ */
+function recursiveSplit(text, separators, maxSize, minSize) {
+  if (text.length <= maxSize) {
+    return text.trim().length >= minSize ? [text.trim()] : [];
+  }
+
+  // Find the first separator that exists in the text
+  let sep = null;
+  for (const s of separators) {
+    if (text.includes(s)) {
+      sep = s;
+      break;
+    }
+  }
+
+  // If no separator found, hard-split by characters
+  if (sep === null) {
+    const chunks = [];
+    for (let i = 0; i < text.length; i += maxSize) {
+      const chunk = text.slice(i, i + maxSize).trim();
+      if (chunk.length >= minSize) chunks.push(chunk);
+    }
+    return chunks;
+  }
+
+  // Split on this separator and greedily merge pieces under maxSize
+  const parts = text.split(sep);
+  const chunks = [];
+  let current = '';
+
+  for (const part of parts) {
+    const candidate = current ? current + sep + part : part;
+
+    if (candidate.length <= maxSize) {
+      current = candidate;
+    } else {
+      // Flush current chunk
+      if (current.trim().length >= minSize) {
+        chunks.push(current.trim());
+      }
+      // If this single part exceeds maxSize, recurse with next separator level
+      if (part.length > maxSize) {
+        const remainingSeps = separators.slice(separators.indexOf(sep) + 1);
+        chunks.push(...recursiveSplit(part, remainingSeps, maxSize, minSize));
+        current = '';
+      } else {
+        current = part;
+      }
+    }
+  }
+
+  // Flush remainder
+  if (current.trim().length >= minSize) {
+    chunks.push(current.trim());
+  }
+
+  return chunks;
+}
+
+/**
+ * Markdown-aware chunking. Splits on headings, preserves structure.
+ * Each heading starts a new chunk; content under it is grouped.
+ * @param {string} text
+ * @param {object} opts
+ * @returns {string[]}
+ */
+function chunkMarkdown(text, opts) {
+  const { size, minSize } = opts;
+
+  // Split on markdown headings (# through ######)
+  const headingPattern = /^(#{1,6}\s.+)$/gm;
+  const sections = [];
+  let lastIndex = 0;
+  let match;
+
+  while ((match = headingPattern.exec(text)) !== null) {
+    // Content before this heading
+    if (match.index > lastIndex) {
+      const content = text.slice(lastIndex, match.index).trim();
+      if (content) {
+        if (sections.length > 0) {
+          // Append to previous section
+          sections[sections.length - 1].content += '\n\n' + content;
+        } else {
+          sections.push({ heading: '', content });
+        }
+      }
+    }
+    sections.push({ heading: match[1], content: '' });
+    lastIndex = match.index + match[0].length;
+  }
+
+  // Remaining content after last heading
+  if (lastIndex < text.length) {
+    const content = text.slice(lastIndex).trim();
+    if (content) {
+      if (sections.length > 0) {
+        sections[sections.length - 1].content += '\n\n' + content;
+      } else {
+        sections.push({ heading: '', content });
+      }
+    }
+  }
+
+  // Build chunks from sections, splitting large sections recursively
+  const chunks = [];
+  for (const section of sections) {
+    const full = section.heading
+      ? section.heading + '\n\n' + section.content.trim()
+      : section.content.trim();
+
+    if (!full || full.length < (minSize || DEFAULTS.minSize)) continue;
+
+    if (full.length <= size) {
+      chunks.push(full);
+    } else {
+      // Section too large — recursively split the content, prepend heading to first chunk
+      const subChunks = chunkRecursive(section.content.trim(), opts);
+      for (let i = 0; i < subChunks.length; i++) {
+        if (i === 0 && section.heading) {
+          chunks.push(section.heading + '\n\n' + subChunks[i]);
+        } else {
+          chunks.push(subChunks[i]);
+        }
+      }
+    }
+  }
+
+  return chunks;
+}
+
+// ── Shared helpers ──
+
+/**
+ * Group text units (sentences, paragraphs) into chunks under a size limit.
+ * Supports overlap by re-including trailing units from the previous chunk.
+ * @param {string[]} units
+ * @param {number} maxSize
+ * @param {number} overlapChars
+ * @param {number} minSize
+ * @returns {string[]}
+ */
+function groupUnits(units, maxSize, overlapChars, minSize) {
+  const chunks = [];
+  let current = [];
+  let currentLen = 0;
+
+  for (const unit of units) {
+    const addLen = current.length > 0 ? unit.length + 1 : unit.length; // +1 for space
+
+    if (currentLen + addLen > maxSize && current.length > 0) {
+      chunks.push(current.join(' ').trim());
+
+      // Overlap: keep trailing units that fit within overlap budget
+      if (overlapChars > 0) {
+        let overlapUnits = [];
+        let overlapLen = 0;
+        for (let i = current.length - 1; i >= 0; i--) {
+          if (overlapLen + current[i].length + 1 > overlapChars) break;
+          overlapUnits.unshift(current[i]);
+          overlapLen += current[i].length + 1;
+        }
+        current = overlapUnits;
+        currentLen = overlapLen;
+      } else {
+        current = [];
+        currentLen = 0;
+      }
+    }
+
+    current.push(unit);
+    currentLen += addLen;
+  }
+
+  // Flush remainder
+  if (current.length > 0) {
+    const text = current.join(' ').trim();
+    if (text.length >= minSize) chunks.push(text);
+  }
+
+  return chunks;
+}
+
+// ── Token estimation ──
+
+/**
+ * Rough token estimate. ~4 chars per token for English text.
+ * @param {string} text
+ * @returns {number}
+ */
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+
+// ── Public API ──
+
+/**
+ * Chunk text using the specified strategy.
+ * @param {string} text - Input text
+ * @param {object} [options]
+ * @param {string} [options.strategy='recursive'] - Chunking strategy
+ * @param {number} [options.size=512] - Target chunk size in characters
+ * @param {number} [options.overlap=50] - Overlap between chunks in characters
+ * @param {number} [options.minSize=20] - Minimum chunk size
+ * @returns {string[]} Array of text chunks
+ */
+function chunk(text, options = {}) {
+  const opts = {
+    strategy: options.strategy || 'recursive',
+    size: options.size || DEFAULTS.size,
+    overlap: options.overlap != null ? options.overlap : DEFAULTS.overlap,
+    minSize: options.minSize || DEFAULTS.minSize,
+  };
+
+  if (!text || text.trim().length === 0) return [];
+
+  switch (opts.strategy) {
+    case 'fixed':
+      return chunkFixed(text, opts);
+    case 'sentence':
+      return chunkSentence(text, opts);
+    case 'paragraph':
+      return chunkParagraph(text, opts);
+    case 'recursive':
+      return chunkRecursive(text, opts);
+    case 'markdown':
+      return chunkMarkdown(text, opts);
+    default:
+      throw new Error(`Unknown chunking strategy: ${opts.strategy}. Available: ${STRATEGIES.join(', ')}`);
+  }
+}
+
+module.exports = {
+  chunk,
+  splitSentences,
+  estimateTokens,
+  STRATEGIES,
+  DEFAULTS,
+};

package/src/lib/project.js

@@ -0,0 +1,122 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const PROJECT_FILE = '.vai.json';
+const PROJECT_VERSION = 1;
+
+/**
+ * Search for .vai.json starting from startDir, walking up to root.
+ * @param {string} [startDir] - Directory to start from (default: cwd)
+ * @returns {string|null} Absolute path to .vai.json or null
+ */
+function findProjectFile(startDir) {
+  let dir = path.resolve(startDir || process.cwd());
+  const root = path.parse(dir).root;
+
+  while (dir !== root) {
+    const candidate = path.join(dir, PROJECT_FILE);
+    if (fs.existsSync(candidate)) return candidate;
+    dir = path.dirname(dir);
+  }
+  // Check root too
+  const rootCandidate = path.join(root, PROJECT_FILE);
+  if (fs.existsSync(rootCandidate)) return rootCandidate;
+
+  return null;
+}
+
+/**
+ * Load project config from .vai.json.
+ * @param {string} [startDir] - Directory to start searching from
+ * @returns {{ config: object, filePath: string|null }}
+ */
+function loadProject(startDir) {
+  const filePath = findProjectFile(startDir);
+  if (!filePath) return { config: {}, filePath: null };
+
+  try {
+    const raw = fs.readFileSync(filePath, 'utf-8');
+    return { config: JSON.parse(raw), filePath };
+  } catch (err) {
+    return { config: {}, filePath };
+  }
+}
+
+/**
+ * Save project config to .vai.json.
+ * @param {object} config - Project configuration
+ * @param {string} [targetPath] - Path to write (default: cwd/.vai.json)
+ */
+function saveProject(config, targetPath) {
+  const filePath = targetPath || path.join(process.cwd(), PROJECT_FILE);
+  const output = { version: PROJECT_VERSION, ...config };
+  fs.writeFileSync(filePath, JSON.stringify(output, null, 2) + '\n', 'utf-8');
+  return filePath;
+}
+
+/**
+ * Merge project config with CLI options. CLI options take precedence.
+ * Only merges known keys — doesn't blindly spread everything.
+ * @param {object} projectConfig - From .vai.json
+ * @param {object} cliOpts - From commander
+ * @returns {object} Merged options
+ */
+function mergeOptions(projectConfig, cliOpts) {
+  const merged = {};
+
+  // Map of project config keys → CLI option keys
+  const keys = [
+    'model', 'db', 'collection', 'field', 'inputType',
+    'dimensions', 'index',
+  ];
+
+  for (const key of keys) {
+    // CLI explicit value wins, then project config, then undefined
+    if (cliOpts[key] !== undefined) {
+      merged[key] = cliOpts[key];
+    } else if (projectConfig[key] !== undefined) {
+      merged[key] = projectConfig[key];
+    }
+  }
+
+  // Chunk config nests under project.chunk
+  if (projectConfig.chunk) {
+    merged.chunk = { ...projectConfig.chunk };
+  }
+
+  return merged;
+}
+
+/**
+ * Default project config scaffold.
+ * @returns {object}
+ */
+function defaultProjectConfig() {
+  return {
+    version: PROJECT_VERSION,
+    model: 'voyage-4-large',
+    db: '',
+    collection: '',
+    field: 'embedding',
+    inputType: 'document',
+    dimensions: 1024,
+    index: 'vector_index',
+    chunk: {
+      strategy: 'recursive',
+      size: 512,
+      overlap: 50,
+    },
+  };
+}
+
+module.exports = {
+  PROJECT_FILE,
+  PROJECT_VERSION,
+  findProjectFile,
+  loadProject,
+  saveProject,
+  mergeOptions,
+  defaultProjectConfig,
+};

package/src/lib/readers.js

@@ -0,0 +1,239 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Supported file extensions and their reader types.
+ */
+const SUPPORTED_EXTENSIONS = {
+  '.txt': 'text',
+  '.md': 'text',
+  '.markdown': 'text',
+  '.rst': 'text',
+  '.html': 'html',
+  '.htm': 'html',
+  '.json': 'json',
+  '.jsonl': 'jsonl',
+  '.ndjson': 'jsonl',
+  '.csv': 'text',
+  '.pdf': 'pdf',
+};
+
+/**
+ * Check if a file extension is supported.
+ * @param {string} filePath
+ * @returns {boolean}
+ */
+function isSupported(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return ext in SUPPORTED_EXTENSIONS;
+}
+
+/**
+ * Get the reader type for a file.
+ * @param {string} filePath
+ * @returns {string|null}
+ */
+function getReaderType(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return SUPPORTED_EXTENSIONS[ext] || null;
+}
+
+/**
+ * Read a text file (txt, md, rst, csv).
+ * @param {string} filePath
+ * @returns {Promise<string>}
+ */
+async function readTextFile(filePath) {
+  return fs.readFileSync(filePath, 'utf-8');
+}
+
+/**
+ * Read an HTML file and strip tags to plain text.
+ * Lightweight — no external dependencies.
+ * @param {string} filePath
+ * @returns {Promise<string>}
+ */
+async function readHtmlFile(filePath) {
+  const html = fs.readFileSync(filePath, 'utf-8');
+  return stripHtml(html);
+}
+
+/**
+ * Strip HTML tags and decode common entities.
+ * @param {string} html
+ * @returns {string}
+ */
+function stripHtml(html) {
+  return html
+    // Remove script and style blocks
+    .replace(/<script[\s\S]*?<\/script>/gi, '')
+    .replace(/<style[\s\S]*?<\/style>/gi, '')
+    // Replace block elements with newlines
+    .replace(/<\/?(p|div|br|h[1-6]|li|tr|blockquote|section|article|header|footer|nav|pre)[^>]*>/gi, '\n')
+    // Remove remaining tags
+    .replace(/<[^>]+>/g, '')
+    // Decode common entities
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    // Collapse whitespace
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+/**
+ * Read a JSON file. Extracts text from objects using a text field.
+ * Supports JSON array of objects or a single object with a text field.
+ * @param {string} filePath
+ * @param {string} [textField='text'] - Field name containing text
+ * @returns {Promise<Array<{text: string, metadata: object}>>}
+ */
+async function readJsonFile(filePath, textField = 'text') {
+  const raw = fs.readFileSync(filePath, 'utf-8');
+  const data = JSON.parse(raw);
+
+  if (Array.isArray(data)) {
+    return data.map((item, i) => {
+      const text = item[textField];
+      if (!text) throw new Error(`Missing "${textField}" field in array item ${i}`);
+      const metadata = { ...item };
+      delete metadata[textField];
+      return { text, metadata };
+    });
+  }
+
+  if (typeof data === 'object' && data[textField]) {
+    const metadata = { ...data };
+    delete metadata[textField];
+    return [{ text: data[textField], metadata }];
+  }
+
+  throw new Error(`JSON file must be an array of objects or an object with a "${textField}" field`);
+}
+
+/**
+ * Read a JSONL/NDJSON file.
+ * @param {string} filePath
+ * @param {string} [textField='text']
+ * @returns {Promise<Array<{text: string, metadata: object}>>}
+ */
+async function readJsonlFile(filePath, textField = 'text') {
+  const raw = fs.readFileSync(filePath, 'utf-8');
+  const lines = raw.split('\n').filter(l => l.trim().length > 0);
+
+  return lines.map((line, i) => {
+    const item = JSON.parse(line);
+    const text = item[textField];
+    if (!text) throw new Error(`Missing "${textField}" field on line ${i + 1}`);
+    const metadata = { ...item };
+    delete metadata[textField];
+    return { text, metadata };
+  });
+}
+
+/**
+ * Read a PDF file. Requires optional `pdf-parse` dependency.
+ * @param {string} filePath
+ * @returns {Promise<string>}
+ */
+async function readPdfFile(filePath) {
+  let pdfParse;
+  try {
+    pdfParse = require('pdf-parse');
+  } catch {
+    throw new Error(
+      'PDF support requires the "pdf-parse" package.\n' +
+      'Install it: npm install pdf-parse\n' +
+      'Then retry your command.'
+    );
+  }
+  const buffer = fs.readFileSync(filePath);
+  const data = await pdfParse(buffer);
+  return data.text;
+}
+
+/**
+ * Read a single file and return its text content.
+ * For structured files (JSON/JSONL), returns array of {text, metadata}.
+ * For text files, returns the raw text string.
+ * @param {string} filePath
+ * @param {object} [opts]
+ * @param {string} [opts.textField='text'] - Field name for JSON/JSONL
+ * @returns {Promise<string|Array<{text: string, metadata: object}>>}
+ */
+async function readFile(filePath, opts = {}) {
+  const type = getReaderType(filePath);
+  if (!type) {
+    throw new Error(`Unsupported file type: ${path.extname(filePath)}. Supported: ${Object.keys(SUPPORTED_EXTENSIONS).join(', ')}`);
+  }
+
+  switch (type) {
+    case 'text':
+      return readTextFile(filePath);
+    case 'html':
+      return readHtmlFile(filePath);
+    case 'json':
+      return readJsonFile(filePath, opts.textField || 'text');
+    case 'jsonl':
+      return readJsonlFile(filePath, opts.textField || 'text');
+    case 'pdf':
+      return readPdfFile(filePath);
+    default:
+      throw new Error(`No reader for type: ${type}`);
+  }
+}
+
+/**
+ * Recursively scan a directory for supported files.
+ * @param {string} dirPath
+ * @param {object} [opts]
+ * @param {string[]} [opts.extensions] - Filter to specific extensions
+ * @param {string[]} [opts.ignore] - Directory names to skip
+ * @returns {string[]} Array of absolute file paths
+ */
+function scanDirectory(dirPath, opts = {}) {
+  const ignore = new Set(opts.ignore || ['node_modules', '.git', '.vai', '__pycache__', '.DS_Store']);
+  const extensions = opts.extensions
+    ? new Set(opts.extensions.map(e => e.startsWith('.') ? e : '.' + e))
+    : null;
+
+  const results = [];
+
+  function walk(dir) {
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.name.startsWith('.') && ignore.has(entry.name)) continue;
+      if (ignore.has(entry.name)) continue;
+
+      const fullPath = path.join(dir, entry.name);
+
+      if (entry.isDirectory()) {
+        walk(fullPath);
+      } else if (entry.isFile()) {
+        const ext = path.extname(entry.name).toLowerCase();
+        if (extensions) {
+          if (extensions.has(ext)) results.push(fullPath);
+        } else if (SUPPORTED_EXTENSIONS[ext]) {
+          results.push(fullPath);
+        }
+      }
+    }
+  }
+
+  walk(path.resolve(dirPath));
+  return results.sort();
+}
+
+module.exports = {
+  SUPPORTED_EXTENSIONS,
+  isSupported,
+  getReaderType,
+  readFile,
+  scanDirectory,
+  stripHtml,
+};