pplx-zero 2.2.2 → 2.3.1

package/README.md

@@ -32,6 +32,7 @@ Query [Perplexity AI](https://perplexity.ai) directly from your terminal. Respon
 - **📝 Export** — Save research to markdown
 - **🎨 Pretty** — Rendered markdown by default
 - **🕐 History** — Browse and search past queries
+- **🧠 Local RAG** — Index your own notes with `--ingest`
 
 ## Install
 
@@ -78,6 +79,11 @@ pplx --raw "explain monads"
 
 # history
 pplx --history
+
+# local knowledge base
+pplx --ingest notes.md           # index a file
+pplx --ingest ./docs/             # index a directory
+pplx -l "my notes on rust"        # search local knowledge
 ```
 
 ## Models
@@ -98,13 +104,15 @@ pplx --history
 | `-i` | Attach image |
 | `-o` | Output to file |
 | `-c` | Continue conversation |
+| `-l` | Search local knowledge |
+| `--ingest` | Index files to local knowledge |
 | `--raw` | Raw output (no markdown) |
 | `--history` | View history |
 | `--json` | JSON output |
 
 ## Philosophy
 
-~400 lines. 1 dependency. No frameworks.
+Minimal. 1 dependency. No frameworks.
 
 ---
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pplx-zero",
-  "version": "2.2.2",
-  "description": "Minimal Perplexity AI CLI - search from terminal",
+  "version": "2.3.1",
+  "description": "Fast, minimal Perplexity AI CLI. Stream answers, analyze PDFs/images, continue conversations. Zero bloat.",
   "author": "kenzo",
   "license": "MIT",
   "type": "module",
@@ -26,7 +26,8 @@
   "files": [
     "src",
     "bin",
-    "!src/**/*.test.ts"
+    "!src/**/*.test.ts",
+    "!**/AGENTS.md"
   ],
   "repository": {
     "type": "git",
@@ -42,6 +43,15 @@
     "search",
     "cli",
     "terminal",
-    "bun"
+    "bun",
+    "llm",
+    "chatgpt",
+    "api",
+    "streaming",
+    "pdf",
+    "image",
+    "multimodal",
+    "research",
+    "assistant"
   ]
 }

package/src/api.ts

@@ -90,6 +90,8 @@ export async function search(
   let buffer = '';
   let citations: SearchResult[] = [];
   let usage = { prompt_tokens: 0, completion_tokens: 0 };
+  let parseErrors = 0;
+  let unexpectedLines = 0;
 
   while (true) {
     const { done, value } = await reader.read();
@@ -100,7 +102,18 @@ export async function search(
     buffer = lines.pop() || '';
 
     for (const line of lines) {
-      if (!line.startsWith('data: ')) continue;
+      if (!line.trim()) continue;
+
+      if (!line.startsWith('data: ')) {
+        if (!line.startsWith(':')) {
+          unexpectedLines++;
+          if (unexpectedLines <= 3) {
+            console.error(`[pplx] Unexpected SSE line: ${line.slice(0, 100)}${line.length > 100 ? '...' : ''}`);
+          }
+        }
+        continue;
+      }
+
       const data = line.slice(6).trim();
       if (data === '[DONE]') continue;
 
@@ -118,10 +131,18 @@ export async function search(
           usage = parsed.usage;
         }
       } catch {
+        parseErrors++;
+        if (parseErrors <= 3) {
+          console.error(`[pplx] Failed to parse JSON: ${data.slice(0, 100)}${data.length > 100 ? '...' : ''}`);
+        }
         continue;
       }
     }
   }
 
+  if (parseErrors > 0 || unexpectedLines > 0) {
+    console.error(`[pplx] Stream completed with ${parseErrors} parse errors, ${unexpectedLines} unexpected lines`);
+  }
+
   callbacks.onDone(citations, usage);
 }

package/src/env.ts

@@ -17,9 +17,9 @@ let _env: z.infer<typeof envSchema> | null = null;
 export function getEnv() {
   if (_env) return _env;
 
-  const key = process.env.PERPLEXITY_API_KEY || process.env.PERPLEXITY_AI_API_KEY;
-  
-  if (!key) {
+  const apiKey = process.env.PERPLEXITY_API_KEY || process.env.PERPLEXITY_AI_API_KEY;
+
+  if (!apiKey) {
     console.error(`
 ${c.red}✗ Missing API Key${c.reset}
 
@@ -27,11 +27,11 @@ Set your Perplexity API key:
 
   ${c.cyan}export PERPLEXITY_API_KEY="pplx-..."${c.reset}
 
-${c.dim}Get one at: https://perplexity.ai/settings/api${c.reset}
+${c.dim}Get one at: https://www.perplexity.ai/settings/api${c.reset}
 `);
-    process.exit(2);
+    process.exit(1);
   }
 
-  _env = { PERPLEXITY_API_KEY: key };
+  _env = { PERPLEXITY_API_KEY: apiKey };
   return _env;
 }

package/src/history.ts

@@ -1,3 +1,6 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { mkdir, appendFile } from 'node:fs/promises';
 import type { Model } from './api';
 
 export interface HistoryEntry {
@@ -8,33 +11,42 @@ export interface HistoryEntry {
   citations?: string[];
 }
 
-const HISTORY_DIR = `${process.env.HOME}/.pplx`;
-const HISTORY_PATH = `${HISTORY_DIR}/history.jsonl`;
-const MAX_ENTRIES = 1000;
+const HISTORY_DIR = join(homedir(), '.pplx');
+const HISTORY_PATH = join(HISTORY_DIR, 'history.jsonl');
+const MAX_ENTRIES_TO_KEEP = 500;
+const ROTATION_SIZE_THRESHOLD = 512 * 1024; // 512KB
 const MAX_ANSWER_LENGTH = 2000;
 
+let appendCount = 0;
+
 async function ensureDir(): Promise<void> {
   const dir = Bun.file(HISTORY_DIR);
   if (!(await dir.exists())) {
-    await Bun.$`mkdir -p ${HISTORY_DIR}`;
+    await mkdir(HISTORY_DIR, { recursive: true });
+  }
+}
+
+export async function maybeRotateHistory(): Promise<void> {
+  const file = Bun.file(HISTORY_PATH);
+  if (!(await file.exists()) || file.size < ROTATION_SIZE_THRESHOLD) return;
+
+  const text = await file.text();
+  const lines = text.trim().split('\n').filter(l => l.length > 0);
+
+  if (lines.length > MAX_ENTRIES_TO_KEEP) {
+    const keep = lines.slice(-MAX_ENTRIES_TO_KEEP).join('\n') + '\n';
+    await Bun.write(HISTORY_PATH, keep);
   }
 }
 
 export async function appendHistory(entry: Omit<HistoryEntry, 'ts'>): Promise<void> {
   await ensureDir();
-  
-  const file = Bun.file(HISTORY_PATH);
-  const exists = await file.exists();
-  
-  if (exists) {
-    const text = await file.text();
-    const lines = text.trim().split('\n').filter(l => l.length > 0);
-    if (lines.length >= MAX_ENTRIES) {
-      const keep = lines.slice(-MAX_ENTRIES + 1).join('\n') + '\n';
-      await Bun.write(HISTORY_PATH, keep);
-    }
+
+  if (appendCount % 10 === 0) {
+    await maybeRotateHistory();
   }
-  
+  appendCount++;
+
   const record: HistoryEntry = {
     ts: Date.now(),
     q: entry.q,
@@ -42,15 +54,9 @@ export async function appendHistory(entry: Omit<HistoryEntry, 'ts'>): Promise<vo
     a: entry.a.slice(0, MAX_ANSWER_LENGTH),
     ...(entry.citations?.length ? { citations: entry.citations } : {}),
   };
-  
+
   const line = JSON.stringify(record) + '\n';
-  
-  if (exists) {
-    const current = await Bun.file(HISTORY_PATH).text();
-    await Bun.write(HISTORY_PATH, current + line);
-  } else {
-    await Bun.write(HISTORY_PATH, line);
-  }
+  await appendFile(HISTORY_PATH, line);
 }
 
 export async function readHistory(limit = 20): Promise<HistoryEntry[]> {

package/src/index.ts

@@ -6,6 +6,7 @@ import { getEnv } from './env';
 import { fmt, write, writeLn } from './output';
 import { appendHistory, readHistory, getLastEntry } from './history';
 import { renderMarkdown, createMarkdownState } from './markdown';
+import { search as ragSearch, searchForRag, ingestDirectory, ingestFile, ingestPath, getDocCount, getKnowledgeDir } from './rag';
 
 getEnv();
 
@@ -20,6 +21,8 @@ const { values, positionals } = parseArgs({
     history: { type: 'boolean', default: false },
     'no-history': { type: 'boolean', default: false },
     continue: { type: 'boolean', short: 'c', default: false },
+    local: { type: 'boolean', short: 'l', default: false },
+    ingest: { type: 'boolean', default: false },
     output: { type: 'string', short: 'o' },
     raw: { type: 'boolean', default: false },
   },
@@ -39,6 +42,8 @@ Options:
   -i, --image <path>   Attach an image (PNG, JPG, etc.)
   -o, --output <path>  Save output to file (.md, .txt)
   -c, --continue       Continue from last query (add context)
+  -l, --local          RAG: search local docs, inject as context
+  --ingest [path]      Index file/dir/glob (default: ~/.pplx/knowledge/)
   --history            Show query history
   --no-history         Don't save this query to history
   --raw                Raw output (no markdown rendering)
@@ -51,6 +56,8 @@ Examples:
   pplx -f report.pdf "summarize this document"
   pplx -c "tell me more about that"
   pplx --history | grep "bun"
+  pplx --ingest
+  pplx -l "my notes on rust"
 `);
   process.exit(0);
 }
@@ -67,7 +74,58 @@ if (values.history) {
   process.exit(0);
 }
 
-if (positionals.length === 0 && !values.continue) {
+if (values.ingest) {
+  const target = positionals[0];
+  
+  try {
+    if (target) {
+      const stats = await ingestPath(target);
+      if (stats.added + stats.updated + stats.skipped === 0) {
+        console.log(`No files found matching: ${target}`);
+      } else {
+        console.log(`Done! Added: ${stats.added}, Updated: ${stats.updated}, Skipped: ${stats.skipped}`);
+      }
+    } else {
+      console.log(`Indexing files from ${getKnowledgeDir()}...`);
+      const stats = await ingestDirectory();
+      console.log(`Done! Added: ${stats.added}, Updated: ${stats.updated}, Skipped: ${stats.skipped}`);
+    }
+    
+    console.log(`Total documents: ${getDocCount()}`);
+  } catch (err) {
+    console.error(fmt.error(err instanceof Error ? err.message : 'Ingest failed'));
+    process.exit(2);
+  }
+  process.exit(0);
+}
+
+let ragContext = '';
+
+if (values.local) {
+  const query = positionals.join(' ');
+  if (!query) {
+    console.error(fmt.error('No query provided for local search.'));
+    process.exit(2);
+  }
+  
+  const results = searchForRag(query);
+  
+  if (results.length === 0) {
+    console.log('No local documents match. Proceeding with Perplexity only...\n');
+  } else {
+    if (!values.json) {
+      console.log(`${fmt.model('local')} Found ${results.length} relevant doc(s), using as context...\n`);
+    }
+    
+    ragContext = 'Context from user\'s knowledge base:\n---\n';
+    for (const r of results) {
+      ragContext += `[${r.title}]:\n${r.content}\n\n`;
+    }
+    ragContext += '---\n\nQuestion: ';
+  }
+}
+
+if (positionals.length === 0 && !values.continue && !values.local) {
   console.error(fmt.error('No query provided. Use -h for help.'));
   process.exit(2);
 }
@@ -81,6 +139,10 @@ const model = values.model as Model;
 
 let query = positionals.join(' ');
 
+if (ragContext) {
+  query = ragContext + query;
+}
+
 if (values.continue) {
   const last = await getLastEntry();
   if (last) {

package/src/rag.ts

@@ -0,0 +1,269 @@
+import { Database } from 'bun:sqlite';
+import { Glob } from 'bun';
+import { homedir } from 'node:os';
+import { join, basename, resolve, extname } from 'node:path';
+import { mkdir, stat, copyFile } from 'node:fs/promises';
+
+const PPLX_DIR = join(homedir(), '.pplx');
+const KNOWLEDGE_DIR = join(PPLX_DIR, 'knowledge');
+const DB_PATH = join(PPLX_DIR, 'rag.db');
+
+let db: Database | null = null;
+
+function getDb(): Database {
+  if (db) return db;
+  
+  db = new Database(DB_PATH);
+  
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(
+      path,
+      title,
+      content,
+      tokenize = 'porter unicode61'
+    );
+  `);
+  
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS docs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      path TEXT UNIQUE NOT NULL,
+      title TEXT,
+      ingested_at INTEGER DEFAULT (unixepoch())
+    );
+  `);
+  
+  return db;
+}
+
+export interface SearchResult {
+  path: string;
+  title: string;
+  snippet: string;
+  score: number;
+}
+
+export interface IngestStats {
+  added: number;
+  updated: number;
+  skipped: number;
+}
+
+export async function ensureKnowledgeDir(): Promise<void> {
+  await mkdir(KNOWLEDGE_DIR, { recursive: true });
+}
+
+export async function ingestFile(filePath: string): Promise<'added' | 'updated' | 'skipped'> {
+  const db = getDb();
+  const content = await Bun.file(filePath).text();
+  
+  if (!content.trim()) return 'skipped';
+  
+  const title = filePath.split('/').pop()?.replace(/\.(md|txt)$/, '') || filePath;
+  
+  const existing = db.query('SELECT id FROM docs WHERE path = ?').get(filePath) as { id: number } | null;
+  
+  if (existing) {
+    db.run('DELETE FROM docs_fts WHERE rowid = ?', [existing.id]);
+    db.run('UPDATE docs SET title = ?, ingested_at = unixepoch() WHERE id = ?', [title, existing.id]);
+    db.run('INSERT INTO docs_fts (rowid, path, title, content) VALUES (?, ?, ?, ?)', 
+      [existing.id, filePath, title, content]);
+    return 'updated';
+  }
+  
+  const result = db.run('INSERT INTO docs (path, title) VALUES (?, ?)', [filePath, title]);
+  const docId = result.lastInsertRowid;
+  db.run('INSERT INTO docs_fts (rowid, path, title, content) VALUES (?, ?, ?, ?)', 
+    [docId, filePath, title, content]);
+  
+  return 'added';
+}
+
+export async function ingestDirectory(dir?: string): Promise<IngestStats> {
+  await ensureKnowledgeDir();
+  
+  const targetDir = dir || KNOWLEDGE_DIR;
+  const glob = new Glob('**/*.{md,txt}');
+  const files: string[] = [];
+  
+  for await (const file of glob.scan({ cwd: targetDir, absolute: true })) {
+    files.push(file);
+  }
+  
+  const stats: IngestStats = { added: 0, updated: 0, skipped: 0 };
+  
+  for (const file of files) {
+    const result = await ingestFile(file);
+    stats[result]++;
+  }
+  
+  return stats;
+}
+
+export function search(query: string, limit = 5): SearchResult[] {
+  const db = getDb();
+  
+  const ftsQuery = query
+    .trim()
+    .split(/\s+/)
+    .map(word => `${word}*`)
+    .join(' OR ');
+  
+  const stmt = db.prepare(`
+    SELECT 
+      path,
+      title,
+      snippet(docs_fts, 2, '>', '<', '...', 40) as snippet,
+      bm25(docs_fts) as score
+    FROM docs_fts
+    WHERE docs_fts MATCH ?
+    ORDER BY score
+    LIMIT ?
+  `);
+  
+  try {
+    return stmt.all(ftsQuery, limit) as SearchResult[];
+  } catch {
+    return [];
+  }
+}
+
+export interface RagContext {
+  title: string;
+  content: string;
+}
+
+function truncateUtf8Safe(str: string, maxLen: number): string {
+  if (str.length <= maxLen) return str;
+  let truncated = str.slice(0, maxLen);
+  const lastChar = truncated.charCodeAt(truncated.length - 1);
+  if (lastChar >= 0xD800 && lastChar <= 0xDBFF) {
+    truncated = truncated.slice(0, -1);
+  }
+  return truncated;
+}
+
+export function searchForRag(query: string, limit = 3, maxChars = 4000): RagContext[] {
+  const db = getDb();
+  
+  const ftsQuery = query
+    .trim()
+    .split(/\s+/)
+    .map(word => `${word}*`)
+    .join(' OR ');
+  
+  const stmt = db.prepare(`
+    SELECT title, content
+    FROM docs_fts
+    WHERE docs_fts MATCH ?
+    ORDER BY bm25(docs_fts)
+    LIMIT ?
+  `);
+  
+  try {
+    const results = stmt.all(ftsQuery, limit) as { title: string; content: string }[];
+    let totalChars = 0;
+    const truncated: RagContext[] = [];
+    
+    for (const r of results) {
+      const remaining = maxChars - totalChars;
+      if (remaining <= 0) break;
+      
+      const content = truncateUtf8Safe(r.content, remaining);
+      truncated.push({
+        title: r.title,
+        content,
+      });
+      totalChars += content.length;
+    }
+    
+    return truncated;
+  } catch {
+    return [];
+  }
+}
+
+export function getDocCount(): number {
+  const db = getDb();
+  const result = db.query('SELECT COUNT(*) as count FROM docs').get() as { count: number };
+  return result.count;
+}
+
+export function getKnowledgeDir(): string {
+  return KNOWLEDGE_DIR;
+}
+
+const SUPPORTED_EXTS = new Set(['.md', '.txt']);
+
+async function copyToKnowledge(filePath: string): Promise<string> {
+  await ensureKnowledgeDir();
+  const ext = extname(filePath);
+  const name = basename(filePath, ext);
+  let dest = join(KNOWLEDGE_DIR, basename(filePath));
+  let counter = 1;
+  while (await Bun.file(dest).exists()) {
+    dest = join(KNOWLEDGE_DIR, `${name}_${counter}${ext}`);
+    counter++;
+  }
+  await copyFile(filePath, dest);
+  return dest;
+}
+
+export async function ingestPath(target: string): Promise<IngestStats> {
+  await ensureKnowledgeDir();
+  const stats: IngestStats = { added: 0, updated: 0, skipped: 0 };
+  const resolved = resolve(target);
+  
+  const isGlob = target.includes('*');
+  
+  if (isGlob) {
+    const glob = new Glob(target);
+    for await (const file of glob.scan({ cwd: process.cwd(), absolute: true })) {
+      const ext = extname(file).toLowerCase();
+      if (!SUPPORTED_EXTS.has(ext)) {
+        console.log(`Skipping unsupported: ${file} (only .md, .txt)`);
+        stats.skipped++;
+        continue;
+      }
+      const dest = await copyToKnowledge(file);
+      const result = await ingestFile(dest);
+      stats[result]++;
+      console.log(`${result}: ${basename(file)}`);
+    }
+    return stats;
+  }
+  
+  let info;
+  try {
+    info = await stat(resolved);
+  } catch {
+    throw new Error(`Path not found: ${target}`);
+  }
+  
+  if (info.isDirectory()) {
+    console.log(`Indexing directory: ${resolved}`);
+    const glob = new Glob('**/*.{md,txt}');
+    for await (const file of glob.scan({ cwd: resolved, absolute: true })) {
+      const dest = await copyToKnowledge(file);
+      const result = await ingestFile(dest);
+      stats[result]++;
+      console.log(`${result}: ${basename(file)}`);
+    }
+    return stats;
+  }
+  
+  const ext = extname(resolved).toLowerCase();
+  if (ext === '.pdf') {
+    throw new Error('PDF indexing not supported (requires text extraction library). Use -f to send PDFs to Perplexity API instead.');
+  }
+  if (!SUPPORTED_EXTS.has(ext)) {
+    throw new Error(`Unsupported file type: ${ext}. Supported: .md, .txt`);
+  }
+  
+  const dest = await copyToKnowledge(resolved);
+  const result = await ingestFile(dest);
+  stats[result]++;
+  console.log(`${result}: ${basename(resolved)} → ~/.pplx/knowledge/`);
+  
+  return stats;
+}