projview 0.4.1 → 0.5.1

package/README.md

@@ -50,10 +50,31 @@ projview search "mermaid" --demo   # quick test on the bundled docs
 | step | detail |
 | --- | --- |
 | walk | recursively finds `.md` / `.mmd`, skips `node_modules`, `dist`, dot-dirs |
-| index | builds an in-memory tree (nothing written to disk) |
+| index | builds an in-memory tree (nothing written to disk by default) |
 | serve | spawns a localhost site + opens your browser |
 | live | edits to files reload in the browser automatically |
 
-> ctrl-c and it's gone - "indexing" is never persisted.
+> ephemeral by default - ctrl-c and it's gone. opt into persistence below.
+
+## persistence (opt-in)
+
+By default the index is in-memory and nothing is written to disk. Opt in for a
+faster warm start, or a queryable store an AI can read:
+
+```bash
+projview docs --persist                                # sqlite cache in ~/.projview/cache
+projview docs --store-url "sqlite:///tmp/docs.sqlite"  # a sqlite file you choose
+projview docs --store-url "$DATABASE_URL"              # your own postgres (needs `pg`)
+```
+
+| backend | how |
+| --- | --- |
+| `memory` (default) | _(nothing)_ - ephemeral |
+| `lite` (sqlite) | `--persist` |
+| `superlite` (JSON) | `--store superlite --persist` |
+| `custom` (sqlite/postgres URL) | `--store-url <url>` (or `DATABASE_URL`) |
+
+On re-run it reloads the cache and only re-reads files whose mtime changed.
+Connection strings come from env/`~/.projview` - never commit credentials.
 
 see [samples/architecture.md](samples/architecture.md) for a mermaid demo.

package/bin/projview.js

@@ -6,6 +6,7 @@ import { readFileSync } from 'node:fs';
 import { startServer } from '../src/server.js';
 import { buildTree, countFiles } from '../src/walk.js';
 import { buildIndex, search } from '../src/search.js';
+import { resolveStoreConfig, createStore } from '../src/store.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const DEMO_DIR = path.join(__dirname, '..', 'docs');   // bundled sample docs
@@ -24,6 +25,9 @@ function parseArgs(argv) {
     if (arg === '--port' || arg === '-p') opts.port = Number(argv[++i]) || opts.port;
     else if (arg === '--no-open') opts.open = false;
     else if (arg === '--demo') opts.demo = true;
+    else if (arg === '--store') opts.store = argv[++i];
+    else if (arg === '--store-url') opts.storeUrl = argv[++i];
+    else if (arg === '--persist') opts.persist = true;
     else if (arg === '--version' || arg === '-v') opts.version = true;
     else if (arg === '--help' || arg === '-h') opts.help = true;
     else if (arg.startsWith('-')) opts.unknown = arg;   // unrecognised flag -> error, don't treat as a path
@@ -36,27 +40,42 @@ function parseArgs(argv) {
 
 // --- machine interface: subcommands that print JSON to stdout and exit ---
 
-// pick the dir a subcommand runs against: --demo wins, else a positional, else cwd
-function rootFrom(positional, args) {
-  if (args.includes('--demo')) return DEMO_DIR;
-  return positional[0] ? path.resolve(process.cwd(), positional[0]) : process.cwd();
+// parse subcommand args: separate positionals from flags (consumes flag values,
+// so e.g. a postgres:// store URL is never mistaken for a path)
+function parseSubArgs(args) {
+  const o = { positional: [] };
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === '--store') o.store = args[++i];
+    else if (a === '--store-url') o.storeUrl = args[++i];
+    else if (a === '--persist') o.persist = true;
+    else if (a === '--demo') o.demo = true;
+    else if (a.startsWith('-')) { /* ignore unknown flag in a subcommand */ }
+    else o.positional.push(a);
+  }
+  return o;
 }
 
+const subRoot = (o, idx) =>
+  o.demo ? DEMO_DIR : (o.positional[idx] ? path.resolve(process.cwd(), o.positional[idx]) : process.cwd());
+
 async function cmdSearch(args) {
-  const positional = args.filter((a) => !a.startsWith('-'));
-  const query = positional[0];
+  const o = parseSubArgs(args);
+  const query = o.positional[0];
   if (!query) {
-    console.error('usage: projview search <query> [path] [--demo]');
+    console.error('usage: projview search <query> [path] [--demo] [--store <kind>|--store-url <url>] [--persist]');
     process.exit(1);
   }
-  const root = rootFrom(positional.slice(1), args);
-  await buildIndex(root);
+  const root = subRoot(o, 1);
+  const store = createStore(resolveStoreConfig(o), root);
+  await buildIndex(root, store);
   process.stdout.write(JSON.stringify(search(query), null, 2) + '\n');
+  await store.close();
 }
 
 function cmdTree(args) {
-  const positional = args.filter((a) => !a.startsWith('-'));
-  const root = rootFrom(positional, args);
+  const o = parseSubArgs(args);
+  const root = subRoot(o, 0);
   const tree = buildTree(root);
   process.stdout.write(JSON.stringify({ root: path.basename(root) || root, files: countFiles(tree), tree }, null, 2) + '\n');
 }
@@ -84,13 +103,17 @@ arguments:
   path              directory to index (default: current directory)
 
 options:
-  -p, --port <n>    preferred port (default: 4321, climbs if taken)
-      --demo        preview projview's bundled sample docs
-      --no-open     do not auto-open the browser
-  -v, --version     print the version and exit
-  -h, --help        show this help
-
-nothing is written to disk - ctrl-c and it's gone.`;
+  -p, --port <n>      preferred port (default: 4321, climbs if taken)
+      --demo          preview projview's bundled sample docs
+      --no-open       do not auto-open the browser
+      --persist       keep the index in ~/.projview/cache and reuse it next run
+      --store <kind>  store backend: memory | superlite | lite  (default: memory)
+      --store-url <url>  custom store: sqlite://<file> or postgres://<...>  (or set DATABASE_URL)
+  -v, --version       print the version and exit
+  -h, --help          show this help
+
+ephemeral by default - nothing is persisted unless you opt in with --persist /
+--store / --store-url. ctrl-c removes any ephemeral store.`;
 
 async function main() {
   const argv = process.argv.slice(2);
@@ -107,12 +130,15 @@ async function main() {
     process.exit(1);
   }
 
-  const { port, close } = await startServer(opts.root, { port: opts.port, host: opts.host });
+  const storeConfig = resolveStoreConfig(opts);
+  const { port, close, storeInfo } = await startServer(opts.root, { port: opts.port, host: opts.host, store: storeConfig });
   const url = `http://${opts.host}:${port}`;
 
   console.log(`\n  projview  ->  ${url}`);
   console.log(`  indexing  ->  ${opts.root}`);
-  console.log(`  (ctrl-c to stop - nothing is persisted)\n`);
+  console.log(`  store     ->  ${storeInfo.describe}`);
+  if (storeInfo.warm) console.log(`  warm      ->  reused cache, reindexed ${storeInfo.reindexed}/${storeInfo.total} changed`);
+  console.log(`  (ctrl-c to stop)\n`);
 
   if (opts.open) openBrowser(url);
 

package/docs/how-to-set-up.md

@@ -94,6 +94,9 @@ stdout, no server, no browser - built for agents and scripts. See
 | `--demo` | use projview's own bundled sample docs (works with the subcommands too) |
 | `-p`, `--port <n>` | preferred port (default `4321`, climbs if taken) |
 | `--no-open` | don't auto-open the browser |
+| `--persist` | keep the index in `~/.projview/cache` and reuse it next run |
+| `--store <kind>` | store backend: `memory` \| `superlite` \| `lite` (default: `memory`; `lite` when `--persist`) |
+| `--store-url <url>` | use a custom store: `sqlite://<file>` or `postgres://<...>` |
 | `-v`, `--version` | print the version and exit |
 | `-h`, `--help` | show help |
 
@@ -103,6 +106,37 @@ stdout, no server, no browser - built for agents and scripts. See
 | --- | --- | --- |
 | `PORT` | `4321` | port to serve on |
 | `HOST` | `127.0.0.1` | interface to bind (`0.0.0.0` to expose) |
+| `PROJVIEW_PERSIST` | `0` | `1` to persist the index (same as `--persist`) |
+| `PROJVIEW_STORE` | `memory` | store backend (same as `--store`) |
+| `PROJVIEW_STORE_URL` / `DATABASE_URL` | — | custom store connection URL (same as `--store-url`) |
+
+## persistence & stores
+
+By default projview is **ephemeral** - the index lives in memory and nothing is
+written to disk. Opt in to persistence when you want a faster warm start or a
+queryable store an AI can read:
+
+| backend | what | how |
+| --- | --- | --- |
+| `memory` | in-memory, nothing persisted (**default**) | _(no flag)_ |
+| `lite` | sqlite via Node's built-in `node:sqlite` | `--persist` (or `--store lite`) |
+| `superlite` | a plain JSON file | `--store superlite --persist` |
+| `custom` | your own DB via a connection URL | `--store-url sqlite://./my.sqlite` or `postgres://…` |
+
+```bash
+projview docs --persist                              # warm-start cache in ~/.projview/cache
+projview docs --store-url "sqlite:///tmp/docs.sqlite"  # a sqlite file you choose
+projview docs --store-url "$DATABASE_URL"            # your postgres (needs the `pg` package)
+```
+
+- **Locations:** persisted stores live in `~/.projview/cache/`; ephemeral ones in
+  `~/.projview/ephemeral/` and are removed on ctrl-c. Defaults can be set in
+  `~/.projview/config.json`.
+- **Warm start:** on re-run, projview loads the cached index and only re-reads
+  files whose modification time changed.
+- **Postgres** needs the optional `pg` package (`npm i pg`). The connection URL
+  comes from `--store-url` / `DATABASE_URL` / your config file - **never commit a
+  connection string with credentials**; keep it in the environment.
 
 ## use the source directly (advanced)
 

package/docs/reference/api.md

@@ -85,9 +85,27 @@ On-demand fetch + render, in-app link routing, anchor jumps.
 
 A chokidar watcher broadcasts change/tree events over SSE.
 
+## stores (persistence)
+
+The index is in-memory by default (ephemeral). Opt into a pluggable store with
+`--persist` / `--store` / `--store-url`:
+
+### backends
+
+`memory` (default) · `superlite` (JSON) · `lite` (sqlite via `node:sqlite`) ·
+`custom` (a `sqlite://` or `postgres://` connection URL; postgres needs the
+optional `pg` package).
+
+### warm start
+
+A persisted store lives in `~/.projview/cache`; on re-run projview reloads it and
+re-reads only files whose mtime changed. Connection strings come from the
+environment / config - never committed.
+
 ## performance notes
 
-Indexing is just a directory walk - no persistence, no cache files.
+Indexing is a directory walk; persistence is opt-in (see stores). Search runs
+in-memory regardless of backend.
 
 ## limitations
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "projview",
-  "version": "0.4.1",
+  "version": "0.5.1",
   "description": "Ephemeral local previewer for markdown + mermaid files in any directory",
   "type": "module",
   "bin": {
@@ -18,7 +18,7 @@
     "docs"
   ],
   "engines": {
-    "node": ">=18"
+    "node": ">=22"
   },
   "keywords": [
     "markdown",
@@ -42,6 +42,9 @@
     "highlight.js": "^11.11.1",
     "markdown-it": "^14.1.0"
   },
+  "optionalDependencies": {
+    "pg": "^8.21.0"
+  },
   "devDependencies": {
     "svg-pan-zoom": "^3.6.2"
   }

package/src/search.js

@@ -1,9 +1,10 @@
 import fsp from 'node:fs/promises';
 import path from 'node:path';
 import { buildTree } from './walk.js';
+import { createStore } from './store.js';
 
-// In-memory index: relPath -> { path, name, content, lower }. Rebuilt per run,
-// kept fresh by the watcher. Nothing is persisted to disk.
+// In-memory index: relPath -> { path, name, content, lower }. Search always runs
+// over this; a Store (optional, opt-in) just persists/warm-starts the entries.
 const index = new Map();
 let theRoot = null;
 
@@ -19,15 +20,37 @@ function addToIndex(rel, content) {
   index.set(rel, { path: rel, name: rel.split('/').pop(), content, lower: content.toLowerCase() });
 }
 
-// Read + index every previewable file under root.
-export async function buildIndex(root) {
+// Build the index for `root`. With a persisted store, loads the cached entries
+// and only re-reads files whose mtime changed (warm start); else reads them all.
+// Returns the store + a small summary, so the caller can close it on shutdown.
+export async function buildIndex(root, store) {
   theRoot = root;
   index.clear();
+  store = store || createStore({ kind: 'memory' }, root);
+
+  const cached = await store.load();
+  const cachedByPath = new Map((cached?.entries || []).map((e) => [e.path, e]));
   const files = flattenFiles(buildTree(root));
-  await Promise.all(files.map(async (f) => {
-    try { addToIndex(f.path, await fsp.readFile(path.join(root, f.path), 'utf8')); }
-    catch { /* unreadable - skip */ }
+  let reindexed = 0;
+
+  const entries = await Promise.all(files.map(async (f) => {
+    const abs = path.join(root, f.path);
+    let mtime = 0;
+    try { mtime = (await fsp.stat(abs)).mtimeMs; } catch { /* gone */ }
+    const prev = cachedByPath.get(f.path);
+    let content;
+    if (prev && prev.mtime === mtime) {
+      content = prev.content;                       // unchanged -> reuse cached
+    } else {
+      reindexed++;
+      try { content = await fsp.readFile(abs, 'utf8'); } catch { content = ''; }
+    }
+    return { path: f.path, name: f.name, ext: f.ext, mtime, content };
   }));
+
+  for (const e of entries) addToIndex(e.path, e.content);
+  await store.save(entries);
+  return { store, total: entries.length, reindexed, warm: !!cached };
 }
 
 export async function updateFile(rel) {

package/src/server.js

@@ -8,6 +8,7 @@ import chokidar from 'chokidar';
 import { buildTree, isIgnoredDir, isPreviewable } from './walk.js';
 import { renderFile } from './render.js';
 import { buildIndex, updateFile, removeFile, search } from './search.js';
+import { createStore } from './store.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const PUBLIC_DIR = path.join(__dirname, '..', 'public');
@@ -46,7 +47,7 @@ function resolveInRoot(root, relPath) {
   return abs;
 }
 
-export async function startServer(root, { port = 4321, host = '127.0.0.1' } = {}) {
+export async function startServer(root, { port = 4321, host = '127.0.0.1', store: storeConfig } = {}) {
   const sseClients = new Set();
 
   function broadcast(event, payload) {
@@ -128,16 +129,22 @@ export async function startServer(root, { port = 4321, host = '127.0.0.1' } = {}
     .on('addDir', () => broadcast('tree'))
     .on('unlinkDir', () => broadcast('tree'));
 
-  await buildIndex(root);   // in-memory content index for search
+  const store = createStore(storeConfig, root);
+  const indexInfo = await buildIndex(root, store);   // load-or-build the search index
   const listenPort = await listenWithRetry(server, port, host);
 
   const close = () => new Promise((resolve) => {
     watcher.close();
     for (const res of sseClients) res.end();
-    server.close(() => resolve());
+    Promise.resolve(store.close()).finally(() => server.close(() => resolve()));
   });
 
-  return { server, port: listenPort, close };
+  return {
+    server,
+    port: listenPort,
+    close,
+    storeInfo: { describe: store.describe(), warm: indexInfo.warm, reindexed: indexInfo.reindexed, total: indexInfo.total }
+  };
 }
 
 // Try `port`, then climb until one is free (give up after a sane range).

package/src/store.js

@@ -0,0 +1,189 @@
+/* pluggable persistence for the index.
+ *
+ * A Store persists indexed entries so a re-spawn can warm-start. Backends:
+ *   memory    - no persistence (ephemeral default; nothing written)
+ *   superlite - a JSON file
+ *   lite      - sqlite via node:sqlite (built-in)
+ *   custom    - a connection URL: sqlite://<file> or postgres://<...> (lazy `pg`)
+ *
+ * An entry is { path, name, ext, mtime, content }. Everything above the store
+ * stays storage-agnostic - search still runs in memory over loaded entries.
+ */
+
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import { createHash } from 'node:crypto';
+
+// node:sqlite is flagged "experimental" and emits a noisy warning on use; we
+// opted into it deliberately, so quiet just that one notice (others pass through).
+const _emitWarning = process.emitWarning.bind(process);
+process.emitWarning = (warning, ...rest) => {
+  if (typeof warning === 'string' && warning.includes('SQLite is an experimental feature')) return;
+  return _emitWarning(warning, ...rest);
+};
+
+const HOME = path.join(os.homedir(), '.projview');
+const CACHE_DIR = path.join(HOME, 'cache');         // retained (--persist)
+const EPHEMERAL_DIR = path.join(HOME, 'ephemeral'); // cleaned up on close
+const CONFIG_FILE = path.join(HOME, 'config.json'); // default flags
+
+const keyFor = (root) => createHash('sha256').update(path.resolve(root)).digest('hex').slice(0, 16);
+const ensureDir = (d) => fs.mkdirSync(d, { recursive: true });
+const redact = (url) => url.replace(/\/\/[^@/]*@/, '//***@');   // hide creds in logs
+
+function readConfigFile() {
+  try { return JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')); } catch { return {}; }
+}
+
+// Resolve store config: CLI flags > env > ~/.projview/config.json > defaults.
+export function resolveStoreConfig(flags = {}) {
+  const env = process.env;
+  const file = readConfigFile();
+  const pick = (...vals) => vals.find((v) => v !== undefined);
+
+  const url = pick(flags.storeUrl, env.PROJVIEW_STORE_URL, env.DATABASE_URL, file.storeUrl);
+  const persist = pick(
+    flags.persist,
+    ['1', 'true'].includes(env.PROJVIEW_PERSIST) ? true : undefined,
+    file.persist
+  ) || false;
+  let kind = pick(flags.store, env.PROJVIEW_STORE, file.store);
+  if (!kind) kind = url ? 'custom' : persist ? 'lite' : 'memory';
+  return { kind, url, persist };
+}
+
+export function createStore(config, root) {
+  const cfg = config || { kind: 'memory' };
+  switch (cfg.kind) {
+    case 'memory': return memoryStore();
+    case 'superlite': return superliteStore(root, cfg.persist);
+    case 'lite': return sqliteAt(fileFor(root, cfg.persist, 'sqlite'), { cleanup: !cfg.persist });
+    case 'custom': return customStore(cfg.url);
+    default: throw new Error(`unknown store kind: ${cfg.kind}`);
+  }
+}
+
+function fileFor(root, persist, ext) {
+  const dir = persist ? CACHE_DIR : EPHEMERAL_DIR;
+  ensureDir(dir);
+  return path.join(dir, `${keyFor(root)}.${ext}`);
+}
+
+/* ---------- backends ---------- */
+
+function memoryStore() {
+  return {
+    kind: 'memory',
+    async load() { return null; },
+    async save() {},
+    async close() {},
+    describe() { return 'in-memory (ephemeral - nothing persisted)'; }
+  };
+}
+
+function superliteStore(root, persist) {
+  const file = fileFor(root, persist, 'json');
+  return {
+    kind: 'superlite',
+    file,
+    async load() {
+      try { return JSON.parse(await fsp.readFile(file, 'utf8')); } catch { return null; }
+    },
+    async save(entries) {
+      await fsp.writeFile(file, JSON.stringify({ builtAt: Date.now(), entries }));
+    },
+    async close({ keep } = {}) {
+      if (!persist && !keep) { try { await fsp.unlink(file); } catch { /* gone */ } }
+    },
+    describe() { return `superlite/json (${persist ? 'persisted' : 'ephemeral'}) ${file}`; }
+  };
+}
+
+// sqlite-backed store at an explicit file. cleanup=true removes the file on close.
+function sqliteAt(file, { cleanup }) {
+  let db = null;
+  const open = async () => {
+    if (db) return db;
+    const { DatabaseSync } = await import('node:sqlite');
+    db = new DatabaseSync(file);
+    db.exec('CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, name TEXT, ext TEXT, mtime REAL, content TEXT)');
+    return db;
+  };
+  return {
+    kind: 'lite',
+    file,
+    async load() {
+      const d = await open();
+      const rows = d.prepare('SELECT path, name, ext, mtime, content FROM files').all();
+      return rows.length ? { entries: rows } : null;
+    },
+    async save(entries) {
+      const d = await open();
+      d.exec('BEGIN');
+      d.exec('DELETE FROM files');
+      const ins = d.prepare('INSERT OR REPLACE INTO files (path, name, ext, mtime, content) VALUES (?, ?, ?, ?, ?)');
+      for (const e of entries) ins.run(e.path, e.name, e.ext, e.mtime, e.content);
+      d.exec('COMMIT');
+    },
+    async close({ keep } = {}) {
+      if (db) { db.close(); db = null; }
+      if (cleanup && !keep) { try { await fsp.unlink(file); } catch { /* gone */ } }
+    },
+    describe() { return `lite/sqlite (${cleanup ? 'ephemeral' : 'persisted'}) ${file}`; }
+  };
+}
+
+// custom: a connection URL the user supplies (--store-url / DATABASE_URL).
+function customStore(url) {
+  if (!url) throw new Error('custom store needs a connection URL (--store-url or DATABASE_URL)');
+  let proto;
+  try { proto = new URL(url).protocol.replace(':', ''); } catch { throw new Error(`invalid store URL: ${url}`); }
+
+  if (proto === 'sqlite' || proto === 'file') {
+    const f = url.replace(/^sqlite:\/\/|^file:\/\/|^sqlite:/, '') || ':memory:';
+    return sqliteAt(path.resolve(f), { cleanup: false });   // the user's own file - never delete
+  }
+  if (proto === 'postgres' || proto === 'postgresql') return postgresStore(url);
+  throw new Error(`unsupported store URL scheme "${proto}://" - use sqlite:// or postgres://`);
+}
+
+function postgresStore(url) {
+  let client = null;
+  const open = async () => {
+    if (client) return client;
+    let pg;
+    try { pg = (await import('pg')).default; }
+    catch { throw new Error('the postgres store needs the "pg" package - run: npm i pg'); }
+    client = new pg.Client({ connectionString: url });
+    await client.connect();
+    await client.query(
+      'CREATE TABLE IF NOT EXISTS projview_files (path TEXT PRIMARY KEY, name TEXT, ext TEXT, mtime DOUBLE PRECISION, content TEXT)'
+    );
+    return client;
+  };
+  return {
+    kind: 'custom',
+    async load() {
+      const c = await open();
+      const { rows } = await c.query('SELECT path, name, ext, mtime, content FROM projview_files');
+      return rows.length ? { entries: rows } : null;
+    },
+    async save(entries) {
+      const c = await open();
+      await c.query('BEGIN');
+      await c.query('DELETE FROM projview_files');
+      for (const e of entries) {
+        await c.query(
+          'INSERT INTO projview_files (path, name, ext, mtime, content) VALUES ($1, $2, $3, $4, $5)' +
+            ' ON CONFLICT (path) DO UPDATE SET name = $2, ext = $3, mtime = $4, content = $5',
+          [e.path, e.name, e.ext, e.mtime, e.content]
+        );
+      }
+      await c.query('COMMIT');
+    },
+    async close() { if (client) { await client.end(); client = null; } },   // never drop the user's table
+    describe() { return `custom/postgres ${redact(url)}`; }
+  };
+}