what-isr 0.10.0

package/src/isr.js

@@ -0,0 +1,125 @@
+// ISR engine — origin-first Incremental Static Regeneration.
+//
+// Stale-while-revalidate at the ORIGIN: a fresh entry is served from cache; a
+// stale entry (past `revalidate`, within `swr`) is served IMMEDIATELY while a
+// single background regeneration refreshes it; a cold/expired entry blocks and
+// renders. Concurrent regenerations of the same key are deduped to ONE render.
+//
+// `render(routeMatch, ctx)` is INJECTED by the adapter (wraps renderPage +
+// serializeState) — the engine never imports what-server, keeping it standalone.
+
+import { cacheKey, normalizePath } from './key.js';
+import { makeEntry, isFresh, isServableStale } from './stores/store-interface.js';
+import { buildCacheHeaders } from './headers.js';
+
+export function createCacheEngine({ store, render, cdn, now = Date.now, logger = console } = {}) {
+  const inFlight = new Map(); // key -> Promise<entry>  (dedupe)
+
+  function keyFor(routeMatch) {
+    return cacheKey({ path: routeMatch.path, query: routeMatch.query, vary: routeMatch.vary });
+  }
+
+  // Render + store, deduping concurrent calls for the same key. `renderOverride`
+  // lets a caller (e.g. the deploy adapter) supply the render for this route
+  // without baking it into the engine — keeps the engine decoupled.
+  function regenerate(key, routeMatch, renderOverride) {
+    const existing = inFlight.get(key);
+    if (existing) return existing;
+    const doRender = renderOverride || render;
+    const p = (async () => {
+      const out = await doRender(routeMatch, {});
+      const entry = makeEntry({ ...out, path: routeMatch.path }, routeMatch.config || {}, now());
+      await store.set(key, entry);
+      return entry;
+    })().finally(() => inFlight.delete(key));
+    inFlight.set(key, p);
+    return p;
+  }
+
+  function serve(entry, cacheStatus, config) {
+    return {
+      html: entry.html,
+      head: entry.head,
+      state: entry.state,
+      status: entry.status || 200,
+      cacheStatus,
+      headers: buildCacheHeaders(entry, config || {}, cacheStatus),
+    };
+  }
+
+  async function handle(routeMatch, renderOverride) {
+    const config = routeMatch.config || {};
+
+    // Uncacheable (server-rendered) routes: always render, never store.
+    if (config.mode === 'server') {
+      const out = await (renderOverride || render)(routeMatch, {});
+      const entry = makeEntry({ ...out, path: routeMatch.path }, config, now());
+      return serve(entry, 'BYPASS', config);
+    }
+
+    const key = keyFor(routeMatch);
+    const entry = await store.get(key);
+    const t = now();
+
+    if (entry && isFresh(entry, t)) {
+      return serve(entry, 'HIT', config);
+    }
+
+    if (entry && isServableStale(entry, t)) {
+      // Serve stale immediately; refresh in the background (deduped, non-blocking).
+      regenerate(key, routeMatch, renderOverride).catch((e) => logger.error?.('[what-isr] background regenerate failed:', e));
+      return serve(entry, 'STALE', config);
+    }
+
+    // Cold miss or expired beyond the swr window.
+    if (entry && config.onMiss === 'stale-if-error') {
+      try {
+        const fresh = await regenerate(key, routeMatch, renderOverride);
+        return serve(fresh, 'MISS', config);
+      } catch (e) {
+        logger.error?.('[what-isr] regenerate failed, serving stale:', e);
+        return serve(entry, 'STALE', config);
+      }
+    }
+
+    const fresh = await regenerate(key, routeMatch, renderOverride);
+    return serve(fresh, 'MISS', config);
+  }
+
+  // --- On-demand invalidation (origin purge + optional CDN fan-out) ---
+
+  async function revalidatePath(path, { regenerate: regen = false, routeResolver } = {}) {
+    const norm = normalizePath(path);
+    const deleted = await store.deleteByPath(norm);
+    if (cdn && cdn.purge) await cdn.purge([path]);
+    if (regen) {
+      const route = routeResolver ? routeResolver(norm) : { path: norm, query: {}, config: {} };
+      await regenerate(keyFor(route), route).catch((e) => logger.error?.('[what-isr] regen after revalidatePath failed:', e));
+    }
+    return deleted;
+  }
+
+  async function revalidateTag(tag, { regenerate: regen = false, routeResolver } = {}) {
+    const deleted = await store.deleteByTag(tag);
+    if (cdn && cdn.purgeTags) await cdn.purgeTags([tag]);
+    if (regen && routeResolver) {
+      for (const key of deleted) {
+        const route = routeResolver(key);
+        if (route) await regenerate(keyFor(route), route).catch(() => {});
+      }
+    }
+    return deleted;
+  }
+
+  return {
+    handle,
+    regenerate: (routeMatch) => regenerate(keyFor(routeMatch), routeMatch),
+    revalidatePath,
+    revalidateTag,
+    keyFor,
+    store,
+    _inFlight: inFlight,
+    _now: now,
+    _cdn: cdn,
+  };
+}

package/src/paths.js

@@ -0,0 +1,43 @@
+// getStaticPaths resolution + fallback decisions for dynamic routes.
+//
+// Known paths (in getStaticPaths) are pre-rendered at build. Unknown params at
+// request time follow the route's `fallback`:
+//   'blocking' -> render on first hit, then cache (ISR)
+//   true       -> serve a skeleton immediately, regenerate in the background
+//   false      -> 404
+
+/** Run a page's getStaticPaths (if any). Returns { paths, fallback }. */
+export async function resolveStaticPaths(getStaticPaths, ctx = {}) {
+  if (typeof getStaticPaths !== 'function') return { paths: [], fallback: false };
+  const result = await getStaticPaths(ctx);
+  return {
+    paths: (result && result.paths) || [],
+    fallback: result && 'fallback' in result ? result.fallback : false,
+  };
+}
+
+/** Build a concrete URL from a route pattern + params. Supports :param and *catchall. */
+export function buildPath(pattern, params = {}) {
+  return pattern.replace(/[:*]([A-Za-z0-9_]+)/g, (_, name) => {
+    const v = params[name];
+    return v == null ? '' : String(v);
+  });
+}
+
+/** Is this param set among the pre-built static paths? */
+export function isKnownParams(staticPaths, params) {
+  return staticPaths.some((entry) => {
+    const p = entry.params || {};
+    const keys = new Set([...Object.keys(p), ...Object.keys(params)]);
+    for (const k of keys) if (String(p[k]) !== String(params[k])) return false;
+    return true;
+  });
+}
+
+/** Decide what to do for a requested dynamic path. */
+export function decideFallback(fallback, isKnown) {
+  if (isKnown) return 'serve';
+  if (fallback === 'blocking') return 'render';
+  if (fallback === true) return 'skeleton';
+  return 'notfound';
+}

package/src/scheduler.js

@@ -0,0 +1,73 @@
+// Poll/scheduled regeneration — keeps the cache warm regardless of traffic.
+// Runs in the long-lived Node adapter process. Self-rescheduling setTimeout
+// (no drift/overlap), jitter (anti-thundering-herd after a restart), a global
+// concurrency cap, and it joins the engine's in-flight lock so a scheduled tick
+// during a traffic-triggered regeneration is a no-op.
+
+export function createScheduler(engine, options = {}) {
+  const {
+    maxConcurrent = 4,
+    random = Math.random,
+    setTimer = setTimeout,
+    clearTimer = clearTimeout,
+    logger = console,
+  } = options;
+
+  const tasks = [];   // { route, intervalMs, timer }
+  const queue = [];
+  let running = false;
+  let active = 0;
+
+  function jittered(intervalMs) {
+    return Math.round(intervalMs * (1 + random() * 0.1));
+  }
+
+  function schedule(task) {
+    task.timer = setTimer(() => fire(task), jittered(task.intervalMs));
+    if (task.timer && typeof task.timer.unref === 'function') task.timer.unref();
+  }
+
+  function fire(task) {
+    if (!running) return;
+    if (active < maxConcurrent) runTask(task);
+    else queue.push(task);
+  }
+
+  async function runTask(task) {
+    active++;
+    try {
+      await engine.regenerate(task.route);
+    } catch (e) {
+      logger.error?.('[what-isr] scheduled regenerate failed:', e);
+    } finally {
+      active--;
+      if (running) schedule(task);
+      drain();
+    }
+  }
+
+  function drain() {
+    while (running && active < maxConcurrent && queue.length) {
+      runTask(queue.shift());
+    }
+  }
+
+  return {
+    register(route, { intervalMs }) {
+      tasks.push({ route, intervalMs, timer: null });
+      return this;
+    },
+    start() {
+      running = true;
+      for (const t of tasks) schedule(t);
+      return this;
+    },
+    stop() {
+      running = false;
+      for (const t of tasks) if (t.timer != null) clearTimer(t.timer);
+      queue.length = 0;
+      return this;
+    },
+    _tasks: tasks,
+  };
+}

package/src/stores/filesystem-store.js

@@ -0,0 +1,127 @@
+// Filesystem cache store — survives restarts and is shareable by multiple
+// worker processes on one box. Entries are sharded JSON files; writes are atomic
+// (write to .tmp then rename). Tag/path reverse indexes are sidecar JSON lists.
+
+import { mkdir, writeFile, readFile, rename, rm, readdir, stat } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { createHash } from 'node:crypto';
+import { hashKey } from '../key.js';
+
+function safeName(s) {
+  return createHash('sha256').update(String(s)).digest('hex');
+}
+
+export function createFilesystemStore({ dir }) {
+  const entriesDir = join(dir, 'entries');
+  const tagsDir = join(dir, 'tags');
+  const pathsDir = join(dir, 'paths');
+
+  const entryFile = (key) => join(entriesDir, hashKey(key) + '.json');
+  const indexFile = (base, name) => join(base, safeName(name) + '.json');
+
+  async function atomicWrite(file, contents) {
+    await mkdir(dirname(file), { recursive: true });
+    const tmp = `${file}.${process.pid}.${safeName(file).slice(0, 8)}.tmp`;
+    await writeFile(tmp, contents);
+    await rename(tmp, file);
+  }
+
+  async function readJson(file) {
+    try {
+      return JSON.parse(await readFile(file, 'utf8'));
+    } catch {
+      return null;
+    }
+  }
+
+  async function addToIndex(base, name, key) {
+    const file = indexFile(base, name);
+    const list = (await readJson(file)) || [];
+    if (!list.includes(key)) {
+      list.push(key);
+      await atomicWrite(file, JSON.stringify(list));
+    }
+  }
+
+  async function removeFromIndex(base, name, key) {
+    const file = indexFile(base, name);
+    const list = await readJson(file);
+    if (!list) return;
+    const next = list.filter((k) => k !== key);
+    if (next.length) await atomicWrite(file, JSON.stringify(next));
+    else await rm(file, { force: true });
+  }
+
+  async function deleteByIndex(base, name) {
+    const file = indexFile(base, name);
+    const keys = await readJson(file);
+    if (!keys) return [];
+    for (const k of keys) await removeKey(k);
+    await rm(file, { force: true });
+    return keys;
+  }
+
+  async function removeKey(key) {
+    const record = await readJson(entryFile(key));
+    if (!record) return false;
+    await rm(entryFile(key), { force: true });
+    const entry = record.entry || {};
+    for (const t of entry.tags || []) await removeFromIndex(tagsDir, t, key);
+    if (entry.path) await removeFromIndex(pathsDir, entry.path, key);
+    return true;
+  }
+
+  async function walkKeys() {
+    const out = [];
+    let shards;
+    try { shards = await readdir(entriesDir); } catch { return out; }
+    for (const a of shards) {
+      const aDir = join(entriesDir, a);
+      let st; try { st = await stat(aDir); } catch { continue; }
+      if (!st.isDirectory()) continue;
+      for (const b of await readdir(aDir)) {
+        const bDir = join(aDir, b);
+        try { if (!(await stat(bDir)).isDirectory()) continue; } catch { continue; }
+        for (const f of await readdir(bDir)) {
+          if (!f.endsWith('.json')) continue;
+          const rec = await readJson(join(bDir, f));
+          if (rec && rec.key != null) out.push(rec.key);
+        }
+      }
+    }
+    return out;
+  }
+
+  return {
+    async get(key) {
+      const rec = await readJson(entryFile(key));
+      return rec ? rec.entry : null;
+    },
+    async set(key, entry) {
+      // de-index any previous version's tags/path first
+      const prev = await readJson(entryFile(key));
+      if (prev && prev.entry) {
+        for (const t of prev.entry.tags || []) await removeFromIndex(tagsDir, t, key);
+        if (prev.entry.path) await removeFromIndex(pathsDir, prev.entry.path, key);
+      }
+      await atomicWrite(entryFile(key), JSON.stringify({ key, entry }));
+      for (const t of entry.tags || []) await addToIndex(tagsDir, t, key);
+      if (entry.path) await addToIndex(pathsDir, entry.path, key);
+    },
+    async delete(key) {
+      return removeKey(key);
+    },
+    async deleteByTag(tag) {
+      return deleteByIndex(tagsDir, tag);
+    },
+    async deleteByPath(path) {
+      return deleteByIndex(pathsDir, path);
+    },
+    async clear() {
+      await rm(dir, { recursive: true, force: true });
+    },
+    async keys() {
+      return walkKeys();
+    },
+  };
+}

package/src/stores/memory-store.js

@@ -0,0 +1,85 @@
+// In-memory cache store — the zero-config default. LRU eviction + reverse
+// indexes (tag -> keys, path -> keys) for O(1) group invalidation.
+//
+// Insertion order of a Map IS the LRU order: get() re-inserts (moves to newest),
+// eviction removes from the front (oldest).
+
+export function createMemoryStore({ maxEntries = 1000 } = {}) {
+  const map = new Map();        // key -> entry
+  const tagIndex = new Map();   // tag -> Set<key>
+  const pathIndex = new Map();  // path -> Set<key>
+
+  function addToIndex(index, name, key) {
+    if (name == null) return;
+    let set = index.get(name);
+    if (!set) index.set(name, (set = new Set()));
+    set.add(key);
+  }
+  function removeFromIndex(index, name, key) {
+    if (name == null) return;
+    const set = index.get(name);
+    if (set) {
+      set.delete(key);
+      if (set.size === 0) index.delete(name);
+    }
+  }
+  function indexEntry(key, entry) {
+    if (entry.tags) for (const t of entry.tags) addToIndex(tagIndex, t, key);
+    addToIndex(pathIndex, entry.path, key);
+  }
+  function deindexEntry(key, entry) {
+    if (entry.tags) for (const t of entry.tags) removeFromIndex(tagIndex, t, key);
+    removeFromIndex(pathIndex, entry.path, key);
+  }
+  function removeKey(key) {
+    const e = map.get(key);
+    if (!e) return false;
+    map.delete(key);
+    deindexEntry(key, e);
+    return true;
+  }
+  function deleteByIndex(index, name) {
+    const set = index.get(name);
+    if (!set) return [];
+    const deleted = [...set];
+    for (const k of deleted) removeKey(k);
+    return deleted;
+  }
+
+  return {
+    async get(key) {
+      const e = map.get(key);
+      if (!e) return null;
+      // LRU touch: move to newest.
+      map.delete(key);
+      map.set(key, e);
+      return e;
+    },
+    async set(key, entry) {
+      if (map.has(key)) deindexEntry(key, map.get(key));
+      map.set(key, entry);
+      indexEntry(key, entry);
+      while (map.size > maxEntries) {
+        const oldest = map.keys().next().value;
+        removeKey(oldest);
+      }
+    },
+    async delete(key) {
+      return removeKey(key);
+    },
+    async deleteByTag(tag) {
+      return deleteByIndex(tagIndex, tag);
+    },
+    async deleteByPath(path) {
+      return deleteByIndex(pathIndex, path);
+    },
+    async clear() {
+      map.clear();
+      tagIndex.clear();
+      pathIndex.clear();
+    },
+    async keys() {
+      return [...map.keys()];
+    },
+  };
+}

package/src/stores/redis-store.js

@@ -0,0 +1,62 @@
+// Redis/KV cache store — the multi-instance story (N app servers share one
+// cache). Takes an INJECTED client (ioredis / node-redis shaped:
+// get/set/del/sadd/srem/smembers, optional keys) so this package keeps zero deps.
+
+export function createRedisStore({ client, namespace = 'what' } = {}) {
+  if (!client) throw new Error('[what-isr] createRedisStore requires { client }');
+
+  const ck = (key) => `${namespace}:cache:${key}`;
+  const tk = (tag) => `${namespace}:tag:${tag}`;
+  const pk = (path) => `${namespace}:path:${path}`;
+
+  async function deindex(key, entry) {
+    if (!entry) return;
+    for (const t of entry.tags || []) await client.srem(tk(t), key);
+    if (entry.path) await client.srem(pk(entry.path), key);
+  }
+
+  async function deleteBySet(setKey) {
+    const keys = (await client.smembers(setKey)) || [];
+    for (const k of keys) await client.del(ck(k));
+    await client.del(setKey);
+    return keys;
+  }
+
+  return {
+    async get(key) {
+      const v = await client.get(ck(key));
+      return v ? JSON.parse(v) : null;
+    },
+    async set(key, entry) {
+      const prev = await this.get(key);
+      if (prev) await deindex(key, prev);
+      await client.set(ck(key), JSON.stringify(entry));
+      for (const t of entry.tags || []) await client.sadd(tk(t), key);
+      if (entry.path) await client.sadd(pk(entry.path), key);
+    },
+    async delete(key) {
+      const entry = await this.get(key);
+      await client.del(ck(key));
+      await deindex(key, entry);
+      return !!entry;
+    },
+    async deleteByTag(tag) {
+      return deleteBySet(tk(tag));
+    },
+    async deleteByPath(path) {
+      return deleteBySet(pk(path));
+    },
+    async clear() {
+      if (typeof client.keys === 'function') {
+        const all = await client.keys(`${namespace}:*`);
+        for (const k of all) await client.del(k);
+      }
+    },
+    async keys() {
+      if (typeof client.keys !== 'function') return [];
+      const prefix = `${namespace}:cache:`;
+      const all = await client.keys(`${prefix}*`);
+      return all.map((k) => k.slice(prefix.length));
+    },
+  };
+}

package/src/stores/store-interface.js

@@ -0,0 +1,60 @@
+// CacheStore contract (JSDoc only — JS source, .d.ts ships types) + helpers.
+//
+// A store maps a cache key -> Entry. All methods are async so memory, filesystem
+// and Redis adapters share one interface (memory just resolves synchronously).
+//
+// @typedef {Object} Entry
+// @property {string} html        Rendered body HTML.
+// @property {string} [head]      Collected <head> HTML.
+// @property {*} [state]          Serialized hydration state (loaderData/resources).
+// @property {string[]} [tags]    Tags for group invalidation (revalidateTag).
+// @property {string} [path]      Normalized path (for revalidatePath).
+// @property {number} renderedAt  ms epoch when rendered.
+// @property {number} maxAge      Revalidate seconds (0 = never time-stale).
+// @property {number} expiresAt   renderedAt + maxAge*1000 (precomputed).
+// @property {number} swrWindow   Grace seconds an expired entry is still served.
+// @property {number} [status]    HTTP status (for 404 stubs / fallback skeletons).
+// @property {boolean} [partial]  True for fallback skeletons (never durable).
+//
+// @typedef {Object} CacheStore
+// @property {(key:string)=>Promise<Entry|null>} get
+// @property {(key:string, entry:Entry)=>Promise<void>} set
+// @property {(key:string)=>Promise<boolean>} delete
+// @property {(tag:string)=>Promise<string[]>} deleteByTag   returns deleted keys
+// @property {(path:string)=>Promise<string[]>} deleteByPath returns deleted keys
+// @property {()=>Promise<void>} clear
+// @property {()=>Promise<string[]>} keys
+
+/**
+ * Fill an Entry's time fields from `now` + a route config. Used by the ISR
+ * engine so every store receives consistent expiry metadata.
+ */
+export function makeEntry(out, config = {}, now = Date.now()) {
+  const maxAge = Number(config.revalidate) || 0;
+  const swrWindow = config.swr != null ? Number(config.swr) : maxAge;
+  return {
+    html: out.html || '',
+    head: out.head || '',
+    state: out.state ?? null,
+    tags: out.tags || config.tags || [],
+    path: out.path || config.path,
+    status: out.status || 200,
+    partial: !!out.partial,
+    renderedAt: now,
+    maxAge,
+    swrWindow,
+    expiresAt: maxAge > 0 ? now + maxAge * 1000 : Infinity,
+  };
+}
+
+/** Freshness check against a clock. */
+export function isFresh(entry, now = Date.now()) {
+  return entry.expiresAt === Infinity || now < entry.expiresAt;
+}
+
+/** Within the stale-while-revalidate grace window (servable while regenerating). */
+export function isServableStale(entry, now = Date.now()) {
+  if (isFresh(entry, now)) return true;
+  if (entry.swrWindow == null) return false;
+  return now < entry.expiresAt + entry.swrWindow * 1000;
+}

package/src/webhook.js

@@ -0,0 +1,55 @@
+// On-demand revalidation webhook — the CMS-trigger path. A POST with a shared
+// secret purges paths/tags so a Sanity/Contentful/WP "published" event can warm
+// or drop cache entries. The adapter mounts this at e.g. /__what_revalidate.
+
+// Constant-time string compare (timing-attack safe). Self-contained so the cache
+// package stays dependency-free and decoupled from what-server.
+function safeEqual(a, b) {
+  if (typeof a !== 'string' || typeof b !== 'string') return false;
+  if (a.length !== b.length) return false;
+  let result = 0;
+  for (let i = 0; i < a.length; i++) result |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return result === 0;
+}
+
+/**
+ * @param engine  cache engine (revalidatePath / revalidateTag)
+ * @param options { secret, header='x-what-revalidate-secret', regenerate=false }
+ * @returns async (reqLike:{headers, body:{paths?,tags?,regenerate?}}) -> { status, body }
+ */
+export function createRevalidateWebhook(engine, options = {}) {
+  const { secret, header = 'x-what-revalidate-secret', regenerate = false } = options;
+
+  return async function handle(reqLike) {
+    const provided = (reqLike.headers || {})[header] || (reqLike.headers || {})[header.toLowerCase()];
+    if (!secret || !safeEqual(provided || '', secret)) {
+      return { status: 401, body: { message: 'Unauthorized' } };
+    }
+
+    const body = reqLike.body;
+    if (!body || typeof body !== 'object') {
+      return { status: 400, body: { message: 'Invalid body' } };
+    }
+
+    const { paths, tags, regenerate: regen = regenerate } = body;
+    if (!Array.isArray(paths) && !Array.isArray(tags)) {
+      return { status: 400, body: { message: 'Provide `paths` and/or `tags` arrays' } };
+    }
+
+    const revalidated = { paths: [], tags: [] };
+    if (Array.isArray(paths)) {
+      for (const p of paths) {
+        await engine.revalidatePath(p, { regenerate: regen });
+        revalidated.paths.push(p);
+      }
+    }
+    if (Array.isArray(tags)) {
+      for (const t of tags) {
+        await engine.revalidateTag(t, { regenerate: regen });
+        revalidated.tags.push(t);
+      }
+    }
+
+    return { status: 200, body: { revalidated: true, ...revalidated } };
+  };
+}