rewritable 0.1.0 → 0.5.0

package/src/atomic-write.mjs

@@ -0,0 +1,38 @@
+// Crash-safe atomic file write for `rwa edit`. Write the new bytes to a sibling
+// temp file, fsync (datasync) its contents, then rename(2) over the target —
+// atomic on the same filesystem, so a reader sees either the old file or the new
+// one, never a half-written stream. (datasync, not sync: a power loss between
+// rename and the kernel flushing dirty pages could otherwise land a renamed file
+// with stale/zero bytes; we don't depend on the temp's metadata being durable.)
+//
+// On ANY failure the temp file is removed. Previously a writeFile/datasync
+// failure left a `.rwa-tmp-<pid>` behind — only the rename-failure path cleaned
+// up. `deps` is injectable so the cleanup-on-failure is unit-testable without an
+// actual disk failure.
+
+import { open, rename, unlink } from 'node:fs/promises';
+
+/**
+ * @param {string} filePath — destination path (overwritten atomically)
+ * @param {string} content — UTF-8 bytes to write
+ * @param {{open:Function, rename:Function, unlink:Function}} [deps] — injectable fs ops
+ */
+export async function atomicWrite(filePath, content, deps = { open, rename, unlink }) {
+  const tmp = `${filePath}.rwa-tmp-${process.pid}`;
+  let handle;
+  try {
+    handle = await deps.open(tmp, 'w');
+    await handle.writeFile(content, 'utf8');
+    await handle.datasync();
+    await handle.close();
+    handle = null;
+    await deps.rename(tmp, filePath);
+  } catch (e) {
+    // Close a still-open handle, then remove the temp so a failed write never
+    // leaks a .rwa-tmp-<pid>. Cleanup errors are swallowed — the original
+    // failure `e` is what the caller needs.
+    if (handle) { try { await handle.close(); } catch { /* already failing */ } }
+    await deps.unlink(tmp).catch(() => {});
+    throw e;
+  }
+}

package/src/backend.mjs

@@ -0,0 +1,43 @@
+// Backend auth resolution for `rwa edit`. Extracted from bin/rwa.mjs so the
+// precedence is unit-testable (the bin entrypoint runs on import and can't be
+// imported cleanly).
+//
+// Only the openrouter backend needs a key — ollama and lmstudio run locally
+// without auth. The key resolves in order: an explicit --api-key flag, then the
+// project-specific RWA_OPENROUTER_KEY (env conventions match the docker-compose
+// deploy in service/), then the CONVENTIONAL OPENROUTER_API_KEY that agents and
+// users normally have exported. Empty strings count as absent.
+
+/**
+ * Resolve the API key for a backend.
+ * @param {string} backendName — 'openrouter' | 'ollama' | 'lmstudio'
+ * @param {string|undefined} flagValue — the --api-key flag value, if any
+ * @param {Record<string,string|undefined>} [env] — environment (injectable for tests)
+ * @returns {string|undefined} the key, or undefined when none applies
+ */
+export function resolveApiKey(backendName, flagValue, env = process.env) {
+  if (flagValue) return flagValue;
+  if (backendName === 'openrouter') {
+    return env.RWA_OPENROUTER_KEY || env.OPENROUTER_API_KEY || undefined;
+  }
+  return undefined;
+}
+
+/**
+ * Default OpenAI-compatible base URL for a backend — mirrors the inline
+ * `envBaseUrl` in bin/rwa.mjs (and seeds/rewritable.html resolveBackendConfig).
+ * ollama and lmstudio honor RWA_*_URL overrides (remote host / non-standard port);
+ * openrouter is fixed (the URL has never drifted in the seed). Shared by `rwa edit`
+ * and `rwa create` so the default never diverges between the two.
+ * @param {string} name — 'openrouter' | 'ollama' | 'lmstudio'
+ * @param {Record<string,string|undefined>} [env] — environment (injectable for tests)
+ * @returns {string|undefined}
+ */
+export function envBaseUrl(name, env = process.env) {
+  switch (name) {
+    case 'openrouter': return 'https://openrouter.ai/api/v1';
+    case 'ollama':     return env.RWA_OLLAMA_URL || 'http://localhost:11434/v1';
+    case 'lmstudio':   return env.RWA_LMSTUDIO_URL || 'http://localhost:1234/v1';
+    default:           return undefined;
+  }
+}

package/src/clone-extract.mjs

@@ -0,0 +1,249 @@
+// clone-extract: locate the main article + title in a fetched webpage's HTML.
+//
+// This is the EXTRACTOR stage of `rwa clone <url>`: fetch (elsewhere) →
+// extract (here) → sanitize (elsewhere) → bootstrap (elsewhere). It only
+// LOCATES content; it does NOT strip scripts/attributes — a later sanitize
+// task owns that. Pure and dependency-free (built-in JS only) so it can be
+// mirrored to the browser /import path the way the rest of the CLI is.
+
+// Decode the entities that show up in titles. Titles come from og:title /
+// <title> / <h1>, which are entity-encoded in source HTML. WordPress wptexturize
+// emits numeric smart-punctuation (&#8217; etc.), so we decode numeric entities
+// generically plus a few common named ones. &amp; is decoded LAST so a
+// double-encoded "&amp;#8217;" doesn't get unescaped into a live entity.
+function decodeEntities(s) {
+  return s
+    .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
+    .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&hellip;/g, '…')
+    .replace(/&mdash;/g, '—')
+    .replace(/&ndash;/g, '–')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&amp;/g, '&');
+}
+
+function stripTags(s) {
+  return s.replace(/<[^>]*>/g, '');
+}
+
+// Title precedence: og:title → <title> (minus a " | Site" / " – Site" /
+// " — Site" suffix) → first <h1> → 'Untitled'. og:title is the cleanest
+// signal on the WordPress/OpenGraph pages we target; the <title> suffix
+// strip removes the site-name tail that most CMSes append.
+function extractTitle(html) {
+  const og = html.match(
+    /<meta[^>]*\bproperty\s*=\s*["']og:title["'][^>]*\bcontent\s*=\s*["']([^"']*)["']/i,
+  ) || html.match(
+    /<meta[^>]*\bcontent\s*=\s*["']([^"']*)["'][^>]*\bproperty\s*=\s*["']og:title["']/i,
+  );
+  if (og) return decodeEntities(og[1].trim());
+
+  const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  if (title) {
+    const raw = decodeEntities(stripTags(title[1]).trim());
+    // Drop a trailing " | Site" / " – Site" / " — Site" separator + tail.
+    const cut = raw.replace(/\s*[|–—]\s*[^|–—]*$/, '').trim();
+    if (cut) return cut;
+    if (raw) return raw;
+  }
+
+  const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
+  if (h1) {
+    const t = decodeEntities(stripTags(h1[1]).trim());
+    if (t) return t;
+  }
+
+  return 'Untitled';
+}
+
+// Find the index of the opening tag for the element whose class attribute
+// contains `cls`. Returns the index of the '<' or -1.
+function findClassOpen(html, cls) {
+  // Bound the class token on the class-list separators (start/space/quote)
+  // rather than \b, so "entry-content" does NOT match inside
+  // "entry-content-wrapper" or "entry-content-meta" — a hyphen is a \w
+  // boundary-free char, so \b would false-match the prefix.
+  const re = new RegExp(
+    `<([a-z][a-z0-9]*)\\b[^>]*\\bclass\\s*=\\s*(["'])(?:[^"']*[\\s])?${cls}(?=[\\s]|\\2)[^"']*\\2[^>]*>`,
+    'i',
+  );
+  const m = re.exec(html);
+  if (!m) return null;
+  // Compute the opening-tag end quote-aware: the regex's trailing [^>]*> stops
+  // at the first '>', which may be inside a later attribute value (e.g.
+  // title="a>b"). tagEnd skips quoted regions so openEnd lands on the real '>'.
+  const gt = tagEnd(html, m.index);
+  return { index: m.index, tag: m[1].toLowerCase(), openEnd: gt === -1 ? m.index + m[0].length : gt + 1 };
+}
+
+// Find the index of the '>' that ends a tag starting at/after `from`, skipping
+// over quoted attribute regions so a '>' inside an attribute value (e.g.
+// <div title="a>b">) is not mistaken for the tag end. Returns -1 if none.
+function tagEnd(html, from) {
+  let i = from, q = null;
+  while (i < html.length) {
+    const ch = html[i];
+    if (q) { if (ch === q) q = null; }
+    else if (ch === '"' || ch === "'") q = ch;
+    else if (ch === '>') return i;
+    i++;
+  }
+  return -1;
+}
+
+// THE CRUX — balanced extraction, parser-free.
+//
+// A naive non-greedy /(<div>)(.*?)(<\/div>)/ truncates at the FIRST nested
+// </div>, which on a real WordPress entry-content (panel-grid wrappers,
+// widget divs, tinymce divs — 6 nested <div>s deep in the fixture) cuts the
+// body off after the first paragraph. So instead we walk forward from the
+// opening tag tracking the open/close depth of `tagName` and stop at the
+// MATCHING close. HTML comments are skipped (so `<!-- .entry-content -->`
+// doesn't confuse the scan) and void/self-closing same-name tags don't
+// bump depth (div/article are never void, but we guard anyway).
+//
+// Returns the INNER HTML of the container (between openEnd and the matching
+// close), or null if no balanced close is found.
+function balancedInner(html, tagName, startIndex, openEnd) {
+  // Locate end of the opening tag at startIndex. Use the caller-supplied
+  // openEnd (from findClassOpen, already quote-aware) when available; else
+  // scan quote-aware so a '>' inside an attribute value doesn't end the tag.
+  const openTagEnd = openEnd != null ? openEnd - 1 : tagEnd(html, startIndex);
+  if (openTagEnd === -1) return null;
+  const innerStart = openTagEnd + 1;
+
+  const openRe = new RegExp(`<${tagName}\\b`, 'gi');
+  const closeRe = new RegExp(`</${tagName}\\s*>`, 'gi');
+  // Raw-text elements: their content is CDATA-like (never markup), so a literal
+  // "</div>" or "<div" inside a <script>/<style> must not perturb the scan.
+  const rawTextRe = /<(script|style)\b/gi;
+
+  let depth = 1;
+  let pos = innerStart;
+  while (pos < html.length) {
+    // Skip over HTML comments so markers like <!-- .entry-content --> and
+    // any commented-out same-name tags don't perturb the depth count.
+    if (html.startsWith('<!--', pos)) {
+      const end = html.indexOf('-->', pos + 4);
+      if (end === -1) break;
+      pos = end + 3;
+      continue;
+    }
+    openRe.lastIndex = pos;
+    closeRe.lastIndex = pos;
+    rawTextRe.lastIndex = pos;
+    const o = openRe.exec(html);
+    const c = closeRe.exec(html);
+    const r = rawTextRe.exec(html);
+    if (!c) break; // unbalanced — no matching close
+    // If a raw-text element opens before the next same-name open/close event,
+    // fast-forward past its matching </script>/</style> (raw-text elements
+    // don't nest), so its text — which may contain "</div>" or "<div" — is
+    // never interpreted as markup.
+    if (r && r.index < c.index && (!o || r.index < o.index)) {
+      const close = new RegExp(`</${r[1]}\\s*>`, 'i');
+      const after = r.index + r[0].length;
+      const m = close.exec(html.slice(after));
+      pos = m ? after + m.index + m[0].length : html.length;
+      continue;
+    }
+    if (o && o.index < c.index) {
+      // Next event is a nested open of the same tag. Self-closing (<tag .../>)
+      // wouldn't add depth; div/article aren't void so we treat all as nesting,
+      // but guard the self-closing form just in case. Quote-aware tag end so a
+      // '>' inside an attribute value doesn't end the tag early.
+      const gt = tagEnd(html, o.index);
+      const selfClosing = gt !== -1 && html[gt - 1] === '/';
+      if (!selfClosing) depth++;
+      pos = (gt === -1 ? html.length : gt + 1);
+    } else {
+      depth--;
+      pos = c.index + c[0].length;
+      if (depth === 0) return html.slice(innerStart, c.index);
+    }
+  }
+  return null;
+}
+
+// Generic density fallback: from <body>, scan top-level block elements and
+// pick the subtree with the highest text-length ÷ tag-count ratio. Chrome
+// (nav/header/footer/aside + class-matched menus/sidebars/comments/etc.) is
+// excluded so the dense content block wins even when the page has no
+// recognised content container. This is a heuristic, not a parser — good
+// enough to find "the main blob of prose" on an unknown layout.
+const CHROME_TAGS = /^(nav|header|footer|aside)$/i;
+const CHROME_CLASS = /\b(nav|menu|sidebar|footer|header|comment|share|related)\b/i;
+
+function isChrome(tag, attrs) {
+  if (CHROME_TAGS.test(tag)) return true;
+  const cls = attrs.match(/\bclass\s*=\s*["']([^"']*)["']/i);
+  if (cls && CHROME_CLASS.test(cls[1])) return true;
+  return false;
+}
+
+function density(inner) {
+  const text = stripTags(inner).replace(/\s+/g, ' ').trim();
+  const tags = (inner.match(/<[a-z][^>]*>/gi) || []).length || 1;
+  return text.length / tags;
+}
+
+function genericFallback(html) {
+  // Restrict to <body> when present.
+  let scope = html;
+  const bodyOpen = html.search(/<body\b[^>]*>/i);
+  if (bodyOpen !== -1) {
+    const bm = html.slice(bodyOpen).match(/<body\b[^>]*>/i);
+    const start = bodyOpen + bm[0].length;
+    const bodyClose = html.toLowerCase().indexOf('</body>', start);
+    scope = html.slice(start, bodyClose === -1 ? html.length : bodyClose);
+  }
+
+  // Walk top-level block elements in scope. We re-scan from each element's
+  // end (using its balanced close) so we only consider siblings, not nested
+  // descendants — that keeps the density comparison apples-to-apples.
+  const blockRe = /<([a-z][a-z0-9]*)\b([^>]*)>/gi;
+  let best = null;
+  let m;
+  let cursor = 0;
+  while ((m = blockRe.exec(scope))) {
+    if (m.index < cursor) continue; // inside a subtree we already consumed
+    const tag = m[1].toLowerCase();
+    const attrs = m[2];
+    const inner = balancedInner(scope, tag, m.index);
+    if (inner == null) { continue; }
+    // advance cursor past this whole element so its descendants are skipped
+    const closeRe = new RegExp(`</${tag}\\s*>`, 'gi');
+    closeRe.lastIndex = m.index + m[0].length + inner.length;
+    const cm = closeRe.exec(scope);
+    cursor = cm ? cm.index + cm[0].length : scope.length;
+    if (isChrome(tag, attrs)) continue;
+    const score = density(inner);
+    if (!best || score > best.score) best = { score, inner };
+  }
+  return best ? best.inner : scope;
+}
+
+export function extractArticle(html) {
+  const title = extractTitle(html);
+
+  // Profile 1 — WordPress/ikangai: the element whose class contains
+  // "entry-content" (appears once per post page).
+  const ec = findClassOpen(html, 'entry-content');
+  if (ec) {
+    const inner = balancedInner(html, ec.tag, ec.index, ec.openEnd);
+    if (inner != null) return { title, html: inner };
+  }
+
+  // Profile 2 — Generic semantic: first <article>…</article>.
+  const art = html.search(/<article\b[^>]*>/i);
+  if (art !== -1) {
+    const inner = balancedInner(html, 'article', art);
+    if (inner != null) return { title, html: inner };
+  }
+
+  // Fallback — density heuristic over <body> top-level blocks.
+  return { title, html: genericFallback(html) };
+}

package/src/clone.mjs

@@ -0,0 +1,161 @@
+// `rwa clone <url>` bootstrap wiring. Turns a fetched web page into a saved
+// rewritable by reusing the EXACT import pipeline: extract the article body,
+// run it through the same sanitiser the import path uses, then drop it into a
+// fresh seed via applySeedSubs + replaceInlineDoc.
+//
+// Why prepend an <h1>: inspectDoc derives the document title from the body's
+// first <h1>, not the <title> tag. A WordPress post's <h1> lives outside
+// .entry-content, so the extracted content carries no heading — without this
+// the cloned doc would render as "Untitled". The extracted page title is the
+// honest source, so we prepend it.
+
+import { access } from 'node:fs/promises';
+import { basename } from 'node:path';
+
+import { extractArticle } from './clone-extract.mjs';
+import { sanitizeImportedHtml } from './import.mjs';
+import { loadSeed, applySeedSubs, replaceInlineDoc } from './seed.mjs';
+import { SEED_CANDIDATES } from './commands.mjs';
+import { atomicWrite } from './atomic-write.mjs';
+import { fetchPage, fetchImageDataUri, CloneError } from './fetch-page.mjs';
+
+function escapeHtml(s) {
+  return String(s)
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+}
+
+// `rwa clone --localize-images`: make a clone truly self-contained by inlining
+// each remote <img src> as a data: URI (the same form the GUI/import produce),
+// so the saved file needs no network. Each image rides the SSRF-guarded
+// fetchImageDataUri (image/* only, raw bytes — the CLI has no canvas to
+// recompress). GRACEFUL by design: a fetch failure, SSRF block, non-image
+// content-type, over-cap image, or exhausted total budget leaves that <img> at
+// its remote URL and records a warning — one bad image never fails the clone.
+// Caps: per-image 2 MB, total 8 MB (under the 10 MB container budget, leaving
+// headroom for the prose). Relative src is resolved against the page URL.
+const LOCALIZE_PER_IMAGE = 2 * 1024 * 1024;
+const LOCALIZE_TOTAL = 8 * 1024 * 1024;
+const IMG_SRC_RE = /(<img\b[^>]*?\bsrc\s*=\s*)(["'])(https?:\/\/[^"']+|\/[^"']*|\.\.?\/[^"']*)\2/gi;
+
+export async function localizeImages(html, sourceUrl, opts = {}) {
+  const deps = opts.deps || {};
+  const perImage = opts.perImage || LOCALIZE_PER_IMAGE;
+  const totalCap = opts.totalCap || LOCALIZE_TOTAL;
+  const warnings = [];
+  let inlined = 0, totalBytes = 0;
+
+  // Collect matches first (regex with async work can't run inside .replace).
+  const matches = [];
+  let m;
+  IMG_SRC_RE.lastIndex = 0;
+  while ((m = IMG_SRC_RE.exec(html)) !== null) {
+    matches.push({ whole: m[0], pre: m[1], quote: m[2], src: m[3], index: m.index });
+  }
+
+  // Resolve + fetch each unique src once (a page often repeats an image).
+  const resolved = new Map(); // original src -> data URI (or null = leave remote)
+  for (const { src } of matches) {
+    if (resolved.has(src)) continue;
+    let abs;
+    try { abs = new URL(src, sourceUrl).href; }
+    catch { resolved.set(src, null); warnings.push(`skipped ${src} (unresolvable URL)`); continue; }
+    if (totalBytes >= totalCap) { resolved.set(src, null); warnings.push(`skipped ${src} (container image budget reached)`); continue; }
+    try {
+      const remaining = Math.min(perImage, totalCap - totalBytes);
+      const dataUri = await fetchImageDataUri(abs, { maxBytes: remaining, deps });
+      // base64 is ~4/3 of the raw bytes; count the raw size toward the budget.
+      totalBytes += Math.floor((dataUri.length - dataUri.indexOf(',') - 1) * 3 / 4);
+      resolved.set(src, dataUri);
+      inlined++;
+    } catch (err) {
+      resolved.set(src, null);
+      warnings.push(`skipped ${src} (${err && err.subcode ? err.subcode : (err && err.message) || 'fetch failed'})`);
+    }
+  }
+
+  const out = html.replace(IMG_SRC_RE, (whole, pre, quote, src) => {
+    const dataUri = resolved.get(src);
+    return dataUri ? pre + quote + dataUri + quote : whole;
+  });
+  return { html: out, inlined, warnings };
+}
+
+export async function cloneFromHtml(html, outPath, sourceUrl, opts = {}) {
+  const { title, html: content } = extractArticle(html);
+  // sanitizeImportedHtml returns { html, warnings }; we only need the cleaned body.
+  let { html: clean } = sanitizeImportedHtml(content);
+
+  // --localize-images: inline remote <img src> as data: URIs so the clone is
+  // truly self-contained. Runs AFTER the sanitizer (its http/https/data:image
+  // src allowlist already passed); swaps surviving remote URLs for data URIs.
+  if (opts.localizeImages) {
+    const r = await localizeImages(clean, sourceUrl, { deps: opts.deps });
+    clean = r.html;
+    if (r.inlined) console.error(`note: inlined ${r.inlined} image(s)`);
+    for (const w of r.warnings) console.error('note: ' + w);
+  }
+
+  // Defence-in-depth: the wired path (cloneCmd → fetchPage) already validates
+  // the scheme, but this exported fn must be safe-by-default for any caller.
+  // Only http/https URLs become a live provenance <a href>; anything else
+  // (javascript:, data:, file:, …) renders as plain escaped text — no href —
+  // so a hostile scheme can never produce a clickable link in the cloned doc.
+  const safeProvenance = /^https?:\/\//i.test(String(sourceUrl));
+  const provenance = safeProvenance
+    ? `<a href="${escapeHtml(sourceUrl)}">${escapeHtml(sourceUrl)}</a>`
+    : escapeHtml(sourceUrl);
+
+  const body = `<article>\n<h1>${escapeHtml(title)}</h1>\n${clean}\n`
+    + `<footer><p><small>Cloned from ${provenance}</small></p></footer>\n</article>`;
+
+  const seed = await loadSeed(SEED_CANDIDATES);
+  // Order matches the `rwa import` lesson: seed-level substitutions on the
+  // pristine seed FIRST, then inject the body — so DOC_UUID can't false-match
+  // inside imported content.
+  const subbed = applySeedSubs(seed, {
+    uuid: crypto.randomUUID(),
+    title,
+    fileMeta: basename(outPath),
+  });
+  const result = replaceInlineDoc(subbed, body);
+  await atomicWrite(outPath, result);
+  return { outPath, title };
+}
+
+// Derive a filename slug from the URL's last non-empty path segment.
+function slugFromUrl(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return 'clone';
+  }
+  const segments = parsed.pathname.split('/').filter(Boolean);
+  const last = segments.length ? segments[segments.length - 1] : '';
+  const slug = last.replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9-]+/gi, '-').replace(/^-+|-+$/g, '');
+  return slug || 'clone';
+}
+
+export async function cloneCmd({ url, outPath, force, localizeImages: localize }) {
+  const html = await fetchPage(url);
+  const resolvedOut = outPath || `./${slugFromUrl(url)}.html`;
+
+  if (!force) {
+    let exists = true;
+    try {
+      await access(resolvedOut);
+    } catch {
+      exists = false;
+    }
+    if (exists) {
+      throw new CloneError(2, 'exists', { path: resolvedOut,
+        message: `destination exists: ${resolvedOut} (use --force to overwrite)` });
+    }
+  }
+
+  await cloneFromHtml(html, resolvedOut, url, { localizeImages: localize });
+  console.log('wrote ' + resolvedOut);
+}