npm - rewritable - Versions diffs - 0.1.0 → 0.5.0 - Mend

rewritable 0.1.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (31) hide show

package/README.md +263 -5
package/bin/rwa.mjs +1033 -6
package/package.json +7 -4
package/seeds/rewritable.html +6989 -156
package/src/agent-loop.mjs +155 -0
package/src/apply-edits.mjs +664 -0
package/src/atomic-write.mjs +38 -0
package/src/backend.mjs +43 -0
package/src/clone-extract.mjs +249 -0
package/src/clone.mjs +161 -0
package/src/commands.mjs +207 -11
package/src/create.mjs +256 -0
package/src/doc.mjs +69 -0
package/src/dsl-compiler.mjs +357 -0
package/src/edit.mjs +300 -0
package/src/fetch-page.mjs +346 -0
package/src/host.mjs +126 -0
package/src/identity.mjs +257 -0
package/src/import-claude.mjs +360 -0
package/src/import-vision.mjs +156 -0
package/src/import.mjs +357 -8
package/src/ls.mjs +105 -0
package/src/publish-site.mjs +85 -0
package/src/publish.mjs +98 -0
package/src/seed-extract.mjs +40 -0
package/src/seed.mjs +1399 -6
package/src/self-contained.mjs +115 -0
package/src/skill-manifest.mjs +227 -0
package/src/skin.mjs +350 -0
package/src/skins.mjs +274 -0
package/src/template.mjs +109 -0

package/src/import-vision.mjs ADDED Viewed

@@ -0,0 +1,156 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+// PDF → HTML via OpenRouter chat completions.
+//
+// Why this exists: pdfjs's text extraction produces flat-paragraph output
+// that loses tables, multi-column layouts, and any text whose font has a
+// broken toUnicode CMap (e.g. "Ü" decoded as "UY"). Sending the raw PDF to
+// a vision-capable model bypasses both — the model reads the rendered
+// content and reconstructs semantic HTML.
+//
+// Trade-off: ~$0.01-$0.05 per page in API costs, network round-trip
+// latency. Opt-in via `rwa import file.pdf --vision`.
+//
+// Wire format: OpenRouter's PDF input docs say content type is "file" with
+// `file_data: "data:application/pdf;base64,..."`. For Anthropic models OR
+// passes this through as a native PDF document block; for others (Gemini,
+// GPT-4o), it's routed through OR's file-parser plugin (engine "native"
+// uses the model's own multimodal capability).
+const OPENROUTER_URL = 'https://openrouter.ai/api/v1/chat/completions';
+const SYSTEM_PROMPT = `You are converting a PDF document into clean, semantic HTML for embedding in a single-file rewritable document container.
+Output requirements:
+- A single <article> element containing all document content.
+- Use semantic HTML: <h1>-<h6> for headings, <p> for paragraphs, <ul>/<ol>/<li> for lists, <table><thead><tbody><tr><td>/<th> for tables, <strong>/<em> for emphasis, <a href="..."> for links.
+- Do NOT output <html>, <head>, <body>, <!doctype>, any preamble, or any explanation before or after the HTML.
+- Do NOT wrap output in markdown code fences (no \`\`\`html).
+- Preserve text content exactly — do not summarize, paraphrase, translate, or reword.
+- Reconstruct multi-column layouts and tables faithfully. Table headers go in <thead>, body rows in <tbody>.
+- Omit <img> entirely; this container is text-focused. If an image carries information, describe it briefly in a <p>.
+- No <script>, <style>, class, or id attributes. Plain semantic HTML only.
+Output ONLY the <article>...</article> element.`;
+const USER_PROMPT = 'Convert this PDF document to a single <article> element of clean semantic HTML, following the rules in the system prompt.';
+/**
+ * @param {Buffer|Uint8Array} bytes  PDF content
+ * @param {object} [opts]
+ * @param {string} [opts.apiKey]     OpenRouter API key. If omitted, read from
+ *                                   process.env.OPENROUTER_API_KEY, then ./.env
+ * @param {string} [opts.model]      OpenRouter model id; default reuses
+ *                                   the rwa container's default
+ * @param {AbortSignal} [opts.signal]
+ * @returns {Promise<{ html: string, warnings: string[] }>}
+ */
+export async function convertPdfViaVision(bytes, { apiKey, model, signal } = {}) {
+  apiKey = apiKey || process.env.OPENROUTER_API_KEY || await readDotEnvKey('OPENROUTER_API_KEY');
+  if (!apiKey) {
+    const e = new Error('vision: OPENROUTER_API_KEY is required (set in env or ./.env)');
+    e.exitCode = 2;
+    throw e;
+  }
+  const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
+  const dataUri = `data:application/pdf;base64,${buf.toString('base64')}`;
+  const body = {
+    model: model || 'google/gemini-3.5-flash',
+    messages: [
+      { role: 'system', content: SYSTEM_PROMPT },
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: USER_PROMPT },
+          { type: 'file', file: { filename: 'document.pdf', file_data: dataUri } },
+        ],
+      },
+    ],
+    // Generous output budget — long PDFs can produce a lot of HTML.
+    // OpenRouter will clamp to model's actual max if smaller.
+    max_tokens: 16384,
+    // Deterministic output — we want the same HTML for the same input.
+    temperature: 0,
+  };
+  const res = await fetch(OPENROUTER_URL, {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+      // Recommended by OpenRouter for tracking, helps with rate-limit accounting.
+      'HTTP-Referer': 'https://github.com/martintreiber/rewritable',
+      'X-Title': 'rwa CLI',
+    },
+    body: JSON.stringify(body),
+    signal,
+  });
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    const e = new Error(`vision: openrouter ${res.status}${text ? ': ' + text.slice(0, 500) : ''}`);
+    e.exitCode = 2;
+    throw e;
+  }
+  const json = await res.json();
+  const content = json?.choices?.[0]?.message?.content;
+  if (typeof content !== 'string' || !content.trim()) {
+    const e = new Error('vision: openrouter returned empty content');
+    e.exitCode = 2;
+    throw e;
+  }
+  const html = extractArticle(content);
+  if (!html) {
+    const e = new Error(
+      `vision: model output did not contain an <article> element. Output preview:\n${content.slice(0, 300)}`
+    );
+    e.exitCode = 2;
+    throw e;
+  }
+  const warnings = [];
+  // Surface usage so the user sees what each import cost.
+  const usage = json?.usage;
+  if (usage) {
+    const tokens = `${usage.prompt_tokens || 0} in / ${usage.completion_tokens || 0} out`;
+    warnings.push(`vision: ${body.model} (${tokens} tokens)`);
+  }
+  return { html, warnings };
+}
+// Minimal .env reader for the OPENROUTER_API_KEY fallback path. Handles
+// KEY=value with optional surrounding whitespace, optional matched quotes,
+// optional `export` prefix. No interpolation, no multi-line values. Returns
+// null if the file or key is missing.
+async function readDotEnvKey(name) {
+  let text;
+  try {
+    text = await fs.readFile(path.join(process.cwd(), '.env'), 'utf8');
+  } catch (_) { return null; }
+  for (const line of text.split('\n')) {
+    const m = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
+    if (!m || m[1] !== name) continue;
+    let v = m[2];
+    if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) {
+      v = v.slice(1, -1);
+    }
+    return v || null;
+  }
+  return null;
+}
+// Extract the outermost <article>...</article>. Models often wrap output in
+// ```html fences or add a "Here is the HTML:" preamble despite the system
+// prompt; pull out only the article element to be robust to that.
+function extractArticle(text) {
+  // Find the first <article (allow attributes) and the LAST </article>.
+  const start = text.search(/<article(?:\s[^>]*)?>/i);
+  if (start < 0) return null;
+  const end = text.lastIndexOf('</article>');
+  if (end < 0 || end < start) return null;
+  return text.slice(start, end + '</article>'.length).trim();
+}

package/src/import.mjs CHANGED Viewed

@@ -1,33 +1,57 @@
 import { marked } from 'marked';
+import Papa from 'papaparse';
+import mammoth from 'mammoth';
+import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs';
-export async function convert(ext, content) {
+// `convert` takes raw bytes (Buffer / Uint8Array). Text formats decode utf8
+// internally; binary formats consume bytes directly. Switching to bytes was
+// driven by docx/pdf — keeping a single signature avoids a fork.
+export async function convert(ext, bytes) {
   switch (ext) {
     case 'md':
     case 'markdown':
-      return convertMd(content);
+      return convertMd(toText(bytes));
     case 'html':
     case 'htm':
-      return convertHtml(content);
+      return convertHtml(toText(bytes));
+    case 'csv':
+      return convertCsv(toText(bytes));
     case 'txt':
     case '':
-      return convertTxt(content);
+      return convertTxt(toText(bytes));
+    case 'docx':
+      return convertDocx(bytes);
+    case 'pdf':
+      return convertPdf(bytes);
     default: {
-      const e = new Error(`unsupported format: .${ext} (supported: .md, .markdown, .html, .htm, .txt)`);
+      const e = new Error(`unsupported format: .${ext} (supported: .md, .markdown, .html, .htm, .csv, .txt, .docx, .pdf)`);
       e.exitCode = 2;
       throw e;
     }
   }
 }
+function toText(bytes) {
+  if (typeof bytes === 'string') return bytes;
+  if (Buffer.isBuffer(bytes)) return bytes.toString('utf8');
+  return Buffer.from(bytes).toString('utf8');
+}
 function convertMd(md) {
-  const html = marked.parse(md, { gfm: true, breaks: false });
-  return { html: `<article>\n${html.trim()}\n</article>`, warnings: [] };
+  const raw = marked.parse(md, { gfm: true, breaks: false });
+  const { html, warnings } = sanitizeImportedHtml(raw);
+  return { html: `<article>\n${html.trim()}\n</article>`, warnings };
 }
 function convertHtml(input) {
   const warnings = [];
-  let body = input.replace(/<!DOCTYPE[^>]*>/gi, '').replace(/<\/?html[^>]*>/gi, '');
+  // Strip HTML comments first. Without this, a comment like <!-- </head> -->
+  // would terminate the non-greedy head match prematurely and let head content
+  // leak into the body. Comments are dropped — acceptable for an offline import
+  // CLI; full preservation would require a real parser.
+  let body = input.replace(/<!--[\s\S]*?-->/g, '');
+  body = body.replace(/<!DOCTYPE[^>]*>/gi, '').replace(/<\/?html[^>]*>/gi, '');
   const headMatch = body.match(/<head[^>]*>([\s\S]*?)<\/head>/i);
   let headStyles = '';
@@ -49,6 +73,42 @@ function convertHtml(input) {
   return { html: headStyles + body, warnings };
 }
+function looksLikeCsv(text) {
+  const probe = Papa.parse(text, { preview: 2, skipEmptyLines: true, header: false });
+  if (probe.errors.length > 0) return false;
+  if (probe.data.length === 0) return false;
+  const cols = probe.data[0].length;
+  if (cols < 2) return false;
+  if (probe.data.length === 2 && probe.data[1].length !== cols) return false;
+  return true;
+}
+function convertCsv(text) {
+  if (!looksLikeCsv(text)) {
+    const e = new Error('csv probe failed: input does not look like CSV (need ≥2 columns with consistent column count)');
+    e.exitCode = 2;
+    throw e;
+  }
+  // skipEmptyLines drops trailing blank rows that csv exporters often emit.
+  // header:false because we own the first-row-as-thead split and want raw rows.
+  const result = Papa.parse(text, { skipEmptyLines: true, header: false });
+  const warnings = result.errors.map(e => {
+    const where = e.row != null ? ` (row ${e.row + 1})` : '';
+    return `csv parse: ${e.message}${where}`;
+  });
+  const rows = result.data;
+  if (rows.length === 0) {
+    return { html: '<article>\n</article>', warnings };
+  }
+  const escape = s => String(s == null ? '' : s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+  const [header, ...body] = rows;
+  const thead = `<thead>\n<tr>${header.map(c => `<th>${escape(c)}</th>`).join('')}</tr>\n</thead>`;
+  const tbody = body.length === 0
+    ? ''
+    : `\n<tbody>\n${body.map(row => `<tr>${row.map(c => `<td>${escape(c)}</td>`).join('')}</tr>`).join('\n')}\n</tbody>`;
+  return { html: `<article>\n<table>\n${thead}${tbody}\n</table>\n</article>`, warnings };
+}
 function convertTxt(text) {
   const escape = s => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
   const blocks = text
@@ -58,3 +118,292 @@ function convertTxt(text) {
     .map(b => `<p>${escape(b)}</p>`);
   return { html: `<article>\n${blocks.join('\n')}\n</article>`, warnings: [] };
 }
+async function convertDocx(bytes) {
+  const buffer = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
+  let result;
+  try {
+    result = await mammoth.convertToHtml({ buffer });
+  } catch (err) {
+    const e = new Error(`docx: ${err && err.message ? err.message : String(err)}`);
+    e.exitCode = 2;
+    throw e;
+  }
+  const raw = (result.value || '').trim();
+  if (!raw) {
+    const e = new Error('docx: produced empty document — input may be corrupt or empty');
+    e.exitCode = 2;
+    throw e;
+  }
+  const { html, skipped } = sanitizeMammothUrls(raw);
+  const warnings = [
+    ...skipped.map(s => `docx: ${s}`),
+    ...(result.messages || []).map(m => `docx: ${m.message}`),
+  ];
+  return { html: `<article>\n${html}\n</article>`, warnings };
+}
+// Mammoth doesn't filter URL schemes — a docx with a `javascript:` hyperlink
+// would land in the imported document and execute on click (stored XSS in the
+// downloaded rwa container). Strip unsafe schemes from href/src and replace
+// with `#`. Mammoth's HTML writer always uses double-quoted attributes and
+// escapes &, ", <, > inside values, so a regex match against `attr="..."` is
+// sufficient — no quote-escape ambiguity to worry about.
+const _SAFE_HREF_SCHEMES = new Set(['http', 'https', 'mailto', 'tel']);
+// Two layers, both required:
+//   1) Strip invisibles before parsing — whitespace + C0/C1 controls (\x00-\x1f,
+//      \x7f-\xa0) + soft hyphen (\xad) + Cf-class format chars (ZWSP/ZWNJ/ZWJ,
+//      LRM/RLM, LRE/RLE/PDF/LRO/RLO, word joiner, BOM, etc.). The previous
+//      regex used JS \s which doesn't match these — they slipped through and
+//      let a docx with `javascript:…` href bypass the scheme check.
+//   2) Parse via WHATWG URL — the same parser the browser uses to navigate.
+//      Resolve against a synthetic base so scheme-less inputs (relative URL,
+//      fragment, path) round-trip back to that base and pass.
+const _ATTR_STRIP_RE = /[\s\x00-\x1f\x7f-\xa0\xad؜᠎-‏‪-‮⁠-⁯]/g;
+const _SANITIZER_BASE = 'http://_rwa_sanitizer_base_/';
+function _attrIsSafe(attr, val) {
+  const normalized = String(val).replace(_ATTR_STRIP_RE, '');
+  let parsed;
+  try { parsed = new URL(normalized, _SANITIZER_BASE); }
+  catch { return true; } // unparseable → cannot be an active URL scheme
+  if (parsed.origin === 'http://_rwa_sanitizer_base_') return true; // resolved relative — no scheme
+  const proto = parsed.protocol.replace(/:$/, '').toLowerCase();
+  if (_SAFE_HREF_SCHEMES.has(proto)) return true;
+  // Mammoth embeds raster images as data:image/...;base64,... — allow on src.
+  // data:image/svg+xml passes here too, but <img src> renders SVG in image-
+  // loading mode with no script execution (HTML spec), so the narrow
+  // 'data:image/*' allowance is still safe for src. Keep scoped to src only.
+  if (attr === 'src' && proto === 'data' && /^data:image\//i.test(parsed.href)) return true;
+  return false;
+}
+function sanitizeMammothUrls(html) {
+  const skipped = [];
+  const stripAttr = (attr) => (full, val) => {
+    if (_attrIsSafe(attr, val)) return full;
+    const m = val.match(/^\s*([a-z][a-z0-9+.\-]*):/i);
+    const scheme = m ? m[1].toLowerCase() : 'unknown';
+    skipped.push(`stripped unsafe ${attr} (scheme: ${scheme}:)`);
+    return `${attr}="#"`;
+  };
+  return {
+    html: html
+      .replace(/href="([^"]*)"/g, stripAttr('href'))
+      .replace(/src="([^"]*)"/g, stripAttr('src')),
+    skipped,
+  };
+}
+// marked v14 explicitly does NOT sanitize HTML — its README points readers at
+// DOMPurify. The seed bootstrap injects INLINE_DOC via m.innerHTML AND
+// re-creates <script> tags so they execute (intended for documents that ship
+// JS), so any active content in the imported HTML runs on container open. An
+// imported .md must not be able to add active content.
+//
+// Regex-based strip (not a parser) for mirror-symmetry with the browser. The
+// rules below are deliberately conservative: when in doubt, strip. Marked's
+// output is well-formed and uses double-quoted attributes, so the regex shape
+// matches reliably. Edge cases (CDATA, malformed nesting) are over-stripped
+// rather than under-stripped — acceptable for an import path.
+const _ACTIVE_TAGS = ['script', 'iframe', 'object', 'embed', 'svg', 'math', 'link', 'meta', 'base'];
+export function sanitizeImportedHtml(html) {
+  const warnings = [];
+  let s = String(html);
+  // 1) Drop active-content tags (open+close blocks, then self-closing/unmatched).
+  for (const tag of _ACTIVE_TAGS) {
+    const block = new RegExp('<' + tag + '\\b[^>]*>[\\s\\S]*?<\\/' + tag + '\\s*>', 'gi');
+    const solo  = new RegExp('<\\/?' + tag + '\\b[^>]*\\/?>', 'gi');
+    if (block.test(s) || solo.test(s)) warnings.push('imported md: stripped <' + tag + '> elements');
+    s = s.replace(block, '').replace(solo, '');
+  }
+  // 2) Drop on*= event-handler attributes from surviving elements.
+  //    Match quoted (double/single) and unquoted-to-whitespace/> forms.
+  let onCount = 0;
+  s = s.replace(/\son[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi, () => { onCount++; return ''; });
+  if (onCount) warnings.push('imported md: stripped ' + onCount + ' event-handler attribute(s)');
+  // 3) Apply scheme allow-list to surviving URL-bearing attributes. Marked's
+  //    output is double-quoted href/src only, but rwa clone feeds ARBITRARY web
+  //    HTML here — single-quoted, unquoted, and other URL attributes (action/
+  //    formaction/poster/xlink:href) are all common and must be checked too, or
+  //    a `href='javascript:…'` survives into the file:// container as a live,
+  //    clickable link. Match all three value forms (mirror of the on*= strip)
+  //    and the full reachable URL-attr set. data:image/* stays allowed on src.
+  let urlSkipped = 0;
+  s = s.replace(
+    /(\s)(xlink:href|formaction|href|src|action|poster)(\s*=\s*)("[^"]*"|'[^']*'|[^\s>]+)/gi,
+    (full, ws, name, eq, rawVal) => {
+      const lname = name.toLowerCase();
+      const attr = (lname === 'src' || lname === 'poster') ? 'src' : 'href';
+      const quoted = rawVal[0] === '"' || rawVal[0] === "'";
+      const val = quoted ? rawVal.slice(1, -1) : rawVal;
+      if (_attrIsSafe(attr, val)) return full;
+      urlSkipped++;
+      return ws + name + eq + '"#"';
+    }
+  );
+  if (urlSkipped) warnings.push('imported md: neutralised ' + urlSkipped + ' unsafe URL attribute(s)');
+  return { html: s, warnings };
+}
+async function convertPdf(bytes) {
+  // pdfjs explicitly rejects Node's Buffer (despite Buffer extending Uint8Array)
+  // and wants a plain Uint8Array view.
+  const src = bytes instanceof Uint8Array ? bytes : Buffer.from(bytes);
+  const data = new Uint8Array(src.buffer, src.byteOffset, src.byteLength);
+  let doc;
+  try {
+    doc = await pdfjs.getDocument({ data, isEvalSupported: false }).promise;
+  } catch (err) {
+    const name = err && err.name;
+    if (name === 'PasswordException') {
+      const e = new Error('pdf: file is password-protected');
+      e.exitCode = 2;
+      throw e;
+    }
+    const e = new Error(`pdf: ${err && err.message ? err.message : String(err)}`);
+    e.exitCode = 2;
+    throw e;
+  }
+  const escape = s => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+  const paragraphs = [];
+  for (let p = 1; p <= doc.numPages; p++) {
+    const page = await doc.getPage(p);
+    const tc = await page.getTextContent();
+    extractParagraphs(tc.items).forEach(line => paragraphs.push(line));
+    paragraphs.push(null); // page break: forces flush of next paragraph
+  }
+  await doc.destroy().catch(() => {});
+  const blocks = [];
+  let buf = [];
+  const flush = () => {
+    const joined = buf.join(' ').replace(/\s+/g, ' ').trim();
+    if (joined) blocks.push(`<p>${escape(joined)}</p>`);
+    buf = [];
+  };
+  for (const line of paragraphs) {
+    if (line === null || line === '') { flush(); continue; }
+    buf.push(line);
+  }
+  flush();
+  if (blocks.length === 0) {
+    const e = new Error('pdf: no extractable text — this looks like a scanned/image PDF; OCR is not supported');
+    e.exitCode = 2;
+    throw e;
+  }
+  return {
+    html: `<article>\n${blocks.join('\n')}\n</article>`,
+    warnings: ['pdf: layout reconstructed by heuristics — review headings/lists manually'],
+  };
+}
+// Group pdf.js text items into paragraph-shaped lines.
+//
+// Two non-obvious problems this handles:
+// (1) Adjacent items inside a word: pdf.js returns text per font run, so a
+//     word like "Aufwände" comes out as ["Aufw", "ä", "nde"] when the umlaut
+//     glyph lives in a different font table from the ASCII letters. Joining
+//     with ' ' produces "Aufw ä nde" — wrong. We concat directly and only
+//     synthesize a space when there's a real positional x-gap.
+// (2) Stacked short lines: an address block (Name / Street / City) has small
+//     y-gaps that fit inside the within-paragraph threshold, so naive logic
+//     would join them into one paragraph "Name Street City". We additionally
+//     break paragraphs when the *previous* line ended significantly short of
+//     the page's typical right margin (a heuristic for "hard line break").
+//
+// Returns an array of strings; '' marks a paragraph break.
+function extractParagraphs(items) {
+  if (!items || items.length === 0) return [];
+  const rows = items.map(it => ({
+    str: it.str,
+    y: it.transform ? it.transform[5] : 0,
+    x: it.transform ? it.transform[4] : 0,
+    w: it.width || 0,
+    h: it.height || (it.transform ? Math.abs(it.transform[3]) : 0) || 12,
+  }));
+  // Sort top-to-bottom (y desc in PDF coords), then left-to-right within a
+  // row. pdfjs's content-stream order is reading order for well-tagged
+  // single-column PDFs, but for multi-column or absolutely-positioned layouts
+  // it interleaves visually-separate lines; sorting first makes the same-y
+  // grouping below tolerant of that.
+  rows.sort((a, b) => b.y - a.y || a.x - b.x);
+  // Group into visual lines by y (within half a line height).
+  const lines = [];
+  let cur = null;
+  for (const r of rows) {
+    if (cur && Math.abs(r.y - cur.y) <= cur.h * 0.5) {
+      cur.parts.push(r);
+      cur.y = (cur.y + r.y) / 2;
+    } else {
+      if (cur) lines.push(cur);
+      cur = { y: r.y, h: r.h, parts: [r] };
+    }
+  }
+  if (cur) lines.push(cur);
+  // For each line: concat parts directly, inserting a synthetic space only
+  // when there's a real positional gap (previous part's right edge to next
+  // part's x). pdf.js often emits explicit space items (str=" ") with tiny
+  // width — those carry the space character themselves, so the position-gap
+  // check below typically sees ~0 distance when they're present and we don't
+  // double-space.
+  const rendered = lines.map(line => {
+    line.parts.sort((a, b) => a.x - b.x);
+    let text = '';
+    let prev = null;
+    for (const p of line.parts) {
+      if (prev) {
+        const gap = p.x - (prev.x + prev.w);
+        const lastChar = text.slice(-1);
+        const firstChar = p.str.charAt(0);
+        // Threshold of 2 user-space units catches inter-word gaps on body
+        // text without false-positives inside words. Skip if the boundary
+        // already has whitespace from either side.
+        if (gap > 2 && !/\s/.test(lastChar) && !/\s/.test(firstChar)) {
+          text += ' ';
+        }
+      }
+      text += p.str;
+      prev = p;
+    }
+    const left = line.parts.length ? Math.min(...line.parts.map(p => p.x)) : 0;
+    const right = line.parts.length
+      ? Math.max(...line.parts.map(p => p.x + p.w))
+      : 0;
+    return { text: text.replace(/\s+/g, ' ').trim(), y: line.y, h: line.h, left, right };
+  });
+  // The page's "typical right margin" — use the 90th-percentile right edge
+  // (more robust than max, which a stray header/page-number could inflate).
+  // Lines ending well short of this are likely hard line-breaks, not soft
+  // wraps to the right margin.
+  const sortedRights = rendered.filter(l => l.text).map(l => l.right).sort((a, b) => a - b);
+  const margin = sortedRights.length
+    ? sortedRights[Math.floor(sortedRights.length * 0.9)]
+    : 0;
+  const out = [];
+  let prev = null;
+  for (const line of rendered) {
+    if (!line.text) continue;  // pdfjs sometimes emits whitespace-only EOL stubs; ignore.
+    if (prev != null) {
+      const yGap = Math.abs(prev.y - line.y);
+      const yJump = yGap > prev.h * 1.5;
+      // Previous line ended significantly short of the page's right margin —
+      // that's the signature of a hard line-break (address line, table cell,
+      // bullet, sender block). Threshold of 1.5× line height (~1-2 chars)
+      // ignores end-of-line whitespace + small justification slop while still
+      // catching genuinely short lines. Soft wraps to the right margin are
+      // within ~few units and don't trigger.
+      const prevShortOfMargin = margin > 0 && (margin - prev.right) > prev.h * 1.5;
+      // Right-aligned blocks have a fixed right edge but varying left edge
+      // per line. A jump of more than a line-height in left position is a
+      // structural change, not text-flow continuation.
+      const leftJump = Math.abs(prev.left - line.left) > line.h;
+      if (yJump || prevShortOfMargin || leftJump) out.push('');
+    }
+    out.push(line.text);
+    prev = line;
+  }
+  return out;
+}

package/src/ls.mjs ADDED Viewed

@@ -0,0 +1,105 @@
+// `rwa ls` — collection-scale self-description. Where inspectDoc answers "what is
+// THIS file?", listRewritables answers "what are all these?": it resolves a set
+// of paths (files, directories, or — by default — the current directory) to
+// candidate .html files and reports each one's self-description/1 projection,
+// flagging non-rewritables. The scan is lenient: a missing path or a
+// non-rewritable is a row in the result, never a thrown error — so one bad entry
+// can't abort the inventory of a whole folder.
+import { readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { inspectDoc } from './doc.mjs';
+const HTML_RE = /\.html?$/i;
+/**
+ * Expand input paths to a flat, ordered list of candidate files. A directory
+ * contributes its (non-recursive) .html children; a file contributes itself; a
+ * path that cannot be stat'd is kept as a `missing` candidate so the caller can
+ * report it rather than silently drop it. No inputs ⇒ scan the current directory.
+ *
+ * @param {string[]} paths
+ * @returns {Promise<Array<{file:string, missing?:boolean}>>}
+ */
+export async function resolveTargets(paths) {
+  const inputs = (paths && paths.length) ? paths : ['.'];
+  const targets = [];
+  for (const p of inputs) {
+    let st;
+    try { st = await stat(p); } catch { targets.push({ file: p, missing: true }); continue; }
+    if (st.isDirectory()) {
+      let names;
+      try { names = await readdir(p); } catch { targets.push({ file: p, missing: true }); continue; }
+      for (const name of names.filter(n => HTML_RE.test(n)).sort()) {
+        targets.push({ file: join(p, name) });
+      }
+    } else {
+      targets.push({ file: p });
+    }
+  }
+  return targets;
+}
+/**
+ * Inspect each candidate and classify it. Each row is one of:
+ *   { file, status:'rewritable', self }            — self-description/1 object
+ *   { file, status:'not_a_rewritable' }            — a plain .html / other file
+ *   { file, status:'error', reason }               — not_found / read_error
+ *
+ * @param {string[]} paths
+ * @returns {Promise<Array<object>>}
+ */
+export async function listRewritables(paths) {
+  const targets = await resolveTargets(paths);
+  const rows = [];
+  for (const t of targets) {
+    if (t.missing) { rows.push({ file: t.file, status: 'error', reason: 'not_found' }); continue; }
+    try {
+      const info = await inspectDoc(t.file);
+      rows.push({ file: t.file, status: 'rewritable', self: info.self });
+    } catch (e) {
+      if (e && e.subcode === 'not_a_rewritable') {
+        rows.push({ file: t.file, status: 'not_a_rewritable' });
+      } else {
+        rows.push({ file: t.file, status: 'error', reason: (e && e.subcode) || 'read_error' });
+      }
+    }
+  }
+  return rows;
+}
+/**
+ * Render the rows as a human-readable, aligned inventory. Rewritables become a
+ * KIND/TITLE/AFFORDANCES/FILE table; a footer counts rewritables vs. other files
+ * and names any errors — nothing is silently dropped (Rule 12).
+ *
+ * @param {Array<object>} rows
+ * @returns {string}
+ */
+export function formatRows(rows) {
+  const rwa = rows.filter(r => r.status === 'rewritable');
+  const other = rows.filter(r => r.status === 'not_a_rewritable');
+  const errors = rows.filter(r => r.status === 'error');
+  const lines = [];
+  if (rwa.length) {
+    const cells = rwa.map(r => ({
+      kind: r.self.kind || '',
+      title: r.self.title || '—',
+      affordances: r.self.affordances.length ? r.self.affordances.map(a => a.kind).join(',') : '—',
+      file: r.file,
+    }));
+    const head = { kind: 'KIND', title: 'TITLE', affordances: 'AFFORDANCES', file: 'FILE' };
+    const w = (k) => Math.max(head[k].length, ...cells.map(c => c[k].length));
+    const wk = w('kind'), wt = w('title'), wa = w('affordances');
+    const row = (c) => `${c.kind.padEnd(wk)}  ${c.title.padEnd(wt)}  ${c.affordances.padEnd(wa)}  ${c.file}`;
+    lines.push(row(head));
+    for (const c of cells) lines.push(row(c));
+    lines.push('');
+  }
+  const parts = [`${rwa.length} rewritable${rwa.length === 1 ? '' : 's'}`];
+  if (other.length) parts.push(`${other.length} other (${other.map(r => r.file).join(', ')})`);
+  if (errors.length) parts.push(`${errors.length} error (${errors.map(r => `${r.file}: ${r.reason}`).join(', ')})`);
+  lines.push(parts.join(', '));
+  return lines.join('\n');
+}