rip-lang 3.15.3 → 3.15.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rip-lang",
-  "version": "3.15.3",
+  "version": "3.15.4",
   "description": "A modern language that compiles to JavaScript",
   "type": "module",
   "main": "src/compiler.js",

package/src/browser.js

@@ -10,6 +10,7 @@ import './schema/loader-browser.js';
 export { Lexer } from './lexer.js';
 export { parser } from './parser.js';
 export { CodeEmitter, Compiler, compile, compileToJS, formatSExpr, getStdlibCode, getReactiveRuntime, getComponentRuntime, RipError, formatError, formatErrorHTML } from './compiler.js';
+import { mergeChunksWithInlineMap } from './sourcemap-merge.js';
 import { getStdlibCode, formatError as _formatError } from './compiler.js';
 
 // Version info (replaced during build)
@@ -224,19 +225,17 @@ async function processRipScripts() {
 
       // Bundle / multi-source path. Components defined in one .rip file
       // need to be visible to siblings (e.g. `WidgetGallery` referencing
-      // sibling widgets like `Toast`, `Dialog`, `Menu`). To make that
-      // work we concatenate every compiled chunk into ONE async-IIFE and
-      // eval it as a single closure — declarations made by one source
-      // are visible to all subsequent sources via lexical scope.
+      // `Toast`, `Dialog`, `Menu`). To make that work we concatenate
+      // every compiled chunk into ONE async-IIFE and eval it as a
+      // single closure — declarations made by one source are visible
+      // to all subsequent sources via lexical scope.
       //
-      // Trade-off: source maps don't work in concat mode (each
-      // `compileToJS` call emits its own map; concatenation would need a
-      // map merger we don't have). Source maps still work for:
-      //   - the data-router path (app.launch component compile, Phase 2)
-      //   - the single-inline-script path (one source = one map)
-      // The bundle no-router path is mostly used by component libraries
-      // (packages/ui itself) where step-debugging is less critical.
-      // Inline source maps are skipped here unconditionally.
+      // For source maps: each compileToJS call emits its own per-chunk
+      // map; mergeChunksWithInlineMap merges them into ONE V3 map with
+      // a multi-entry `sources` / `sourcesContent` array. DevTools
+      // reads the single merged map and shows each `.rip` file as a
+      // navigable source — same UX as the data-router path, no
+      // per-component IIFE split that would break lexical scope.
       const debug = runtimeTag?.getAttribute('data-debug') !== 'false';
       if (globalThis.__ripDebug) globalThis.__ripDebug.enabled = debug;
       const baseOpts = { skipRuntimes: true, skipExports: true, skipImports: true };
@@ -245,8 +244,11 @@ async function processRipScripts() {
       for (const s of expanded) {
         if (!s.code) continue;
         const ripName = s.url || `inline-${++inlineCounter}.rip`;
+        const opts = debug
+          ? { ...baseOpts, sourceMap: 'inline', filename: ripName }
+          : baseOpts;
         try {
-          const js = compileToJS(s.code, baseOpts);
+          const js = compileToJS(s.code, opts);
           compiled.push({ js, url: ripName });
         } catch (e) {
           console.error(_formatError(e, { source: s.code, file: ripName, color: false }));
@@ -280,14 +282,27 @@ async function processRipScripts() {
       // this, each chunk's `let Foo = class ...` dies when its IIFE
       // returns and `WidgetGallery` (defined in one chunk) can't see
       // `Toast` (defined in another).
+      //
+      // mergeChunksWithInlineMap concatenates the per-chunk JS bodies
+      // (stripping their individual `//# sourceMappingURL=` pragmas)
+      // and emits ONE V3 map at the end whose `sources` array holds
+      // every chunk's original `.rip` filename. DevTools shows each
+      // `.rip` file as its own source even though everything runs in
+      // one eval'd closure.
       if (compiled.length > 0) {
         const mount = runtimeTag?.getAttribute('data-mount');
         const target = runtimeTag?.getAttribute('data-target') || 'body';
         const mountSnippet = mount ? `\n${mount}.mount(${JSON.stringify(target)});\n` : '';
-        const merged = compiled.map(c => c.js).join('\n;\n');
+        const mergedBody = mergeChunksWithInlineMap(compiled.map(c => ({ js: c.js })));
+        // Inject the mount call BEFORE the trailing sourceMappingURL pragma
+        // so the pragma stays at the end of the eval'd chunk (DevTools
+        // requires it there). If no mount, mergedBody is emitted as-is.
+        const wrapped = mount
+          ? mergedBody.replace(/(\n\/\/# sourceMappingURL=[^\n]*\n?)?$/, mountSnippet + '$1')
+          : mergedBody;
         let ok = true;
         try {
-          await (0, eval)(`(async()=>{\n${merged}${mountSnippet}\n})()`);
+          await (0, eval)(`(async()=>{\n${wrapped}\n})()`);
         } catch (e) {
           ok = false;
           if (e instanceof SyntaxError) console.error(`Rip syntax error: ${e.message}`);

package/src/compiler.js

@@ -354,21 +354,45 @@ export class CodeEmitter {
 
   // Walk the s-expression tree and record source map entries for
   // sub-expressions that carry .loc, giving column-level precision.
+  //
+  // Performance: the inner loop runs M × N times where M = sub-expressions
+  // and N = regex matches per sub-expression. Computing genLine/genCol via
+  // `code.substring(0, m.index).split('\n')` was O(N) per match, making
+  // the function O(M × N²) overall and catastrophic on large generated
+  // blocks (a 100KB statement was taking 36 seconds in the browser).
+  //
+  // Fix: precompute a sorted `lineStarts` array (offset of each line's
+  // first character), then binary-search to convert offset → line/col in
+  // O(log N) per match. Brings the inline-gallery compile from 36s → ~30ms.
   recordSubMappings(code, sexpr, lineOffset) {
     let stmtOrigLine = sexpr.loc ? sexpr.loc.r : 0;
     let subs = [];
     this.collectSubExprs(sexpr, subs);
     let codeLines = code.split('\n');
+    // lineStarts[i] = offset in `code` of the first char on line i.
+    // Length is codeLines.length; lineStarts[0] is 0.
+    const lineStarts = [0];
+    for (let i = 0; i < code.length; i++) {
+      if (code.charCodeAt(i) === 10) lineStarts.push(i + 1);
+    }
+    // Binary-search the largest lineStart <= offset; that gives the line.
+    const offsetToLine = (offset) => {
+      let lo = 0, hi = lineStarts.length - 1;
+      while (lo <= hi) {
+        const mid = (lo + hi) >> 1;
+        if (lineStarts[mid] <= offset) lo = mid + 1;
+        else hi = mid - 1;
+      }
+      return hi;
+    };
     for (let { name, origLine, origCol } of subs) {
       let escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
       let re = new RegExp('\\b' + escaped + '\\b', 'g');
       let m, bestMatch = null, bestDist = Infinity;
       let origLineInStmt = origLine - stmtOrigLine;
       while ((m = re.exec(code)) !== null) {
-        let before = code.substring(0, m.index);
-        let nl = before.split('\n');
-        let genLineInStmt = nl.length - 1;
-        let genCol = nl[nl.length - 1].length;
+        const genLineInStmt = offsetToLine(m.index);
+        const genCol = m.index - lineStarts[genLineInStmt];
         // Skip matches inside string literals — prevents false mappings when
         // an identifier also appears as a string value (e.g. union type member)
         let lineText = codeLines[genLineInStmt];

package/src/sourcemap-merge.js

@@ -0,0 +1,287 @@
+// Source Map V3 — multi-chunk merger.
+//
+// Concatenates N JS chunks (each with its own inline source map) into one
+// JS string + one merged source map. Used by `processRipScripts` to produce
+// a single eval'd chunk that DevTools can map back to multiple original
+// `.rip` files.
+//
+// Why this exists: DevTools only honours the LAST `//# sourceMappingURL`
+// inside one eval'd chunk. So when N chunks share lexical scope (concat
+// + single eval, the bundle-no-router execution model), we must produce
+// ONE map that covers all N sources. Source Map V3 supports this natively
+// via a multi-entry `sources` / `sourcesContent` array and source-index
+// fields in the VLQ mappings — every production JS bundler does this.
+//
+// Algorithm:
+//   1. Strip per-chunk `//# sourceMappingURL=` pragmas from each chunk's JS
+//   2. Decode each chunk's `mappings` to absolute coordinates
+//   3. Concatenate JS with `\n;\n` separators (statement-safe, +2 newlines
+//      between chunks)
+//   4. Offset each chunk's generated line numbers by the cumulative line
+//      count of preceding chunks + separators
+//   5. Dedupe `sources`, `sourcesContent`, `names` into merged arrays;
+//      remap each chunk's source-index / name-index fields
+//   6. Re-encode as relative VLQ across the full concatenated mapping
+//
+// Returns { js, mapJSON } where mapJSON is a JSON string ready to embed
+// as a base64 data URL via a single `//# sourceMappingURL=` at the end.
+
+import { vlqEncode, vlqDecode } from './sourcemaps.js';
+
+// `\n;\n` is statement-safe between chunks (the `;` ensures the previous
+// line's expression is terminated). Counts as 2 newlines for line-offset.
+const SEPARATOR = '\n;\n';
+const SEPARATOR_LINES = 2;
+
+// Strip a trailing `//# sourceMappingURL=...` pragma from a JS string so
+// concatenated chunks don't carry stray maps that DevTools would honor
+// in unexpected order. Returns { js, mapJSON } where mapJSON is the
+// JSON string extracted from the inline data URL (or null if no pragma).
+function extractMap(js) {
+  const re = /\n?\/\/# sourceMappingURL=data:application\/json(?:;charset=[^;,]+)?;base64,([A-Za-z0-9+/=]+)\s*$/;
+  const m = js.match(re);
+  if (!m) return { js, mapJSON: null };
+  let mapJSON;
+  try {
+    // UTF-8-safe decode (counterpart of the encode in compiler.js).
+    const bin = atob(m[1]);
+    const bytes = new Uint8Array(bin.length);
+    for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+    mapJSON = new TextDecoder().decode(bytes);
+  } catch {
+    return { js: js.slice(0, m.index), mapJSON: null };
+  }
+  return { js: js.slice(0, m.index), mapJSON };
+}
+
+// Count the number of newline characters in a string. Used for line-offset
+// math when concatenating chunks; one chunk's line count = newline count
+// (a chunk starting at line offset L and containing N newlines occupies
+// generated lines [L, L+N], so the next chunk starts at L+N+1 if the
+// chunk had a trailing newline, or L+N if not — handled via SEPARATOR).
+function countNewlines(str) {
+  let n = 0;
+  for (let i = 0; i < str.length; i++) if (str.charCodeAt(i) === 10) n++;
+  return n;
+}
+
+// Decode a Source Map V3 `mappings` field into an array of arrays of
+// absolute-coordinate segments. Each segment is one of:
+//   { genCol }                              — unmapped (1 field)
+//   { genCol, src, origLine, origCol }      — mapped (4 fields)
+//   { genCol, src, origLine, origCol, name } — mapped + name (5 fields)
+// State: genCol resets to 0 at each line; src/origLine/origCol/name
+// persist across lines per the V3 spec.
+function decodeMappings(mappingsStr) {
+  const lines = mappingsStr.split(';');
+  const result = [];
+  let src = 0, origLine = 0, origCol = 0, nameIdx = 0;
+  for (const line of lines) {
+    const segs = [];
+    let genCol = 0;
+    if (line.length > 0) {
+      for (const segStr of line.split(',')) {
+        if (segStr.length === 0) continue;
+        const fields = vlqDecode(segStr);
+        genCol += fields[0];
+        if (fields.length === 1) {
+          segs.push({ genCol });
+        } else {
+          src += fields[1];
+          origLine += fields[2];
+          origCol += fields[3];
+          const seg = { genCol, src, origLine, origCol };
+          if (fields.length >= 5) {
+            nameIdx += fields[4];
+            seg.name = nameIdx;
+          }
+          segs.push(seg);
+        }
+      }
+    }
+    result.push(segs);
+  }
+  return result;
+}
+
+// Re-encode an array of arrays of absolute-coordinate segments back to
+// V3 `mappings` string. Counterpart of decodeMappings.
+function encodeMappings(perLineSegs) {
+  let prevGenCol = 0; // resets per line
+  let prevSrc = 0, prevOrigLine = 0, prevOrigCol = 0, prevNameIdx = 0;
+  const out = [];
+  for (const segs of perLineSegs) {
+    prevGenCol = 0;
+    const lineParts = [];
+    for (const seg of segs) {
+      if (seg.src == null) {
+        lineParts.push(vlqEncode(seg.genCol - prevGenCol));
+      } else {
+        let s = vlqEncode(seg.genCol - prevGenCol);
+        s += vlqEncode(seg.src - prevSrc);
+        s += vlqEncode(seg.origLine - prevOrigLine);
+        s += vlqEncode(seg.origCol - prevOrigCol);
+        if (seg.name != null) {
+          s += vlqEncode(seg.name - prevNameIdx);
+          prevNameIdx = seg.name;
+        }
+        prevSrc = seg.src;
+        prevOrigLine = seg.origLine;
+        prevOrigCol = seg.origCol;
+        lineParts.push(s);
+      }
+      prevGenCol = seg.genCol;
+    }
+    out.push(lineParts.join(','));
+  }
+  return out.join(';');
+}
+
+// Given an array of `{js, mapJSON}` chunks (where mapJSON is the source-map
+// V3 JSON string for that chunk, or null/undefined if the chunk has no
+// map), produce a single merged result.
+//
+// Returns:
+//   {
+//     js,        // concatenated JS, with all per-chunk sourceMappingURL
+//                // pragmas stripped — caller appends ONE merged pragma
+//     mapJSON,   // V3 source-map JSON string covering all chunks, or
+//                // null if no chunk had a map
+//   }
+//
+// The chunks must be in the order they should be eval'd. Each chunk's JS
+// is treated as opaque — we don't parse it, we only count newlines for
+// line-offset math.
+export function mergeChunks(chunks) {
+  // Strip per-chunk pragmas first; we'll append exactly one at the end.
+  const stripped = chunks.map((c) => {
+    if (c.mapJSON !== undefined) return { js: c.js, mapJSON: c.mapJSON };
+    return extractMap(c.js);
+  });
+
+  // If no chunk has a map, the merged JS is just concatenated; no map.
+  const anyMap = stripped.some((c) => c.mapJSON);
+  const mergedJS = stripped.map((c) => c.js).join(SEPARATOR);
+  if (!anyMap) return { js: mergedJS, mapJSON: null };
+
+  // Build merged sources / sourcesContent / names with deduplication
+  // by string identity. Per-chunk remap tables let us rewrite each
+  // chunk's source-index / name-index fields into the merged arrays.
+  const mergedSources = [];
+  const mergedSourcesContent = [];
+  const sourceIndexByName = new Map();
+  const mergedNames = [];
+  const nameIndexByName = new Map();
+
+  function addSource(name, content) {
+    if (sourceIndexByName.has(name)) return sourceIndexByName.get(name);
+    const idx = mergedSources.length;
+    mergedSources.push(name);
+    mergedSourcesContent.push(content ?? null);
+    sourceIndexByName.set(name, idx);
+    return idx;
+  }
+  function addName(name) {
+    if (nameIndexByName.has(name)) return nameIndexByName.get(name);
+    const idx = mergedNames.length;
+    mergedNames.push(name);
+    nameIndexByName.set(name, idx);
+    return idx;
+  }
+
+  // Per-chunk: line offset (cumulative gen-line shift), source-index
+  // remap, name-index remap, decoded segments.
+  const chunkInfos = [];
+  let lineOffset = 0;
+  for (let i = 0; i < stripped.length; i++) {
+    const c = stripped[i];
+    const info = { lineOffset, srcRemap: [], nameRemap: [], perLineSegs: null };
+
+    if (c.mapJSON) {
+      let map;
+      try { map = JSON.parse(c.mapJSON); }
+      catch { map = null; }
+      if (map && typeof map.mappings === 'string') {
+        const sources = map.sources || [];
+        const sourcesContent = map.sourcesContent || [];
+        info.srcRemap = sources.map((s, idx) => addSource(s, sourcesContent[idx]));
+        const names = map.names || [];
+        info.nameRemap = names.map((n) => addName(n));
+        info.perLineSegs = decodeMappings(map.mappings);
+      }
+    }
+
+    chunkInfos.push(info);
+
+    // Advance lineOffset by this chunk's line count + separator (except
+    // after the last chunk). Use the JS string we'll actually emit so
+    // line counts stay consistent with the merged output.
+    const chunkLines = countNewlines(c.js);
+    const sepLines = (i < stripped.length - 1) ? SEPARATOR_LINES : 0;
+    lineOffset += chunkLines + sepLines;
+  }
+
+  // Walk all chunks and emit their segments into the merged per-line
+  // segment array, with each chunk's gen-line indices shifted by the
+  // chunk's lineOffset and source/name indices remapped.
+  const totalLines = lineOffset + 1; // last chunk's lines fit within
+  const mergedPerLineSegs = new Array(totalLines).fill(null).map(() => []);
+
+  for (const info of chunkInfos) {
+    if (!info.perLineSegs) continue;
+    for (let li = 0; li < info.perLineSegs.length; li++) {
+      const target = info.lineOffset + li;
+      if (target >= mergedPerLineSegs.length) {
+        // Defensive: extend if a chunk has more lines than we counted.
+        while (mergedPerLineSegs.length <= target) mergedPerLineSegs.push([]);
+      }
+      for (const seg of info.perLineSegs[li]) {
+        if (seg.src == null) {
+          mergedPerLineSegs[target].push({ genCol: seg.genCol });
+        } else {
+          const remapped = {
+            genCol: seg.genCol,
+            src: info.srcRemap[seg.src],
+            origLine: seg.origLine,
+            origCol: seg.origCol,
+          };
+          if (seg.name != null) remapped.name = info.nameRemap[seg.name];
+          mergedPerLineSegs[target].push(remapped);
+        }
+      }
+    }
+  }
+
+  const mergedMappings = encodeMappings(mergedPerLineSegs);
+  const mergedMap = {
+    version: 3,
+    sources: mergedSources,
+    sourcesContent: mergedSourcesContent,
+    names: mergedNames,
+    mappings: mergedMappings,
+  };
+  return { js: mergedJS, mapJSON: JSON.stringify(mergedMap) };
+}
+
+// UTF-8-safe base64 encode for embedding mapJSON as a data URL pragma.
+// Counterpart of the encode used in src/compiler.js.
+function utf8ToBase64(str) {
+  if (typeof Buffer !== 'undefined') {
+    return Buffer.from(str, 'utf8').toString('base64');
+  }
+  // Browser path: TextEncoder + btoa over latin1 bytes.
+  const bytes = new TextEncoder().encode(str);
+  let bin = '';
+  for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);
+  return btoa(bin);
+}
+
+// Convenience: take chunks, run mergeChunks, append a single
+// `//# sourceMappingURL=` data-URL pragma to the merged JS, return it.
+// If no chunk had a map, returns the concat'd JS unchanged.
+export function mergeChunksWithInlineMap(chunks) {
+  const { js, mapJSON } = mergeChunks(chunks);
+  if (!mapJSON) return js;
+  const b64 = utf8ToBase64(mapJSON);
+  return js + '\n//# sourceMappingURL=data:application/json;base64,' + b64 + '\n';
+}