parseman 0.58.2 → 0.59.0

package/src/cst/plain.js

@@ -0,0 +1,74 @@
+// Constructors for the objects a parse keeps until it ends. Plain JavaScript on
+// purpose: a function constructor whose `prototype` is `Object.prototype` is the only
+// spelling that builds a plain object with a `new` call, and TypeScript cannot declare
+// one in a `.ts` file without a cast. `plain.d.ts` types them. Why constructors at all,
+// and why this is ONE module shared by every bundle: see `plain.d.ts`.
+
+export function PlainSpan(start, end) {
+  this.start = start
+  this.end = end
+}
+
+export function PlainLeaf(value, span) {
+  this._tag = 'leaf'
+  this.value = value
+  this.span = span
+}
+
+export function PlainLineSpan(start, end, startLine, startColumn, endLine, endColumn) {
+  this.start = start
+  this.end = end
+  this.startLine = startLine
+  this.startColumn = startColumn
+  this.endLine = endLine
+  this.endColumn = endColumn
+}
+
+export function PlainNode(type, span, state, children) {
+  this._tag = 'node'
+  this.type = type
+  this.span = span
+  this.state = state
+  this.children = children
+}
+
+export function PlainTaggedNode(type, tags, span, state, children) {
+  this._tag = 'node'
+  this.type = type
+  this.tags = tags
+  this.span = span
+  this.state = state
+  this.children = children
+}
+
+export function PlainField(name, value, span) {
+  this.name = name
+  this.value = value
+  this.span = span
+}
+
+export function PlainTriviaField(name, value, span, leading, trailing) {
+  this.name = name
+  this.value = value
+  this.span = span
+  this.leading = leading
+  this.trailing = trailing
+}
+
+// Every field assigned, so the three buffer kinds share one shape. `rawLen` is always a
+// count: a field that held `undefined` in most buffers and a number in some would
+// generalize and deoptimize the pushes that read it.
+export function PlainBuf(rawOnly, noRaw, rawLen) {
+  this.rawOnly = rawOnly
+  this.noRaw = noRaw
+  this.rawLen = rawLen
+  this.single = undefined
+  this.ch = undefined
+  this.rawSingle = undefined
+  this.raw = undefined
+  this.tl = undefined
+}
+
+for (const f of [PlainSpan, PlainLineSpan, PlainLeaf, PlainNode, PlainTaggedNode, PlainField, PlainTriviaField, PlainBuf]) {
+  f.prototype = Object.prototype
+}

package/src/lead-ranges.ts

@@ -0,0 +1,53 @@
+/**
+ * How a first set's ranges answer for an input's leading code point (`lead()`).
+ * A leaf module: the minimal run entry reaches it through recovery sentinels.
+ */
+import type { CharRange } from './types.ts'
+
+/**
+ * Whether `ranges` admit an input led by the code point `code`: in a range, or,
+ * for an astral lead, through its high surrogate. Outside Unicode mode a regex
+ * reads code units, so a set holding a high surrogate starts every astral code
+ * point that surrogate begins. Exact for such a set; for a `u` set holding a
+ * lone high surrogate it only over-approximates.
+ *
+ * The interpreter's two hot lead tests (`startsFirstSet`, choice `inFirstSet`)
+ * repeat this body inline: called, it costs them about 6% of the instructions per
+ * parse, measured on the CSS example.
+ */
+export function rangesHaveLead(ranges: readonly CharRange[], code: number): boolean {
+  for (const r of ranges) if (code >= r.lo && code <= r.hi) return true
+  if (code <= 0xffff) return false
+  const high = 0xd800 + ((code - 0x10000) >> 10)
+  for (const r of ranges) if (high >= r.lo && high <= r.hi) return true
+  return false
+}
+
+/** `ranges` with the astral code points `rangesHaveLead` admits through their
+ * high surrogates, for a test that reads a lead by range alone. */
+export function leadRanges(ranges: readonly CharRange[]): readonly CharRange[] {
+  let out: CharRange[] | undefined
+  for (const r of ranges) {
+    const lo = Math.max(r.lo, 0xd800), hi = Math.min(r.hi, 0xdbff)
+    if (lo <= hi) (out ??= [...ranges]).push({ lo: 0x10000 + ((lo - 0xd800) << 10), hi: 0x10000 + ((hi - 0xd800) << 10) + 0x3ff })
+  }
+  return out ?? ranges
+}
+
+/** The inverse of `leadRanges`: drop the astral code points that `ranges`' own high
+ * surrogates already admit, so a set read through `leadRanges` (an intersection)
+ * keeps the code-unit form a table encodes narrowly. `leadRanges` of the result
+ * equals `leadRanges` of the input. */
+export function implicitLeadsDropped(ranges: readonly CharRange[]): CharRange[] {
+  const implied = leadRanges(ranges.filter(r => r.lo <= 0xffff)).filter(r => r.lo > 0xffff)
+  const out: CharRange[] = []
+  for (const r of ranges) {
+    let pieces: CharRange[] = [{ lo: r.lo, hi: r.hi }]
+    if (r.hi > 0xffff) for (const c of implied) {
+      pieces = pieces.flatMap(q => q.hi < c.lo || q.lo > c.hi ? [q]
+        : [...(q.lo < c.lo ? [{ lo: q.lo, hi: c.lo - 1 }] : []), ...(q.hi > c.hi ? [{ lo: c.hi + 1, hi: q.hi }] : [])])
+    }
+    out.push(...pieces)
+  }
+  return out
+}

package/src/line-index.ts

@@ -1,4 +1,5 @@
 import type { Span } from './types.ts'
+import { PlainLineSpan, PlainSpan } from './cst/plain.js'
 
 /**
  * Precomputed index of newline positions for O(log n) offset->line/col lookup.
@@ -98,6 +99,42 @@ export function annotateSpan(span: Span, index: LineIndex): Span {
   }
 }
 
+const SPAN_KEYS = ['start', 'end', 'startLine', 'startColumn', 'endLine', 'endColumn'] as const
+
+/**
+ * A copy of a span, exactly as given: the same keys in the same order. The two shapes
+ * parseman builds itself, `{ start, end }` and the six-key line span, are rebuilt by their
+ * constructors (`src/cst/plain.d.ts`); any other shape a caller passes (an extra key, a
+ * different order, a partial line span) is spread, as before.
+ */
+export function copySpan(span: Span): Span {
+  // `for…in` walks the keys in order without allocating a key list.
+  let n = 0
+  for (const key in span) {
+    if (key !== SPAN_KEYS[n]) return { ...span }
+    n++
+  }
+  if (n === 2) return new PlainSpan(span.start, span.end)
+  if (n === 6 && span.startLine !== undefined && span.startColumn !== undefined && span.endLine !== undefined && span.endColumn !== undefined) {
+    return new PlainLineSpan(span.start, span.end, span.startLine, span.startColumn, span.endLine, span.endColumn)
+  }
+  return { ...span }
+}
+
+/**
+ * `annotateSpanFromLineContext` for a span parseman built itself (`{ start, end }` only),
+ * returning a constructor-built line span (`src/cst/plain.d.ts`). The public form keeps
+ * any extra keys a caller's span carries, so it spreads.
+ */
+export function lineSpanFromContext(span: Span, ctx: LineTrackingContext): Span {
+  if (!ctx.trackLines) return span
+  const index = ctx._lineIndex ?? (ctx._lineStarts ? { lineStarts: ctx._lineStarts } : undefined)
+  if (!index) return span
+  const s = offsetToLineCol(index, span.start)
+  const e = offsetToLineCol(index, span.end)
+  return new PlainLineSpan(span.start, span.end, s.line, s.col, e.line, e.col)
+}
+
 export function annotateSpanFromLineContext(
   span: Span,
   ctx: LineTrackingContext,

package/src/plugin/entry-supercompiler.ts

@@ -12,6 +12,7 @@ import { OP_ATTEMPT, OP_NODE, OP_NODE_TRACK, OP_REPV, OP_SCAN } from '../table/o
 import { emitAssemblySource, type EmitResult } from '../table/emit-assembly.ts'
 import { defaultAssemblyCfgs } from '../table/emit.ts'
 import { assemblyExtraIps, resolveTable, type TableProgram } from '../table/program.ts'
+import { FACTORY_ARITY_GUARD } from '../table/emit.ts'
 
 type Ast = { type: string; start?: number; end?: number; [key: string]: unknown }
 
@@ -271,8 +272,8 @@ export function supercompileRuleMapReplacement(
   if (typeof body.start !== 'number' || typeof body.end !== 'number') return replacement
   const start = body.start - prefix.length + 1
   const end = body.end - prefix.length - 1
-  if (replacement.slice(start, end) !== canonical.source + '\n') return replacement
-  const transformed = replacement.slice(0, start) + candidate.source + '\n' + replacement.slice(end)
+  if (replacement.slice(start, end) !== FACTORY_ARITY_GUARD + canonical.source + '\n') return replacement
+  const transformed = replacement.slice(0, start) + FACTORY_ARITY_GUARD + candidate.source + '\n' + replacement.slice(end)
   const checked = parseSync('parseman-entry-supercompile-replacement-result.js', prefix + transformed)
   return checked.errors.length === 0 ? transformed : replacement
 }

package/src/recovery/scan.ts

@@ -1,6 +1,8 @@
 import type { Combinator, FirstSet, ParseContext, ParseError, ParseResult, RecoveryHelpers } from '../types.ts'
 import { cstCaptureActive, pushCstChild } from '../cst/capture-buffer.ts'
+import { rangesHaveLead } from '../lead-ranges.ts'
 import { annotateSpanFromLineContext, recordLineRangeFromContext } from '../line-index.ts'
+import { PlainSpan } from '../cst/plain.js'
 
 /**
  * Recovery MECHANISM — sync-source-agnostic. These primitives implement "scan
@@ -83,9 +85,7 @@ export function firstSetSentinel(fs: FirstSet): Combinator<null> | null {
     parse(input: string, pos: number): ParseResult<null> {
       const code = pos < input.length ? input.codePointAt(pos) : undefined
       if (code !== undefined) {
-        for (const r of ranges) {
-          if (code >= r.lo && code <= r.hi) return { ok: true, value: null, span: { start: pos, end: pos } }
-        }
+        if (rangesHaveLead(ranges, code)) return { ok: true, value: null, span: { start: pos, end: pos } }
       }
       return { ok: false, expected: [], span: { start: pos, end: pos } }
     },
@@ -113,7 +113,7 @@ export function recoverScan(
 ): { error: ParseError; end: number } {
   let scanPos = from
   while (scanPos < input.length && !probeAt(sync, input, scanPos, ctx)) scanPos++
-  const span = { start: from, end: scanPos }
+  const span = new PlainSpan(from, scanPos)
   if (ctx.trackLines) recordLineRangeFromContext(ctx, input, from, scanPos)
   const error: ParseError = { _tag: 'parseError', span: ctx.trackLines ? annotateSpanFromLineContext(span, ctx) : span, expected }
   ctx._errors?.push(error)

package/src/regex/classes.ts

@@ -38,12 +38,20 @@ export function shorthandRanges(ch: 'd' | 'w' | 's'): Array<[number, number]> {
   return [[48, 57], [65, 90], [97, 122], [95, 95]]
 }
 
-/** `\uXXXX` at `body[i]` → its code point and the index past it, or null. */
-export function readUnicodeEscape(body: string, i: number): { cp: number; next: number } | null {
+/** `\uXXXX` at `body[i]` → its code point and the index past it, or null. Under
+ * `unicode`, an escaped high surrogate followed by an escaped low one is one
+ * code point (`\uD83D\uDE00` is U+1F600). */
+export function readUnicodeEscape(body: string, i: number, unicode = false): { cp: number; next: number } | null {
   if (body[i] !== '\\' || body[i + 1] !== 'u') return null
   const hex = body.slice(i + 2, i + 6)
   if (!/^[0-9a-fA-F]{4}$/.test(hex)) return null
-  return { cp: Number.parseInt(hex, 16), next: i + 6 }
+  const cp = Number.parseInt(hex, 16)
+  if (unicode && cp >= 0xd800 && cp <= 0xdbff && body[i + 6] === '\\' && body[i + 7] === 'u') {
+    const lowHex = body.slice(i + 8, i + 12)
+    const low = /^[0-9a-fA-F]{4}$/.test(lowHex) ? Number.parseInt(lowHex, 16) : -1
+    if (low >= 0xdc00 && low <= 0xdfff) return { cp: 0x10000 + ((cp - 0xd800) << 10) + (low - 0xdc00), next: i + 12 }
+  }
+  return { cp, next: i + 6 }
 }
 
 /** Regex syntax characters that cannot appear in a LITERAL fragment unescaped. */
@@ -66,6 +74,8 @@ export function literalCodePoints(frag: string): number[] | null {
       // Any other word-character escape is a class (`\d`), an assertion (`\b`), a
       // backreference or a hex, unicode or control escape, never the character itself.
       if (!(e in CLASS_ESCAPES) && /\w/.test(e)) return null
+      // `\0` then a digit is a legacy octal escape, not NUL followed by a digit.
+      if (e === '0' && frag[i + 2] !== undefined && frag[i + 2]! >= '0' && frag[i + 2]! <= '9') return null
       out.push(e in CLASS_ESCAPES ? CLASS_ESCAPES[e]! : e.codePointAt(0)!)
       i += 2
       continue
@@ -142,14 +152,14 @@ type ClassAtom = { cp: number } | { set: Array<[number, number]> }
  * than being mis-read as a literal letter, so callers fall back to a safe
  * over-approximation instead of a wrong set.
  */
-export function parseClassRanges(body: string): Array<[number, number]> | null {
+export function parseClassRanges(body: string, unicode = false): Array<[number, number]> | null {
   const ranges: Array<[number, number]> = []
   let i = 0
   const readAtom = (): ClassAtom | null => {
     const ch = body[i]
     if (ch === undefined) return null
     if (ch === '\\') {
-      const uni = readUnicodeEscape(body, i)
+      const uni = readUnicodeEscape(body, i, unicode)
       if (uni) {
         i = uni.next
         return { cp: uni.cp }
@@ -157,14 +167,19 @@ export function parseClassRanges(body: string): Array<[number, number]> | null {
       const e = body[i + 1]
       if (e === undefined) return null
       i += 2
+      // A digit after `\0`, or any other digit escape, is a legacy octal escape
+      // (`[\012]` holds a newline), which this does not decode.
+      if ((e === '0' && body[i] !== undefined && body[i]! >= '0' && body[i]! <= '9') || (e >= '1' && e <= '9')) return null
       if (e in CLASS_ESCAPES) return { cp: CLASS_ESCAPES[e]! }
       if (e === 'd' || e === 'w' || e === 's') return { set: shorthandRanges(e) }
       // Any other letter escape is a class we can't safely lower (\D, \W, \S, …).
       if ((e >= 'a' && e <= 'z') || (e >= 'A' && e <= 'Z')) return null
       return { cp: e.codePointAt(0)! }
     }
-    i += ch.length
-    return { cp: ch.codePointAt(0)! }
+    // Unicode mode reads a member as one code point; otherwise as one code unit.
+    const cp = unicode ? body.codePointAt(i)! : body.charCodeAt(i)
+    i += cp > 0xffff ? 2 : 1
+    return { cp }
   }
   while (i < body.length) {
     const lo = readAtom()

package/src/regex/first-set.ts

@@ -23,7 +23,7 @@
  */
 import type { FirstSet } from '../types.ts'
 import { any, empty, fromRange, union } from '../combinators/first-set.ts'
-import { shorthandRanges, parseClassRanges } from './classes.ts'
+import { shorthandRanges, parseClassRanges, readUnicodeEscape } from './classes.ts'
 
 // A minimal regex AST. Each node answers two questions: what first chars can a
 // non-empty match begin with (`firstSet`), and can the node match zero chars
@@ -72,7 +72,7 @@ const BAIL = Symbol('regex-bail')
  * Closure over a single cursor `i` — recursive-descent: alt → seq → quantified
  * → atom.
  */
-function parseRegex(src: string): Node {
+function parseRegex(src: string, unicode = false, sets = false): Node {
   let i = 0
 
   const parseAlt = (): Node => {
@@ -128,9 +128,13 @@ function parseRegex(src: string): Node {
       case '^':
       case '$': i++; return EMPTY // anchors are zero-width
       case '{': i++; return charNode(fromRange(0x7b, 0x7b), false) // a `{` with nothing to quantify is a literal brace
-      default:
-        i += ch.length
-        return charNode(fromRange(ch.codePointAt(0)!, ch.codePointAt(0)!), ch === '\n')
+      default: {
+        // Unicode mode reads a literal as one code point (an astral one is two
+        // code units); otherwise each code unit is its own atom.
+        const cp = unicode ? src.codePointAt(i)! : src.charCodeAt(i)
+        i += cp > 0xffff ? 2 : 1
+        return charNode(fromRange(cp, cp), ch === '\n')
+      }
     }
   }
 
@@ -163,6 +167,24 @@ function parseRegex(src: string): Node {
     let k = i + 1
     const negated = src[k] === '^'
     if (negated) k++
+    if (sets) {
+      // `v` mode nests classes and adds set operations (`[[^a]--b]`, `[a&&b]`) and
+      // string members (`\q{…}`). Only a flat class is enumerated here; the rest
+      // match one character (or string) of a set this does not compute.
+      let depth = 1
+      let nested = false
+      while (k < src.length && depth > 0) {
+        const c = src[k]!
+        if (c === '\\') { if (src[k + 1] === 'q') nested = true; k += 2; continue }
+        if (c === '[') { depth++; nested = true }
+        else if (c === ']') { depth--; if (depth === 0) break }
+        else if ((c === '-' || c === '&') && src[k + 1] === c) nested = true
+        k++
+      }
+      if (src[k] !== ']') throw BAIL
+      if (nested) { i = k + 1; return charNode(any(), true) }
+      k = i + 1 + (negated ? 1 : 0)
+    }
     let body = ''
     while (k < src.length && src[k] !== ']') {
       if (src[k] === '\\') { body += src[k]! + (src[k + 1] ?? ''); k += 2 }
@@ -170,7 +192,7 @@ function parseRegex(src: string): Node {
     }
     if (src[k] !== ']') throw BAIL
     i = k + 1
-    const ranges = parseClassRanges(body)
+    const ranges = parseClassRanges(body, unicode)
     // A negated class can start with (almost) any char, so the first-set widens to
     // `any`, but its newline capability is still cheap and useful to know:
     // `[^,\r\n]*` is a workhorse "field until line end" regex and must not force
@@ -184,9 +206,19 @@ function parseRegex(src: string): Node {
     const e = src[i + 1]
     if (e === undefined) throw BAIL
     // \uXXXX
-    if (e === 'u' && /^[0-9a-fA-F]{4}$/.test(src.slice(i + 2, i + 6))) {
-      const cp = Number.parseInt(src.slice(i + 2, i + 6), 16)
-      i += 6
+    const uni = readUnicodeEscape(src, i, unicode)
+    if (uni !== null) {
+      i = uni.next
+      return charNode(fromRange(uni.cp, uni.cp), uni.cp === 0x0a)
+    }
+    // \u{X…}: a code point in Unicode mode. Without `u` it is a literal `u` under
+    // a `{…}` quantifier, which may repeat it zero times, so it widens.
+    if (e === 'u' && src[i + 2] === '{') {
+      const close = src.indexOf('}', i + 3)
+      const hex = close < 0 ? '' : src.slice(i + 3, close)
+      if (!unicode || !/^[0-9a-fA-F]{1,6}$/.test(hex)) { i += 2; return ANY }
+      const cp = Number.parseInt(hex, 16)
+      i = close + 1
       return charNode(fromRange(cp, cp), cp === 0x0a)
     }
     // \xHH
@@ -195,6 +227,17 @@ function parseRegex(src: string): Node {
       i += 4
       return charNode(fromRange(cp, cp), cp === 0x0a)
     }
+    // \cX: the control character X names. Anything else after `\c` is not one.
+    if (e === 'c') {
+      const x = src[i + 2]
+      if (x !== undefined && /^[A-Za-z]$/.test(x)) {
+        const cp = x.charCodeAt(0) % 32
+        i += 3
+        return charNode(fromRange(cp, cp), cp === 0x0a)
+      }
+      i += 2
+      return ANY
+    }
     i += 2
     switch (e) {
       case 'd': return charNode(rangesToSet(shorthandRanges('d')), false)
@@ -207,7 +250,12 @@ function parseRegex(src: string): Node {
       case 't': return charNode(fromRange(0x09, 0x09), false)
       case 'f': return charNode(fromRange(0x0c, 0x0c), false)
       case 'v': return charNode(fromRange(0x0b, 0x0b), false)
-      case '0': return charNode(fromRange(0, 0), false)
+      // `\0` then a digit is a legacy octal escape (`\012` is a newline), not NUL.
+      case '0': return src[i] !== undefined && src[i]! >= '0' && src[i]! <= '9' ? ANY : charNode(fromRange(0, 0), false)
+      // A property escape (`\p{…}`/`\P{…}`) matches one character of a class this
+      // analyzer does not enumerate; `\k<name>` is a backreference, which may match empty.
+      case 'p': case 'P': return charNode(any(), true)
+      case 'k': return ANY
       default:
         // A digit escape is a backreference we don't resolve → widen.
         if (e >= '1' && e <= '9') return ANY
@@ -300,10 +348,16 @@ export function regexCanMatchEmpty(source: string): boolean {
   try { return nullable(parseRegex(source)) } catch { return true }
 }
 
-export function firstSetFromRegex(source: string): { firstSet: FirstSet; canMatchNewline: boolean } {
+/**
+ * `flags` decides how atoms read: under `u`/`v` each is one code point, else one
+ * code unit, so a non-`u` set holds a high surrogate where an astral character
+ * can start a match (`rangesHaveLead` reads an astral lead through it). Case
+ * folding under `i` is the caller's, since only it knows what it gates.
+ */
+export function firstSetFromRegex(source: string, flags = ''): { firstSet: FirstSet; canMatchNewline: boolean } {
   let ast: Node
   try {
-    ast = parseRegex(source)
+    ast = parseRegex(source, flags.includes('u') || flags.includes('v'), flags.includes('v'))
   } catch {
     return { firstSet: any(), canMatchNewline: true }
   }
@@ -314,3 +368,4 @@ export function firstSetFromRegex(source: string): { firstSet: FirstSet; canMatc
   if (nullable(ast)) return { firstSet: any(), canMatchNewline: canConsumeNewline(ast) }
   return { firstSet: firstSet(ast), canMatchNewline: canConsumeNewline(ast) }
 }
+