parseman 0.59.3 → 0.60.1

package/src/combinators/regex.ts

@@ -4,7 +4,7 @@ import { failAt } from './probe.ts'
 import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
 import { recordLineRangeFromContext } from '../line-index.ts'
 import { shorthandRanges, parseClassRanges } from '../regex/classes.ts'
-import { firstSetFromRegex } from '../regex/first-set.ts'
+import { consumingFirstSetFromRegex, firstSetFromRegex, regexCanMatchEmpty } from '../regex/first-set.ts'
 import { directTerminalFailureExpected } from './expected.ts'
 import type { ScalarParser } from './scalar.ts'
 import { PlainLeaf, PlainSpan } from '../cst/plain.js'
@@ -156,13 +156,18 @@ export function regex(pattern: string | RegExp, flags = ''): Combinator<string>
   //     pairs, so gating on the ASCII set would false-exclude them. Fall back to `any()`
   //     (always-try) — sound, and only forfeits gating for the rare Unicode-mode `/i`
   //     recognizer, never the ASCII-only `/i` at-keywords the fix targets.
-  const firstSet = !resolvedFlags.includes('i')
-    ? raw.firstSet
+  const forFlags = (fs: FirstSet): FirstSet => !resolvedFlags.includes('i')
+    ? fs
     : (resolvedFlags.includes('u') || resolvedFlags.includes('v'))
       ? any()
-      : asciiCaseFold(raw.firstSet)
+      : asciiCaseFold(fs)
+  const firstSet = forFlags(raw.firstSet)
   const canMatchNewline = raw.canMatchNewline
   const meta: ParserMeta = { firstSet, canMatchNewline, isTrivia: false }
+  if (firstSet.kind === 'any' && regexCanMatchEmpty(source)) {
+    const consuming = forFlags(consumingFirstSetFromRegex(source, resolvedFlags))
+    if (consuming.kind !== 'any') meta.consumingFirstSet = consuming
+  }
   const def = { tag: 'regex', source, flags: resolvedFlags } as const
   const expected = directTerminalFailureExpected(def)
   const parseScalar: ScalarParser = (input, pos, ctx) => {

package/src/combinators/repeat.ts

@@ -154,11 +154,11 @@ export function many<T>(combinator: Combinator<T>, opts: RepeatOptions = {}): Co
     canMatchNewline: combinator._meta.canMatchNewline,
     isTrivia: false,
   }
-  const def: { tag: 'many'; parser: Combinator<unknown>; min: 0; max?: number; valueUnused?: boolean } =
-    { tag: 'many', parser: combinator as Combinator<unknown>, min: 0, ...(max === Infinity ? {} : { max }) }
-  let expected: string[] | undefined
   // A non-nullable body can be first-set-gated per loop iteration (see repItem).
   const guardable = combinator._meta.firstSet.kind !== 'any' && !matchesEmpty(combinator)
+  const def: { tag: 'many'; parser: Combinator<unknown>; min: 0; max?: number; valueUnused?: boolean; guarded?: true } =
+    { tag: 'many', parser: combinator as Combinator<unknown>, min: 0, ...(max === Infinity ? {} : { max }), ...(guardable ? { guarded: true } : {}) }
+  let expected: string[] | undefined
   const scalarChild = scalarOf(combinator)
   const parseScalar = (input: string, pos: number, ctx: ParseContext): number =>
     repeatScalarTail(combinator, scalarChild, input, pos, ctx, guardable, def.valueUnused ? undefined : [], max)
@@ -229,11 +229,11 @@ function atLeast<T>(combinator: Combinator<T>, min: number, max: number): Combin
     canMatchNewline: combinator._meta.canMatchNewline,
     isTrivia: false,
   }
-  const def: { tag: 'oneOrMore'; parser: Combinator<unknown>; min: number; max?: number; valueUnused?: boolean } =
-    { tag: 'oneOrMore', parser: combinator as Combinator<unknown>, min, ...(max === Infinity ? {} : { max }) }
-  let expected: string[] | undefined
   // A non-nullable body can be first-set-gated per loop iteration (see repItem).
   const guardable = combinator._meta.firstSet.kind !== 'any' && !matchesEmpty(combinator)
+  const def: { tag: 'oneOrMore'; parser: Combinator<unknown>; min: number; max?: number; valueUnused?: boolean; guarded?: true } =
+    { tag: 'oneOrMore', parser: combinator as Combinator<unknown>, min, ...(max === Infinity ? {} : { max }), ...(guardable ? { guarded: true } : {}) }
+  let expected: string[] | undefined
   const scalarChild = scalarOf(combinator)
   const parseScalar = (input: string, pos: number, ctx: ParseContext): number => {
     let cur = scalarChild(input, pos, ctx)

package/src/combinators/sequence.ts

@@ -108,7 +108,7 @@ export function sequence<T extends [Combinator<unknown>, ...Combinator<unknown>[
       if (tolerant) {
         followSentinels ??= parsers.map((_, i) => firstSetSentinel(
           parsers.slice(i + 1).reduce<ReturnType<typeof firstSetOf>>(
-            (set, parser) => union(set, firstSetOf(parser)), { kind: 'empty' },
+            (set, parser) => union(set, firstSetOf(parser, new Set(), undefined, undefined, false)), { kind: 'empty' },
           ),
         ))
       }

package/src/combinators/trivia-boundary.ts

@@ -46,3 +46,21 @@ export function hasOwnTriviaBoundary(p: Combinator<unknown>, depth: number = MAX
     default: return false
   }
 }
+
+/**
+ * The trivia `target` runs under when it is reached through the reference `ref`
+ * from a scope whose live trivia is `trivia` (`null` or `undefined`: none).
+ *
+ * A reference made from a region that cleared trivia re-installs the rule's own
+ * ambient trivia, where the rule has a boundary for it to repair (`ref.ts`,
+ * `Encoder.scopedRef`). Anything derived statically through a reference — a
+ * first set, a gate, a gap — must take the same scope, or it describes the
+ * caller's scope where the body runs under its own.
+ */
+export function triviaThroughRef(
+  ref: Combinator<unknown>, target: Combinator<unknown>, trivia: Combinator<unknown> | null | undefined,
+): Combinator<unknown> | null | undefined {
+  if (trivia !== undefined && trivia !== null) return trivia
+  const amb = ref._meta.grammarTrivia ?? target._meta.grammarTrivia
+  return amb !== undefined && hasOwnTriviaBoundary(target) ? amb : trivia
+}

package/src/combinators/trivia-skip.ts

@@ -413,7 +413,7 @@ export function scanTriviaCompact(input: string, cur: number, ctx: ParseContext)
   return { end: tr.ok ? tr.span.end : cur, commit: NOOP_COMMIT }
 }
 
-/** Legacy object-shaped contract used by previously emitted assembly factories. */
+/** The object-shaped form of `scanTriviaCompact`. No engine calls it; the trivia-skip tests compare the two. */
 export function scanTrivia(input: string, cur: number, ctx: ParseContext): TriviaScan {
   const scan = scanTriviaCompact(input, cur, ctx)
   return typeof scan === 'number' ? { end: scan, commit: NOOP_COMMIT } : scan

package/src/compiler/ir-serialize.ts

@@ -461,7 +461,7 @@ class Serializer {
             arms.push(`when(${matcherExpr(entry)}, ${kid(entry.parser)}${entry.caseInsensitive ? ', { caseInsensitive: true }' : ''})`)
           }
         }
-        if (def.otherwise) arms.push(`otherwise(${kid(def.otherwise)})`)
+        if (def.otherwise) arms.push(`otherwise(${kid(def.otherwise)}${def.otherwiseCommit === false ? ', { commit: false }' : ''})`)
         return `dispatch(${kid(def.selector)}${arms.length === 0 ? '' : `, ${arms.join(', ')}`})`
       }
       case 'many':      return `many(${kid(def.parser)}${repeatOpts(def.min, def.max)})`

package/src/compiler/token-alphabet.ts

@@ -140,6 +140,8 @@ export type ExecutableLexNode =
     readonly parsers: readonly Combinator<unknown>[]
     /** Inside a no-trivia scope of the token, where the encoder's trivia is none. */
     readonly cleared: boolean
+    /** The authored choice's own `disjoint`, which decides how the encoder classifies its arms. */
+    readonly disjoint: boolean
     readonly armExpected: readonly (readonly string[])[]
     readonly expected: readonly string[]
   }
@@ -509,7 +511,7 @@ function directFixedLexNode(
       if (arms.some(arm => arm === undefined)) return undefined
       return {
         kind: 'choice', arms: arms as ExecutableLexNode[],
-        parsers: def.parsers, cleared,
+        parsers: def.parsers, cleared, disjoint: def.disjoint,
         armExpected: def.parsers.map(deriveExpected),
         expected: def.parsers.flatMap(deriveExpected),
       }

package/src/coverage.ts

@@ -309,7 +309,7 @@ function coverageEntry(entry: Combinator<unknown>, collector: GrammarCoverageCol
               arms.push(when(dispatchMatcher(entry), wrapArm(entry.parser, ids[armIndex++]), entry.caseInsensitive ? { caseInsensitive: true } : undefined))
             }
           }
-          if (def.otherwise) arms.push(otherwise(wrapArm(def.otherwise, ids[armIndex++])))
+          if (def.otherwise) arms.push(otherwise(wrapArm(def.otherwise, ids[armIndex++]), def.otherwiseCommit === false ? { commit: false } : undefined))
           const unsafeDispatch = dispatch as (selector: Combinator<string>, ...items: DispatchArm<unknown>[]) => Combinator<unknown>
           const base = unsafeDispatch(build(def.selector) as Combinator<string>, ...arms)
           return {

package/src/index.ts

@@ -17,7 +17,7 @@ export type { KeywordsOptions, WordOptions } from './combinators/keywords.ts'
 export { sequence } from './combinators/sequence.ts'
 export { choice } from './combinators/choice.ts'
 export { dispatch, endsWith, makeWhen, matches, otherwise, routed, startsWith, when } from './combinators/dispatch.ts'
-export type { DispatchArm, DispatchOtherwise, DispatchStringMatcher, DispatchWhen, DispatchWhenFactory, DispatchWhenOptions, DispatchWhenMatcher } from './combinators/dispatch.ts'
+export type { DispatchArm, DispatchOtherwise, DispatchOtherwiseOptions, DispatchStringMatcher, DispatchWhen, DispatchWhenFactory, DispatchWhenOptions, DispatchWhenMatcher } from './combinators/dispatch.ts'
 export { attempt } from './combinators/attempt.ts'
 export type { AttemptOptions } from './combinators/attempt.ts'
 export { many, oneOrMore, optional, sepBy, oneOrMoreSep, keepSeparator } from './combinators/repeat.ts'

package/src/plugin/evaluator.ts

@@ -431,11 +431,16 @@ function dispatchArmValue(node: Expression, scope: XScope, code?: string, mfs?:
   }
 
   if (callee.name === 'otherwise') {
-    const [parserArg] = node.arguments
-    if (!parserArg || parserArg.type === 'SpreadElement') return null
+    const [parserArg, optsArg] = node.arguments
+    if (!parserArg || parserArg.type === 'SpreadElement' || optsArg?.type === 'SpreadElement') return null
     const parserValue = anyValue(parserArg as Expression, scope, code, mfs)
     if (!isCombinator(parserValue)) return null
-    return parseman.otherwise(parserValue)
+    if (optsArg === undefined) return parseman.otherwise(parserValue)
+    const opts = anyValue(optsArg as Expression, scope, code, mfs)
+    if (typeof opts !== 'object' || opts === null) return null
+    const commit = (opts as { commit?: unknown }).commit
+    if (commit !== undefined && typeof commit !== 'boolean') return null
+    return parseman.otherwise(parserValue, commit === undefined ? {} : { commit })
   }
 
   if (callee.name === 'when') {

package/src/regex/first-set.ts

@@ -32,7 +32,7 @@ import { shorthandRanges, parseClassRanges, readUnicodeEscape } from './classes.
 type Node =
   | { t: 'char'; set: FirstSet; nl: boolean } // matches exactly one char from `set`
   | { t: 'any' }                              // unknown 1-char-ish construct: could start with anything
-  | { t: 'empty' }                            // zero-width: anchors, lookaround, word boundary
+  | { t: 'empty'; look?: Node }               // zero-width: anchors, lookaround, word boundary (`look`: a positive lookahead's body)
   | { t: 'seq'; parts: Node[] }
   | { t: 'alt'; arms: Node[] }
   | { t: 'rep'; node: Node; min: number }     // quantified; min 0 ⇒ nullable
@@ -142,10 +142,11 @@ function parseRegex(src: string, unicode = false, sets = false): Node {
     i++ // consume '('
     // Group prefixes: (?:…) (?=…) (?!…) (?<=…) (?<!…) (?<name>…)
     let lookaround = false
+    let lookahead = false
     if (src[i] === '?') {
       const c1 = src[i + 1]
       if (c1 === ':') i += 2
-      else if (c1 === '=' || c1 === '!') { i += 2; lookaround = true }
+      else if (c1 === '=' || c1 === '!') { lookahead = c1 === '='; i += 2; lookaround = true }
       else if (c1 === '<') {
         const c2 = src[i + 2]
         if (c2 === '=' || c2 === '!') { i += 3; lookaround = true } // lookbehind
@@ -159,7 +160,10 @@ function parseRegex(src: string, unicode = false, sets = false): Node {
     const inner = parseAlt()
     if (src[i] !== ')') throw BAIL
     i++ // consume ')'
-    return lookaround ? EMPTY : inner // lookaround is zero-width
+    // Lookaround is zero-width. A positive LOOKAHEAD keeps its body for
+    // `regexFirstTwo`; every other walker reads it as `EMPTY`.
+    if (lookaround) return lookahead ? { t: 'empty', look: inner } : EMPTY
+    return inner
   }
 
   const parseClass = (): Node => {
@@ -348,6 +352,17 @@ export function regexCanMatchEmpty(source: string): boolean {
   try { return nullable(parseRegex(source)) } catch { return true }
 }
 
+/**
+ * The first chars of a NON-EMPTY match of `source`, nullable or not: the set
+ * `firstSetFromRegex` reports for a pattern that must consume, without its
+ * widening of a nullable one to `any()`. Only a caller that accounts for the
+ * empty match itself may use it (`sequenceSet`). `any()` when unparseable.
+ * `flags` reads atoms as `firstSetFromRegex` does.
+ */
+export function consumingFirstSetFromRegex(source: string, flags = ''): FirstSet {
+  try { return firstSet(parseRegex(source, flags.includes('u') || flags.includes('v'), flags.includes('v'))) } catch { return any() }
+}
+
 /**
  * `flags` decides how atoms read: under `u`/`v` each is one code point, else one
  * code unit, so a non-`u` set holds a high surrogate where an astral character
@@ -369,3 +384,22 @@ export function firstSetFromRegex(source: string, flags = ''): { firstSet: First
   return { firstSet: firstSet(ast), canMatchNewline: canConsumeNewline(ast) }
 }
 
+/**
+ * The first TWO characters every match of `source` begins with, where the
+ * pattern says them outright: a one-character class and then another, or a
+ * positive lookahead over one (`:(?=[ \t])`). `undefined` otherwise. Every
+ * match that starts with a character of `first` has a second character, and it
+ * is in `second`. `flags` reads atoms as `firstSetFromRegex` does.
+ */
+export function regexFirstTwo(source: string, flags = ''): { first: FirstSet; second: FirstSet } | undefined {
+  let ast: Node
+  try { ast = parseRegex(source, flags.includes('u') || flags.includes('v'), flags.includes('v')) } catch { return undefined }
+  const parts = ast.t === 'seq' ? ast.parts : [ast]
+  const a = parts[0]
+  const b = parts[1]
+  if (a === undefined || a.t !== 'char' || a.set.kind !== 'ranges' || b === undefined) return undefined
+  const second = b.t === 'char'
+    ? b.set
+    : b.t === 'empty' && b.look !== undefined && !nullable(b.look) ? firstSet(b.look) : undefined
+  return second !== undefined && second.kind === 'ranges' ? { first: a.set, second } : undefined
+}