tex2typst 0.3.21 → 0.3.23

package/src/convert.ts

@@ -1,8 +1,9 @@
-import { TexNode, TypstNode, TexSupsubData, TypstSupsubData, TexSqrtData, Tex2TypstOptions, TYPST_NONE, TypstPrimitiveValue, TypstLrData, TexArrayData, TypstNamedParams } from "./types";
+import { TexNode, TypstNode, TexSupsubData, TypstSupsubData, TexSqrtData, Tex2TypstOptions, TYPST_NONE, TypstLrData, TexArrayData, TypstNamedParams } from "./types";
 import { TypstWriterError } from "./typst-writer";
 import { symbolMap, reverseSymbolMap } from "./map";
-import { array_join } from "./generic";
+import { array_intersperse } from "./generic";
 import { assert } from "./util";
+import { TEX_BINARY_COMMANDS, TEX_UNARY_COMMANDS } from "./tex-tokenizer";
 
 
 // native textual operators in Typst
@@ -94,6 +95,44 @@ function convert_underset(node: TexNode, options: Tex2TypstOptions): TypstNode {
     });
 }
 
+function convert_tex_array_align_literal(alignLiteral: string): TypstNamedParams {
+    const np: TypstNamedParams = {};
+    const alignMap: Record<string, string> = { l: '#left', c: '#center', r: '#right' };
+    const chars = Array.from(alignLiteral);
+
+    const vlinePositions: number[] = [];
+    let columnIndex = 0;
+    for (const c of chars) {
+        if (c === '|') {
+            vlinePositions.push(columnIndex);
+        } else if (c === 'l' || c === 'c' || c === 'r') {
+            columnIndex++;
+        }
+    }
+
+    if (vlinePositions.length > 0) {
+        let augment_str: string;
+        if (vlinePositions.length === 1) {
+            augment_str = `#${vlinePositions[0]}`;
+        } else {
+            augment_str = `#(vline: (${vlinePositions.join(', ')}))`;
+        }
+
+        np['augment'] = new TypstNode('literal', augment_str);
+    }
+
+    const alignments = chars
+        .map(c => alignMap[c])
+        .filter((x) => x !== undefined)
+        .map(s => new TypstNode('literal', s!));
+
+    if (alignments.length > 0) {
+        const all_same = alignments.every(item => item.eq(alignments[0]));
+        np['align'] = all_same ? alignments[0] : new TypstNode('literal', '#center');
+    }
+    return np;
+}
+
 
 export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptions = {}): TypstNode {
     switch (node.type) {
@@ -121,6 +160,9 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
             }
             return new TypstNode('text', node.content);
         }
+        case 'literal':
+            // This happens, for example, node={type: 'literal', content: 'myop'} as in `\operatorname{myop}`
+            return new TypstNode('literal', node.content);
         case 'comment':
             return new TypstNode('comment', node.content);
         case 'supsub': {
@@ -272,31 +314,19 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
             }
             // \operatorname{opname} -> op("opname")
             if (node.content === '\\operatorname') {
+                // arg0 must be of type 'literal' in this situation
                 if (options.optimize) {
                     if (TYPST_INTRINSIC_OP.includes(arg0.content)) {
                         return new TypstNode('symbol', arg0.content);
                     }
                 }
-                return new TypstNode('funcCall', 'op', [arg0]);
-            }
-            // \hspace{1cm} -> #h(1cm)
-            // TODO: reverse conversion support for this
-            if (node.content === '\\hspace') {
-                const text = arg0.content;
-                return new TypstNode(
-                    'funcCall',
-                    '#h',
-                    [new TypstNode('symbol', text)]
-                );
+                return new TypstNode('funcCall', 'op', [new TypstNode('text', arg0.content)]);
             }
 
+            // \substack{a \\ b} -> `a \ b`
+            // as in translation from \sum_{\substack{a \\ b}} to sum_(a \ b)
             if (node.content === '\\substack') {
-                // \sum_{\substack{a \\ b}} -> sum_(a \ b)
-                return new TypstNode(
-                    'group',
-                    '',
-                    [arg0]
-                );
+                return arg0;
             }
 
             if(options.optimize) {
@@ -321,97 +351,57 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
             const matrix = node.data as TexNode[][];
             const data = matrix.map((row) => row.map((n) => convert_tex_node_to_typst(n, options)));
 
-            if (node.content!.startsWith('align')) {
+            if (node.content.startsWith('align')) {
                 // align, align*, alignat, alignat*, aligned, etc.
                 return new TypstNode('align', '', [], data);
             }
-            if (node.content! === 'cases') {
+            if (node.content === 'cases') {
                 return new TypstNode('cases', '', [], data);
             }
-            if (node.content! === 'subarray') {
+            if (node.content === 'subarray') {
                 const align_node = node.args![0];
-                if (align_node.content == 'r') {
-                    data.forEach(row => row[0].args!.push(new TypstNode('symbol', '&')));
-                }
-                if (align_node.content == 'l') {
-                    data.forEach(row => row[0].args!.unshift(new TypstNode('symbol', '&')));
+                switch (align_node.content) {
+                    case 'r':
+                        data.forEach(row => row[0].args!.push(new TypstNode('symbol', '&')));
+                        break;
+                    case 'l':
+                        data.forEach(row => row[0].args!.unshift(new TypstNode('symbol', '&')));
+                        break;
+                    default:
+                        break;
                 }
-                return new TypstNode(
-                    'group',
-                    '',
-                    [new TypstNode('align', '', [], data)]
-                );
+                return new TypstNode('align', '', [], data);
             }
-            if (node.content! === 'array') {
-                const res = new TypstNode('matrix', '', [], data);
-                const options: TypstNamedParams = { 'delim': TYPST_NONE };
-
-                const align_args = node.args!;
-                if (align_args.length > 0) {
-                    const align_node = align_args[0];
-                    const align_str = (() => {
-                        if (align_node.type === 'element') return align_node.content;
-                        if (align_node.type === 'ordgroup') {
-                            return align_node.args!.map(n => n.type === 'element' ? n.content : '').join('');
-                        }
-                        return '';
-                    })();
-
-                    if (align_str) {
-                        const alignMap: Record<string, string> = { l: '#left', c: '#center', r: '#right' };
-                        const chars = Array.from(align_str);
-
-                        const alignments = chars
-                            .map(c => alignMap[c])
-                            .filter(Boolean)
-                            .map(s => new TypstNode('symbol', s!));
-
-                        const vlinePositions: number[] = [];
-                        let columnIndex = 0;
-                        for (const c of chars) {
-                            if (c === '|') {
-                                vlinePositions.push(columnIndex);
-                            } else if (c === 'l' || c === 'c' || c === 'r') {
-                                columnIndex++;
-                            }
-                        }
-
-                        if (vlinePositions.length > 0) {
-                            if (vlinePositions.length === 1) {
-                                options['augment'] = new TypstNode('symbol', `#${vlinePositions[0]}`);
-                            } else {
-                                options['augment'] = new TypstNode('symbol', `#(vline: (${vlinePositions.join(', ')}))`);
-                            }
-                        }
-
-                        if (alignments.length > 0) {
-                            const first_align = alignments[0].content;
-                            const all_same = alignments.every(item => item.content === first_align);
-                            options['align'] = all_same ? alignments[0] : new TypstNode('symbol', '#center');
-                        }
-                    }
-                }
+            if (node.content === 'array') {
+                const np: TypstNamedParams = { 'delim': TYPST_NONE };
 
-                res.setOptions(options);
+                assert(node.args!.length > 0 && node.args![0].type === 'literal');
+                const np_new = convert_tex_array_align_literal(node.args![0].content);
+                Object.assign(np, np_new);
+
+                const res = new TypstNode('matrix', '', [], data);
+                res.setOptions(np);
                 return res;
             }
-            if (node.content!.endsWith('matrix')) {
-                let delim: TypstPrimitiveValue;
+            if (node.content.endsWith('matrix')) {
+                const res = new TypstNode('matrix', '', [], data);
+                let delim: TypstNode;
                 switch (node.content) {
                     case 'matrix':
                         delim = TYPST_NONE;
                         break;
                     case 'pmatrix':
-                        delim = '(';
-                        break;
+                        // delim = new TypstNode('text', '(');
+                        // break;
+                        return res; // typst mat use delim:"(" by default
                     case 'bmatrix':
-                        delim = '[';
+                        delim = new TypstNode('text', '[');
                         break;
                     case 'Bmatrix':
-                        delim = '{';
+                        delim = new TypstNode('text', '{');
                         break;
                     case 'vmatrix':
-                        delim = '|';
+                        delim = new TypstNode('text', '|');
                         break;
                     case 'Vmatrix': {
                         delim = new TypstNode('symbol', 'bar.v.double');
@@ -420,7 +410,6 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
                     default:
                         throw new TypstWriterError(`Unimplemented beginend: ${node.content}`, node);
                 }
-                const res = new TypstNode('matrix', '', [], data);
                 res.setOptions({ 'delim': delim });
                 return res;
             }
@@ -430,7 +419,13 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
             return new TypstNode('unknown', tex_token_to_typst(node.content));
         case 'control':
             if (node.content === '\\\\') {
+                // \\ -> \
                 return new TypstNode('symbol', '\\');
+            } else if (node.content === '\\!') {
+                // \! -> #h(-math.thin.amount)
+                return new TypstNode('funcCall', '#h', [
+                    new TypstNode('literal', '-math.thin.amount')
+                ]);
             } else if (symbolMap.has(node.content.substring(1))) {
                 // node.content is one of \, \: \;
                 const typst_symbol = symbolMap.get(node.content.substring(1))!;
@@ -444,7 +439,7 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
 }
 
 
-
+/*
 const TYPST_UNARY_FUNCTIONS: string[] = [
     'sqrt',
     'bold',
@@ -466,6 +461,7 @@ const TYPST_UNARY_FUNCTIONS: string[] = [
     'ceil',
     'norm',
     'limits',
+    '#h',
 ];
 
 const TYPST_BINARY_FUNCTIONS: string[] = [
@@ -474,6 +470,7 @@ const TYPST_BINARY_FUNCTIONS: string[] = [
     'overbrace',
     'underbrace',
 ];
+*/
 
 function apply_escape_if_needed(c: string) {
     if (['{', '}', '%'].includes(c)) {
@@ -510,18 +507,29 @@ export function convert_typst_node_to_tex(node: TypstNode): TexNode {
             return new TexNode('whitespace', node.content);
         case 'atom':
             return new TexNode('element', node.content);
-        case 'symbol':
-            switch (node.content) {
-                // special hook for comma
-                case 'comma':
-                    return new TexNode('element', ',');
-                // special hook for hyph and hyph.minus
-                case 'hyph':
-                case 'hyph.minus':
-                    return new TexNode('text', '-');
-                default:
-                    return new TexNode('symbol', typst_token_to_tex(node.content));
+        case 'symbol': {
+            // special hook for comma
+            if(node.content === 'comma') {
+                return new TexNode('element', ',');
+            }
+            // special hook for dif
+            if(node.content === 'dif') {
+                return new TexNode('unaryFunc', '\\mathrm', [new TexNode('element', 'd')]);
+            }
+            // special hook for hyph and hyph.minus
+            if(node.content === 'hyph' || node.content === 'hyph.minus') {
+                return new TexNode('text', '-');
+            }
+            // special hook for mathbb{R} <-- RR
+            if(/^([A-Z])\1$/.test(node.content)) {
+                return new TexNode('unaryFunc', '\\mathbb', [
+                    new TexNode('element', node.content[0])
+                ]);
             }
+            return new TexNode('symbol', typst_token_to_tex(node.content));
+        }
+        case 'literal':
+            return new TexNode('literal', node.content);
         case 'text':
             return new TexNode('text', node.content);
         case 'comment':
@@ -538,83 +546,85 @@ export function convert_typst_node_to_tex(node: TypstNode): TexNode {
             return new TexNode('ordgroup', node.content, args);
         }
         case 'funcCall': {
-            if (TYPST_UNARY_FUNCTIONS.includes(node.content)) {
-                // special hook for lr
-                if (node.content === 'lr') {
-                    const data = node.data as TypstLrData;
-                    if (data.leftDelim !== null) {
-                        let left_delim = apply_escape_if_needed(data.leftDelim);
-                        assert(data.rightDelim !== null, "leftDelim has value but rightDelim not");
-                        let right_delim = apply_escape_if_needed(data.rightDelim!);
-                        // TODO: should be TeXNode('leftright', ...)
-                        // But currently writer will output `\left |` while people commonly prefer `\left|`.
-                        return new TexNode('ordgroup', '', [
-                            new TexNode('element', '\\left' + left_delim),
-                            ...node.args!.map(convert_typst_node_to_tex),
-                            new TexNode('element', '\\right' + right_delim)
-                        ]);
-                    } else {
-                        return new TexNode('ordgroup', '', node.args!.map(convert_typst_node_to_tex));
-                    }
-                }
-                // special hook for norm
-                // `\| a  \|` <- `norm(a)`
-                // `\left\| a + \frac{1}{3} \right\|` <- `norm(a + 1/3)`
-                if (node.content === 'norm') {
-                    const arg0 = node.args![0];
-                    const tex_node_type = node.isOverHigh() ? 'leftright' : 'ordgroup';
-                    return new TexNode(tex_node_type, '', [
-                        new TexNode('symbol', "\\|"),
-                        convert_typst_node_to_tex(arg0),
-                        new TexNode('symbol', "\\|")
-                    ]);
-                }
-                // special hook for floor, ceil
-                // `\lfloor a \rfloor` <- `floor(a)`
-                // `\lceil a \rceil` <- `ceil(a)`
-                // `\left\lfloor a \right\rfloor` <- `floor(a)`
-                // `\left\lceil a \right\rceil` <- `ceil(a)`
-                if (node.content === 'floor' || node.content === 'ceil') {
-                    const left = "\\l" + node.content;
-                    const right = "\\r" + node.content;
-                    const arg0 = node.args![0];
-                    const tex_node_type = node.isOverHigh() ? 'leftright' : 'ordgroup';
-                    return new TexNode(tex_node_type, '', [
-                        new TexNode('symbol', left),
-                        convert_typst_node_to_tex(arg0),
-                        new TexNode('symbol', right)
+            // special hook for lr
+            if (node.content === 'lr') {
+                const data = node.data as TypstLrData;
+                if (data.leftDelim !== null) {
+                    let left_delim = apply_escape_if_needed(data.leftDelim);
+                    assert(data.rightDelim !== null, "leftDelim has value but rightDelim not");
+                    let right_delim = apply_escape_if_needed(data.rightDelim!);
+                    // TODO: should be TeXNode('leftright', ...)
+                    // But currently writer will output `\left |` while people commonly prefer `\left|`.
+                    return new TexNode('ordgroup', '', [
+                        new TexNode('element', '\\left' + left_delim),
+                        ...node.args!.map(convert_typst_node_to_tex),
+                        new TexNode('element', '\\right' + right_delim)
                     ]);
+                } else {
+                    return new TexNode('ordgroup', '', node.args!.map(convert_typst_node_to_tex));
                 }
-                const command = typst_token_to_tex(node.content);
-                return new TexNode('unaryFunc', command, node.args!.map(convert_typst_node_to_tex));
-            } else if (TYPST_BINARY_FUNCTIONS.includes(node.content)) {
-                // special hook for root
-                if (node.content === 'root') {
-                    const [degree, radicand] = node.args!;
-                    const data: TexSqrtData = convert_typst_node_to_tex(degree);
-                    return new TexNode('unaryFunc', '\\sqrt', [convert_typst_node_to_tex(radicand)], data);
-                }
-                // special hook for overbrace and underbrace
-                if (node.content === 'overbrace' || node.content === 'underbrace') {
-                    const [body, label] = node.args!;
-                    const base = new TexNode('unaryFunc', '\\' + node.content, [convert_typst_node_to_tex(body)]);
-                    const script = convert_typst_node_to_tex(label);
-                    const data = node.content === 'overbrace' ? { base, sup: script } : { base, sub: script };
-                    return new TexNode('supsub', '', [], data);
-                }
-                const command = typst_token_to_tex(node.content);
-                return new TexNode('binaryFunc', command, node.args!.map(convert_typst_node_to_tex));
+            }
+            // special hook for norm
+            // `\| a  \|` <- `norm(a)`
+            // `\left\| a + \frac{1}{3} \right\|` <- `norm(a + 1/3)`
+            if (node.content === 'norm') {
+                const arg0 = node.args![0];
+                const tex_node_type = node.isOverHigh() ? 'leftright' : 'ordgroup';
+                return new TexNode(tex_node_type, '', [
+                    new TexNode('symbol', "\\|"),
+                    convert_typst_node_to_tex(arg0),
+                    new TexNode('symbol', "\\|")
+                ]);
+            }
+            // special hook for floor, ceil
+            // `\lfloor a \rfloor` <- `floor(a)`
+            // `\lceil a \rceil` <- `ceil(a)`
+            // `\left\lfloor a \right\rfloor` <- `floor(a)`
+            // `\left\lceil a \right\rceil` <- `ceil(a)`
+            if (node.content === 'floor' || node.content === 'ceil') {
+                const left = "\\l" + node.content;
+                const right = "\\r" + node.content;
+                const arg0 = node.args![0];
+                const tex_node_type = node.isOverHigh() ? 'leftright' : 'ordgroup';
+                return new TexNode(tex_node_type, '', [
+                    new TexNode('symbol', left),
+                    convert_typst_node_to_tex(arg0),
+                    new TexNode('symbol', right)
+                ]);
+            }
+            // special hook for root
+            if (node.content === 'root') {
+                const [degree, radicand] = node.args!;
+                const data: TexSqrtData = convert_typst_node_to_tex(degree);
+                return new TexNode('unaryFunc', '\\sqrt', [convert_typst_node_to_tex(radicand)], data);
+            }
+            // special hook for overbrace and underbrace
+            if (node.content === 'overbrace' || node.content === 'underbrace') {
+                const [body, label] = node.args!;
+                const base = new TexNode('unaryFunc', '\\' + node.content, [convert_typst_node_to_tex(body)]);
+                const script = convert_typst_node_to_tex(label);
+                const data = node.content === 'overbrace' ? { base, sup: script } : { base, sub: script };
+                return new TexNode('supsub', '', [], data);
+            }
+
+            // special hook for vec
+            // "vec(a, b, c)" -> "\begin{pmatrix}a\\ b\\ c\end{pmatrix}"
+            if (node.content === 'vec') {
+                const tex_data = node.args!.map(convert_typst_node_to_tex).map((n) => [n]) as TexArrayData;
+                return new TexNode('beginend', 'pmatrix', [], tex_data);
+            }
+
+            // general case
+            const func_name_tex = typst_token_to_tex(node.content);
+            if (func_name_tex.length > 0 && TEX_UNARY_COMMANDS.includes(func_name_tex.substring(1))) {
+                return new TexNode('unaryFunc', func_name_tex, node.args!.map(convert_typst_node_to_tex));
+            } else if (func_name_tex.length > 0 && TEX_BINARY_COMMANDS.includes(func_name_tex.substring(1))) {
+                return new TexNode('binaryFunc', func_name_tex, node.args!.map(convert_typst_node_to_tex));
             } else {
-                // special hook for vec
-                // "vec(a, b, c)" -> "\begin{pmatrix}a\\ b\\ c\end{pmatrix}"
-                if (node.content === 'vec') {
-                    const tex_data = node.args!.map(convert_typst_node_to_tex).map((n) => [n]) as TexArrayData;
-                    return new TexNode('beginend', 'pmatrix', [], tex_data);
-                }
                 return new TexNode('ordgroup', '', [
                     new TexNode('symbol', typst_token_to_tex(node.content)),
                     new TexNode('element', '('),
-                    ...array_join(node.args!.map(convert_typst_node_to_tex), TEX_NODE_COMMA),
+                    ...array_intersperse(node.args!.map(convert_typst_node_to_tex), TEX_NODE_COMMA),
                     new TexNode('element', ')')
                 ]);
             }
@@ -659,47 +669,38 @@ export function convert_typst_node_to_tex(node: TypstNode): TexNode {
         case 'matrix': {
             const typst_data = node.data as TypstNode[][];
             const tex_data = typst_data.map(row => row.map(convert_typst_node_to_tex));
-            let env_type = 'pmatrix';
+            let env_type = 'pmatrix'; // typst mat use delim:"(" by default
             if (node.options) {
                 if ('delim' in node.options) {
                     const delim = node.options.delim;
-                    if (delim instanceof TypstNode) {
-                        switch (delim.content) {
-                            case '#none':
-                                env_type = 'matrix';
-                                break;
-                            case 'bar.v.double':
-                                env_type = 'Vmatrix';
-                                break;
-                            case 'bar':
-                            case 'bar.v':
-                                env_type = 'vmatrix';
-                                break;
-                            default:
-                                throw new Error(`Unexpected delimiter ${delim.content}`);
-                        }
-                    } else {
-                        switch (delim) {
-                            case '[':
-                                env_type = 'bmatrix';
-                                break;
-                            case ']':
-                                env_type = 'bmatrix';
-                                break;
-                            case '{':
-                                env_type = 'Bmatrix';
-                                break;
-                            case '}':
-                                env_type = 'Bmatrix';
-                                break;
-                            case '|':
-                                env_type = 'vmatrix';
-                                break;
-                            case ')':
-                            case '(':
-                            default:
-                                env_type = 'pmatrix';
-                        }
+                    switch (delim.content) {
+                        case '#none':
+                            env_type = 'matrix';
+                            break;
+                        case '[':
+                        case ']':
+                            env_type = 'bmatrix';
+                            break;
+                        case '(':
+                        case ')':
+                            env_type = 'pmatrix';
+                            break;
+                        case '{':
+                        case '}':
+                            env_type = 'Bmatrix';
+                            break;
+                        case '|':
+                            env_type = 'vmatrix';
+                            break;
+                        case 'bar':
+                        case 'bar.v':
+                            env_type = 'vmatrix';
+                            break;
+                        case 'bar.v.double':
+                            env_type = 'Vmatrix';
+                            break;
+                        default:
+                            throw new Error(`Unexpected delimiter ${delim.content}`);
                     }
                 }
             }

package/src/generic.ts

@@ -38,7 +38,7 @@ export function array_split<T extends IEquatable>(array: T[], sep: T): T[][] {
 
 // e.g. input array=['a', 'b', 'c'], sep = '+'
 // return ['a','+', 'b', '+','c']
-export function array_join<T>(array: T[], sep: T): T[] {
+export function array_intersperse<T>(array: T[], sep: T): T[] {
     const res: T[] = [];
     for (let i = 0; i < array.length; i++) {
         res.push(array[i]);

package/src/jslex.ts

@@ -15,9 +15,8 @@ interface IRule<T> {
 
 interface IMatch<T> {
     index: number;
-    text: string;
-    len: number;
     rule: IRule<T>;
+    reMatchArray: RegExpMatchArray;
 }
 
 
@@ -31,8 +30,10 @@ const EOF = {};
  * @return {int} Difference between the matches.
  */
 function matchcompare<T>(m1: IMatch<T>, m2: IMatch<T>): number {
-    if(m2.len !== m1.len) {
-        return m2.len - m1.len;
+    const m1_len = m1.reMatchArray[0].length;
+    const m2_len = m2.reMatchArray[0].length;
+    if(m2_len !== m1_len) {
+        return m2_len - m1_len;
     } else {
         return m1.index - m2.index;
     }
@@ -59,6 +60,7 @@ export class Scanner<T> {
 
     private _text: string | null = null;
     private _leng: number | null = null;
+    private _reMatchArray: RegExpMatchArray | null = null;
 
     constructor(input: string, lexer: JSLex<T>) {
         this._input = input;
@@ -77,6 +79,10 @@ export class Scanner<T> {
         return this._leng;
     }
 
+    public reMatchArray(): RegExpMatchArray | null {
+        return this._reMatchArray;
+    }
+
     /**
      * Position of in stream, line number and column number of match.
      */
@@ -180,9 +186,8 @@ export class Scanner<T> {
             if (mt !== null && mt[0].length > 0) {
                 matches.push({
                     index: i,
-                    text: mt[0],
-                    len: mt[0].length,
-                    rule: rule
+                    rule: rule,
+                    reMatchArray: mt,
                 });
             }
         }
@@ -193,22 +198,24 @@ export class Scanner<T> {
         this._go = true;
 
         let result: T | T[];
-        let m: IMatch<T>;
+        let matched_text: string;
         for (let j = 0, n = matches.length; j < n && this._go; j++) {
             this._offset = 0;
             this._less = null;
             this._go = false;
             this._newstate = null;
-            m = matches[j];
-            this._text = m.text;
-            this._leng = m.len;
+            const m = matches[j];
+            matched_text = m.reMatchArray[0];
+            this._text = matched_text;
+            this._leng = matched_text.length;
+            this._reMatchArray = m.reMatchArray;
             result = m.rule.action(this);
             if (this._newstate && this._newstate != this._state) {
                 this._state = this._newstate;
                 break;
             }
         }
-        const text = this._less === null ? m!.text : m!.text.substring(0, this._less);
+        const text = this._less === null ? matched_text! : matched_text!.substring(0, this._less);
         const len = text.length;
         this._pos += len + this._offset;