securemark 0.296.2 → 0.296.4

package/src/parser/segment.test.ts

@@ -4,108 +4,108 @@ import { Command } from './context';
 describe('Unit: parser/segment', () => {
   describe('segment', () => {
     it('huge input', () => {
-      const result = segment(`${'\n'.repeat(1e6 + 1)}`).next().value?.split('\n', 1)[0];
+      const result = segment(`${'\n'.repeat(1e6 + 1)}`).next().value?.[0].split('\n', 1)[0];
       assert(result?.startsWith(`${Command.Error}Too large input`));
     });
 
     it('huge segment', () => {
-      const result = segment(`${'\n'.repeat(1e5 + 1)}`).next().value?.split('\n', 1)[0];
+      const result = segment(`${'\n'.repeat(1e5 + 1)}`).next().value?.[0].split('\n', 1)[0];
       assert(result?.startsWith(`${Command.Error}Too large segment`));
     });
 
     it('basic', () => {
-      assert.deepStrictEqual([...segment('')], []);
-      assert.deepStrictEqual([...segment('a')], ['a']);
-      assert.deepStrictEqual([...segment('a\n')], ['a\n']);
-      assert.deepStrictEqual([...segment('a\n\n')], ['a\n', '\n']);
-      assert.deepStrictEqual([...segment('a\n\n\n')], ['a\n', '\n\n']);
-      assert.deepStrictEqual([...segment('a\nb')], ['a\nb']);
-      assert.deepStrictEqual([...segment('a\nb\n')], ['a\nb\n']);
-      assert.deepStrictEqual([...segment('a\nb\n\n')], ['a\nb\n', '\n']);
-      assert.deepStrictEqual([...segment('a\nb\n\n\n')], ['a\nb\n', '\n\n']);
-      assert.deepStrictEqual([...segment('a\nb\n\nc\n\nd')], ['a\nb\n', '\n', 'c\n', '\n', 'd']);
-      assert.deepStrictEqual([...segment('a\n\\\nb')], ['a\n\\\nb']);
-      assert.deepStrictEqual([...segment('a ')], ['a ']);
-      assert.deepStrictEqual([...segment(' a')], [' a']);
-      assert.deepStrictEqual([...segment(' a ')], [' a ']);
+      assert.deepStrictEqual([...segment('')].map(t => t[0]), []);
+      assert.deepStrictEqual([...segment('a')].map(t => t[0]), ['a']);
+      assert.deepStrictEqual([...segment('a\n')].map(t => t[0]), ['a\n']);
+      assert.deepStrictEqual([...segment('a\n\n')].map(t => t[0]), ['a\n', '\n']);
+      assert.deepStrictEqual([...segment('a\n\n\n')].map(t => t[0]), ['a\n', '\n\n']);
+      assert.deepStrictEqual([...segment('a\nb')].map(t => t[0]), ['a\nb']);
+      assert.deepStrictEqual([...segment('a\nb\n')].map(t => t[0]), ['a\nb\n']);
+      assert.deepStrictEqual([...segment('a\nb\n\n')].map(t => t[0]), ['a\nb\n', '\n']);
+      assert.deepStrictEqual([...segment('a\nb\n\n\n')].map(t => t[0]), ['a\nb\n', '\n\n']);
+      assert.deepStrictEqual([...segment('a\nb\n\nc\n\nd')].map(t => t[0]), ['a\nb\n', '\n', 'c\n', '\n', 'd']);
+      assert.deepStrictEqual([...segment('a\n\\\nb')].map(t => t[0]), ['a\n\\\nb']);
+      assert.deepStrictEqual([...segment('a ')].map(t => t[0]), ['a ']);
+      assert.deepStrictEqual([...segment(' a')].map(t => t[0]), [' a']);
+      assert.deepStrictEqual([...segment(' a ')].map(t => t[0]), [' a ']);
     });
 
     it('linebreak', () => {
-      assert.deepStrictEqual([...segment('\n')], ['\n']);
-      assert.deepStrictEqual([...segment('\n\n')], ['\n\n']);
-      assert.deepStrictEqual([...segment('\n\n\n')], ['\n\n\n']);
-      assert.deepStrictEqual([...segment('\n\n\n\n')], ['\n\n\n\n']);
-      assert.deepStrictEqual([...segment(' ')], [' ']);
-      assert.deepStrictEqual([...segment(' \n')], [' \n']);
-      assert.deepStrictEqual([...segment(' \n \n \n ')], [' \n \n \n ']);
-      assert.deepStrictEqual([...segment('a\n')], ['a\n']);
-      assert.deepStrictEqual([...segment('a\n ')], ['a\n', ' ']);
-      assert.deepStrictEqual([...segment('a\n\n ')], ['a\n', '\n ']);
-      assert.deepStrictEqual([...segment('a\n\n\n ')], ['a\n', '\n\n ']);
-      assert.deepStrictEqual([...segment('a\n\n\n\n ')], ['a\n', '\n\n\n ']);
-      assert.deepStrictEqual([...segment('a\n\n\n\n\n ')], ['a\n', '\n\n\n\n ']);
-      assert.deepStrictEqual([...segment('a\n\n\n\n\n\n ')], ['a\n', '\n\n\n\n\n ']);
+      assert.deepStrictEqual([...segment('\n')].map(t => t[0]), ['\n']);
+      assert.deepStrictEqual([...segment('\n\n')].map(t => t[0]), ['\n\n']);
+      assert.deepStrictEqual([...segment('\n\n\n')].map(t => t[0]), ['\n\n\n']);
+      assert.deepStrictEqual([...segment('\n\n\n\n')].map(t => t[0]), ['\n\n\n\n']);
+      assert.deepStrictEqual([...segment(' ')].map(t => t[0]), [' ']);
+      assert.deepStrictEqual([...segment(' \n')].map(t => t[0]), [' \n']);
+      assert.deepStrictEqual([...segment(' \n \n \n ')].map(t => t[0]), [' \n \n \n ']);
+      assert.deepStrictEqual([...segment('a\n')].map(t => t[0]), ['a\n']);
+      assert.deepStrictEqual([...segment('a\n ')].map(t => t[0]), ['a\n', ' ']);
+      assert.deepStrictEqual([...segment('a\n\n ')].map(t => t[0]), ['a\n', '\n ']);
+      assert.deepStrictEqual([...segment('a\n\n\n ')].map(t => t[0]), ['a\n', '\n\n ']);
+      assert.deepStrictEqual([...segment('a\n\n\n\n ')].map(t => t[0]), ['a\n', '\n\n\n ']);
+      assert.deepStrictEqual([...segment('a\n\n\n\n\n ')].map(t => t[0]), ['a\n', '\n\n\n\n ']);
+      assert.deepStrictEqual([...segment('a\n\n\n\n\n\n ')].map(t => t[0]), ['a\n', '\n\n\n\n\n ']);
     });
 
     it('codeblock', () => {
-      assert.deepStrictEqual([...segment('```')], ['```']);
-      assert.deepStrictEqual([...segment('```\n```')], ['```\n```']);
-      assert.deepStrictEqual([...segment('```\n\n```')], ['```\n\n```']);
-      assert.deepStrictEqual([...segment('```\n\n\n```')], ['```\n', '\n\n', '```']);
-      assert.deepStrictEqual([...segment('```\n\n0\n```')], ['```\n', '\n', '0\n```']);
-      assert.deepStrictEqual([...segment('```\n0\n\n```')], ['```\n0\n\n```']);
-      assert.deepStrictEqual([...segment('```\n````\n```')], ['```\n````\n```']);
-      assert.deepStrictEqual([...segment('````\n```\n````')], ['````\n```\n````']);
-      assert.deepStrictEqual([...segment('```\n\n```\n\n')], ['```\n\n```\n', '\n']);
+      assert.deepStrictEqual([...segment('```')].map(t => t[0]), ['```']);
+      assert.deepStrictEqual([...segment('```\n```')].map(t => t[0]), ['```\n```']);
+      assert.deepStrictEqual([...segment('```\n\n```')].map(t => t[0]), ['```\n\n```']);
+      assert.deepStrictEqual([...segment('```\n\n\n```')].map(t => t[0]), ['```\n', '\n\n', '```']);
+      assert.deepStrictEqual([...segment('```\n\n0\n```')].map(t => t[0]), ['```\n', '\n', '0\n```']);
+      assert.deepStrictEqual([...segment('```\n0\n\n```')].map(t => t[0]), ['```\n0\n\n```']);
+      assert.deepStrictEqual([...segment('```\n````\n```')].map(t => t[0]), ['```\n````\n```']);
+      assert.deepStrictEqual([...segment('````\n```\n````')].map(t => t[0]), ['````\n```\n````']);
+      assert.deepStrictEqual([...segment('```\n\n```\n\n')].map(t => t[0]), ['```\n\n```\n', '\n']);
     });
 
     it('mathblock', () => {
-      assert.deepStrictEqual([...segment('$$')], ['$$']);
-      assert.deepStrictEqual([...segment('$$\n$$')], ['$$\n$$']);
-      assert.deepStrictEqual([...segment('$$\n\n$$')], ['$$\n\n$$']);
-      assert.deepStrictEqual([...segment('$$\n\n\n$$')], ['$$\n', '\n\n', '$$']);
-      assert.deepStrictEqual([...segment('$$\n\n0\n$$')], ['$$\n', '\n', '0\n$$']);
-      assert.deepStrictEqual([...segment('$$\n0\n\n$$')], ['$$\n0\n\n$$']);
-      assert.deepStrictEqual([...segment('$$\n\n$$\n\n')], ['$$\n\n$$\n', '\n']);
+      assert.deepStrictEqual([...segment('$$')].map(t => t[0]), ['$$']);
+      assert.deepStrictEqual([...segment('$$\n$$')].map(t => t[0]), ['$$\n$$']);
+      assert.deepStrictEqual([...segment('$$\n\n$$')].map(t => t[0]), ['$$\n\n$$']);
+      assert.deepStrictEqual([...segment('$$\n\n\n$$')].map(t => t[0]), ['$$\n', '\n\n', '$$']);
+      assert.deepStrictEqual([...segment('$$\n\n0\n$$')].map(t => t[0]), ['$$\n', '\n', '0\n$$']);
+      assert.deepStrictEqual([...segment('$$\n0\n\n$$')].map(t => t[0]), ['$$\n0\n\n$$']);
+      assert.deepStrictEqual([...segment('$$\n\n$$\n\n')].map(t => t[0]), ['$$\n\n$$\n', '\n']);
     });
 
     it('extension', () => {
-      assert.deepStrictEqual([...segment('~~~')], ['~~~']);
-      assert.deepStrictEqual([...segment('~~~\n~~~')], ['~~~\n~~~']);
-      assert.deepStrictEqual([...segment('~~~\n\n~~~')], ['~~~\n\n~~~']);
-      assert.deepStrictEqual([...segment('~~~\n\n\n~~~')], ['~~~\n', '\n\n', '~~~']);
-      assert.deepStrictEqual([...segment('~~~\n\n0\n~~~')], ['~~~\n', '\n', '0\n~~~']);
-      assert.deepStrictEqual([...segment('~~~\n0\n\n~~~')], ['~~~\n0\n\n~~~']);
-      assert.deepStrictEqual([...segment('~~~\n~~~~\n~~~')], ['~~~\n~~~~\n~~~']);
-      assert.deepStrictEqual([...segment('~~~~\n~~~\n~~~~')], ['~~~~\n~~~\n~~~~']);
-      assert.deepStrictEqual([...segment('~~~\n\n~~~\n\n')], ['~~~\n\n~~~\n', '\n']);
-      assert.deepStrictEqual([...segment('~~~\n```\n~~~\n```\n~~~')], ['~~~\n```\n~~~\n```\n~~~']);
-      assert.deepStrictEqual([...segment('~~~\ninvalid\n~~~')], ['~~~\ninvalid\n~~~']);
-      assert.deepStrictEqual([...segment('~~~\ninvalid\n\ncaption\n~~~')], ['~~~\ninvalid\n\ncaption\n~~~']);
-      assert.deepStrictEqual([...segment('~~~figure [$-name]\n$$\n$$\n\n```\n~~~')], ['~~~figure [$-name]\n$$\n$$\n\n```\n~~~']);
+      assert.deepStrictEqual([...segment('~~~')].map(t => t[0]), ['~~~']);
+      assert.deepStrictEqual([...segment('~~~\n~~~')].map(t => t[0]), ['~~~\n~~~']);
+      assert.deepStrictEqual([...segment('~~~\n\n~~~')].map(t => t[0]), ['~~~\n\n~~~']);
+      assert.deepStrictEqual([...segment('~~~\n\n\n~~~')].map(t => t[0]), ['~~~\n', '\n\n', '~~~']);
+      assert.deepStrictEqual([...segment('~~~\n\n0\n~~~')].map(t => t[0]), ['~~~\n', '\n', '0\n~~~']);
+      assert.deepStrictEqual([...segment('~~~\n0\n\n~~~')].map(t => t[0]), ['~~~\n0\n\n~~~']);
+      assert.deepStrictEqual([...segment('~~~\n~~~~\n~~~')].map(t => t[0]), ['~~~\n~~~~\n~~~']);
+      assert.deepStrictEqual([...segment('~~~~\n~~~\n~~~~')].map(t => t[0]), ['~~~~\n~~~\n~~~~']);
+      assert.deepStrictEqual([...segment('~~~\n\n~~~\n\n')].map(t => t[0]), ['~~~\n\n~~~\n', '\n']);
+      assert.deepStrictEqual([...segment('~~~\n```\n~~~\n```\n~~~')].map(t => t[0]), ['~~~\n```\n~~~\n```\n~~~']);
+      assert.deepStrictEqual([...segment('~~~\ninvalid\n~~~')].map(t => t[0]), ['~~~\ninvalid\n~~~']);
+      assert.deepStrictEqual([...segment('~~~\ninvalid\n\ncaption\n~~~')].map(t => t[0]), ['~~~\ninvalid\n\ncaption\n~~~']);
+      assert.deepStrictEqual([...segment('~~~figure [$-name]\n$$\n$$\n\n```\n~~~')].map(t => t[0]), ['~~~figure [$-name]\n$$\n$$\n\n```\n~~~']);
     });
 
     it('mixed', () => {
-      assert.deepStrictEqual([...segment('```\n\n\n')], ['```\n', '\n\n']);
-      assert.deepStrictEqual([...segment('~~~\n\n\n')], ['~~~\n', '\n\n']);
-      assert.deepStrictEqual([...segment('```\n~~~\n```')], ['```\n~~~\n```']);
-      assert.deepStrictEqual([...segment('~~~\n```\n~~~')], ['~~~\n```\n~~~']);
-      assert.deepStrictEqual([...segment('```\n```\n\n~~~\n~~~')], ['```\n```\n', '\n', '~~~\n~~~']);
-      assert.deepStrictEqual([...segment('~~~\n~~~\n\n```\n```')], ['~~~\n~~~\n', '\n', '```\n```']);
-      assert.deepStrictEqual([...segment(' ```\n\n```')], [' ```\n', '\n', '```']);
-      assert.deepStrictEqual([...segment(' ~~~\n\n~~~')], [' ~~~\n', '\n', '~~~']);
+      assert.deepStrictEqual([...segment('```\n\n\n')].map(t => t[0]), ['```\n', '\n\n']);
+      assert.deepStrictEqual([...segment('~~~\n\n\n')].map(t => t[0]), ['~~~\n', '\n\n']);
+      assert.deepStrictEqual([...segment('```\n~~~\n```')].map(t => t[0]), ['```\n~~~\n```']);
+      assert.deepStrictEqual([...segment('~~~\n```\n~~~')].map(t => t[0]), ['~~~\n```\n~~~']);
+      assert.deepStrictEqual([...segment('```\n```\n\n~~~\n~~~')].map(t => t[0]), ['```\n```\n', '\n', '~~~\n~~~']);
+      assert.deepStrictEqual([...segment('~~~\n~~~\n\n```\n```')].map(t => t[0]), ['~~~\n~~~\n', '\n', '```\n```']);
+      assert.deepStrictEqual([...segment(' ```\n\n```')].map(t => t[0]), [' ```\n', '\n', '```']);
+      assert.deepStrictEqual([...segment(' ~~~\n\n~~~')].map(t => t[0]), [' ~~~\n', '\n', '~~~']);
     });
 
     it('blockquote', () => {
-      assert.deepStrictEqual([...segment('> ```\n\n```')], ['> ```\n', '\n', '```']);
-      assert.deepStrictEqual([...segment('!> ```\n\n```')], ['!> ```\n', '\n', '```']);
+      assert.deepStrictEqual([...segment('> ```\n\n```')].map(t => t[0]), ['> ```\n', '\n', '```']);
+      assert.deepStrictEqual([...segment('!> ```\n\n```')].map(t => t[0]), ['!> ```\n', '\n', '```']);
     });
 
     it('heading', () => {
-      assert.deepStrictEqual([...segment('# a\n\n# b\n')], ['# a\n', '\n', '# b\n']);
-      assert.deepStrictEqual([...segment('# a\n# b\n')], ['# a\n', '# b\n']);
-      assert.deepStrictEqual([...segment('# a\n# b')], ['# a\n', '# b']);
-      assert.deepStrictEqual([...segment('# a\n # b')], ['# a\n # b']);
+      assert.deepStrictEqual([...segment('# a\n\n# b\n')].map(t => t[0]), ['# a\n', '\n', '# b\n']);
+      assert.deepStrictEqual([...segment('# a\n# b\n')].map(t => t[0]), ['# a\n', '# b\n']);
+      assert.deepStrictEqual([...segment('# a\n# b')].map(t => t[0]), ['# a\n', '# b']);
+      assert.deepStrictEqual([...segment('# a\n # b')].map(t => t[0]), ['# a\n # b']);
     });
 
   });

package/src/parser/segment.ts

@@ -1,11 +1,11 @@
 import { MarkdownParser } from '../../markdown';
-import { Context, Command } from './context';
+import { Context, Segment, Command } from './context';
 import { union, some } from '../combinator';
 import { segment as heading } from './block/heading';
 import { segment as codeblock } from './block/codeblock';
 import { segment as mathblock } from './block/mathblock';
 import { segment as extension } from './block/extension';
-import { contentline, emptyline } from './source';
+import { contentline, emptysegment } from './source';
 
 import SegmentParser = MarkdownParser.SegmentParser;
 
@@ -13,7 +13,7 @@ export const MAX_SEGMENT_SIZE = 100_000; // 100,000 bytes (Max value size of FDB
 export const MAX_INPUT_SIZE = MAX_SEGMENT_SIZE * 10;
 
 const parser: SegmentParser = union([
-  some(emptyline, MAX_SEGMENT_SIZE + 1),
+  some(emptysegment, MAX_SEGMENT_SIZE + 1),
   input => {
     const { context: { source, position } } = input;
     if (position === source.length) return;
@@ -26,21 +26,19 @@ const parser: SegmentParser = union([
         break;
       case '$':
         if (source[position + 1] === '$') return mathblock(input);
-        break;
+        return extension(input);
       case '[':
         if (source[position + 1] === '$') return extension(input);
         break;
       case '#':
         return heading(input);
-      case '$':
-        return extension(input);
     }
   },
   some(contentline, MAX_SEGMENT_SIZE + 1),
 ]) as any;
 
-export function* segment(source: string): Generator<string, undefined, undefined> {
-  if (!validate(source, MAX_INPUT_SIZE)) return yield `${Command.Error}Too large input over ${MAX_INPUT_SIZE.toLocaleString('en')} bytes.\n${source.slice(0, 1001)}`;
+export function* segment(source: string): Generator<readonly [string, Segment], undefined, undefined> {
+  if (!validate(source, MAX_INPUT_SIZE)) return yield [`${Command.Error}Too large input over ${MAX_INPUT_SIZE.toLocaleString('en')} bytes.\n${source.slice(0, 1001)}`, Segment.unknown];
   assert(source.length < Number.MAX_SAFE_INTEGER);
   for (let position = 0; position < source.length;) {
     const context = new Context({ source, position });
@@ -51,13 +49,13 @@ export function* segment(source: string): Generator<string, undefined, undefined
       ? result.foldl<string[]>((acc, { value }) => void acc.push(value) || acc, [])
       : [source.slice(position, context.position)];
     assert(segs.join('') === source.slice(position, context.position));
+    position = context.position;
     for (let i = 0; i < segs.length; ++i) {
       const seg = segs[i];
       validate(seg, MAX_SEGMENT_SIZE)
-        ? yield seg
-        : yield `${Command.Error}Too large segment over ${MAX_SEGMENT_SIZE.toLocaleString('en')} bytes.\n${seg}`
+        ? yield [seg, context.segment]
+        : yield [`${Command.Error}Too large segment over ${MAX_SEGMENT_SIZE.toLocaleString('en')} bytes.\n${seg}`, Segment.unknown];
     }
-    position = context.position;
   }
 }
 

package/src/parser/source/line.ts

@@ -1,4 +1,5 @@
-import { AnyLineParser, EmptyLineParser, ContentLineParser } from '../source';
+import { AnyLineParser, EmptyLineParser, EmptySegmentParser, ContentLineParser } from '../source';
+import { Segment } from '../context';
 import { List } from '../../combinator/data/parser';
 
 export const anyline: AnyLineParser = input => {
@@ -14,14 +15,32 @@ export const emptyline: EmptyLineParser = input => {
   const { context } = input;
   const { source, position } = context;
   if (position === source.length) return;
-  if (source[position] === '\n') return ++context.position, new List();
-  regEmptyline.lastIndex = position;
-  regEmptyline.test(source);
-  const i = regEmptyline.lastIndex;
-  if (i === 0) return;
+  const i = eoel(source, position);
+  if (i === position) return;
+  context.position = i;
+  return new List();
+};
+export const emptysegment: EmptySegmentParser = input => {
+  const { context } = input;
+  const { source, position, segment } = context;
+  if (position === source.length) return;
+  if (segment & Segment.write) {
+    if (segment !== (Segment.empty | Segment.write)) return;
+    context.position = source.length;
+    return new List();
+  }
+  const i = eoel(source, position);
+  if (i === position) return;
   context.position = i;
+  context.segment = Segment.empty;
   return new List();
 };
+function eoel(source: string, position: number): number {
+  if (source[position] === '\n') return position + 1;
+  regEmptyline.lastIndex = position;
+  regEmptyline.test(source);
+  return regEmptyline.lastIndex || position;
+}
 
 const regContentline = /[^\S\n]*\S[^\n]*(?:$|\n)/y;
 export const contentline: ContentLineParser = input => {

package/src/parser/source/str.ts

@@ -7,18 +7,17 @@ export function str(pattern: string | RegExp, after?: string | RegExp): Parser<s
   return matcher(pattern, true, after ? tester(after, false) : undefined);
 }
 
-export function strs(pattern: string, limit?: number): StrParser;
-export function strs(pattern: string, limit: number = -1): Parser<string> {
-  assert(pattern);
+export function strs(char: string, min?: number, max?: number): StrParser;
+export function strs(char: string, min: number = 1, max: number = -1): Parser<string> {
+  assert(char.length === 1);
   return ({ context }) => {
-    const { source } = context;
-    let acc = '';
-    for (let i = 0; i !== limit && context.position < source.length && source.startsWith(pattern, context.position); ++i) {
-      acc += pattern;
-      context.position += pattern.length;
+    const { source, position } = context;
+    let cnt = 0;
+    for (; cnt !== max && context.position < source.length && source[context.position] === char; ++cnt) {
+      context.position += char.length;
     }
-    return acc
-      ? new List([new Node(acc)])
+    return cnt >= min
+      ? new List([new Node(source.slice(position, context.position))])
       : undefined;
   };
 }

package/src/parser/source.ts

@@ -8,10 +8,11 @@ export import UnescapableSourceParser = SourceParser.UnescapableSourceParser;
 export import StrParser = SourceParser.StrParser;
 export import ContentLineParser = SourceParser.ContentLineParser;
 export import EmptyLineParser = SourceParser.EmptyLineParser;
+export import EmptySegmentParser = SourceParser.EmptySegmentParser;
 export import AnyLineParser = SourceParser.AnyLineParser;
 
 export { text, txt } from './source/text';
 export { escsource } from './source/escapable';
 export { unescsource } from './source/unescapable';
 export { str, strs } from './source/str';
-export { contentline, emptyline, anyline } from './source/line';
+export { contentline, emptyline, emptysegment, anyline } from './source/line';