simple-customize-markdown-converter 1.1.0 → 1.2.0

package/dist/core/lexer/handler.d.ts

@@ -0,0 +1,23 @@
+import { TokenizerStrategy } from "../../types/token";
+declare const EscapeCharacterHandler: TokenizerStrategy;
+declare const CommentHandler: TokenizerStrategy;
+declare const HtmlHandler: TokenizerStrategy;
+declare const HorizontalLineHandler: TokenizerStrategy;
+declare const CodeBlockHandler: TokenizerStrategy;
+declare const BoldHandler: TokenizerStrategy;
+declare const StrikethroughHandler: TokenizerStrategy;
+declare const FootnoteDefHandler: TokenizerStrategy;
+declare const FootnoteRefHandler: TokenizerStrategy;
+declare const TaskListHandler: TokenizerStrategy;
+declare const UnorderedListHandler: TokenizerStrategy;
+declare const OrderedListHandler: TokenizerStrategy;
+declare const EndListHandler: TokenizerStrategy;
+declare const TableHandler: TokenizerStrategy;
+declare const InlineCodeHandler: TokenizerStrategy;
+declare const HeaderHandler: TokenizerStrategy;
+declare const ItalicHandler: TokenizerStrategy;
+declare const QuoteHandler: TokenizerStrategy;
+declare const LinkHandler: TokenizerStrategy;
+declare const ImageHandler: TokenizerStrategy;
+declare const NewLineHandler: TokenizerStrategy;
+export { BoldHandler, CodeBlockHandler, CommentHandler, EndListHandler, EscapeCharacterHandler, FootnoteDefHandler, FootnoteRefHandler, HeaderHandler, HorizontalLineHandler, HtmlHandler, ImageHandler, InlineCodeHandler, ItalicHandler, LinkHandler, NewLineHandler, OrderedListHandler, QuoteHandler, StrikethroughHandler, TableHandler, TaskListHandler, UnorderedListHandler };

package/dist/core/lexer/handler.js

@@ -0,0 +1,272 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UnorderedListHandler = exports.TaskListHandler = exports.TableHandler = exports.StrikethroughHandler = exports.QuoteHandler = exports.OrderedListHandler = exports.NewLineHandler = exports.LinkHandler = exports.ItalicHandler = exports.InlineCodeHandler = exports.ImageHandler = exports.HtmlHandler = exports.HorizontalLineHandler = exports.HeaderHandler = exports.FootnoteRefHandler = exports.FootnoteDefHandler = exports.EscapeCharacterHandler = exports.EndListHandler = exports.CommentHandler = exports.CodeBlockHandler = exports.BoldHandler = void 0;
+const utils = __importStar(require("../../utilities/tokenizer-utils"));
+const EscapeCharacterHandler = {
+    name: "EscapeCharacter",
+    match: (lex) => lex.peek() === "\\" && lex.peek(1) !== undefined,
+    emit: (lex) => {
+        lex.next(1);
+        utils.handleTextBlock(lex);
+    }
+};
+exports.EscapeCharacterHandler = EscapeCharacterHandler;
+const CommentHandler = {
+    name: "Comment",
+    match: (lex) => lex.startsWith("<!--"),
+    emit: (lex) => lex.readUntilMatchString("-->", true)
+};
+exports.CommentHandler = CommentHandler;
+const HtmlHandler = {
+    name: "HTML",
+    match: (lex) => lex.peek() === "<",
+    emit: (lex) => {
+        //Handle comment
+        const line = lex.peekUntil(">");
+        const blockRegex = /^<(h[1-6]|div|table|pre|blockquote|ul|ol|li|p|section|article|header|footer|nav|aside|hr|form|iframe)\b/i;
+        if (blockRegex.test(line)) {
+            utils.handleHtmlBlock(lex);
+        }
+        else {
+            utils.handleHtmlInline(lex);
+        }
+    }
+};
+exports.HtmlHandler = HtmlHandler;
+const HorizontalLineHandler = {
+    name: "HorizontalLine",
+    match: (lex) => /^([-*_])\1{2,}$/.test(lex.peekUntil("\n").trim()) && lex.getLastToken()?.type === "NewLine",
+    emit: (lex) => {
+        lex.next(2); //Skip two first characters, remain will be skiped after loop
+        lex.listToken.push({ type: "HorizontalLine" });
+    }
+};
+exports.HorizontalLineHandler = HorizontalLineHandler;
+const CodeBlockHandler = {
+    name: "CodeBlock",
+    match: (lex) => lex.startsWith("```"),
+    emit: (lex) => {
+        let lang = "";
+        let content = "";
+        lex.next(3); //Skip open block
+        while (!lex.isEndOfFile() && lex.peek() !== "\n") {
+            lang += lex.peek();
+            lex.next();
+        }
+        lex.next(); //Skip \n
+        while (!lex.isEndOfFile() && !lex.startsWith("```")) {
+            content += lex.peek();
+            lex.next();
+        }
+        lex.next(2); //Skip close block (due to next() after each tokenize iteration)
+        lex.listToken.push({ "type": "CodeBlock", lang: lang.trim(), content: content.trimEnd() });
+    }
+};
+exports.CodeBlockHandler = CodeBlockHandler;
+const BoldHandler = {
+    name: "Bold",
+    match: (lex) => lex.startsWith("**"),
+    emit: (lex) => {
+        lex.listToken.push({ type: "Bold" });
+        lex.next(); //Skip remain *
+    }
+};
+exports.BoldHandler = BoldHandler;
+const StrikethroughHandler = {
+    name: "Strikethrough",
+    match: (lex) => lex.startsWith("~~"),
+    emit: (lex) => {
+        lex.listToken.push({ type: "Strikethrough" });
+        lex.next(); //Skip remain ~
+    }
+};
+exports.StrikethroughHandler = StrikethroughHandler;
+// Footnote
+const FootnoteDefHandler = {
+    name: "FootnoteDef",
+    match: (lex) => lex.isStartOfLine() && /^\[\^[^\]]+\]:/.test(lex.peekUntil("\n")),
+    emit: (lex) => {
+        const line = lex.readUntil("\n");
+        const match = line.match(/^\[\^([^\]]+)\]:\s*(.*)$/);
+        if (match) {
+            const id = match[1];
+            const content = match[2];
+            lex.listToken.push({ type: "FootnoteDef", id, content });
+        }
+    }
+};
+exports.FootnoteDefHandler = FootnoteDefHandler;
+const FootnoteRefHandler = {
+    name: "FootnoteRef",
+    match: (lex) => lex.startsWith("[^"),
+    emit: (lex) => {
+        lex.next(2); //Skip [^
+        const id = lex.readUntil("]");
+        lex.listToken.push({ type: "FootnoteRef", id });
+    }
+};
+exports.FootnoteRefHandler = FootnoteRefHandler;
+//List
+const TaskListHandler = {
+    name: "TaskList",
+    match: (lex) => lex.isStartOfLine() && /^(\s*)([-*+]) \[( |x|X)\] /.test(lex.peekUntil("\n")),
+    emit: (lex) => utils.handleList(lex, false, true)
+};
+exports.TaskListHandler = TaskListHandler;
+const UnorderedListHandler = {
+    name: "UnorderList",
+    match: (lex) => lex.isStartOfLine() && /^(\s*)([-*+]) /.test(lex.peekUntil("\n")),
+    emit: (lex) => utils.handleList(lex, false, false)
+};
+exports.UnorderedListHandler = UnorderedListHandler;
+const OrderedListHandler = {
+    name: "OrderedList",
+    match: (lex) => lex.isStartOfLine() && /^(\s*)(\d+)\. /.test(lex.peekUntil("\n")),
+    emit: (lex) => utils.handleList(lex, true, false)
+};
+exports.OrderedListHandler = OrderedListHandler;
+const EndListHandler = {
+    name: "EndList",
+    match: (lex) => lex.listLevelFlag > 0 && lex.isStartOfLine() && !/^(\s*)([-+*]|\d+\.) /.test(lex.peekUntil("\n")),
+    emit: (lex) => {
+        while (lex.listLevelFlag > 0) {
+            utils.handleEndList(lex);
+        }
+    }
+};
+exports.EndListHandler = EndListHandler;
+//Table
+const TableHandler = {
+    name: "Table",
+    match: (lex) => lex.isStartOfLine() && /^\s*\|.*\|\s*$/.test(lex.peekUntil("\n")),
+    emit: (lex) => utils.handleTable(lex)
+};
+exports.TableHandler = TableHandler;
+//Other common syntax
+const InlineCodeHandler = {
+    name: "InlineCode",
+    match: (lex) => lex.peek() === "`",
+    emit: (lex) => {
+        let content = "";
+        lex.next(); //Skip open block
+        while (!lex.isEndOfFile() && !lex.startsWith("`")) {
+            content += lex.peek();
+            lex.next();
+        }
+        lex.listToken.push({ "type": "InlineCode", content: content });
+    }
+};
+exports.InlineCodeHandler = InlineCodeHandler;
+const HeaderHandler = {
+    name: "Header",
+    match: (lex) => lex.peek() === "#",
+    emit: (lex) => {
+        let level = 0;
+        while (lex.peek() === "#") {
+            level++;
+            lex.next();
+        }
+        if (lex.peek() === " ") {
+            lex.next();
+            lex.pos--;
+        }
+        lex.listToken.push({ type: "Header", level });
+    }
+};
+exports.HeaderHandler = HeaderHandler;
+const ItalicHandler = {
+    name: "Italic",
+    match: (lex) => lex.peek() === "*" || lex.peek() === "_",
+    emit: (lex) => {
+        lex.listToken.push({ type: "Italic" });
+    }
+};
+exports.ItalicHandler = ItalicHandler;
+const QuoteHandler = {
+    name: "Quote",
+    match: (lex) => lex.peek() === ">",
+    emit: (lex) => {
+        lex.listToken.push({ type: "Quote" });
+    }
+};
+exports.QuoteHandler = QuoteHandler;
+const LinkHandler = {
+    name: "Link",
+    match: (lex) => lex.peek() === "[",
+    emit: (lex) => {
+        lex.next(); //Skip [
+        const text = lex.readUntil("]");
+        lex.next(); //Skip ]
+        if (lex.peek() === "(") {
+            lex.next(); //Skip (
+            const url = lex.readUntil(")");
+            //Don't skip ) due to auto skip on while loop
+            lex.listToken.push({ type: "Link", text: text, href: url });
+        }
+        else
+            lex.listToken.push({ type: "Text", value: `[${text}]` });
+    }
+};
+exports.LinkHandler = LinkHandler;
+const ImageHandler = {
+    name: "Image",
+    match: (lex) => lex.peek() === "!" && lex.peek(1) === "[",
+    emit: (lex) => {
+        lex.next(); //Skip !
+        if (lex.peek() !== "[")
+            return;
+        lex.next(); //Skip [
+        const alt = lex.readUntil("]");
+        lex.next(); //Skip ]
+        if (lex.peek() === "(") {
+            lex.next(); //Skip (
+            const src = lex.readUntil(")");
+            lex.next(); //Skip )
+            lex.listToken.push({ type: "Image", alt: alt, src: src });
+        }
+        else
+            lex.listToken.push({ type: "Text", value: `![${alt}]` });
+    }
+};
+exports.ImageHandler = ImageHandler;
+const NewLineHandler = {
+    name: "NewLine",
+    match: (lex) => lex.peek() === "\n",
+    emit: (lex) => {
+        lex.listToken.push({ type: "NewLine" });
+    }
+};
+exports.NewLineHandler = NewLineHandler;

package/dist/core/lexer/index.d.ts

@@ -0,0 +1,42 @@
+import { Token } from "../../types/token";
+export interface ILexer {
+    pos: number;
+    input: string;
+    listToken: Token[];
+    listLevelFlag: number;
+    peek(offset?: number): string | null;
+    next(amount?: number): void;
+    startsWith(str: string): boolean;
+    readUntil(char: string, isConsume?: boolean): string;
+    readUntilMatchString(str: string, isConsume: boolean): string;
+    peekUntil(char: string): string;
+    peekUntilByOffset(offset: number): string;
+    isEndOfFile(): boolean;
+    isStartOfLine(): boolean;
+    getLastToken(): Token;
+}
+export default class Lexer implements ILexer {
+    input: string;
+    pos: number;
+    listToken: Token[];
+    listLevelFlag: number;
+    private strategies;
+    constructor(input: string);
+    setInput(input: string): void;
+    peek(offset?: number): string | null;
+    next(amount?: number): void;
+    startsWith(str: string): boolean;
+    isEndOfFile(): boolean;
+    getLastToken(): Token;
+    readUntil(char: string, isConsumeChar?: boolean): string;
+    peekUntil(char: string): string;
+    peekUntilByOffset(offset: number): string;
+    isStartOfLine(): boolean;
+    readUntilMatchString(str: string, isConsume?: boolean): string;
+    /**
+     * Tokenize the markdown into a list of tokens.
+     * @param isEof - `True` when input is whole markdown, `False` if input is just a part of markdown.
+     * @returns List of tokens
+     */
+    tokenize(isEof?: boolean): Token[];
+}

package/dist/core/lexer/index.js

@@ -0,0 +1,177 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const Handlers = __importStar(require("./handler"));
+const utils = __importStar(require("../../utilities/tokenizer-utils"));
+class Lexer {
+    constructor(input) {
+        this.pos = 0;
+        this.listToken = [];
+        // Flag for handle special syntax
+        this.listLevelFlag = 0;
+        this.input = input;
+        this.strategies = [
+            Handlers.EscapeCharacterHandler,
+            Handlers.CommentHandler,
+            Handlers.HtmlHandler,
+            Handlers.HorizontalLineHandler,
+            Handlers.CodeBlockHandler,
+            Handlers.BoldHandler,
+            Handlers.StrikethroughHandler,
+            Handlers.FootnoteDefHandler,
+            Handlers.FootnoteRefHandler,
+            Handlers.TaskListHandler,
+            Handlers.UnorderedListHandler,
+            Handlers.OrderedListHandler,
+            Handlers.EndListHandler,
+            Handlers.TableHandler,
+            Handlers.InlineCodeHandler,
+            Handlers.HeaderHandler,
+            Handlers.ItalicHandler,
+            Handlers.QuoteHandler,
+            Handlers.LinkHandler,
+            Handlers.ImageHandler,
+            Handlers.NewLineHandler,
+        ];
+    }
+    //Reset input and other attribute
+    setInput(input) {
+        this.input = input;
+        this.pos = 0;
+        this.listLevelFlag = 0;
+        this.listToken = [];
+    }
+    //Get current character with offset
+    peek(offset = 0) {
+        const i = this.pos + offset;
+        return i < this.input.length ? this.input[i] : null;
+    }
+    //Move cursor by amount
+    next(amount = 1) {
+        this.pos += amount;
+    }
+    //If current cursor startsWith given str
+    startsWith(str) {
+        return this.input.slice(this.pos, this.pos + str.length) === str;
+    }
+    isEndOfFile() {
+        return this.pos >= this.input.length;
+    }
+    getLastToken() {
+        return this.listToken[this.listToken.length - 1];
+    }
+    readUntil(char, isConsumeChar = false) {
+        let result = "";
+        while (this.peek() !== char) {
+            result += this.peek();
+            this.next();
+            if (this.isEndOfFile())
+                break;
+        }
+        if (isConsumeChar)
+            this.next(char.length); //Make cursor skip the char
+        return result;
+    }
+    peekUntil(char) {
+        let result = "";
+        let i = 0;
+        while (true) {
+            const current = this.peek(i++);
+            if (current == null)
+                break;
+            if (current == char)
+                break;
+            result += current;
+        }
+        return result;
+    }
+    peekUntilByOffset(offset) {
+        let result = "";
+        let i = 0;
+        while (i !== offset) {
+            const current = this.peek(i++);
+            if (current == null)
+                break;
+            if (this.isEndOfFile())
+                break;
+            result += current;
+        }
+        return result;
+    }
+    isStartOfLine() {
+        return this.pos === 0 || this.peek(-1) === "\n";
+    }
+    readUntilMatchString(str, isConsume = false) {
+        let result = "";
+        while (!this.isEndOfFile()) {
+            if (this.peekUntilByOffset(str.length) === str) {
+                if (isConsume)
+                    this.next(str.length);
+                break;
+            }
+            result += this.peek();
+            this.next();
+        }
+        return result;
+    }
+    /**
+     * Tokenize the markdown into a list of tokens.
+     * @param isEof - `True` when input is whole markdown, `False` if input is just a part of markdown.
+     * @returns List of tokens
+     */
+    tokenize(isEof = true) {
+        while (!this.isEndOfFile()) {
+            let matched = false;
+            for (const strategy of this.strategies) {
+                if (strategy.match(this)) {
+                    strategy.emit(this);
+                    matched = true;
+                    break;
+                }
+            }
+            if (!matched) {
+                utils.handleTextBlock(this);
+            }
+            this.next();
+        }
+        while (this.listLevelFlag > 0) {
+            utils.handleEndList(this);
+        }
+        if (isEof)
+            this.listToken.push({ type: "EOF" });
+        return this.listToken;
+    }
+}
+exports.default = Lexer;

package/dist/core/parser/handler.d.ts

@@ -0,0 +1,19 @@
+import { ParsingStrategy } from "../../types/parser";
+declare const CodeBlockHandler: ParsingStrategy;
+declare const HeaderHandler: ParsingStrategy;
+declare const BoldHandler: ParsingStrategy;
+declare const ItalicHandler: ParsingStrategy;
+declare const StrikethroughHandler: ParsingStrategy;
+declare const InlineHandler: ParsingStrategy;
+declare const QuoteHandler: ParsingStrategy;
+declare const ListHandler: ParsingStrategy;
+declare const LinkHandler: ParsingStrategy;
+declare const ImageHandler: ParsingStrategy;
+declare const TableHandler: ParsingStrategy;
+declare const HtmlBlockHandler: ParsingStrategy;
+declare const HtmlInlineHandler: ParsingStrategy;
+declare const HorizontalLineHandler: ParsingStrategy;
+declare const FootnoteDefHandler: ParsingStrategy;
+declare const FootnoteRefHandler: ParsingStrategy;
+declare const NewLineHandler: ParsingStrategy;
+export { BoldHandler, CodeBlockHandler, FootnoteDefHandler, FootnoteRefHandler, HeaderHandler, HorizontalLineHandler, HtmlBlockHandler, HtmlInlineHandler, ImageHandler, InlineHandler, ItalicHandler, LinkHandler, ListHandler, NewLineHandler, QuoteHandler, StrikethroughHandler, TableHandler, };