pantsdown 2.2.7 → 2.2.8

package/src/tokenizer.ts

@@ -1,15 +1,17 @@
 import type { Lexer } from "./lexer.ts";
 import { block } from "./rules/block.ts";
 import { inline } from "./rules/inline.ts";
-import { type Links, type Tokens } from "./types.ts";
+import { other } from "./rules/other.ts";
+import { type Links, type Token, type Tokens } from "./types.ts";
 import {
    ALERTS,
-   escape,
+   expandTabs,
    findClosingBracket,
    indentCodeCompensation,
    outputLink,
    rtrim,
    splitCells,
+   trimTrailingBlankLines,
 } from "./utils.ts";
 
 /**
@@ -40,13 +42,14 @@ export class Tokenizer {
       const cap = block.code.exec(src);
       if (!cap) return undefined;
 
-      const text = cap[0].replace(/^ {1,4}/gm, "");
+      const raw = trimTrailingBlankLines(cap[0]);
+      const text = raw.replace(other.codeRemoveIndent, "");
       return {
          type: "code",
-         raw: cap[0],
+         raw,
          codeBlockStyle: "indented",
-         text: rtrim(text, "\n"),
-         sourceMap: this.lexer.getSourceMap(cap[0]),
+         text,
+         sourceMap: this.lexer.getSourceMap(raw),
       };
    }
 
@@ -80,13 +83,14 @@ export class Tokenizer {
          }
       }
 
+      const raw = rtrim(cap[0], "\n");
       return {
          type: "heading",
-         raw: cap[0],
+         raw,
          depth: cap[1]!.length,
          text,
          tokens: this.lexer.inline(text),
-         sourceMap: this.lexer.getSourceMap(cap[0]),
+         sourceMap: this.lexer.getSourceMap(raw),
       };
    }
 
@@ -94,10 +98,11 @@ export class Tokenizer {
       const cap = block.hr.exec(src);
       if (!cap) return undefined;
 
+      const raw = rtrim(cap[0], "\n");
       return {
          type: "hr",
-         raw: cap[0],
-         sourceMap: this.lexer.getSourceMap(cap[0]),
+         raw,
+         sourceMap: this.lexer.getSourceMap(raw),
       };
    }
 
@@ -105,18 +110,95 @@ export class Tokenizer {
       const cap = block.blockquote.exec(src);
       if (!cap) return undefined;
 
-      // precede setext continuation with 4 spaces so it isn't a setext
-      let text = cap[0].replace(/\n {0,3}((?:=+|-+) *)(?=\n|$)/g, "\n    $1");
-      text = rtrim(text.replace(/^ *>[ \t]?/gm, ""), "\n");
-      const top = this.lexer.state.top;
-      this.lexer.state.top = true;
-      const tokens = this.lexer.blockTokens(text, []);
-      this.lexer.state.top = top;
-      this.lexer.line++;
+      let lines = rtrim(cap[0], "\n").split("\n");
+      let raw = "";
+      let text = "";
+      const tokens: Token[] = [];
+      // the accumulated `raw` can end up with garbled *content* at continuation-merge
+      // boundaries (upstream does the same), but its length and newline count always
+      // match the consumed source — which is all the lexer and sourcemaps rely on
+      const startLine = this.lexer.line;
+
+      while (lines.length > 0) {
+         let inBlockquote = false;
+         const currentLines = [];
+
+         let i;
+         for (i = 0; i < lines.length; i++) {
+            // get lines up to a continuation
+            if (other.blockquoteStart.test(lines[i]!)) {
+               currentLines.push(lines[i]!);
+               inBlockquote = true;
+            } else if (!inBlockquote) {
+               currentLines.push(lines[i]!);
+            } else {
+               break;
+            }
+         }
+         lines = lines.slice(i);
+
+         const currentRaw = currentLines.join("\n");
+         const currentText = currentRaw
+            // precede setext continuation with 4 spaces so it isn't a setext
+            .replace(other.blockquoteSetextReplace, "\n    $1")
+            .replace(other.blockquoteSetextReplace2, "");
+         // this round starts right after the lines already accumulated in `raw`
+         this.lexer.line = startLine + (raw ? raw.split("\n").length : 0);
+         raw = raw ? `${raw}\n${currentRaw}` : currentRaw;
+         text = text ? `${text}\n${currentText}` : currentText;
+
+         // parse blockquote lines as top level tokens
+         // merge paragraphs if this is a continuation
+         const top = this.lexer.state.top;
+         this.lexer.state.top = true;
+         this.lexer.blockTokens(currentText, tokens, true);
+         this.lexer.state.top = top;
+
+         // if there is no continuation then we are done
+         if (lines.length === 0) {
+            break;
+         }
+
+         const lastToken = tokens[tokens.length - 1];
+
+         if (lastToken?.type === "code") {
+            // blockquote continuation cannot be preceded by a code block
+            break;
+         } else if (lastToken?.type === "blockquote") {
+            // include continuation in nested blockquote
+            const oldToken = lastToken;
+            const newText = oldToken.raw + "\n" + lines.join("\n");
+            // re-lexing the nested blockquote restarts at its first source line
+            this.lexer.line = startLine + raw.split("\n").length - oldToken.raw.split("\n").length;
+            const newToken = this.blockquote(newText)!;
+            tokens[tokens.length - 1] = newToken;
+
+            raw = raw.substring(0, raw.length - oldToken.raw.length) + newToken.raw;
+            text = text.substring(0, text.length - oldToken.text.length) + newToken.text;
+            break;
+         } else if (lastToken?.type === "list") {
+            // include continuation in nested list
+            const oldToken = lastToken;
+            const newText = oldToken.raw + "\n" + lines.join("\n");
+            // re-lexing the nested list restarts at its first source line
+            this.lexer.line = startLine + raw.split("\n").length - oldToken.raw.split("\n").length;
+            const newToken = this.list(newText)!;
+            tokens[tokens.length - 1] = newToken;
+
+            raw = raw.substring(0, raw.length - lastToken.raw.length) + newToken.raw;
+            text = text.substring(0, text.length - oldToken.raw.length) + newToken.raw;
+            lines = newText.substring(tokens[tokens.length - 1]!.raw.length).split("\n");
+            continue;
+         }
+      }
+
+      // leave the counter on the blockquote's last line; `raw` has no trailing
+      // newline, so the pending line terminator is counted by the next space token
+      this.lexer.line = startLine + raw.split("\n").length - 1;
 
       const blockquoteToken: Tokens["Blockquote"] = {
          type: "blockquote",
-         raw: cap[0],
+         raw,
          tokens,
          text,
       };
@@ -169,13 +251,13 @@ export class Tokenizer {
       bull = isordered ? `\\d{1,9}\\${bull.slice(-1)}` : `\\${bull}`;
 
       // Get next list item
-      const itemRegex = new RegExp(`^( {0,3}${bull})((?:[\t ][^\\n]*)?(?:\\n|$))`);
-      let raw = "";
-      let itemContents = "";
+      const itemRegex = other.listItemRegex(bull);
       let endsWithBlankLine = false;
       // Check if current bullet point can start a new List Item
       while (src) {
          let endEarly = false;
+         let raw = "";
+         let itemContents = "";
          if (!(cap = itemRegex.exec(src))) {
             break;
          }
@@ -188,20 +270,22 @@ export class Tokenizer {
          raw = cap[0];
          src = src.substring(raw.length);
 
-         let line = cap[2]!
-            .split("\n", 1)[0]!
-            .replace(/^\t+/, (t: string) => " ".repeat(3 * t.length));
+         let line = expandTabs(cap[2]!.split("\n", 1)[0]!, cap[1]!.length);
          let nextLine = src.split("\n", 1)[0] ?? "";
 
-         let indent = 0;
-         indent = cap[2]!.search(/[^ ]/); // Find first non-space char
-         indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent
-         itemContents = line.slice(indent);
-         indent += cap[1]!.length;
+         let blankLine = !line.trim();
 
-         let blankLine = false;
+         let indent = 0;
+         if (blankLine) {
+            indent = cap[1]!.length + 1;
+         } else {
+            indent = line.search(other.nonSpaceChar); // Find first non-space char
+            indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent
+            itemContents = line.slice(indent);
+            indent += cap[1]!.length;
+         }
 
-         if (!line && /^ *$/.test(nextLine)) {
+         if (blankLine && other.blankLine.test(nextLine)) {
             // Items begin with at most one blank line
             raw += nextLine + "\n";
             src = src.substring(nextLine.length + 1);
@@ -209,19 +293,18 @@ export class Tokenizer {
          }
 
          if (!endEarly) {
-            const nextBulletRegex = new RegExp(
-               `^ {0,${Math.min(3, indent - 1)}}(?:[*+-]|\\d{1,9}[.)])((?:[ \t][^\\n]*)?(?:\\n|$))`,
-            );
-            const hrRegex = new RegExp(
-               `^ {0,${Math.min(3, indent - 1)}}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$)`,
-            );
-            const fencesBeginRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:\`\`\`|~~~)`);
-            const headingBeginRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}#`);
+            const nextBulletRegex = other.nextBulletRegex(indent);
+            const hrRegex = other.hrRegex(indent);
+            const fencesBeginRegex = other.fencesBeginRegex(indent);
+            const headingBeginRegex = other.headingBeginRegex(indent);
+            const htmlBeginRegex = other.htmlBeginRegex(indent);
+            const blockquoteBeginRegex = other.blockquoteBeginRegex(indent);
 
             // Check if following lines should be included in List Item
             while (src) {
                const rawLine = src.split("\n", 1)[0] ?? "";
                nextLine = rawLine;
+               const nextLineWithoutTabs = nextLine.replace(other.tabCharGlobal, "    ");
 
                // End list item if found code fences
                if (fencesBeginRegex.test(nextLine)) {
@@ -233,19 +316,29 @@ export class Tokenizer {
                   break;
                }
 
+               // End list item if found start of html block
+               if (htmlBeginRegex.test(nextLine)) {
+                  break;
+               }
+
+               // End list item if found start of blockquote
+               if (blockquoteBeginRegex.test(nextLine)) {
+                  break;
+               }
+
                // End list item if found start of new bullet
                if (nextBulletRegex.test(nextLine)) {
                   break;
                }
 
                // Horizontal rule found
-               if (hrRegex.test(src)) {
+               if (hrRegex.test(nextLine)) {
                   break;
                }
 
-               if (nextLine.search(/[^ ]/) >= indent || !nextLine.trim()) {
+               if (nextLineWithoutTabs.search(other.nonSpaceChar) >= indent || !nextLine.trim()) {
                   // Dedent if possible
-                  itemContents += "\n" + nextLine.slice(indent);
+                  itemContents += "\n" + nextLineWithoutTabs.slice(indent);
                } else {
                   // not enough indentation
                   if (blankLine) {
@@ -253,7 +346,7 @@ export class Tokenizer {
                   }
 
                   // paragraph continuation unless last line was a different block level element
-                  if (line.search(/[^ ]/) >= 4) {
+                  if (line.replace(other.tabCharGlobal, "    ").search(other.nonSpaceChar) >= 4) {
                      // indented code block
                      break;
                   }
@@ -270,14 +363,12 @@ export class Tokenizer {
                   itemContents += "\n" + nextLine;
                }
 
-               if (!blankLine && !nextLine.trim()) {
-                  // Check if current line is blank
-                  blankLine = true;
-               }
+               // Check if current line is blank
+               blankLine = !nextLine.trim();
 
                raw += rawLine + "\n";
                src = src.substring(rawLine.length + 1);
-               line = nextLine.slice(indent);
+               line = nextLineWithoutTabs.slice(indent);
             }
          }
 
@@ -285,25 +376,15 @@ export class Tokenizer {
             // If the previous item ended with a blank line, the list is loose
             if (endsWithBlankLine) {
                list.loose = true;
-            } else if (/\n *\n *$/.test(raw)) {
+            } else if (other.doubleBlankLine.test(raw)) {
                endsWithBlankLine = true;
             }
          }
 
-         let istask: RegExpExecArray | null = null;
-         let ischecked: boolean | undefined;
-         // Check for task list items
-         istask = /^\[[ xX]\] /.exec(itemContents);
-         if (istask) {
-            ischecked = istask[0] !== "[ ] ";
-            itemContents = itemContents.replace(/^\[[ xX]\] +/, "");
-         }
-
          list.items.push({
             type: "list_item",
             raw,
-            task: Boolean(istask),
-            checked: ischecked,
+            task: other.listIsTask.test(itemContents),
             loose: false,
             text: itemContents,
             tokens: [],
@@ -313,37 +394,103 @@ export class Tokenizer {
          list.raw += raw;
       }
 
+      const lastItem = list.items[list.items.length - 1];
+      if (!lastItem) {
+         // not a list since there were no items
+         return undefined;
+      }
+
       // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic
-      const lastTrimmed = raw.trimEnd();
+      const lastTrimmed = lastItem.raw.trimEnd();
 
-      if (list.items[list.items.length - 1]!.sourceMap) {
+      if (lastItem.sourceMap) {
          // give back the trimmed trailing newlines; the counter tracks lines, not chars
-         this.lexer.line -= (raw.slice(lastTrimmed.length).match(/\n/g) ?? []).length;
+         this.lexer.line -= (lastItem.raw.slice(lastTrimmed.length).match(/\n/g) ?? []).length;
       }
 
-      list.items[list.items.length - 1]!.raw = lastTrimmed;
-      list.items[list.items.length - 1]!.text = itemContents.trimEnd();
+      lastItem.raw = lastTrimmed;
+      lastItem.text = lastItem.text.trimEnd();
       list.raw = list.raw.trimEnd();
 
       // Item child tokens handled here at end because we needed to have the final item to trim it first
-      for (let i = 0, listItemsLen = list.items.length; i < listItemsLen; i++) {
+      // save/restore top: blockTokens resets it to true on exit, and a nested list
+      // lexed with a stale top=true would advance the sourcemap line counter
+      const top = this.lexer.state.top;
+      for (const item of list.items) {
          this.lexer.state.top = false;
-         list.items[i]!.tokens = this.lexer.blockTokens(list.items[i]!.text, []);
+         item.tokens = this.lexer.blockTokens(item.text, []);
+
+         const itemToken = item.tokens[0];
+         if (item.task && (itemToken?.type === "text" || itemToken?.type === "paragraph")) {
+            // Remove checkbox markdown from item tokens
+            item.text = item.text.replace(other.listReplaceTask, "");
+            itemToken.raw = itemToken.raw.replace(other.listReplaceTask, "");
+            itemToken.text = itemToken.text.replace(other.listReplaceTask, "");
+            for (let i = this.lexer.inlineQueue.length - 1; i >= 0; i--) {
+               if (other.listIsTask.test(this.lexer.inlineQueue[i]!.src)) {
+                  this.lexer.inlineQueue[i]!.src = this.lexer.inlineQueue[i]!.src.replace(
+                     other.listReplaceTask,
+                     "",
+                  );
+                  break;
+               }
+            }
+
+            const taskRaw = other.listTaskCheckbox.exec(item.raw);
+            if (taskRaw) {
+               const checkboxToken: Tokens["Checkbox"] = {
+                  type: "checkbox",
+                  raw: taskRaw[0] + " ",
+                  checked: taskRaw[0] !== "[ ]",
+               };
+               item.checked = checkboxToken.checked;
+               const firstToken = item.tokens[0];
+               if (list.loose) {
+                  if (
+                     firstToken &&
+                     (firstToken.type === "paragraph" || firstToken.type === "text") &&
+                     "tokens" in firstToken
+                  ) {
+                     firstToken.raw = checkboxToken.raw + firstToken.raw;
+                     firstToken.text = checkboxToken.raw + firstToken.text;
+                     firstToken.tokens.unshift(checkboxToken);
+                  } else {
+                     item.tokens.unshift({
+                        type: "paragraph",
+                        raw: checkboxToken.raw,
+                        text: checkboxToken.raw,
+                        tokens: [checkboxToken],
+                        sourceMap: undefined,
+                     });
+                  }
+               } else {
+                  item.tokens.unshift(checkboxToken);
+               }
+            }
+         } else if (item.task) {
+            item.task = false;
+         }
 
          if (!list.loose) {
             // Check if list should be loose
-            const spacers = list.items[i]!.tokens.filter((t) => t.type === "space");
+            const spacers = item.tokens.filter((t) => t.type === "space");
             const hasMultipleLineBreaks =
-               spacers.length > 0 && spacers.some((t) => /\n.*\n/.test(t.raw));
+               spacers.length > 0 && spacers.some((t) => other.anyLine.test(t.raw));
 
             list.loose = hasMultipleLineBreaks;
          }
       }
+      this.lexer.state.top = top;
 
       // Set all items to loose if list is loose
       if (list.loose) {
-         for (let i = 0, listItemsLen = list.items.length; i < listItemsLen; i++) {
-            list.items[i]!.loose = true;
+         for (const item of list.items) {
+            item.loose = true;
+            for (const token of item.tokens) {
+               if (token.type === "text") {
+                  (token as unknown as Tokens["Paragraph"]).type = "paragraph";
+               }
+            }
          }
       }
 
@@ -380,13 +527,14 @@ export class Tokenizer {
       const cap = block.html.exec(src);
       if (!cap) return undefined;
 
+      const raw = trimTrailingBlankLines(cap[0]);
       const token: Tokens["HTML"] = {
          type: "html",
          block: true,
-         raw: cap[0],
+         raw,
          pre: cap[1] === "pre" || cap[1] === "script" || cap[1] === "style",
-         text: cap[0],
-         sourceMap: this.lexer.getSourceMap(cap[0]),
+         text: raw,
+         sourceMap: this.lexer.getSourceMap(raw),
       };
 
       /*
@@ -454,20 +602,21 @@ export class Tokenizer {
       const cap = block.def.exec(src);
       if (!cap) return undefined;
 
-      const tag = cap[1]!.toLowerCase().replace(/\s+/g, " ");
+      const tag = cap[1]!.toLowerCase().replace(other.multipleSpaceGlobal, " ");
       const href = cap[2]
-         ? cap[2].replace(/^<(.*)>$/, "$1").replace(inline.anyPunctuation, "$1")
+         ? cap[2].replace(other.hrefBrackets, "$1").replace(inline.anyPunctuation, "$1")
          : "";
       const title = cap[3]
          ? cap[3].substring(1, cap[3].length - 1).replace(inline.anyPunctuation, "$1")
          : "";
+      const raw = rtrim(cap[0], "\n");
       return {
          type: "def",
          tag,
-         raw: cap[0],
+         raw,
          href,
          title,
-         sourceMap: this.lexer.getSourceMap(cap[0]),
+         sourceMap: this.lexer.getSourceMap(raw),
       };
    }
 
@@ -475,76 +624,62 @@ export class Tokenizer {
       const cap = block.table.exec(src);
       if (!cap?.[2]) return;
 
-      if (!/[:|]/.test(cap[2])) {
+      if (!other.tableDelimiter.test(cap[2])) {
          // delimiter row must have a pipe (|) or colon (:) otherwise it is a setext heading
          return;
       }
 
+      const headers = splitCells(cap[1]!);
+      const aligns = cap[2].replace(other.tableAlignChars, "").split("|");
+      const rows = cap[3]?.trim() ? cap[3].replace(other.tableRowBlankLine, "").split("\n") : [];
+
+      if (headers.length !== aligns.length) return;
+
       const item: Tokens["Table"] = {
          type: "table",
-         raw: cap[0],
-         header: splitCells(cap[1]!).map((c) => ({
-            type: "tablecell",
-            raw: c,
-            text: c,
-            tokens: [],
-         })),
+         raw: rtrim(cap[0], "\n"),
+         header: [],
          align: [],
          rows: [],
-         sourceMap: this.lexer.getSourceMap(cap[0]),
+         sourceMap: this.lexer.getSourceMap(rtrim(cap[0], "\n")),
       };
 
-      const align = cap[2].replace(/^\||\| *$/g, "").split("|") as (string | null)[];
-      const rows = cap[3]?.trim() ? cap[3].replace(/\n[ \t]*$/, "").split("\n") : [];
-
-      if (item.header.length !== align.length) return;
-
-      let l = align.length;
-      let i, j, k, row;
-      for (i = 0; i < l; i++) {
-         const alignStr = align[i];
-         if (alignStr) {
-            if (/^ *-+: *$/.test(alignStr)) {
-               item.align.push("right");
-            } else if (/^ *:-+: *$/.test(alignStr)) {
-               item.align.push("center");
-            } else if (/^ *:-+ *$/.test(alignStr)) {
-               item.align.push("left");
-            } else {
-               item.align.push(null);
-            }
+      for (const align of aligns) {
+         if (other.tableAlignRight.test(align)) {
+            item.align.push("right");
+         } else if (other.tableAlignCenter.test(align)) {
+            item.align.push("center");
+         } else if (other.tableAlignLeft.test(align)) {
+            item.align.push("left");
+         } else {
+            item.align.push(null);
          }
       }
 
-      l = rows.length;
-      for (i = 0; i < l; i++) {
+      for (let i = 0, len = headers.length; i < len; i++) {
+         item.header.push({
+            type: "tablecell",
+            raw: headers[i]!,
+            text: headers[i]!,
+            tokens: this.lexer.inline(headers[i]!),
+            header: true,
+            align: item.align[i]!,
+         });
+      }
+
+      for (const row of rows) {
          item.rows.push(
-            splitCells(rows[i] as unknown as string, item.header.length).map((c) => ({
-               type: "tablecell",
-               raw: c,
-               text: c,
-               tokens: [],
+            splitCells(row, headers.length).map((cell, i) => ({
+               type: "tablecell" as const,
+               raw: cell,
+               text: cell,
+               tokens: this.lexer.inline(cell),
+               header: false,
+               align: item.align[i]!,
             })),
          );
       }
 
-      // parse child tokens inside headers and cells
-
-      // header child tokens
-      l = item.header.length;
-      for (j = 0; j < l; j++) {
-         item.header[j]!.tokens = this.lexer.inline(item.header[j]!.text);
-      }
-
-      // cell child tokens
-      l = item.rows.length;
-      for (j = 0; j < l; j++) {
-         row = item.rows[j]!;
-         for (k = 0; k < row.length; k++) {
-            row[k]!.tokens = this.lexer.inline(row[k]!.text);
-         }
-      }
-
       return item;
    }
 
@@ -552,13 +687,15 @@ export class Tokenizer {
       const cap = block.lheading.exec(src);
       if (!cap) return undefined;
 
+      const text = cap[1]!.trim();
+      const raw = rtrim(cap[0], "\n");
       return {
          type: "heading",
-         raw: cap[0],
+         raw,
          depth: cap[2]!.startsWith("=") ? 1 : 2,
-         text: cap[1]!,
-         tokens: this.lexer.inline(cap[1]!),
-         sourceMap: this.lexer.getSourceMap(cap[0]),
+         text,
+         tokens: this.lexer.inline(text),
+         sourceMap: this.lexer.getSourceMap(raw),
       };
    }
 
@@ -596,7 +733,7 @@ export class Tokenizer {
       return {
          type: "escape",
          raw: cap[0],
-         text: escape(cap[1]!),
+         text: cap[1]!,
       };
    }
 
@@ -604,14 +741,14 @@ export class Tokenizer {
       const cap = inline.tag.exec(src);
       if (!cap) return undefined;
 
-      if (!this.lexer.state.inLink && /^<a /i.test(cap[0])) {
+      if (!this.lexer.state.inLink && other.startATag.test(cap[0])) {
          this.lexer.state.inLink = true;
-      } else if (this.lexer.state.inLink && /^<\/a>/i.test(cap[0])) {
+      } else if (this.lexer.state.inLink && other.endATag.test(cap[0])) {
          this.lexer.state.inLink = false;
       }
-      if (!this.lexer.state.inRawBlock && /^<(pre|code|kbd|script)(\s|>)/i.test(cap[0])) {
+      if (!this.lexer.state.inRawBlock && other.startPreScriptTag.test(cap[0])) {
          this.lexer.state.inRawBlock = true;
-      } else if (this.lexer.state.inRawBlock && /^<\/(pre|code|kbd|script)(\s|>)/i.test(cap[0])) {
+      } else if (this.lexer.state.inRawBlock && other.endPreScriptTag.test(cap[0])) {
          this.lexer.state.inRawBlock = false;
       }
 
@@ -644,6 +781,10 @@ export class Tokenizer {
       } else {
          // find closing parenthesis
          const lastParenIndex = findClosingBracket(cap[2]!, "()");
+         if (lastParenIndex === -2) {
+            // more open parens than closed
+            return;
+         }
          if (lastParenIndex > -1) {
             const start = cap[0].startsWith("!") ? 5 : 4;
             const linkLen = start + cap[1]!.length + lastParenIndex;
@@ -699,11 +840,12 @@ export class Tokenizer {
    ): Tokens["Em"] | Tokens["Strong"] | undefined {
       let match = inline.emStrong.lDelim.exec(src);
       if (!match) return;
+      if (!match[1] && !match[2] && !match[3] && !match[4]) return;
 
       // _ can't be between two alphanumerics. \p{L}\p{N} includes non-english alphabet/numbers as well
-      if (match[3] && /[\p{L}\p{N}]/u.exec(prevChar)) return;
+      if (match[4] && other.unicodeAlphaNumeric.exec(prevChar)) return;
 
-      const nextChar = match[1] || match[2] || "";
+      const nextChar = match[1] || match[3] || "";
 
       if (!nextChar || !prevChar || inline.punctuation.exec(prevChar)) {
          // unicode Regex counts emoji as 1 char; convert to array for proper count (used multiple times below)
@@ -790,13 +932,12 @@ export class Tokenizer {
       const cap = inline.code.exec(src);
       if (!cap) return undefined;
 
-      let text = cap[2]!.replace(/\n/g, " ");
-      const hasNonSpaceChars = /[^ ]/.test(text);
+      let text = cap[2]!.replace(other.newLineCharGlobal, " ");
+      const hasNonSpaceChars = other.nonSpaceChar.test(text);
       const hasSpaceCharsOnBothEnds = text.startsWith(" ") && text.endsWith(" ");
       if (hasNonSpaceChars && hasSpaceCharsOnBothEnds) {
          text = text.substring(1, text.length - 1);
       }
-      text = escape(text, true);
       return {
          type: "codespan",
          raw: cap[0],
@@ -814,16 +955,62 @@ export class Tokenizer {
       };
    }
 
-   del(src: string): Tokens["Del"] | undefined {
-      const cap = inline.del.exec(src);
-      if (!cap) return undefined;
+   del(src: string, maskedSrc: string, prevChar = ""): Tokens["Del"] | undefined {
+      let match = inline.delLDelim.exec(src);
+      if (!match) return;
 
-      return {
-         type: "del",
-         raw: cap[0],
-         text: cap[2]!,
-         tokens: this.lexer.inlineTokens(cap[2]!),
-      };
+      const nextChar = match[1] || "";
+
+      if (!nextChar || !prevChar || inline.punctuation.exec(prevChar)) {
+         // unicode Regex counts emoji as 1 char; spread into array for proper count
+         const lLength = Array.from(match[0]).length - 1;
+         let rDelim,
+            rLength,
+            delimTotal = lLength;
+
+         const endReg = inline.delRDelim;
+         endReg.lastIndex = 0;
+
+         // Clip maskedSrc to same section of string as src
+         maskedSrc = maskedSrc.slice(-1 * src.length + lLength);
+
+         while ((match = endReg.exec(maskedSrc)) !== null) {
+            rDelim = match[1] || match[2] || match[3] || match[4] || match[5] || match[6];
+
+            if (!rDelim) continue;
+
+            rLength = Array.from(rDelim).length;
+
+            if (rLength !== lLength) continue;
+
+            if (match[3] || match[4]) {
+               // found another Left Delim
+               delimTotal += rLength;
+               continue;
+            }
+
+            delimTotal -= rLength;
+
+            if (delimTotal > 0) continue; // Haven't found enough closing delimiters
+
+            // Remove extra characters
+            rLength = Math.min(rLength, rLength + delimTotal);
+            // char length can be >1 for unicode characters
+            const lastCharLength = Array.from(match[0])[0]!.length;
+            const raw = src.slice(0, lLength + match.index + lastCharLength + rLength);
+
+            // Create del token - only single ~ or double ~~ supported
+            const text = raw.slice(lLength, -lLength);
+            return {
+               type: "del",
+               raw,
+               text,
+               tokens: this.lexer.inlineTokens(text),
+            };
+         }
+      }
+
+      return undefined;
    }
 
    autolink(src: string): Tokens["Link"] | undefined {
@@ -832,10 +1019,10 @@ export class Tokenizer {
 
       let text, href;
       if (cap[2] === "@") {
-         text = escape(cap[1]!);
+         text = cap[1]!;
          href = "mailto:" + text;
       } else {
-         text = escape(cap[1]!);
+         text = cap[1]!;
          href = text;
       }
 
@@ -860,7 +1047,7 @@ export class Tokenizer {
       if ((cap = inline.url.exec(src))) {
          let text, href;
          if (cap[2] === "@") {
-            text = escape(cap[0]);
+            text = cap[0];
             href = "mailto:" + text;
          } else {
             // do extended autolink path validation
@@ -869,7 +1056,7 @@ export class Tokenizer {
                prevCapZero = cap[0];
                cap[0] = inline.backpedal.exec(cap[0])![0];
             } while (prevCapZero !== cap[0]);
-            text = escape(cap[0]);
+            text = cap[0];
             if (cap[1] === "www.") {
                href = "http://" + cap[0];
             } else {
@@ -898,16 +1085,11 @@ export class Tokenizer {
       const cap = inline.text.exec(src);
       if (!cap) return undefined;
 
-      let text;
-      if (this.lexer.state.inRawBlock) {
-         text = cap[0];
-      } else {
-         text = escape(cap[0]);
-      }
       return {
          type: "text",
          raw: cap[0],
-         text,
+         text: cap[0],
+         escaped: this.lexer.state.inRawBlock,
       };
    }