pantsdown 2.3.3 → 2.3.4

package/README.md

@@ -200,4 +200,4 @@ console.log(html, javascript);
 Pantsdown is based on [Marked](https://github.com/markedjs/marked). Without their hard work,
 Pantsdown would not exist.
 
-Last synced with Marked [v18.0.7](https://github.com/markedjs/marked/releases/tag/v18.0.7).
+Last synced with Marked [v18.0.14](https://github.com/markedjs/marked/releases/tag/v18.0.14).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pantsdown",
-  "version": "2.3.3",
+  "version": "2.3.4",
   "description": "Markdown to 'GitHub HTML' parser",
   "license": "MIT",
   "author": "wallpants",
@@ -27,7 +27,6 @@
     }
   },
   "scripts": {
-    "commit": "cz",
     "format": "oxfmt .",
     "typecheck": "tsc",
     "lint": "oxlint .",
@@ -37,40 +36,22 @@
   },
   "dependencies": {
     "github-slugger": "^2.0.0",
-    "highlight.js": "^11.11.1",
-    "katex": "^0.18.1"
+    "highlight.js": "^11.12.0",
+    "katex": "^0.18.9"
   },
   "devDependencies": {
-    "@commitlint/cli": "^21.2.1",
-    "@commitlint/config-conventional": "^21.2.0",
-    "@commitlint/cz-commitlint": "^21.2.0",
-    "@happy-dom/global-registrator": "^20.11.1",
-    "@types/bun": "^1.3.14",
+    "@commitlint/cli": "^21.2.3",
+    "@commitlint/config-conventional": "^21.2.3",
+    "@happy-dom/global-registrator": "^20.14.5",
+    "@types/bun": "^1.4.2",
     "@types/katex": "^0.16.8",
-    "commitizen": "^4.3.2",
     "husky": "^9.1.7",
-    "inquirer": "^12.11.1",
-    "oxfmt": "^0.62.0",
-    "oxlint": "^1.77.0",
-    "oxlint-tsgolint": "^7.0.2001",
-    "semantic-release": "^25.0.8",
+    "oxfmt": "^0.71.0",
+    "oxlint": "^1.86.0",
+    "oxlint-tsgolint": "^7.0.2003",
+    "semantic-release": "^25.0.9",
     "typescript": "7.0.2"
   },
-  "commitlint": {
-    "extends": [
-      "@commitlint/config-conventional"
-    ],
-    "rules": {
-      "body-max-line-length": [
-        0
-      ]
-    }
-  },
-  "config": {
-    "commitizen": {
-      "path": "@commitlint/cz-commitlint"
-    }
-  },
   "release": {
     "branches": [
       "main"

package/src/lexer.ts

@@ -2,6 +2,7 @@ import { inline } from "./rules/inline.ts";
 import { other } from "./rules/other.ts";
 import { Tokenizer } from "./tokenizer.ts";
 import { type Links, type SourceMap, type Token, type Tokens } from "./types.ts";
+import { normalizeLabel } from "./utils.ts";
 
 export class Lexer {
    private tokenizer: Tokenizer;
@@ -14,6 +15,8 @@ export class Lexer {
    state = {
       inLink: false,
       inRawBlock: false,
+      /** a link was produced in the inline run currently being scanned */
+      linkEmitted: false,
       top: true,
    };
 
@@ -35,6 +38,7 @@ export class Lexer {
       this.state = {
          inLink: false,
          inRawBlock: false,
+         linkEmitted: false,
          top: true,
       };
 
@@ -215,6 +219,9 @@ export class Lexer {
             if (lastParagraphClipped && lastToken?.type === "paragraph") {
                lastToken.raw += (lastToken.raw.endsWith("\n") ? "" : "\n") + token.raw;
                lastToken.text += "\n" + token.text;
+               if (lastToken.sourceMap && token.sourceMap) {
+                  lastToken.sourceMap[1] = token.sourceMap[1];
+               }
                this.inlineQueue.pop();
                const lastInline = this.inlineQueue[this.inlineQueue.length - 1];
                if (lastInline) lastInline.src = lastToken.text;
@@ -256,6 +263,42 @@ export class Lexer {
       return tokens;
    }
 
+   /**
+    * Does this link text hold a link already? An image does not count: an image
+    * may hold a link, a link may not.
+    */
+   private linkInText(text: string): boolean {
+      if (!text.includes("[")) {
+         return false;
+      }
+
+      for (const match of text.matchAll(inline.blockSkip)) {
+         // blockSkip also matches code spans and html, and the `!` of an image is
+         // left out of the match, so read the character before it.
+         if (inline.link.test(match[0]) && text.charAt(match.index - 1) !== "!") {
+            return true;
+         }
+      }
+
+      for (const match of text.matchAll(inline.reflinkSearch)) {
+         const match0 = match[0];
+         const refStart = match0.lastIndexOf("[");
+         if (
+            match0.startsWith("!") ||
+            !Object.hasOwn(this.links, normalizeLabel(match0.slice(refStart + 1, -1)))
+         ) {
+            continue;
+         }
+         // a candidate holding a link is not a link either, so it does not count
+         if (refStart > 1 && this.linkInText(match0.slice(1, refStart - 1))) {
+            continue;
+         }
+         return true;
+      }
+
+      return false;
+   }
+
    /**
     * Lexing/Compiling
     */
@@ -267,16 +310,38 @@ export class Lexer {
       let keepPrevChar, prevChar;
 
       // Mask out reflinks
-      const links = Object.keys(this.links);
-      if (links.length > 0) {
-         maskedSrc = maskedSrc.replace(inline.reflinkSearch, (match0) =>
-            links.includes(match0.slice(match0.lastIndexOf("[") + 1, -1))
-               ? "[" + "a".repeat(match0.length - 2) + "]"
-               : match0,
-         );
+      if (src.includes("[")) {
+         const maskReflink = (match0: string): string => {
+            const refStart = match0.lastIndexOf("[");
+            if (!Object.hasOwn(this.links, normalizeLabel(match0.slice(refStart + 1, -1)))) {
+               return match0;
+            }
+            // CommonMark: "Links may not contain other links, at any level of
+            // nesting." A candidate whose text already holds one never becomes a
+            // link, so flattening the whole span would hide the emphasis that
+            // does still apply inside it. Mask the links it holds instead.
+            // Images are exempt: their text is flattened into an alt attribute.
+            if (refStart > 1 && !match0.startsWith("!")) {
+               const text = match0.slice(1, refStart - 1);
+               if (this.linkInText(text)) {
+                  return (
+                     "[" +
+                     text.replace(inline.reflinkSearch, maskReflink) +
+                     "][" +
+                     "a".repeat(match0.length - refStart - 2) +
+                     "]"
+                  );
+               }
+            }
+            return "[" + "a".repeat(match0.length - 2) + "]";
+         };
+         maskedSrc = maskedSrc.replace(inline.reflinkSearch, maskReflink);
       }
-      // Mask out escaped characters
-      maskedSrc = maskedSrc.replace(inline.anyPunctuation, "++");
+      // Mask out escaped characters.
+      // Every mask must keep the length it replaces: emStrong and del line
+      // maskedSrc up with src by slicing from the end. `anyPunctuation` matches
+      // unicode punctuation, so an escaped astral character is 3 code units.
+      maskedSrc = maskedSrc.replace(inline.anyPunctuation, (match0) => "+".repeat(match0.length));
 
       // Mask out other blocks
       maskedSrc = maskedSrc.replace(

package/src/renderer.ts

@@ -60,7 +60,8 @@ export class Renderer {
       }
 
       const language = lang && hljs.getLanguage(lang) ? lang : "plaintext";
-      code = hljs.highlight(code + "\n", { language }).value;
+      // An empty code block has no content, so it must not gain a newline.
+      code = hljs.highlight(code ? code + "\n" : "", { language }).value;
       code = `<code class="hljs language-${escape(language)}">${code}</code>`;
 
       const result = `<pre style="position: relative;">` + code + `</pre>`;
@@ -234,15 +235,17 @@ export class Renderer {
       return `<del>${this.parser.parseInline(tokens)}</del>`;
    }
 
-   link({ href, title, tokens }: Tokens["Link"]): string {
-      const text = this.parser.parseInline(tokens);
+   link({ href, title, text, tokens, autolink }: Tokens["Link"]): string {
+      // References are not resolved inside an autolink, so every `&` there is
+      // literal. Elsewhere only an `&` that cannot start one needs escaping.
+      const parsedText = autolink ? escape(text, true) : this.parser.parseInline(tokens);
       const cleanHref = cleanUrl(href);
       if (cleanHref === null) {
-         return text;
+         return parsedText;
       }
-      const attrs: HTMLAttrs = [["href", cleanHref]];
+      const attrs: HTMLAttrs = [["href", escape(cleanHref, autolink)]];
       if (title) attrs.push(["title", escape(title)]);
-      return injectHtmlAttributes(`<a>${text}</a>`, attrs);
+      return injectHtmlAttributes(`<a>${parsedText}</a>`, attrs);
    }
 
    image({ href, title, tokens }: Tokens["Image"]): string {
@@ -252,7 +255,7 @@ export class Renderer {
          return escape(text);
       }
       const attrs: HTMLAttrs = [
-         ["src", fixLocalImageHref(cleanHref, this.pantsdown.config)],
+         ["src", escape(fixLocalImageHref(cleanHref, this.pantsdown.config))],
          ["alt", escape(text)],
       ];
       if (title) attrs.push(["title", escape(title)]);

package/src/rules/block.ts

@@ -21,7 +21,7 @@ type BlockRuleNames =
 
 export const label = /(?!\s*\])(?:\\[\s\S]|[^\[\]\\])+/;
 
-const tag =
+export const tag =
    "address|article|aside|base|basefont|blockquote|body|caption" +
    "|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption" +
    "|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe" +
@@ -60,8 +60,8 @@ const block_html = edit(
       "|<![A-Z][\\s\\S]*?(?:>[^\\n]*\\n*|$)" + // (4)
       "|<!\\[CDATA\\[[\\s\\S]*?(?:\\]\\]>[^\\n]*\\n*|$)" + // (5)
       "|</?(tag)(?: +|\\n|/?>)[\\s\\S]*?(?:(?:\\n[ \\t]*)+\\n|$)" + // (6)
-      "|<(?!script|pre|style|textarea)([a-z][\\w-]*)(?:attribute)*? */?>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ \\t]*)+\\n|$)" + // (7) open tag
-      "|</(?!script|pre|style|textarea)[a-z][\\w-]*\\s*>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ \\t]*)+\\n|$)" + // (7) closing tag
+      "|<(?!script|pre|style|textarea)([a-z][a-z0-9-]*)(?:attribute)*? */?>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ \\t]*)+\\n|$)" + // (7) open tag
+      "|</(?!script|pre|style|textarea)[a-z][a-z0-9-]*\\s*>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ \\t]*)+\\n|$)" + // (7) closing tag
       ")",
    "i",
 )
@@ -72,13 +72,14 @@ const block_html = edit(
 
 // upstream's lheadingGfm variant (we are GFM-only; the commonmark variant drops |table)
 const block_lheading = edit(
-   /^(?!bull |blockCode|fences|blockquote|heading|html|table)((?:.|\n(?!\s*?\n|bull |blockCode|fences|blockquote|heading|html|table))+?)\n {0,3}(=+|-+) *(?:\n+|$)/,
+   /^(?!bull |blockCode|fences|blockquote|heading|html|table)((?:.|\n(?!\s*?\n|bull |fences|blockquote|heading|hr|html|table))+?)\n {0,3}(=+|-+) *(?:\n+|$)/,
 )
    .replace(/bull/g, block_bullet) // lists can interrupt
-   .replace(/blockCode/g, /(?: {4}| {0,3}\t)/) // indented code blocks can interrupt
+   .replace(/blockCode/g, /(?: {4}| {0,3}\t)/) // indented code can start a block but cannot interrupt a paragraph
    .replace(/fences/g, / {0,3}(?:`{3,}|~{3,})/) // fenced code blocks can interrupt
    .replace(/blockquote/g, / {0,3}>/) // blockquote can interrupt
    .replace(/heading/g, / {0,3}#{1,6}(?:\s|$)/) // ATX heading can interrupt
+   .replace(/hr/g, / {0,3}(?:(?:-[\t ]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})(?:\n+|$)/) // thematic break can interrupt
    .replace(/html/g, / {0,3}<[^\n>]+>\n/) // block html can interrupt
    .replace(/table/g, / {0,3}\|?(?:[:\- ]*\|)+[\:\- ]*\n/) // table can interrupt
    .getRegex();
@@ -94,7 +95,7 @@ const block_table = edit(
    .replace("heading", " {0,3}#{1,6}(?:\\s|$)")
    .replace("blockquote", " {0,3}>")
    .replace("code", "(?: {4}| {0,3}\\t)[^\\n]")
-   .replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~~~)[^\\n]*\\n")
+   .replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*(?:\\n|$))|~~~)[^\\n]*(?:\\n|$)")
    .replace("list", " {0,3}(?:[*+-]|1[.)])[ \\t]") // any bullet ends the table rows
    .replace("html", "</?(?:tag)(?: +|\\n|/?>)|<(?:script|pre|style|textarea|!--)")
    .replace("tag", tag) // tables can be interrupted by type (6) html blocks
@@ -107,7 +108,7 @@ const createParagraph = (listInterrupt: string) =>
       .replace("|lheading", "") // setext headings don't interrupt commonmark paragraphs
       .replace("table", block_table) // interrupt paragraphs with table
       .replace("blockquote", " {0,3}>")
-      .replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~~~)[^\\n]*\\n")
+      .replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*(?:\\n|$))|~~~)[^\\n]*(?:\\n|$)")
       .replace("list", listInterrupt)
       .replace("html", "</?(?:tag)(?: +|\\n|/?>)|<(?:script|pre|style|textarea|!--)")
       .replace("tag", tag) // pars can be interrupted by type (6) html blocks

package/src/rules/inline.ts

@@ -39,12 +39,25 @@ const href = /<(?:\\.|[^\n<>\\])+>|[^ \t\n\x00-\x1f]+|(?=\))/;
 const scheme = /[a-zA-Z][a-zA-Z0-9+.-]{1,31}/;
 const comment = edit(blockComment).replace("(?:-->|$)", "-->").getRegex();
 const attribute = /\s+[a-zA-Z:_][\w.:-]*(?:\s*=\s*"[^"]*"|\s*=\s*'[^']*'|\s*=\s*[^\s"'=<>`]+)?/;
+// One matched pair of brackets, holding no bracket of its own.
+const labelBrackets = /\[(?:\\[\s\S]|[^\[\]\\])*\]/;
+// CommonMark lets the brackets in a link label nest to any depth, which a regex
+// cannot follow, so it stops at a fixed one. Two levels is what the spec suite
+// asks for: a third and a fourth flip no further example.
 // codespan branches carry the #3918 ReDoS fix (`+(?!`) head, ``+(?=\]) tail)
-const label =
-   /(?:\[(?:\\[\s\S]|[^\[\]\\])*\]|\\[\s\S]|`+(?!`)[^`]*?`+(?!`)|``+(?=\])|[^\[\]\\`])*?/;
+const label = edit(
+   /(?:\[(?:brackets|\\[\s\S]|[^\[\]\\])*\]|\\[\s\S]|`+(?!`)[^`]*?`+(?!`)|``+(?=\])|[^\[\]\\`])*?/,
+)
+   .replace("brackets", labelBrackets)
+   .getRegex();
 const email =
    /[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+(@)[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+(?![-_])/;
-const extended_email = /[A-Za-z0-9._+-]+(@)[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![-_])/;
+const extended_email = /[A-Za-z0-9._+-]+(@)[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![\w-])/;
+// GFM protocol autolinks (`mailto:`/`xmpp:`); the email here has no `(@)` group,
+// so the url tokenizer treats a match as a plain url (href = text)
+const extended_email_protocol = edit(/(?:mailto:email|xmpp:email(?:\/[A-Za-z0-9@.]+)?)/)
+   .replace(/email/g, /[A-Za-z0-9._+-]+@[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![\w-])/)
+   .getRegex();
 
 const inline_punctuation = edit(/^((?![*_])punctSpace)/, "u")
    .replace(/punctSpace/g, _punctuationOrSpace)
@@ -106,8 +119,8 @@ const inline_autolink = edit(/^<(scheme:[^\s\x00-\x1f<>]*|email)>/)
 
 const inline_tag = edit(
    "^comment" +
-      "|^</[a-zA-Z][\\w:-]*\\s*>" + // self-closing tag
-      "|^<[a-zA-Z][\\w-]*(?:attribute)*?\\s*/?>" + // open tag
+      "|^</[a-zA-Z][a-zA-Z0-9-]*\\s*>" + // self-closing tag
+      "|^<[a-zA-Z][a-zA-Z0-9-]*(?:attribute)*?\\s*/?>" + // open tag
       "|^<\\?[\\s\\S]*?\\?>" + // processing instruction, e.g. <?php ?>
       "|^<![a-zA-Z]+\\s[\\s\\S]*?>" + // declaration, e.g. <!DOCTYPE html>
       "|^<!\\[CDATA\\[[\\s\\S]*?\\]\\]>", // CDATA section
@@ -133,9 +146,40 @@ const inline_nolink = edit(/^!?\[(ref)\](?:\[\])?/)
    .replace("ref", blockLabel)
    .getRegex();
 
+// `reflink` and `nolink` are anchored, so the tokenizer tries each of them at a
+// single position. `reflinkSearch` drops the anchors and runs with the global
+// flag, which makes every '[' in the source a start position. A label crosses a
+// bracket only by escaping it, and nothing caps how often it does, so a
+// candidate that can never match still scans to the end of the source and the
+// whole pass costs O(n^2).
+//
+// The labels below bound that scan. `boundedInlineLabel` limits how many
+// escapes, code spans and nested brackets a link text may hold, leaving runs of
+// ordinary characters unbounded so link text of any length is still found.
+// `boundedBlockLabel` limits the label itself to 999 items, which is every
+// label CommonMark allows: "A link label can have at most 999 characters
+// inside the square brackets."
+const boundedBlockLabel = /(?!\s*\])(?:\\[\s\S]|[^\[\]\\]){1,999}/;
+const boundedInlineLabel = edit(
+   /(?:[^\[\]\\`]*(?:\[(?:brackets|\\[\s\S]|[^\[\]\\])*\]|\\[\s\S]|`+(?!`)[^`]*?`+(?!`)|``+(?=\]))){0,999}?[^\[\]\\`]*?/,
+)
+   .replace("brackets", labelBrackets)
+   .getRegex();
+
 const inline_reflinkSearch = edit("reflink|nolink(?!\\()", "g")
-   .replace("reflink", inline_reflink)
-   .replace("nolink", inline_nolink)
+   .replace(
+      "reflink",
+      edit(/^!?\[(label)\]\[(ref)\]/)
+         .replace("label", boundedInlineLabel)
+         .replace("ref", boundedBlockLabel)
+         .getRegex(),
+   )
+   .replace(
+      "nolink",
+      edit(/^!?\[(ref)\](?:\[\])?/)
+         .replace("ref", boundedBlockLabel)
+         .getRegex(),
+   )
    .getRegex();
 
 const inline_escape = /^\\([!"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])/;
@@ -165,9 +209,10 @@ const inline_delRDelim = edit(delRDelimCore, "gu")
    .getRegex();
 
 const inline_text = edit(
-   /^(`+|~+|[^`~])(?:(?=[`~])|(?= {2,}\n)|(?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)|[\s\S]*?(?:(?=[\\<!\[`*~_$]|\b_|protocol:\/\/|www\.|$)|[^ ](?= {2,}\n)|[^a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-](?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)))/,
+   /^(?:[^a-zA-Z0-9](?=emailProtocol)|(`+|~+|[^`~])(?:(?=[`~])|(?= {2,}\n)|(?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)|[\s\S]*?(?:(?=[\\<!\[`*~_$]|\b_|protocol:\/\/|www\.|$)|[^ ](?= {2,}\n)|[^a-zA-Z0-9](?=emailProtocol)|[^a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-](?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@))))/,
 )
    .replace("protocol", /[hH][tT][tT][pP][sS]?|[fF][tT][pP]/)
+   .replace(/emailProtocol/g, /(?:mailto|xmpp):/)
    .getRegex();
 
 // DEVIATION from upstream `(?:[a-zA-Z0-9\-]+\.?)+`: the nested quantifier
@@ -176,8 +221,9 @@ const inline_text = edit(
 // domain form matches the exact same language (fuzz-verified over 200k
 // samples) — preserve it when porting upstream changes to this rule.
 const inline_url = edit(
-   /^((?:protocol):\/\/|www\.)(?:[a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.?[^\s<]*|^email/,
+   /^emailProtocol|^((?:protocol):\/\/|www\.)(?:[a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.?[^\s<]*|^email/,
 )
+   .replace("emailProtocol", extended_email_protocol)
    .replace("protocol", /[fF][tT][pP]|[hH][tT][tT][pP][sS]?/)
    .replace("email", extended_email)
    .getRegex();
@@ -196,7 +242,7 @@ export const inline: Omit<Record<InlineRuleNames, RegExp>, "emStrong"> & {
    anyPunctuation: inline_anyPunctuation,
    emStrong: inline_emStrong,
    code: /^(`+)([^`]|[^`][\s\S]*?[^`])\1(?!`)/,
-   br: /^( {2,}|\\)\n(?!\s*$)/,
+   br: /^( {2,}|\\)\n(?!\s*$)[ \t]*/,
    delLDelim: inline_delLDelim,
    delRDelim: inline_delRDelim,
    text: inline_text,

package/src/rules/other.ts

@@ -1,9 +1,11 @@
+import { tag } from "./block.ts";
+
 /**
  * Regexes that don't belong to the block or inline grammars.
- * Names and values mirror marked's `other` rules object (src/rules.ts,
- * currently marked v18.0.7 — see "Last synced" in the root README) so
- * future syncs stay diffable.
+ * Names and values mirror marked's `other` rules object (src/rules.ts, as of
+ * the "Last synced" version in the root README) so future syncs stay diffable.
  */
+
 function cachedIndentRegex(createRegex: (indent: number) => RegExp) {
    const cache: RegExp[] = [];
    return (indent: number) => {
@@ -18,11 +20,13 @@ function cachedIndentRegex(createRegex: (indent: number) => RegExp) {
 }
 
 export const other = {
-   codeRemoveIndent: /^(?: {1,4}| {0,3}\t)/gm,
+   codeRemoveIndent: /^(?: {0,3}\t| {1,4})/gm,
    tabCharGlobal: /\t/g,
+   leadingSpaceTab: /^[ \t]+/,
    outputLinkReplace: /\\([\[\]])/g,
    indentCodeCompensation: /^(\s+)(?:```)/,
    beginningSpace: /^\s+/,
+   endingSpaceTabChar: /[ \t]$/,
    nonSpaceChar: /[^ ]/,
    newLineCharGlobal: /\n/g,
    multipleSpaceGlobal: /\s+/g,
@@ -42,11 +46,17 @@ export const other = {
    tableAlignRight: /^ *-+: *$/,
    tableAlignCenter: /^ *:-+: *$/,
    tableAlignLeft: /^ *:-+ *$/,
+   // Pantsdown: setext heading continuation lines (see Tokenizer.lheading)
+   continuationIndent: /\n[ \t]+/g,
+   // Pantsdown: html block sourcemaps (see Tokenizer.html)
+   htmlOpenTagName: /^ {0,3}<([a-zA-Z][a-zA-Z0-9-]*)/,
+   htmlEndingCloseTagName: /<\/([a-zA-Z][a-zA-Z0-9-]*)\s*>$/,
    startATag: /^<a /i,
    endATag: /^<\/a>/i,
    startPreScriptTag: /^<(pre|code|kbd|script)(\s|>)/i,
    endPreScriptTag: /^<\/(pre|code|kbd|script)(\s|>)/i,
    unicodeAlphaNumeric: /[\p{L}\p{N}]/u,
+   numericCharacterReference: /&#(?:(\d{1,7})|[Xx]([A-Fa-f0-9]{1,6}));/g,
    escapeTest: /[&<>"']/,
    escapeReplace: /[&<>"']/g,
    escapeTestNoEncode: /[<>"']|&(?!(#\d{1,7}|#[Xx][a-fA-F0-9]{1,6}|\w+);)/,
@@ -66,14 +76,20 @@ export const other = {
    ),
    hrRegex: cachedIndentRegex(
       (indent: number) =>
-         new RegExp(`^ {0,${indent}}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$)`),
+         new RegExp(`^ {0,${indent}}((?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\\*[ \t]*){3,})(?:\\n+|$)`),
    ),
    fencesBeginRegex: cachedIndentRegex(
       (indent: number) => new RegExp(`^ {0,${indent}}(?:\`\`\`|~~~)`),
    ),
    headingBeginRegex: cachedIndentRegex((indent: number) => new RegExp(`^ {0,${indent}}#`)),
+   // a list item ends where a paragraph would be interrupted, so this mirrors the
+   // html start conditions in the paragraph rule; type 7 is excluded there
    htmlBeginRegex: cachedIndentRegex(
-      (indent: number) => new RegExp(`^ {0,${indent}}<(?:[a-z].*>|!--)`, "i"),
+      (indent: number) =>
+         new RegExp(
+            `^ {0,${indent}}(?:</?(?:${tag})(?: +|$|/?>)|<(?:script|pre|style|textarea|!--))`,
+            "i",
+         ),
    ),
    blockquoteBeginRegex: cachedIndentRegex((indent: number) => new RegExp(`^ {0,${indent}}>`)),
 };

package/src/tokenizer.ts

@@ -5,9 +5,12 @@ import { other } from "./rules/other.ts";
 import { type Links, type Token, type Tokens } from "./types.ts";
 import {
    ALERTS,
+   decodeNumericCharacterReferences,
    expandTabs,
    findClosingBracket,
    indentCodeCompensation,
+   isLabelEndInsideToken,
+   normalizeLabel,
    outputLink,
    rtrim,
    splitCells,
@@ -19,7 +22,7 @@ import {
  */
 export class Tokenizer {
    private lexer: Lexer;
-   pendingHtmlClose: [tag: string, index: number][] = [];
+   pendingHtmlClose: { tag: string; token: Tokens["HTML"] }[] = [];
 
    constructor(lexer: Lexer) {
       this.lexer = lexer;
@@ -77,8 +80,8 @@ export class Tokenizer {
       // remove trailing #s
       if (text.endsWith("#")) {
          const trimmed = rtrim(text, "#");
-         if (!trimmed || trimmed.endsWith(" ")) {
-            // CommonMark requires space before trailing #s
+         if (!trimmed || other.endingSpaceTabChar.test(trimmed)) {
+            // CommonMark requires a space or tab before trailing #s
             text = trimmed.trim();
          }
       }
@@ -167,13 +170,27 @@ export class Tokenizer {
          } else if (lastToken?.type === "blockquote") {
             // include continuation in nested blockquote
             const oldToken = lastToken;
-            const newText = oldToken.raw + "\n" + lines.join("\n");
+            // The continuation lines belong to the same nesting frame as the
+            // nested blockquote, which already had one '>' marker stripped, so
+            // strip one marker from them too. Otherwise a restated marker after a
+            // lazy line is re-parsed as a spurious deeper blockquote.
+            const continuation = lines.join("\n");
+            const newText =
+               oldToken.raw + "\n" + continuation.replace(other.blockquoteSetextReplace2, "");
             // re-lexing the nested blockquote restarts at its first source line
             this.lexer.line = startLine + raw.split("\n").length - oldToken.raw.split("\n").length;
             const newToken = this.blockquote(newText)!;
             tokens[tokens.length - 1] = newToken;
 
-            raw = raw.substring(0, raw.length - oldToken.raw.length) + newToken.raw;
+            // Only include continuation lines the nested blockquote actually
+            // consumed. Unconsumed trailing lines must stay out of `raw` so the
+            // lexer can still tokenize them (paragraph, next blockquote, etc.).
+            const leftover = newText.substring(newToken.raw.length).replace(/^\n/, "");
+            const leftoverCount = leftover ? leftover.split("\n").length : 0;
+            const consumed = leftoverCount ? lines.slice(0, -leftoverCount) : lines;
+            if (consumed.length > 0) {
+               raw = `${raw}\n${consumed.join("\n")}`;
+            }
             text = text.substring(0, text.length - oldToken.text.length) + newToken.text;
             break;
          } else if (lastToken?.type === "list") {
@@ -270,19 +287,23 @@ export class Tokenizer {
          raw = cap[0];
          src = src.substring(raw.length);
 
-         let line = expandTabs(cap[2]!.split("\n", 1)[0]!, cap[1]!.length);
+         const firstLine = cap[2]!.split("\n", 1)[0]!;
+         const bulletIndent = cap[1]!.length;
+         let line = firstLine.replace(other.leadingSpaceTab, (whitespace) =>
+            expandTabs(whitespace, bulletIndent),
+         );
          let nextLine = src.split("\n", 1)[0] ?? "";
 
          let blankLine = !line.trim();
 
          let indent = 0;
          if (blankLine) {
-            indent = cap[1]!.length + 1;
+            indent = bulletIndent + 1;
          } else {
             indent = line.search(other.nonSpaceChar); // Find first non-space char
             indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent
             itemContents = line.slice(indent);
-            indent += cap[1]!.length;
+            indent += bulletIndent;
          }
 
          if (blankLine && other.blankLine.test(nextLine)) {
@@ -304,7 +325,9 @@ export class Tokenizer {
             while (src) {
                const rawLine = src.split("\n", 1)[0] ?? "";
                nextLine = rawLine;
-               const nextLineWithoutTabs = nextLine.replace(other.tabCharGlobal, "    ");
+               const nextLineWithoutTabs = nextLine.replace(other.leadingSpaceTab, (whitespace) =>
+                  whitespace.replace(other.tabCharGlobal, "    "),
+               );
 
                // End list item if found code fences
                if (fencesBeginRegex.test(nextLine)) {
@@ -416,25 +439,37 @@ export class Tokenizer {
       // save/restore top: blockTokens resets it to true on exit, and a nested list
       // lexed with a stale top=true would advance the sourcemap line counter
       const top = this.lexer.state.top;
+      // First pass: tokenize items and finalize list.loose from spacers before placing checkboxes
       for (const item of list.items) {
          this.lexer.state.top = false;
          item.tokens = this.lexer.blockTokens(item.text, []);
 
+         if (!list.loose) {
+            // Check if list should be loose
+            const spacers = item.tokens.filter((t) => t.type === "space");
+            const hasMultipleLineBreaks =
+               spacers.length > 0 && spacers.some((t) => other.anyLine.test(t.raw));
+
+            list.loose = hasMultipleLineBreaks;
+         }
+      }
+      this.lexer.state.top = top;
+
+      // Second pass: place task checkboxes using the final list.loose
+      for (const item of list.items) {
          const itemToken = item.tokens[0];
          if (item.task && (itemToken?.type === "text" || itemToken?.type === "paragraph")) {
             // Remove checkbox markdown from item tokens
             item.text = item.text.replace(other.listReplaceTask, "");
             itemToken.raw = itemToken.raw.replace(other.listReplaceTask, "");
             itemToken.text = itemToken.text.replace(other.listReplaceTask, "");
-            for (let i = this.lexer.inlineQueue.length - 1; i >= 0; i--) {
-               if (other.listIsTask.test(this.lexer.inlineQueue[i]!.src)) {
-                  this.lexer.inlineQueue[i]!.src = this.lexer.inlineQueue[i]!.src.replace(
-                     other.listReplaceTask,
-                     "",
-                  );
-                  break;
-               }
-            }
+            // DEVIATION: upstream strips the last queued src that looks like a task,
+            // which can be a later paragraph of the same item (`- [ ] a\n\n  [ ] b`).
+            // The item token's own queue entry shares its tokens array.
+            const queued = this.lexer.inlineQueue.findLast(
+               (entry) => entry.tokens === itemToken.tokens,
+            );
+            if (queued) queued.src = queued.src.replace(other.listReplaceTask, "");
 
             const taskRaw = other.listTaskCheckbox.exec(item.raw);
             if (taskRaw) {
@@ -470,17 +505,7 @@ export class Tokenizer {
          } else if (item.task) {
             item.task = false;
          }
-
-         if (!list.loose) {
-            // Check if list should be loose
-            const spacers = item.tokens.filter((t) => t.type === "space");
-            const hasMultipleLineBreaks =
-               spacers.length > 0 && spacers.some((t) => other.anyLine.test(t.raw));
-
-            list.loose = hasMultipleLineBreaks;
-         }
       }
-      this.lexer.state.top = top;
 
       // Set all items to loose if list is loose
       if (list.loose) {
@@ -573,26 +598,26 @@ export class Tokenizer {
        * and update their sourceMap once they're closed.
        */
 
-      const capEndsWith = (str?: string) => str && cap[0].trimEnd().endsWith(str);
-
-      const tag = inline.tag.exec(src);
-      const isHtmlClosed = capEndsWith(tag?.[0].slice(1));
-
-      if (tag?.[0] && !isHtmlClosed) {
-         // index where the token we just created will be inserted
-         const tokenIdx = this.lexer.tokens.length;
-         // first in last out
-         this.pendingHtmlClose.unshift([tag[0], tokenIdx]);
-      } else if (this.pendingHtmlClose.length) {
-         for (const [pendingTag, index] of this.pendingHtmlClose) {
-            if (capEndsWith(pendingTag.slice(1))) {
-               const updateToken = this.lexer.tokens[index] as Tokens["HTML"];
-               if (updateToken.sourceMap?.[1] && token.sourceMap?.[1]) {
-                  updateToken.sourceMap[1] = token.sourceMap[1];
-               }
-               this.pendingHtmlClose.shift();
-            }
+      const openTag = other.htmlOpenTagName.exec(raw)?.[1]?.toLowerCase();
+      const closeTag = other.htmlEndingCloseTagName.exec(raw.trimEnd())?.[1]?.toLowerCase();
+
+      // a token ending in the closing tag of a pending html token closes it, even
+      // when it opens other tags itself (`  <dt>a</dt>\n</dl>`); one that closes
+      // the tag it opens (`<div>a</div>`) is self-contained
+      const index =
+         closeTag === undefined || closeTag === openTag
+            ? -1
+            : this.pendingHtmlClose.findIndex((pending) => pending.tag === closeTag);
+      const pending = this.pendingHtmlClose[index];
+      if (pending) {
+         if (pending.token.sourceMap && token.sourceMap) {
+            pending.token.sourceMap[1] = token.sourceMap[1];
          }
+         // tags opened after the one just closed can no longer be closed
+         this.pendingHtmlClose.splice(0, index + 1);
+      } else if (openTag && openTag !== closeTag) {
+         // first in last out
+         this.pendingHtmlClose.unshift({ tag: openTag, token });
       }
 
       return token;
@@ -602,7 +627,7 @@ export class Tokenizer {
       const cap = block.def.exec(src);
       if (!cap) return undefined;
 
-      const tag = cap[1]!.toLowerCase().replace(other.multipleSpaceGlobal, " ");
+      const tag = normalizeLabel(cap[1]!).replace(other.multipleSpaceGlobal, " ");
       const href = cap[2]
          ? cap[2].replace(other.hrefBrackets, "$1").replace(inline.anyPunctuation, "$1")
          : "";
@@ -687,7 +712,9 @@ export class Tokenizer {
       const cap = block.lheading.exec(src);
       if (!cap) return undefined;
 
-      const text = cap[1]!.trim();
+      // DEVIATION: upstream keeps continuation lines' indentation; CommonMark (and
+      // GitHub, whose heading slugs would otherwise gain a dash per space) strips it
+      const text = cap[1]!.trim().replace(other.continuationIndent, "\n");
       const raw = rtrim(cap[0], "\n");
       return {
          type: "heading",
@@ -766,6 +793,10 @@ export class Tokenizer {
       const cap = inline.link.exec(src);
       if (!cap) return undefined;
 
+      if (isLabelEndInsideToken(src, cap[1]!, cap[0].startsWith("!") ? 2 : 1)) {
+         return;
+      }
+
       const trimmedUrl = cap[2]!.trim();
       if (trimmedUrl.startsWith("<")) {
          // commonmark requires matching angle brackets
@@ -818,8 +849,12 @@ export class Tokenizer {
    ): Tokens["Link"] | Tokens["Image"] | Tokens["Text"] | undefined {
       let cap;
       if ((cap = inline.reflink.exec(src)) ?? (cap = inline.nolink.exec(src))) {
+         if (isLabelEndInsideToken(src, cap[1]!, cap[0].startsWith("!") ? 2 : 1)) {
+            return;
+         }
+
          const linkStr = (cap[2] ?? cap[1])!.replace(/\s+/g, " ");
-         const link = links[linkStr.toLowerCase()];
+         const link = links[normalizeLabel(linkStr)];
          if (!link) {
             const text = cap[0].charAt(0);
             return {
@@ -855,9 +890,12 @@ export class Tokenizer {
             delimTotal = lLength,
             midDelimTotal = 0;
 
-         const endReg = match[0].startsWith("*")
-            ? inline.emStrong.rDelimAst
-            : inline.emStrong.rDelimUnd;
+         const delimChar = match[0][0];
+         // A mid-run opener (for example the second star of an unmatched `**`) must
+         // only pair with a delimiter that can only close, otherwise it steals the
+         // opener of a later span (`**a*b*c` must be `**a<em>b</em>c`).
+         const midRun = prevChar === delimChar;
+         const endReg = delimChar === "*" ? inline.emStrong.rDelimAst : inline.emStrong.rDelimUnd;
          endReg.lastIndex = 0;
 
          // Clip maskedSrc to same section of string as src (move to lexer?)
@@ -880,6 +918,11 @@ export class Tokenizer {
                   midDelimTotal += rLength;
                   continue; // CommonMark Emphasis Rules 9-10
                }
+               if (midRun) {
+                  // A mid-run opener cannot close against an ambiguous delimiter that
+                  // can also open; that delimiter opens its own emphasis span instead.
+                  break;
+               }
             }
 
             delimTotal -= rLength;
@@ -1032,6 +1075,7 @@ export class Tokenizer {
          raw: cap[0],
          text,
          href,
+         autolink: true,
          tokens: [
             {
                type: "text",
@@ -1069,6 +1113,7 @@ export class Tokenizer {
             raw: cap[0],
             text,
             href,
+            autolink: true,
             tokens: [
                {
                   type: "text",
@@ -1085,11 +1130,14 @@ export class Tokenizer {
       const cap = inline.text.exec(src);
       if (!cap) return undefined;
 
+      const escaped = this.lexer.state.inRawBlock;
       return {
          type: "text",
          raw: cap[0],
-         text: cap[0],
-         escaped: this.lexer.state.inRawBlock,
+         // raw HTML keeps whatever it was written with, everywhere else a numeric
+         // character reference stands for the character itself
+         text: escaped ? cap[0] : decodeNumericCharacterReferences(cap[0]),
+         escaped,
       };
    }
 

package/src/types.ts

@@ -141,6 +141,11 @@ export interface Tokens extends Record<string, BaseToken> {
       href: string;
       title: string | null;
       tokens: Token[];
+      /**
+       * Set for autolinks and extended (GFM) urls, where character references are
+       * not resolved, so the destination and text are literal.
+       */
+      autolink?: boolean;
    };
    Image: {
       type: "image";

package/src/utils.ts

@@ -1,4 +1,5 @@
 import { type Lexer } from "./lexer.ts";
+import { inline } from "./rules/inline.ts";
 import { other } from "./rules/other.ts";
 import { type HTMLAttrs, type PantsdownConfig, type SourceMap, type Tokens } from "./types.ts";
 
@@ -154,6 +155,24 @@ export function fixLocalImageHref(href: string, config: PantsdownConfig): string
    }
 }
 
+/**
+ * Numeric character references are recognized outside code and are equivalent to the
+ * character they name. Values that are zero, out of range, or a surrogate become the
+ * replacement character.
+ */
+export function decodeNumericCharacterReferences(text: string) {
+   return text.replace(
+      other.numericCharacterReference,
+      (_, dec: string | undefined, hex: string | undefined) => {
+         const code = dec === undefined ? Number.parseInt(hex!, 16) : Number.parseInt(dec, 10);
+         if (code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) {
+            return "\uFFFD";
+         }
+         return String.fromCodePoint(code);
+      },
+   );
+}
+
 export function cleanUrl(href: string) {
    try {
       href = encodeURI(href).replace(other.percentDecode, "%");
@@ -252,6 +271,20 @@ export function trimTrailingBlankLines(str: string) {
    return lines.slice(0, end + 1).join("\n");
 }
 
+/**
+ * Normalizes a link label so definitions and references can be matched.
+ * CommonMark asks for a Unicode case fold, which `toLowerCase()` does not
+ * reach: `ẞ` lowercases to `ß` and so never meets `SS`. Round-tripping
+ * through upper case does, as in commonmark.js; the final `toLowerCase()`
+ * keeps the folded label lower case, the form `def.tag` has always used.
+ */
+export function normalizeLabel(label: string) {
+   // The spec also asks for leading and trailing spaces, tabs and line endings
+   // to be stripped. Doing it here keeps every call site in agreement: the
+   // definition and the reference have to normalize to the same key.
+   return label.trim().toLowerCase().toUpperCase().toLowerCase();
+}
+
 export function findClosingBracket(str: string, b: string) {
    if (!b[1] || !str.includes(b[1])) {
       return -1;
@@ -281,22 +314,43 @@ export function outputLink(
    link: Pick<Tokens["Link"], "href" | "title">,
    raw: string,
    lexer: Lexer,
-): Tokens["Link"] | Tokens["Image"] {
+): Tokens["Link"] | Tokens["Image"] | undefined {
    const href = link.href;
    const title = link.title || null;
    const text = cap[1]?.replace(other.outputLinkReplace, "$1") ?? "";
+   const isImage = cap[0]?.charAt(0) === "!";
 
    lexer.state.inLink = true;
-   const token: Tokens["Link"] | Tokens["Image"] = {
-      type: cap[0]?.charAt(0) === "!" ? "image" : "link",
+   const outerLinkEmitted = lexer.state.linkEmitted;
+   const outerInRawBlock = lexer.state.inRawBlock;
+   lexer.state.linkEmitted = false;
+   const tokens = lexer.inlineTokens(text);
+   // widen: TS keeps the `= false` narrowing across the inlineTokens call that sets it
+   const textHasLink = lexer.state.linkEmitted as boolean;
+   lexer.state.linkEmitted = outerLinkEmitted;
+   lexer.state.inLink = false;
+
+   if (!isImage) {
+      // CommonMark: "Links may not contain other links, at any level of nesting."
+      // Bail so the caller falls through to text and the inner link is the one kept.
+      // Images are exempt: their text is flattened into an alt attribute.
+      if (textHasLink) {
+         // these tokens are discarded, so undo the raw-block state they opened;
+         // leaving it set would suppress escaping for the text that is re-scanned
+         lexer.state.inRawBlock = outerInRawBlock;
+         return;
+      }
+      lexer.state.linkEmitted = true;
+   }
+
+   return {
+      type: isImage ? "image" : "link",
       raw,
       href,
       title,
       text,
-      tokens: lexer.inlineTokens(text),
+      tokens,
    };
-   lexer.state.inLink = false;
-   return token;
 }
 
 export function indentCodeCompensation(raw: string, text: string) {
@@ -318,15 +372,55 @@ export function indentCodeCompensation(raw: string, text: string) {
 
          const [indentInNode] = matchIndentInNode;
 
-         if (indentToCode && indentInNode.length >= indentToCode.length) {
-            return node.slice(indentToCode.length);
-         }
-
-         return node;
+         // Up to the fence's own indentation is removed from each line, so a line
+         // indented less than the fence loses whatever indentation it has.
+         return node.slice(Math.min(indentInNode.length, indentToCode?.length ?? 0));
       })
       .join("\n");
 }
 
+/**
+ * Does the link label end inside a raw token (html tag or autolink) that starts
+ * within it? Such a token takes precedence over the link, so its closing `]`
+ * does not close the label.
+ */
+export function isLabelEndInsideToken(src: string, label: string, labelStart: number) {
+   if (!label.includes("<")) {
+      return false;
+   }
+
+   for (let i = 0; i < label.length; i++) {
+      if (label[i] === "\\") {
+         i++;
+         continue;
+      }
+
+      if (label[i] === "`") {
+         const code = inline.code.exec(label.slice(i));
+         if (code) {
+            i += code[0].length - 1;
+            continue;
+         }
+      }
+
+      if (label[i] !== "<") {
+         continue;
+      }
+
+      const tokenSrc = src.slice(labelStart + i);
+      const token = inline.tag.exec(tokenSrc) ?? inline.autolink.exec(tokenSrc);
+      if (!token) {
+         continue;
+      }
+
+      if (token[0].length > label.length - i) {
+         return true;
+      }
+      i += token[0].length - 1;
+   }
+   return false;
+}
+
 function makeAlertRegex(type: string) {
    return new RegExp(`^(?:\\[\\!${type.toUpperCase()}\\]|[\\*]{2}${type}[\\*]{2})[ \\t]*\\n?`);
 }