verybot 0.1.8

package/dist/markdown/ir.js

@@ -0,0 +1,529 @@
+/**
+ * Markdown Intermediate Representation (IR) parser.
+ * Markdown IR parser — standalone with no external deps
+ * beyond markdown-it.
+ */
+import MarkdownIt from "markdown-it";
+import { chunkText } from "./chunk.js";
+import { initTableState, initCellTarget, finishTableCell, renderTableAsBullets, renderTableAsCode, } from "./table-render.js";
+function createMarkdownIt(options) {
+    const md = new MarkdownIt({
+        html: false,
+        linkify: options.linkify ?? true,
+        breaks: false,
+        typographer: false,
+    });
+    md.enable("strikethrough");
+    if (options.tableMode && options.tableMode !== "off") {
+        md.enable("table");
+    }
+    else {
+        md.disable("table");
+    }
+    if (options.autolink === false) {
+        md.disable("autolink");
+    }
+    return md;
+}
+function getAttr(token, name) {
+    if (token.attrGet) {
+        return token.attrGet(name);
+    }
+    if (token.attrs) {
+        for (const [key, value] of token.attrs) {
+            if (key === name) {
+                return value;
+            }
+        }
+    }
+    return null;
+}
+function createTextToken(base, content) {
+    return { ...base, type: "text", content, children: undefined };
+}
+function applySpoilerTokens(tokens) {
+    for (const token of tokens) {
+        if (token.children && token.children.length > 0) {
+            token.children = injectSpoilersIntoInline(token.children);
+        }
+    }
+}
+function injectSpoilersIntoInline(tokens) {
+    const result = [];
+    const state = { spoilerOpen: false };
+    for (const token of tokens) {
+        if (token.type !== "text") {
+            result.push(token);
+            continue;
+        }
+        const content = token.content ?? "";
+        if (!content.includes("||")) {
+            result.push(token);
+            continue;
+        }
+        let index = 0;
+        while (index < content.length) {
+            const next = content.indexOf("||", index);
+            if (next === -1) {
+                if (index < content.length) {
+                    result.push(createTextToken(token, content.slice(index)));
+                }
+                break;
+            }
+            if (next > index) {
+                result.push(createTextToken(token, content.slice(index, next)));
+            }
+            state.spoilerOpen = !state.spoilerOpen;
+            result.push({
+                type: state.spoilerOpen ? "spoiler_open" : "spoiler_close",
+            });
+            index = next + 2;
+        }
+    }
+    return result;
+}
+function initRenderTarget() {
+    return {
+        text: "",
+        styles: [],
+        openStyles: [],
+        links: [],
+        linkStack: [],
+    };
+}
+function resolveRenderTarget(state) {
+    return state.table?.currentCell ?? state;
+}
+function appendText(state, value) {
+    if (!value) {
+        return;
+    }
+    const target = resolveRenderTarget(state);
+    target.text += value;
+}
+function openStyle(state, style) {
+    const target = resolveRenderTarget(state);
+    target.openStyles.push({ style, start: target.text.length });
+}
+function closeStyle(state, style) {
+    const target = resolveRenderTarget(state);
+    for (let i = target.openStyles.length - 1; i >= 0; i -= 1) {
+        if (target.openStyles[i]?.style === style) {
+            const start = target.openStyles[i].start;
+            target.openStyles.splice(i, 1);
+            const end = target.text.length;
+            if (end > start) {
+                target.styles.push({ start, end, style });
+            }
+            return;
+        }
+    }
+}
+function appendParagraphSeparator(state) {
+    if (state.env.listStack.length > 0) {
+        return;
+    }
+    if (state.table) {
+        return;
+    }
+    state.text += "\n\n";
+}
+function appendListPrefix(state) {
+    const stack = state.env.listStack;
+    const top = stack[stack.length - 1];
+    if (!top) {
+        return;
+    }
+    top.index += 1;
+    const indent = "  ".repeat(Math.max(0, stack.length - 1));
+    const prefix = top.type === "ordered" ? `${top.index}. ` : "\u2022 ";
+    state.text += `${indent}${prefix}`;
+}
+function renderInlineCode(state, content) {
+    if (!content) {
+        return;
+    }
+    const target = resolveRenderTarget(state);
+    const start = target.text.length;
+    target.text += content;
+    target.styles.push({ start, end: start + content.length, style: "code" });
+}
+function renderCodeBlock(state, content) {
+    let code = content ?? "";
+    if (!code.endsWith("\n")) {
+        code = `${code}\n`;
+    }
+    const target = resolveRenderTarget(state);
+    const start = target.text.length;
+    target.text += code;
+    target.styles.push({ start, end: start + code.length, style: "code_block" });
+    if (state.env.listStack.length === 0) {
+        target.text += "\n";
+    }
+}
+function handleLinkClose(state) {
+    const target = resolveRenderTarget(state);
+    const link = target.linkStack.pop();
+    if (!link?.href) {
+        return;
+    }
+    const href = link.href.trim();
+    if (!href) {
+        return;
+    }
+    const start = link.labelStart;
+    const end = target.text.length;
+    target.links.push({ start, end, href });
+}
+function closeRemainingStyles(target) {
+    for (let i = target.openStyles.length - 1; i >= 0; i -= 1) {
+        const open = target.openStyles[i];
+        const end = target.text.length;
+        if (end > open.start) {
+            target.styles.push({
+                start: open.start,
+                end,
+                style: open.style,
+            });
+        }
+    }
+    target.openStyles = [];
+}
+function renderTokens(tokens, state) {
+    for (const token of tokens) {
+        switch (token.type) {
+            case "inline":
+                if (token.children) {
+                    renderTokens(token.children, state);
+                }
+                break;
+            case "text":
+                appendText(state, token.content ?? "");
+                break;
+            case "em_open":
+                openStyle(state, "italic");
+                break;
+            case "em_close":
+                closeStyle(state, "italic");
+                break;
+            case "strong_open":
+                openStyle(state, "bold");
+                break;
+            case "strong_close":
+                closeStyle(state, "bold");
+                break;
+            case "s_open":
+                openStyle(state, "strikethrough");
+                break;
+            case "s_close":
+                closeStyle(state, "strikethrough");
+                break;
+            case "code_inline":
+                renderInlineCode(state, token.content ?? "");
+                break;
+            case "spoiler_open":
+                if (state.enableSpoilers) {
+                    openStyle(state, "spoiler");
+                }
+                break;
+            case "spoiler_close":
+                if (state.enableSpoilers) {
+                    closeStyle(state, "spoiler");
+                }
+                break;
+            case "link_open": {
+                const href = getAttr(token, "href") ?? "";
+                const target = resolveRenderTarget(state);
+                target.linkStack.push({ href, labelStart: target.text.length });
+                break;
+            }
+            case "link_close":
+                handleLinkClose(state);
+                break;
+            case "image":
+                appendText(state, token.content ?? "");
+                break;
+            case "softbreak":
+            case "hardbreak":
+                appendText(state, "\n");
+                break;
+            case "paragraph_close":
+                appendParagraphSeparator(state);
+                break;
+            case "heading_open":
+                if (state.headingStyle === "bold") {
+                    openStyle(state, "bold");
+                }
+                break;
+            case "heading_close":
+                if (state.headingStyle === "bold") {
+                    closeStyle(state, "bold");
+                }
+                appendParagraphSeparator(state);
+                break;
+            case "blockquote_open":
+                if (state.blockquotePrefix) {
+                    state.text += state.blockquotePrefix;
+                }
+                break;
+            case "blockquote_close":
+                state.text += "\n";
+                break;
+            case "bullet_list_open":
+                state.env.listStack.push({ type: "bullet", index: 0 });
+                break;
+            case "bullet_list_close":
+                state.env.listStack.pop();
+                break;
+            case "ordered_list_open": {
+                const start = Number(getAttr(token, "start") ?? "1");
+                state.env.listStack.push({ type: "ordered", index: start - 1 });
+                break;
+            }
+            case "ordered_list_close":
+                state.env.listStack.pop();
+                break;
+            case "list_item_open":
+                appendListPrefix(state);
+                break;
+            case "list_item_close":
+                state.text += "\n";
+                break;
+            case "code_block":
+            case "fence":
+                renderCodeBlock(state, token.content ?? "");
+                break;
+            case "html_block":
+            case "html_inline":
+                appendText(state, token.content ?? "");
+                break;
+            // Table handling
+            case "table_open":
+                if (state.tableMode !== "off") {
+                    state.table = initTableState();
+                    state.hasTables = true;
+                }
+                break;
+            case "table_close":
+                if (state.table) {
+                    const inList = state.env.listStack.length > 0;
+                    if (state.tableMode === "bullets") {
+                        renderTableAsBullets(state, state.table);
+                    }
+                    else if (state.tableMode === "code") {
+                        renderTableAsCode(state, state.table, inList);
+                    }
+                }
+                state.table = null;
+                break;
+            case "thead_open":
+                if (state.table) {
+                    state.table.inHeader = true;
+                }
+                break;
+            case "thead_close":
+                if (state.table) {
+                    state.table.inHeader = false;
+                }
+                break;
+            case "tbody_open":
+            case "tbody_close":
+                break;
+            case "tr_open":
+                if (state.table) {
+                    state.table.currentRow = [];
+                }
+                break;
+            case "tr_close":
+                if (state.table) {
+                    if (state.table.inHeader) {
+                        state.table.headers = state.table.currentRow;
+                    }
+                    else {
+                        state.table.rows.push(state.table.currentRow);
+                    }
+                    state.table.currentRow = [];
+                }
+                break;
+            case "th_open":
+            case "td_open":
+                if (state.table) {
+                    state.table.currentCell = initCellTarget();
+                }
+                break;
+            case "th_close":
+            case "td_close":
+                if (state.table?.currentCell) {
+                    state.table.currentRow.push(finishTableCell(state.table.currentCell));
+                    state.table.currentCell = null;
+                }
+                break;
+            case "hr":
+                state.text += "\n";
+                break;
+            default:
+                if (token.children) {
+                    renderTokens(token.children, state);
+                }
+                break;
+        }
+    }
+}
+function clampStyleSpans(spans, maxLength) {
+    const clamped = [];
+    for (const span of spans) {
+        const start = Math.max(0, Math.min(span.start, maxLength));
+        const end = Math.max(start, Math.min(span.end, maxLength));
+        if (end > start) {
+            clamped.push({ start, end, style: span.style });
+        }
+    }
+    return clamped;
+}
+function clampLinkSpans(spans, maxLength) {
+    const clamped = [];
+    for (const span of spans) {
+        const start = Math.max(0, Math.min(span.start, maxLength));
+        const end = Math.max(start, Math.min(span.end, maxLength));
+        if (end > start) {
+            clamped.push({ start, end, href: span.href });
+        }
+    }
+    return clamped;
+}
+function mergeStyleSpans(spans) {
+    const sorted = [...spans].sort((a, b) => {
+        if (a.start !== b.start) {
+            return a.start - b.start;
+        }
+        if (a.end !== b.end) {
+            return a.end - b.end;
+        }
+        return a.style.localeCompare(b.style);
+    });
+    const merged = [];
+    for (const span of sorted) {
+        const prev = merged[merged.length - 1];
+        if (prev && prev.style === span.style && span.start <= prev.end) {
+            prev.end = Math.max(prev.end, span.end);
+            continue;
+        }
+        merged.push({ ...span });
+    }
+    return merged;
+}
+function sliceStyleSpans(spans, start, end) {
+    if (spans.length === 0) {
+        return [];
+    }
+    const sliced = [];
+    for (const span of spans) {
+        const sliceStart = Math.max(span.start, start);
+        const sliceEnd = Math.min(span.end, end);
+        if (sliceEnd > sliceStart) {
+            sliced.push({
+                start: sliceStart - start,
+                end: sliceEnd - start,
+                style: span.style,
+            });
+        }
+    }
+    return mergeStyleSpans(sliced);
+}
+function sliceLinkSpans(spans, start, end) {
+    if (spans.length === 0) {
+        return [];
+    }
+    const sliced = [];
+    for (const span of spans) {
+        const sliceStart = Math.max(span.start, start);
+        const sliceEnd = Math.min(span.end, end);
+        if (sliceEnd > sliceStart) {
+            sliced.push({
+                start: sliceStart - start,
+                end: sliceEnd - start,
+                href: span.href,
+            });
+        }
+    }
+    return sliced;
+}
+export function markdownToIR(markdown, options = {}) {
+    return markdownToIRWithMeta(markdown, options).ir;
+}
+export function markdownToIRWithMeta(markdown, options = {}) {
+    const env = { listStack: [] };
+    const md = createMarkdownIt(options);
+    const tokens = md.parse(markdown ?? "", env);
+    if (options.enableSpoilers) {
+        applySpoilerTokens(tokens);
+    }
+    const tableMode = options.tableMode ?? "off";
+    const state = {
+        text: "",
+        styles: [],
+        openStyles: [],
+        links: [],
+        linkStack: [],
+        env,
+        headingStyle: options.headingStyle ?? "none",
+        blockquotePrefix: options.blockquotePrefix ?? "",
+        enableSpoilers: options.enableSpoilers ?? false,
+        tableMode,
+        table: null,
+        hasTables: false,
+    };
+    renderTokens(tokens, state);
+    closeRemainingStyles(state);
+    const trimmedText = state.text.trimEnd();
+    const trimmedLength = trimmedText.length;
+    let codeBlockEnd = 0;
+    for (const span of state.styles) {
+        if (span.style !== "code_block") {
+            continue;
+        }
+        if (span.end > codeBlockEnd) {
+            codeBlockEnd = span.end;
+        }
+    }
+    const finalLength = Math.max(trimmedLength, codeBlockEnd);
+    const finalText = finalLength === state.text.length ? state.text : state.text.slice(0, finalLength);
+    return {
+        ir: {
+            text: finalText,
+            styles: mergeStyleSpans(clampStyleSpans(state.styles, finalLength)),
+            links: clampLinkSpans(state.links, finalLength),
+        },
+        hasTables: state.hasTables,
+    };
+}
+export function chunkMarkdownIR(ir, limit) {
+    if (!ir.text) {
+        return [];
+    }
+    if (limit <= 0 || ir.text.length <= limit) {
+        return [ir];
+    }
+    const chunks = chunkText(ir.text, limit);
+    const results = [];
+    let cursor = 0;
+    chunks.forEach((chunk, index) => {
+        if (!chunk) {
+            return;
+        }
+        if (index > 0) {
+            while (cursor < ir.text.length && /\s/.test(ir.text[cursor] ?? "")) {
+                cursor += 1;
+            }
+        }
+        const start = cursor;
+        const end = Math.min(ir.text.length, start + chunk.length);
+        results.push({
+            text: chunk,
+            styles: sliceStyleSpans(ir.styles, start, end),
+            links: sliceLinkSpans(ir.links, start, end),
+        });
+        cursor = end;
+    });
+    return results;
+}

package/dist/markdown/render.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Generic marker-based renderer for Markdown IR.
+ * Generic marker-based renderer for Markdown IR.
+ */
+import type { MarkdownIR, MarkdownLinkSpan, MarkdownStyle } from "./ir.js";
+export type RenderStyleMarker = {
+    open: string;
+    close: string;
+};
+export type RenderStyleMap = Partial<Record<MarkdownStyle, RenderStyleMarker>>;
+export type RenderLink = {
+    start: number;
+    end: number;
+    open: string;
+    close: string;
+};
+export type RenderOptions = {
+    styleMarkers: RenderStyleMap;
+    escapeText: (text: string) => string;
+    buildLink?: (link: MarkdownLinkSpan, text: string) => RenderLink | null;
+};
+export declare function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions): string;

package/dist/markdown/render.js

@@ -0,0 +1,148 @@
+/**
+ * Generic marker-based renderer for Markdown IR.
+ * Generic marker-based renderer for Markdown IR.
+ */
+const STYLE_ORDER = [
+    "code_block",
+    "code",
+    "bold",
+    "italic",
+    "strikethrough",
+    "spoiler",
+];
+const STYLE_RANK = new Map(STYLE_ORDER.map((style, index) => [style, index]));
+function sortStyleSpans(spans) {
+    return [...spans].sort((a, b) => {
+        if (a.start !== b.start) {
+            return a.start - b.start;
+        }
+        if (a.end !== b.end) {
+            return b.end - a.end;
+        }
+        return (STYLE_RANK.get(a.style) ?? 0) - (STYLE_RANK.get(b.style) ?? 0);
+    });
+}
+export function renderMarkdownWithMarkers(ir, options) {
+    const text = ir.text ?? "";
+    if (!text) {
+        return "";
+    }
+    const styleMarkers = options.styleMarkers;
+    const styled = sortStyleSpans(ir.styles.filter((span) => Boolean(styleMarkers[span.style])));
+    const boundaries = new Set();
+    boundaries.add(0);
+    boundaries.add(text.length);
+    const startsAt = new Map();
+    for (const span of styled) {
+        if (span.start === span.end) {
+            continue;
+        }
+        boundaries.add(span.start);
+        boundaries.add(span.end);
+        const bucket = startsAt.get(span.start);
+        if (bucket) {
+            bucket.push(span);
+        }
+        else {
+            startsAt.set(span.start, [span]);
+        }
+    }
+    for (const spans of startsAt.values()) {
+        spans.sort((a, b) => {
+            if (a.end !== b.end) {
+                return b.end - a.end;
+            }
+            return (STYLE_RANK.get(a.style) ?? 0) - (STYLE_RANK.get(b.style) ?? 0);
+        });
+    }
+    const linkStarts = new Map();
+    if (options.buildLink) {
+        for (const link of ir.links) {
+            if (link.start === link.end) {
+                continue;
+            }
+            const rendered = options.buildLink(link, text);
+            if (!rendered) {
+                continue;
+            }
+            boundaries.add(rendered.start);
+            boundaries.add(rendered.end);
+            const openBucket = linkStarts.get(rendered.start);
+            if (openBucket) {
+                openBucket.push(rendered);
+            }
+            else {
+                linkStarts.set(rendered.start, [rendered]);
+            }
+        }
+    }
+    const points = [...boundaries].sort((a, b) => a - b);
+    const stack = [];
+    let out = "";
+    for (let i = 0; i < points.length; i += 1) {
+        const pos = points[i];
+        // Close ALL elements in LIFO order at this position
+        while (stack.length && stack[stack.length - 1]?.end === pos) {
+            const item = stack.pop();
+            if (item) {
+                out += item.close;
+            }
+        }
+        const openingItems = [];
+        const openingLinks = linkStarts.get(pos);
+        if (openingLinks && openingLinks.length > 0) {
+            for (const [index, link] of openingLinks.entries()) {
+                openingItems.push({
+                    end: link.end,
+                    open: link.open,
+                    close: link.close,
+                    kind: "link",
+                    index,
+                });
+            }
+        }
+        const openingStyles = startsAt.get(pos);
+        if (openingStyles) {
+            for (const [index, span] of openingStyles.entries()) {
+                const marker = styleMarkers[span.style];
+                if (!marker) {
+                    continue;
+                }
+                openingItems.push({
+                    end: span.end,
+                    open: marker.open,
+                    close: marker.close,
+                    kind: "style",
+                    style: span.style,
+                    index,
+                });
+            }
+        }
+        if (openingItems.length > 0) {
+            openingItems.sort((a, b) => {
+                if (a.end !== b.end) {
+                    return b.end - a.end;
+                }
+                if (a.kind !== b.kind) {
+                    return a.kind === "link" ? -1 : 1;
+                }
+                if (a.kind === "style" && b.kind === "style") {
+                    return (STYLE_RANK.get(a.style) ?? 0) - (STYLE_RANK.get(b.style) ?? 0);
+                }
+                return a.index - b.index;
+            });
+            for (const item of openingItems) {
+                out += item.open;
+                stack.push({ close: item.close, end: item.end });
+            }
+        }
+        const next = points[i + 1];
+        if (next === undefined) {
+            break;
+        }
+        if (next > pos) {
+            out += options.escapeText(text.slice(pos, next));
+        }
+    }
+    return out;
+}