yuque-cookie-plugin 0.1.0

package/dist/downloader.js

@@ -0,0 +1,388 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { writeReport } from "./reports.js";
+import { buildResourceManifest, buildTocMaps, fixInlineCode, fixLatex, fixMarkdownImage, fixPath, formatDate, getMarkdownImageList, handleMarkdownFooter, localizeAttachments, localizeFileCards, localizeImages, localizeMedia, parseSheet, readProgress, shouldSkipByIncremental, writeProgress, writeSummary } from "./download-utils.js";
+export async function downloadBook(client, url, options) {
+    const book = await client.getBookInfo(url);
+    if (!book.tocList.length)
+        throw new Error('No found toc list');
+    const bookPath = path.resolve(options.distDir, book.bookName ? fixPath(book.bookName) : String(book.bookId));
+    await mkdir(bookPath, { recursive: true });
+    const previous = new Map((await readProgress(bookPath)).map((item) => [item.toc.uuid, item]));
+    const tocMap = buildTocMaps(book.tocList);
+    const progressItems = [];
+    const articleUrlPrefix = url.replace(new RegExp(`(.*?/${book.bookSlug}).*`), '$1');
+    const failures = [];
+    const warnings = [];
+    let downloaded = 0;
+    let skipped = 0;
+    let handledDocs = 0;
+    const docTotal = book.tocList.filter((item) => item.type?.toLowerCase() !== 'title' && item.type?.toLowerCase() !== 'link' && item.url).length;
+    logProgress(options, `Downloading book "${book.bookName || book.bookId}" (${docTotal} docs) -> ${bookPath}`);
+    for (const item of book.tocList) {
+        const progressItem = buildProgressItem(item, tocMap);
+        if (!progressItem)
+            continue;
+        progressItems.push(progressItem);
+        const itemType = item.type.toLowerCase();
+        if (itemType === 'title' && !item.url) {
+            await mkdir(path.resolve(bookPath, progressItem.path), { recursive: true });
+            continue;
+        }
+        if (itemType === 'link') {
+            warnings.push({
+                type: 'link',
+                title: item.title,
+                url: item.url,
+                error: 'TOC link node is not a Yuque document and was not downloaded.'
+            });
+            continue;
+        }
+        if (!item.url)
+            continue;
+        try {
+            const articleUrl = `${articleUrlPrefix}/${item.url}`;
+            logProgress(options, `[${handledDocs + 1}/${docTotal}] ${item.title}`);
+            const articleResult = await downloadArticleForTest(client, {
+                bookId: book.bookId,
+                itemUrl: item.url,
+                savePath: path.resolve(bookPath, progressItem.savePath || ''),
+                saveFilePath: path.resolve(bookPath, progressItem.path),
+                uuid: item.uuid,
+                articleTitle: item.title,
+                articleUrl,
+                host: book.host,
+                imageServiceDomains: book.imageServiceDomains,
+                options,
+                progressItem,
+                previousProgressItem: previous.get(item.uuid),
+                warnings
+            });
+            if (articleResult.skipped) {
+                skipped += 1;
+                logProgress(options, `  skipped unchanged: ${progressItem.path}`);
+            }
+            else {
+                downloaded += 1;
+                logProgress(options, `  saved: ${progressItem.path}`);
+            }
+        }
+        catch (error) {
+            logProgress(options, `  failed: ${item.title}`);
+            failures.push({
+                title: item.title,
+                url: item.url,
+                retry_url: `${articleUrlPrefix}/${item.url}`,
+                error: error instanceof Error ? error.message : String(error)
+            });
+        }
+        finally {
+            handledDocs += 1;
+        }
+        await writeProgress(bookPath, progressItems);
+    }
+    await writeSummary({
+        bookPath,
+        bookName: book.bookName,
+        bookDesc: book.bookDesc,
+        progressItems
+    });
+    await writeProgress(bookPath, progressItems);
+    const resourceManifest = await buildResourceManifest(bookPath);
+    const summary = {
+        ok: failures.length === 0,
+        book_path: bookPath,
+        total: progressItems.length,
+        docs: progressItems.filter((item) => {
+            const type = item.toc.type?.toLowerCase();
+            return type !== 'title' && type !== 'link' && item.path.endsWith('.md');
+        }).length,
+        downloaded,
+        skipped,
+        incremental: options.incremental,
+        options,
+        resources: resourceManifest,
+        warnings,
+        warning_summary: buildWarningSummary(warnings),
+        failures,
+        retry: buildRetryPlan('download-doc', failures.map((item) => item.retry_url || item.url).filter(isString), options)
+    };
+    const report = await writeReport('download-book', summary);
+    return {
+        ...summary,
+        report
+    };
+}
+export async function downloadDocs(client, urls, options) {
+    const distPath = path.resolve(options.distDir);
+    await mkdir(distPath, { recursive: true });
+    const failures = [];
+    const warnings = [];
+    const files = [];
+    let downloaded = 0;
+    logProgress(options, `Downloading ${urls.length} doc(s) -> ${distPath}`);
+    for (const [index, url] of urls.entries()) {
+        try {
+            const doc = await client.getDocInfoFromUrl(url);
+            if (!doc.docSlug || !doc.bookId)
+                throw new Error('Failed to get document info from URL');
+            const fileName = fixPath(doc.docTitle || doc.docSlug);
+            const saveFilePath = path.resolve(distPath, `${fileName}.md`);
+            logProgress(options, `[${index + 1}/${urls.length}] ${doc.docTitle || doc.docSlug}`);
+            const progressItem = {
+                path: `${fileName}.md`,
+                savePath: '',
+                pathTitleList: [fileName],
+                pathIdList: [String(doc.docId || doc.docSlug)],
+                toc: {
+                    type: 'DOC',
+                    title: doc.docTitle || doc.docSlug,
+                    uuid: String(doc.docId || doc.docSlug),
+                    url: doc.docSlug,
+                    parent_uuid: '',
+                    child_uuid: ''
+                }
+            };
+            await downloadArticleForTest(client, {
+                bookId: doc.bookId,
+                itemUrl: doc.docSlug,
+                savePath: distPath,
+                saveFilePath,
+                uuid: String(doc.docId || doc.docSlug),
+                articleTitle: doc.docTitle || doc.docSlug,
+                articleUrl: url,
+                host: doc.host,
+                imageServiceDomains: doc.imageServiceDomains,
+                options,
+                progressItem,
+                warnings
+            });
+            downloaded += 1;
+            files.push(saveFilePath);
+            logProgress(options, `  saved: ${saveFilePath}`);
+        }
+        catch (error) {
+            logProgress(options, `  failed: ${url}`);
+            failures.push({
+                url,
+                error: error instanceof Error ? error.message : String(error)
+            });
+        }
+    }
+    const resourceManifest = await buildResourceManifest(distPath);
+    const summary = {
+        ok: failures.length === 0,
+        dist_path: distPath,
+        files,
+        downloaded,
+        options,
+        resources: resourceManifest,
+        warnings,
+        warning_summary: buildWarningSummary(warnings),
+        failures,
+        retry: buildRetryPlan('download-doc', failures.map((item) => item.url).filter(isString), options)
+    };
+    const report = await writeReport('download-doc', summary);
+    return {
+        ...summary,
+        report
+    };
+}
+export async function downloadArticleForTest(client, params) {
+    const { response, apiUrl, httpStatus } = await client.getDocMarkdownData({
+        articleUrl: params.itemUrl,
+        bookId: params.bookId,
+        host: params.host,
+        isMarkdown: true
+    });
+    const data = response?.data || {};
+    params.progressItem.createAt = data.created_at || '';
+    params.progressItem.contentUpdatedAt = data.content_updated_at || '';
+    params.progressItem.publishedAt = data.published_at || '';
+    params.progressItem.firstPublishedAt = data.first_published_at || '';
+    if (shouldSkipByIncremental(params.progressItem, params.previousProgressItem, params.options.incremental)) {
+        return { skipped: true };
+    }
+    let markdown = '';
+    const type = String(data.type || '').toLowerCase();
+    if (type === 'sheet') {
+        const raw = await client.getDocMarkdownData({
+            articleUrl: params.itemUrl,
+            bookId: params.bookId,
+            host: params.host,
+            isMarkdown: false
+        });
+        const rawContent = raw.response?.data?.content;
+        const content = rawContent ? JSON.parse(rawContent) : {};
+        markdown = content?.sheet ? parseSheet(content.sheet) : '';
+    }
+    else if (type === 'board' || type === 'table') {
+        markdown = `[Unsupported Yuque document type: ${type}]\n`;
+    }
+    else if (typeof data.sourcecode === 'string') {
+        markdown = fixLatex(data.sourcecode);
+    }
+    else {
+        throw new Error(`download article Error: ${apiUrl}, http status ${httpStatus}`);
+    }
+    const raw = await client.getDocMarkdownData({
+        articleUrl: params.itemUrl,
+        bookId: params.bookId,
+        host: params.host,
+        isMarkdown: false
+    });
+    const htmlData = raw.response?.data?.content || '';
+    const imgList = getMarkdownImageList(markdown);
+    if (imgList.length && !params.options.ignoreImg) {
+        markdown = fixMarkdownImage(imgList, markdown, htmlData);
+    }
+    if (params.options.ignoreAttachments !== true) {
+        markdown = await localizeAttachments(markdown, {
+            savePath: params.savePath,
+            attachmentsDir: `attachments/${fixPath(params.uuid)}`,
+            headers: client.htmlHeaders(),
+            ignoreAttachments: params.options.ignoreAttachments,
+            warnings: params.warnings,
+            title: params.articleTitle
+        });
+    }
+    if (params.options.ignoreAttachments !== true) {
+        markdown = await localizeFileCards(markdown, htmlData, {
+            savePath: params.savePath,
+            attachmentsDir: `attachments/${fixPath(params.uuid)}`,
+            headers: client.htmlHeaders(),
+            ignoreAttachments: params.options.ignoreAttachments,
+            warnings: params.warnings,
+            title: params.articleTitle
+        });
+    }
+    if (params.options.ignoreAttachments !== true) {
+        markdown = await localizeMedia(markdown, htmlData, {
+            savePath: params.savePath,
+            attachmentsDir: `attachments/${fixPath(params.uuid)}`,
+            headers: client.htmlHeaders(),
+            ignoreAttachments: params.options.ignoreAttachments,
+            getVideoInfo: (videoId) => client.getVideoInfo(videoId),
+            warnings: params.warnings,
+            title: params.articleTitle
+        });
+    }
+    if (!params.options.ignoreImg) {
+        markdown = await localizeImages(markdown, {
+            savePath: params.savePath,
+            imageDir: `img/${fixPath(params.uuid)}`,
+            referer: params.articleUrl,
+            headers: client.htmlHeaders(),
+            imageServiceDomains: params.imageServiceDomains,
+            warnings: params.warnings,
+            title: params.articleTitle
+        });
+    }
+    markdown = fixInlineCode(markdown, htmlData);
+    markdown = handleMarkdownFooter(markdown, {
+        articleTitle: params.articleTitle,
+        articleUrl: params.articleUrl,
+        toc: params.options.toc,
+        articleUpdateTime: formatDate(data.content_updated_at || ''),
+        hideFooter: params.options.hideFooter,
+        convertMarkdownVideoLinks: params.options.convertMarkdownVideoLinks
+    });
+    await mkdir(path.dirname(params.saveFilePath), { recursive: true });
+    await writeFile(params.saveFilePath, markdown);
+    return { skipped: false };
+}
+function buildProgressItem(item, tocMap) {
+    if (!item.type)
+        return null;
+    const pathItems = getPathItems(item, tocMap);
+    const pathTitleList = pathItems.map((toc) => fixPath(toc.title));
+    const pathIdList = pathItems.map((toc) => toc.uuid);
+    const type = item.type.toLowerCase();
+    if (type === 'title' && !item.url) {
+        return {
+            path: pathTitleList.join('/'),
+            pathTitleList,
+            pathIdList,
+            toc: item
+        };
+    }
+    if (!item.url)
+        return null;
+    if (type === 'link') {
+        return {
+            path: pathTitleList.join('/'),
+            savePath: pathTitleList.slice(0, -1).join('/'),
+            pathTitleList,
+            pathIdList,
+            toc: item
+        };
+    }
+    let mdPath = `${pathTitleList.join('/')}.md`;
+    let savePath = pathTitleList.slice(0, -1).join('/');
+    if (type === 'doc' && item.child_uuid) {
+        mdPath = `${pathTitleList.join('/')}/index.md`;
+        savePath = pathTitleList.join('/');
+    }
+    return {
+        path: mdPath,
+        savePath,
+        pathTitleList,
+        pathIdList,
+        toc: item
+    };
+}
+function getPathItems(item, tocMap) {
+    const items = [item];
+    let current = item;
+    while (current.parent_uuid && tocMap.has(current.parent_uuid)) {
+        current = tocMap.get(current.parent_uuid);
+        items.unshift(current);
+    }
+    return items;
+}
+function logProgress(options, message) {
+    if (options.quiet)
+        return;
+    process.stderr.write(`${message}\n`);
+}
+function buildRetryPlan(command, urls, options) {
+    return {
+        command,
+        urls,
+        count: urls.length,
+        args: [
+            command,
+            ...urls,
+            '--dist-dir',
+            options.distDir,
+            ...(options.ignoreImg ? ['--ignore-img'] : []),
+            ...(options.ignoreAttachments === true ? ['--ignore-attachments'] : []),
+            ...(typeof options.ignoreAttachments === 'string' ? ['--ignore-attachments', options.ignoreAttachments] : []),
+            ...(options.toc ? ['--toc'] : []),
+            ...(options.hideFooter ? ['--hide-footer'] : []),
+            ...(options.quiet ? ['--quiet'] : [])
+        ]
+    };
+}
+function isString(value) {
+    return typeof value === 'string' && value.length > 0;
+}
+function buildWarningSummary(warnings) {
+    const byType = {};
+    for (const warning of warnings) {
+        byType[warning.type] = (byType[warning.type] || 0) + 1;
+    }
+    return {
+        total: warnings.length,
+        by_type: byType,
+        retryable_resources: warnings
+            .filter((warning) => warning.type !== 'link' && Boolean(warning.url))
+            .map((warning) => ({
+            type: warning.type,
+            title: warning.title,
+            url: warning.url,
+            file: warning.file,
+            error: warning.error
+        }))
+    };
+}

package/dist/editor-bridge.js

@@ -0,0 +1,240 @@
+import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const VENDOR_DIR = path.join(PACKAGE_ROOT, 'vendor/lake-editor');
+export async function serializeHtmlToLake({ html, out, keepTemp = false }) {
+    const { chromium } = await importPlaywright();
+    const tempDir = await makeTempDir();
+    const pagePath = path.join(tempDir, 'editor-bridge.html');
+    const vendorAvailable = await hasOfficialEditorAssets();
+    await writeFile(pagePath, bridgeHtml({ vendorAvailable }), 'utf8');
+    const browser = await chromium.launch({ headless: true });
+    try {
+        const page = await browser.newPage();
+        await page.goto(pathToFileURL(pagePath).href);
+        const result = await page.evaluate(async (input) => {
+            return window.__yuqueBridge.serialize(input);
+        }, html);
+        if (!result?.lake?.trim())
+            throw new Error('Editor bridge returned empty Lake content.');
+        if (out) {
+            await mkdir(path.dirname(path.resolve(out)), { recursive: true });
+            await writeFile(out, result.lake, 'utf8');
+        }
+        return result;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (/Executable doesn't exist|browserType.launch/.test(message)) {
+            throw new Error(`${message}\nRun "npx playwright install chromium" and try again.`);
+        }
+        throw error;
+    }
+    finally {
+        await browser.close();
+        if (!keepTemp)
+            await rm(tempDir, { recursive: true, force: true });
+    }
+}
+async function importPlaywright() {
+    try {
+        return await import('playwright');
+    }
+    catch {
+        throw new Error('Missing Playwright. Run "npm install --save-dev playwright" first.');
+    }
+}
+async function makeTempDir() {
+    await mkdir(path.join(os.tmpdir(), 'yuque-editor-bridge-'), { recursive: true });
+    return mkdtemp(path.join(os.tmpdir(), 'yuque-editor-bridge-'));
+}
+async function hasOfficialEditorAssets() {
+    try {
+        await access(path.join(VENDOR_DIR, 'doc.umd.js'));
+        await access(path.join(VENDOR_DIR, 'doc.css'));
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function bridgeHtml({ vendorAvailable }) {
+    const vendorUrl = pathToFileURL(VENDOR_DIR).href;
+    return `<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Yuque Editor Bridge</title>
+  ${vendorAvailable ? `<link rel="stylesheet" href="${vendorUrl}/antd.4.24.13.css"><link rel="stylesheet" href="${vendorUrl}/doc.css">` : ''}
+</head>
+<body>
+  <div id="root"></div>
+  ${vendorAvailable ? `<script>window.KITCHEN_URL = ${JSON.stringify(`${vendorUrl}/lake-editor-icon.js`)};</script><script src="${vendorUrl}/react.production.min.js"></script><script src="${vendorUrl}/react-dom.production.min.js"></script><script src="${vendorUrl}/doc.umd.js"></script>` : ''}
+  <script>
+    const vendorAvailable = ${JSON.stringify(vendorAvailable)};
+    const vendorUrl = ${JSON.stringify(vendorUrl)};
+
+    window.__yuqueBridge = {
+      async serialize(html) {
+        if (vendorAvailable && window.Doc && typeof window.Doc.createOpenEditor === 'function') {
+          return serializeWithOfficialEditor(html);
+        }
+        return {
+          serializer: 'fallback',
+          lake: fallbackSerialize(html)
+        };
+      }
+    };
+
+    async function serializeWithOfficialEditor(html) {
+      const root = document.getElementById('root');
+      const editor = window.Doc.createOpenEditor(root, {
+        placeholder: '',
+        defaultFontsize: 14,
+        codeblock: {
+          codemirrorURL: vendorUrl + '/CodeMirror.js',
+          supportCustomStyle: true
+        },
+        math: {
+          KaTexURL: vendorUrl + '/katex.js'
+        },
+        image: {
+          isCaptureImageURL() {
+            return false;
+          },
+          async createUploadPromise(request) {
+            if (request.type === 'base64') {
+              return {
+                url: request.data,
+                size: request.data.length * 0.75,
+                name: 'image.png'
+              };
+            }
+            return new Promise((resolve, reject) => {
+              const reader = new FileReader();
+              reader.onload = () => resolve({
+                url: reader.result,
+                size: request.data.size,
+                name: request.data.name || 'image.png'
+              });
+              reader.onerror = reject;
+              reader.readAsDataURL(request.data);
+            });
+          }
+        },
+        bookmark: {
+          recognizeYuque: true,
+          fetchDetailHandler: async (url) => ({ url, title: url })
+        },
+        link: {
+          isValidURL() {
+            return true;
+          },
+          sanitizeURL(url) {
+            return url;
+          }
+        }
+      });
+      editor.setDocument('text/html', html);
+      let attempts = 0;
+      while (editor.canGetDocument && !editor.canGetDocument()) {
+        if (attempts > 100) throw new Error('Official editor was not ready after 10s.');
+        attempts += 1;
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+      return {
+        serializer: 'official-lake-editor',
+        lake: editor.getDocument('text/lake', { includeMeta: true })
+      };
+    }
+
+    function fallbackSerialize(html) {
+      const doc = new DOMParser().parseFromString(html, 'text/html');
+      const blocks = [];
+      for (const node of Array.from(doc.body.childNodes)) {
+        const rendered = renderBlock(node);
+        if (rendered) blocks.push(rendered);
+      }
+      return '<meta name="serializer" content="yuque-cookie-plugin-fallback">' + blocks.join('');
+    }
+
+    function renderBlock(node) {
+      if (node.nodeType === Node.TEXT_NODE) {
+        const text = node.textContent.trim();
+        return text ? '<p>' + escapeHtml(text) + '</p>' : '';
+      }
+      if (node.nodeType !== Node.ELEMENT_NODE) return '';
+      const tag = node.tagName.toLowerCase();
+      if (/^h[1-6]$/.test(tag)) return '<' + tag + '>' + renderInlineChildren(node) + '</' + tag + '>';
+      if (tag === 'p') return '<p>' + renderInlineChildren(node) + '</p>';
+      if (tag === 'blockquote') return '<blockquote>' + renderInlineChildren(node) + '</blockquote>';
+      if (tag === 'hr') return '<card name="hr" value="data:%7B%7D"></card>';
+      if (tag === 'pre') return renderCodeBlock(node);
+      if (tag === 'ul' || tag === 'ol') return renderList(node, tag);
+      if (tag === 'table') return renderTable(node);
+      if (tag === 'div' || tag === 'section' || tag === 'article') {
+        return Array.from(node.childNodes).map(renderBlock).filter(Boolean).join('');
+      }
+      return '<p>' + renderInline(node) + '</p>';
+    }
+
+    function renderInlineChildren(node) {
+      return Array.from(node.childNodes).map(renderInline).join('');
+    }
+
+    function renderInline(node) {
+      if (node.nodeType === Node.TEXT_NODE) return escapeHtml(node.textContent);
+      if (node.nodeType !== Node.ELEMENT_NODE) return '';
+      const tag = node.tagName.toLowerCase();
+      const content = renderInlineChildren(node);
+      if (tag === 'strong' || tag === 'b') return '<strong>' + content + '</strong>';
+      if (tag === 'em' || tag === 'i') return '<em>' + content + '</em>';
+      if (tag === 'del' || tag === 's') return '<del>' + content + '</del>';
+      if (tag === 'u') return '<u>' + content + '</u>';
+      if (tag === 'code') return '<code>' + content + '</code>';
+      if (tag === 'a') return '<a href="' + escapeAttr(node.getAttribute('href') || '') + '">' + content + '</a>';
+      if (tag === 'br') return '<br>';
+      if (tag === 'img') {
+        const data = {
+          src: node.getAttribute('src') || '',
+          name: node.getAttribute('alt') || 'image-placeholder'
+        };
+        return '<card name="image" value="data:' + encodeURIComponent(JSON.stringify(data)) + '"></card>';
+      }
+      return content;
+    }
+
+    function renderCodeBlock(node) {
+      const codeNode = node.querySelector('code') || node;
+      const className = codeNode.getAttribute('class') || '';
+      const language = (className.match(/language-([\\w-]+)/) || [])[1] || 'plain';
+      const data = { mode: language, code: codeNode.textContent || '', name: '' };
+      return '<card name="codeblock" value="data:' + encodeURIComponent(JSON.stringify(data)) + '"></card>';
+    }
+
+    function renderList(node, tag) {
+      return '<' + tag + '>' + Array.from(node.children).filter(child => child.tagName.toLowerCase() === 'li').map((li) => {
+        return '<li>' + renderInlineChildren(li) + '</li>';
+      }).join('') + '</' + tag + '>';
+    }
+
+    function renderTable(node) {
+      return '<table>' + node.innerHTML + '</table>';
+    }
+
+    function escapeHtml(value) {
+      return String(value)
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;');
+    }
+
+    function escapeAttr(value) {
+      return escapeHtml(value).replace(/"/g, '&quot;');
+    }
+  </script>
+</body>
+</html>`;
+}

package/dist/fs-utils.js

@@ -0,0 +1,9 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+export async function writeJson(file, data) {
+    await mkdir(path.dirname(path.resolve(file)), { recursive: true });
+    await writeFile(file, `${JSON.stringify(data, null, 2)}\n`);
+}
+export function timestampForFile(date = new Date()) {
+    return date.toISOString().replace(/[:.]/g, '-');
+}