yuque-dl 1.0.0

package/.editorconfig

@@ -0,0 +1,16 @@
+# http://editorconfig.org
+
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+end_of_line = lf
+# 最后一行换行 取消掉
+insert_final_newline = false
+trim_trailing_whitespace = true
+
+[*.md]
+insert_final_newline = false
+trim_trailing_whitespace = false

package/.eslintignore

@@ -0,0 +1,5 @@
+dist
+types
+node_modules
+bin
+test

package/.eslintrc.js

@@ -0,0 +1,21 @@
+module.exports = {
+  env: {
+    node: true,
+  },
+  parser: '@typescript-eslint/parser',
+  plugins: [
+    '@typescript-eslint',
+  ],
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+  ],
+  rules: {
+    semi: ['error', 'never'],
+    'no-console': 'off',
+    'no-plusplus': ['error', { allowForLoopAfterthoughts: true }],
+    // TODO
+    '@typescript-eslint/no-explicit-any': 'off',
+    '@typescript-eslint/no-non-null-assertion': 'off'
+  },
+}

package/.prettierrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+  semi: false,
+  endOfLine: 'lf',
+  singleQuote: true,
+  tabWidth: 2,
+  useTabs: false,
+}

package/README.md

@@ -0,0 +1,3 @@
+# yuque-dl
+
+yuque 知识库下载

package/bin/index.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+
+function run() {
+  return import('../dist/es/cli.js')
+}
+run()

package/dist/cjs/cli.js

@@ -0,0 +1,44 @@
+'use strict';
+
+var cac = require('cac');
+var index = require('./index-0e9279f8.js');
+require('node:fs/promises');
+require('node:path');
+require('axios');
+require('log4js');
+require('progress');
+require('rand-user-agent');
+require('pull-md-img');
+
+var version = "1.0.0";
+
+const cli = cac.cac('yuque-dl');
+cli
+    .command('<url>', 'request url')
+    .option('-d, --distDir <dir>', '下载的目录 eg: -d download', {
+    default: 'download',
+})
+    .action(async (url, options) => {
+    try {
+        await index.main(url, options);
+    }
+    catch (err) {
+        index.logger.error(err.message || 'unknown exception');
+    }
+});
+cli.help();
+cli.version(version);
+try {
+    cli.parse();
+}
+catch (err) {
+    index.logger.error(err.message || 'unknown exception');
+    process.exit(1);
+}
+// try {
+//   cli.parse(process.argv, { run: false })
+//   await cli.runMatchedCommand()
+// } catch (error) {
+//   logger.error(error.message || 'unknown exception')
+//   process.exit(1)
+// }

package/dist/cjs/index-0e9279f8.js

@@ -0,0 +1,405 @@
+'use strict';
+
+var fs = require('node:fs/promises');
+var path = require('node:path');
+var axios = require('axios');
+var log4js = require('log4js');
+var progress = require('progress');
+var randUserAgentLib = require('rand-user-agent');
+var mdImg = require('pull-md-img');
+
+var config = {
+    knowledgeBaseReg: /decodeURIComponent\(\"(.+)\"\)\);/m,
+    dist: './download'
+};
+// data = re.findall(r"decodeURIComponent\(\"(.+)\"\)\);", docsdata.content.decode('utf-8'))
+// const config = {
+//   path: '',
+//   suffix: '',
+//   dist: './res/',
+//   imgDir:`./img/${Date.now()}`,
+//   // markdown img 正则 注意多行匹配
+//   mdImgReg: /!\[(.*?)\]\((.*?)\)/gm,
+//   isIgnoreConsole: false
+// }
+
+/**
+ * log 初始化
+ */
+const getLogger = () => {
+    log4js.configure({
+        appenders: {
+            // cheeseLog: { type: 'file', filename: 'cheese.log' },
+            cheese: {
+                type: 'console',
+                layout: {
+                    // type: 'messagePassThrough',
+                    type: 'pattern',
+                    // pattern: '%[%d{yyyy-MM-dd hh:mm:ss} [%p] %c -%] %m%n'
+                    pattern: '%[%c [%p]:%] %m%n'
+                }
+            }
+        },
+        categories: { default: { appenders: ['cheese'], level: 'trace' } }
+    });
+    return log4js.getLogger('yuque-dl');
+};
+var logger = getLogger();
+
+class Progress {
+    constructor(bookPath, total) {
+        this.bookPath = '';
+        this.processFilePath = '';
+        this.processInfo = [];
+        this.curr = 0;
+        this.total = 0;
+        this.isDownloadInterrupted = false;
+        this.bar = null;
+        this.completePromise = null;
+        this.bookPath = bookPath;
+        this.processFilePath = `${bookPath}/process.json`;
+        this.total = total;
+    }
+    async init() {
+        this.processInfo = await this.getProgress();
+        this.curr = this.processInfo.length;
+        let completeResolve;
+        this.completePromise = new Promise(resolve => {
+            completeResolve = resolve;
+        });
+        if (this.curr > 0 && this.curr !== this.total) {
+            this.isDownloadInterrupted = true;
+            logger.info('断点下载');
+        }
+        this.bar = new progress('[:bar] :rate/bps :percent :etas', {
+            // width: 20,
+            total: this.total,
+            curr: this.curr,
+            callback() {
+                completeResolve();
+            }
+        });
+    }
+    async getProgress() {
+        let processInfo = [];
+        try {
+            const processInfoStr = await fs.readFile(this.processFilePath, { encoding: 'utf8' });
+            processInfo = JSON.parse(processInfoStr);
+        }
+        catch (err) {
+            if (err && err.code === 'ENOENT') {
+                await fs.writeFile(this.processFilePath, JSON.stringify(processInfo), { encoding: 'utf8' });
+            }
+        }
+        return processInfo;
+    }
+    async updateProgress(progressItem) {
+        this.processInfo.push(progressItem);
+        await fs.writeFile(this.processFilePath, JSON.stringify(this.processInfo), { encoding: 'utf8' });
+        this.curr = this.curr + 1;
+        if (this.bar) {
+            this.bar?.tick();
+        }
+    }
+}
+
+function randUserAgent({ browser = 'chrome', os = 'mac os', device = 'desktop' }) {
+    device = device.toLowerCase();
+    browser = browser.toLowerCase();
+    os = os.toLowerCase();
+    let UA = randUserAgentLib(device, browser, os);
+    if (browser === 'chrome') {
+        while (UA.includes('Chrome-Lighthouse')
+            || UA.includes('Gener8')
+            || UA.includes('HeadlessChrome')
+            || UA.includes('SMTBot')) {
+            UA = randUserAgentLib(device, browser, os);
+        }
+    }
+    if (browser === 'safari') {
+        while (UA.includes('Applebot')) {
+            UA = randUserAgentLib(device, browser, os);
+        }
+    }
+    return UA;
+}
+
+// SUMMARY.md
+function getKnowledgeBaseInfo(url) {
+    return axios.get(url, {
+        headers: {
+            "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
+        }
+    }).then(({ data = '', status }) => {
+        if (status === 200)
+            return data;
+        return '';
+    }).then(html => {
+        const data = html.match(config.knowledgeBaseReg) || '';
+        if (!data[1])
+            return {};
+        const jsonData = JSON.parse(decodeURIComponent(data[1]));
+        const info = {
+            bookId: jsonData.book.id,
+            tocList: jsonData.book.toc || [],
+            bookName: jsonData.book.name || '',
+            bookDesc: jsonData.book.description || '',
+        };
+        return info;
+    });
+}
+async function main(url, options) {
+    const { bookId, tocList, bookName, bookDesc } = await getKnowledgeBaseInfo(url);
+    if (!bookId)
+        throw new Error('No found book id');
+    if (!tocList || tocList.length === 0)
+        throw new Error('No found toc list');
+    const bookPath = `${options.distDir}/${bookId}`;
+    await fs.mkdir(bookPath, { recursive: true });
+    const total = tocList.length;
+    const progress = new Progress(bookPath, total);
+    await progress.init();
+    if (progress.curr === total) {
+        logger.info('√ 已完成');
+        return;
+    }
+    const uuidMap = new Map();
+    // 下载中断 重新获取下载进度数据
+    if (progress.isDownloadInterrupted) {
+        progress.processInfo.forEach(item => {
+            uuidMap.set(item.toc.uuid, item);
+        });
+    }
+    const articleUrlPrefix = path.parse(url).dir;
+    // const url.split('/')
+    for (let i = 0; i < total; i++) {
+        const item = tocList[i];
+        if (typeof item.type !== 'string')
+            continue;
+        if (uuidMap.get(item.uuid))
+            continue;
+        console.log(item.uuid);
+        const itemType = item.type.toLocaleLowerCase();
+        const dirNameReg = /[\\\/:\*\?"<>\|\n\r]/g;
+        // 目录
+        if (itemType === 'title' || item['child_uuid'] !== '') {
+            let tempItem = item;
+            let pathTitleList = [];
+            let pathIdList = [];
+            while (tempItem) {
+                pathTitleList.unshift(tempItem.title.replace(dirNameReg, '_'));
+                pathIdList.unshift(tempItem.uuid);
+                if (uuidMap.get(tempItem['parent_uuid'])) {
+                    tempItem = uuidMap.get(tempItem['parent_uuid']).toc;
+                }
+                else {
+                    tempItem = undefined;
+                }
+            }
+            const progressItem = {
+                path: pathTitleList.join('/'),
+                pathTitleList,
+                pathIdList,
+                toc: item
+            };
+            await fs.mkdir(`${bookPath}/${pathTitleList.join('/')}`, { recursive: true });
+            uuidMap.set(item.uuid, progressItem);
+            progress.updateProgress(progressItem);
+        }
+        else if (item.url) {
+            let preItem = {
+                path: '',
+                pathTitleList: [],
+                pathIdList: []
+            };
+            if (uuidMap.get(item['parent_uuid'])) {
+                preItem = uuidMap.get(item['parent_uuid']);
+            }
+            const fileName = item.title.replace(dirNameReg, '_');
+            const pathTitleList = [...preItem.pathTitleList, `${fileName}.md`];
+            const pathIdList = [...preItem.pathIdList, item.uuid];
+            const progressItem = {
+                path: pathTitleList.join('/'),
+                pathTitleList,
+                pathIdList,
+                toc: item
+            };
+            const isSuccess = await downloadArticle({
+                bookId,
+                itemUrl: item.url,
+                savePath: `${bookPath}/${preItem.path}`,
+                saveFilePath: `${bookPath}/${progressItem.path}`,
+                uuid: item.uuid,
+                articleUrl: `${articleUrlPrefix}/${item.url}`,
+                articleTitle: item.title
+            });
+            if (isSuccess) {
+                uuidMap.set(item.uuid, progressItem);
+                progress.updateProgress(progressItem);
+            }
+        }
+        console.log('完成', item.uuid);
+    }
+    await progress.completePromise;
+    logger.info('生成目录 SUMMARY.md');
+    await genSummaryFile({
+        bookPath,
+        bookName,
+        bookDesc,
+        uuidMap
+    });
+    if (progress.curr === total) {
+        logger.info('√ 已完成');
+        return;
+    }
+}
+async function downloadArticle(params) {
+    const { bookId, itemUrl, savePath, saveFilePath, uuid, articleUrl, articleTitle } = params;
+    const apiUrl = `https://www.yuque.com/api/docs/${itemUrl}`;
+    const response = await axios.get(apiUrl, {
+        headers: {
+            "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
+        },
+        params: {
+            'book_id': bookId,
+            'merge_dynamic_data': false,
+            mode: 'markdown'
+        }
+    }).then(({ data, status }) => {
+        if (status === 200)
+            return data;
+        logger.error(`download article error: ${apiUrl} http status ${status}`);
+        return null;
+    });
+    if (!response || !response.data || !response.data.sourcecode)
+        return false;
+    let mdData = response.data.sourcecode;
+    try {
+        mdData = await mdImg.run(mdData, {
+            dist: savePath,
+            imgDir: `img/${uuid}`,
+            isIgnoreConsole: true
+        });
+    }
+    catch (err) {
+        logger.error(`download article img Error (api: ${apiUrl})`);
+        return false;
+    }
+    mdData = mdData.replace(/<br(\s?)\/>/gm, '\n');
+    if (articleTitle) {
+        mdData = `# ${articleTitle}\n<!--page header-->\n\n${mdData}\n\n`;
+    }
+    if (articleUrl) {
+        mdData += `<!--page footer-->\n- 原文: <${articleUrl}>`;
+    }
+    try {
+        await fs.writeFile(saveFilePath, mdData);
+        return true;
+    }
+    catch (e) {
+        logger.error(`download article write file Error (api: ${apiUrl})`);
+        return false;
+    }
+}
+async function genSummaryFile(params) {
+    const { bookName, bookDesc, bookPath, uuidMap } = params;
+    const header = `# ${bookName}\n\n > ${bookDesc}\n\n`;
+    let mdContent = header;
+    const summary = [];
+    uuidMap.forEach(processItem => {
+        const toc = processItem.toc;
+        const parentId = toc['parent_uuid'];
+        const findRes = findTree(summary, parentId);
+        if (toc.type.toLocaleLowerCase() === 'title' || toc['child_uuid'] !== '') {
+            if (typeof findRes !== 'boolean') {
+                if (!Array.isArray(findRes.children))
+                    findRes.children = [];
+                findRes.children.push({
+                    id: toc.uuid,
+                    type: 'title',
+                    level: findRes.level + 1,
+                    text: toc.title
+                });
+            }
+            else {
+                summary.push({
+                    id: toc.uuid,
+                    type: 'title',
+                    text: toc.title,
+                    level: 1
+                });
+            }
+        }
+        else {
+            if (typeof findRes !== 'boolean') {
+                if (!Array.isArray(findRes.children))
+                    findRes.children = [];
+                findRes.children.push({
+                    id: toc.uuid,
+                    type: 'link',
+                    level: findRes.level + 1,
+                    link: processItem.path,
+                    text: toc.title
+                });
+            }
+            else {
+                summary.push({
+                    id: toc.uuid,
+                    type: 'link',
+                    text: toc.title,
+                    link: processItem.path,
+                    level: 1
+                });
+            }
+        }
+    });
+    let summaryContent = genSummaryContent(summary, '');
+    mdContent += summaryContent;
+    try {
+        await fs.writeFile(`${bookPath}/SUMMARY.md`, mdContent);
+    }
+    catch (err) {
+        logger.error('Generate Summary Error');
+    }
+}
+function genSummaryContent(summary, summaryContent) {
+    for (let i = 0; i < summary.length; i++) {
+        const item = summary[i];
+        if (item.type === 'title') {
+            summaryContent += `\n${''.padStart(item.level + 1, '#')} ${item.text}\n\n`;
+        }
+        else if (item.type === 'link') {
+            const link = item.link ? item.link.replace(/\s/g, '%20') : item.link;
+            summaryContent += `- [${item.text}](${link})\n`;
+        }
+        if (Array.isArray(item.children)) {
+            summaryContent += genSummaryContent(item.children, '');
+        }
+    }
+    return summaryContent;
+}
+function findIdItem(node, id) {
+    if (node.id === id) {
+        return node;
+    }
+    else if (node.children) {
+        const findRes = findTree(node.children, id);
+        if (findRes)
+            return findRes;
+    }
+    return false;
+}
+function findTree(tree, id) {
+    if (!id)
+        return false;
+    for (let i = 0; i < tree.length; i++) {
+        const findRes = findIdItem(tree[i], id);
+        if (findRes)
+            return findRes;
+    }
+    return false;
+}
+
+exports.getKnowledgeBaseInfo = getKnowledgeBaseInfo;
+exports.logger = logger;
+exports.main = main;

package/dist/cjs/index.js

@@ -0,0 +1,15 @@
+'use strict';
+
+require('node:fs/promises');
+require('node:path');
+require('axios');
+var index = require('./index-0e9279f8.js');
+require('pull-md-img');
+require('log4js');
+require('progress');
+require('rand-user-agent');
+
+
+
+exports.getKnowledgeBaseInfo = index.getKnowledgeBaseInfo;
+exports.main = index.main;

package/dist/es/cli.js

@@ -0,0 +1,42 @@
+import { cac } from 'cac';
+import { m as main, l as logger } from './index-be42878f.js';
+import 'node:fs/promises';
+import 'node:path';
+import 'axios';
+import 'log4js';
+import 'progress';
+import 'rand-user-agent';
+import 'pull-md-img';
+
+var version = "1.0.0";
+
+const cli = cac('yuque-dl');
+cli
+    .command('<url>', 'request url')
+    .option('-d, --distDir <dir>', '下载的目录 eg: -d download', {
+    default: 'download',
+})
+    .action(async (url, options) => {
+    try {
+        await main(url, options);
+    }
+    catch (err) {
+        logger.error(err.message || 'unknown exception');
+    }
+});
+cli.help();
+cli.version(version);
+try {
+    cli.parse();
+}
+catch (err) {
+    logger.error(err.message || 'unknown exception');
+    process.exit(1);
+}
+// try {
+//   cli.parse(process.argv, { run: false })
+//   await cli.runMatchedCommand()
+// } catch (error) {
+//   logger.error(error.message || 'unknown exception')
+//   process.exit(1)
+// }

package/dist/es/index-be42878f.js

@@ -0,0 +1,401 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import axios from 'axios';
+import log4js from 'log4js';
+import progress from 'progress';
+import randUserAgentLib from 'rand-user-agent';
+import mdImg from 'pull-md-img';
+
+var config = {
+    knowledgeBaseReg: /decodeURIComponent\(\"(.+)\"\)\);/m,
+    dist: './download'
+};
+// data = re.findall(r"decodeURIComponent\(\"(.+)\"\)\);", docsdata.content.decode('utf-8'))
+// const config = {
+//   path: '',
+//   suffix: '',
+//   dist: './res/',
+//   imgDir:`./img/${Date.now()}`,
+//   // markdown img 正则 注意多行匹配
+//   mdImgReg: /!\[(.*?)\]\((.*?)\)/gm,
+//   isIgnoreConsole: false
+// }
+
+/**
+ * log 初始化
+ */
+const getLogger = () => {
+    log4js.configure({
+        appenders: {
+            // cheeseLog: { type: 'file', filename: 'cheese.log' },
+            cheese: {
+                type: 'console',
+                layout: {
+                    // type: 'messagePassThrough',
+                    type: 'pattern',
+                    // pattern: '%[%d{yyyy-MM-dd hh:mm:ss} [%p] %c -%] %m%n'
+                    pattern: '%[%c [%p]:%] %m%n'
+                }
+            }
+        },
+        categories: { default: { appenders: ['cheese'], level: 'trace' } }
+    });
+    return log4js.getLogger('yuque-dl');
+};
+var logger = getLogger();
+
+class Progress {
+    constructor(bookPath, total) {
+        this.bookPath = '';
+        this.processFilePath = '';
+        this.processInfo = [];
+        this.curr = 0;
+        this.total = 0;
+        this.isDownloadInterrupted = false;
+        this.bar = null;
+        this.completePromise = null;
+        this.bookPath = bookPath;
+        this.processFilePath = `${bookPath}/process.json`;
+        this.total = total;
+    }
+    async init() {
+        this.processInfo = await this.getProgress();
+        this.curr = this.processInfo.length;
+        let completeResolve;
+        this.completePromise = new Promise(resolve => {
+            completeResolve = resolve;
+        });
+        if (this.curr > 0 && this.curr !== this.total) {
+            this.isDownloadInterrupted = true;
+            logger.info('断点下载');
+        }
+        this.bar = new progress('[:bar] :rate/bps :percent :etas', {
+            // width: 20,
+            total: this.total,
+            curr: this.curr,
+            callback() {
+                completeResolve();
+            }
+        });
+    }
+    async getProgress() {
+        let processInfo = [];
+        try {
+            const processInfoStr = await fs.readFile(this.processFilePath, { encoding: 'utf8' });
+            processInfo = JSON.parse(processInfoStr);
+        }
+        catch (err) {
+            if (err && err.code === 'ENOENT') {
+                await fs.writeFile(this.processFilePath, JSON.stringify(processInfo), { encoding: 'utf8' });
+            }
+        }
+        return processInfo;
+    }
+    async updateProgress(progressItem) {
+        this.processInfo.push(progressItem);
+        await fs.writeFile(this.processFilePath, JSON.stringify(this.processInfo), { encoding: 'utf8' });
+        this.curr = this.curr + 1;
+        if (this.bar) {
+            this.bar?.tick();
+        }
+    }
+}
+
+function randUserAgent({ browser = 'chrome', os = 'mac os', device = 'desktop' }) {
+    device = device.toLowerCase();
+    browser = browser.toLowerCase();
+    os = os.toLowerCase();
+    let UA = randUserAgentLib(device, browser, os);
+    if (browser === 'chrome') {
+        while (UA.includes('Chrome-Lighthouse')
+            || UA.includes('Gener8')
+            || UA.includes('HeadlessChrome')
+            || UA.includes('SMTBot')) {
+            UA = randUserAgentLib(device, browser, os);
+        }
+    }
+    if (browser === 'safari') {
+        while (UA.includes('Applebot')) {
+            UA = randUserAgentLib(device, browser, os);
+        }
+    }
+    return UA;
+}
+
+// SUMMARY.md
+function getKnowledgeBaseInfo(url) {
+    return axios.get(url, {
+        headers: {
+            "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
+        }
+    }).then(({ data = '', status }) => {
+        if (status === 200)
+            return data;
+        return '';
+    }).then(html => {
+        const data = html.match(config.knowledgeBaseReg) || '';
+        if (!data[1])
+            return {};
+        const jsonData = JSON.parse(decodeURIComponent(data[1]));
+        const info = {
+            bookId: jsonData.book.id,
+            tocList: jsonData.book.toc || [],
+            bookName: jsonData.book.name || '',
+            bookDesc: jsonData.book.description || '',
+        };
+        return info;
+    });
+}
+async function main(url, options) {
+    const { bookId, tocList, bookName, bookDesc } = await getKnowledgeBaseInfo(url);
+    if (!bookId)
+        throw new Error('No found book id');
+    if (!tocList || tocList.length === 0)
+        throw new Error('No found toc list');
+    const bookPath = `${options.distDir}/${bookId}`;
+    await fs.mkdir(bookPath, { recursive: true });
+    const total = tocList.length;
+    const progress = new Progress(bookPath, total);
+    await progress.init();
+    if (progress.curr === total) {
+        logger.info('√ 已完成');
+        return;
+    }
+    const uuidMap = new Map();
+    // 下载中断 重新获取下载进度数据
+    if (progress.isDownloadInterrupted) {
+        progress.processInfo.forEach(item => {
+            uuidMap.set(item.toc.uuid, item);
+        });
+    }
+    const articleUrlPrefix = path.parse(url).dir;
+    // const url.split('/')
+    for (let i = 0; i < total; i++) {
+        const item = tocList[i];
+        if (typeof item.type !== 'string')
+            continue;
+        if (uuidMap.get(item.uuid))
+            continue;
+        console.log(item.uuid);
+        const itemType = item.type.toLocaleLowerCase();
+        const dirNameReg = /[\\\/:\*\?"<>\|\n\r]/g;
+        // 目录
+        if (itemType === 'title' || item['child_uuid'] !== '') {
+            let tempItem = item;
+            let pathTitleList = [];
+            let pathIdList = [];
+            while (tempItem) {
+                pathTitleList.unshift(tempItem.title.replace(dirNameReg, '_'));
+                pathIdList.unshift(tempItem.uuid);
+                if (uuidMap.get(tempItem['parent_uuid'])) {
+                    tempItem = uuidMap.get(tempItem['parent_uuid']).toc;
+                }
+                else {
+                    tempItem = undefined;
+                }
+            }
+            const progressItem = {
+                path: pathTitleList.join('/'),
+                pathTitleList,
+                pathIdList,
+                toc: item
+            };
+            await fs.mkdir(`${bookPath}/${pathTitleList.join('/')}`, { recursive: true });
+            uuidMap.set(item.uuid, progressItem);
+            progress.updateProgress(progressItem);
+        }
+        else if (item.url) {
+            let preItem = {
+                path: '',
+                pathTitleList: [],
+                pathIdList: []
+            };
+            if (uuidMap.get(item['parent_uuid'])) {
+                preItem = uuidMap.get(item['parent_uuid']);
+            }
+            const fileName = item.title.replace(dirNameReg, '_');
+            const pathTitleList = [...preItem.pathTitleList, `${fileName}.md`];
+            const pathIdList = [...preItem.pathIdList, item.uuid];
+            const progressItem = {
+                path: pathTitleList.join('/'),
+                pathTitleList,
+                pathIdList,
+                toc: item
+            };
+            const isSuccess = await downloadArticle({
+                bookId,
+                itemUrl: item.url,
+                savePath: `${bookPath}/${preItem.path}`,
+                saveFilePath: `${bookPath}/${progressItem.path}`,
+                uuid: item.uuid,
+                articleUrl: `${articleUrlPrefix}/${item.url}`,
+                articleTitle: item.title
+            });
+            if (isSuccess) {
+                uuidMap.set(item.uuid, progressItem);
+                progress.updateProgress(progressItem);
+            }
+        }
+        console.log('完成', item.uuid);
+    }
+    await progress.completePromise;
+    logger.info('生成目录 SUMMARY.md');
+    await genSummaryFile({
+        bookPath,
+        bookName,
+        bookDesc,
+        uuidMap
+    });
+    if (progress.curr === total) {
+        logger.info('√ 已完成');
+        return;
+    }
+}
+async function downloadArticle(params) {
+    const { bookId, itemUrl, savePath, saveFilePath, uuid, articleUrl, articleTitle } = params;
+    const apiUrl = `https://www.yuque.com/api/docs/${itemUrl}`;
+    const response = await axios.get(apiUrl, {
+        headers: {
+            "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
+        },
+        params: {
+            'book_id': bookId,
+            'merge_dynamic_data': false,
+            mode: 'markdown'
+        }
+    }).then(({ data, status }) => {
+        if (status === 200)
+            return data;
+        logger.error(`download article error: ${apiUrl} http status ${status}`);
+        return null;
+    });
+    if (!response || !response.data || !response.data.sourcecode)
+        return false;
+    let mdData = response.data.sourcecode;
+    try {
+        mdData = await mdImg.run(mdData, {
+            dist: savePath,
+            imgDir: `img/${uuid}`,
+            isIgnoreConsole: true
+        });
+    }
+    catch (err) {
+        logger.error(`download article img Error (api: ${apiUrl})`);
+        return false;
+    }
+    mdData = mdData.replace(/<br(\s?)\/>/gm, '\n');
+    if (articleTitle) {
+        mdData = `# ${articleTitle}\n<!--page header-->\n\n${mdData}\n\n`;
+    }
+    if (articleUrl) {
+        mdData += `<!--page footer-->\n- 原文: <${articleUrl}>`;
+    }
+    try {
+        await fs.writeFile(saveFilePath, mdData);
+        return true;
+    }
+    catch (e) {
+        logger.error(`download article write file Error (api: ${apiUrl})`);
+        return false;
+    }
+}
+async function genSummaryFile(params) {
+    const { bookName, bookDesc, bookPath, uuidMap } = params;
+    const header = `# ${bookName}\n\n > ${bookDesc}\n\n`;
+    let mdContent = header;
+    const summary = [];
+    uuidMap.forEach(processItem => {
+        const toc = processItem.toc;
+        const parentId = toc['parent_uuid'];
+        const findRes = findTree(summary, parentId);
+        if (toc.type.toLocaleLowerCase() === 'title' || toc['child_uuid'] !== '') {
+            if (typeof findRes !== 'boolean') {
+                if (!Array.isArray(findRes.children))
+                    findRes.children = [];
+                findRes.children.push({
+                    id: toc.uuid,
+                    type: 'title',
+                    level: findRes.level + 1,
+                    text: toc.title
+                });
+            }
+            else {
+                summary.push({
+                    id: toc.uuid,
+                    type: 'title',
+                    text: toc.title,
+                    level: 1
+                });
+            }
+        }
+        else {
+            if (typeof findRes !== 'boolean') {
+                if (!Array.isArray(findRes.children))
+                    findRes.children = [];
+                findRes.children.push({
+                    id: toc.uuid,
+                    type: 'link',
+                    level: findRes.level + 1,
+                    link: processItem.path,
+                    text: toc.title
+                });
+            }
+            else {
+                summary.push({
+                    id: toc.uuid,
+                    type: 'link',
+                    text: toc.title,
+                    link: processItem.path,
+                    level: 1
+                });
+            }
+        }
+    });
+    let summaryContent = genSummaryContent(summary, '');
+    mdContent += summaryContent;
+    try {
+        await fs.writeFile(`${bookPath}/SUMMARY.md`, mdContent);
+    }
+    catch (err) {
+        logger.error('Generate Summary Error');
+    }
+}
+function genSummaryContent(summary, summaryContent) {
+    for (let i = 0; i < summary.length; i++) {
+        const item = summary[i];
+        if (item.type === 'title') {
+            summaryContent += `\n${''.padStart(item.level + 1, '#')} ${item.text}\n\n`;
+        }
+        else if (item.type === 'link') {
+            const link = item.link ? item.link.replace(/\s/g, '%20') : item.link;
+            summaryContent += `- [${item.text}](${link})\n`;
+        }
+        if (Array.isArray(item.children)) {
+            summaryContent += genSummaryContent(item.children, '');
+        }
+    }
+    return summaryContent;
+}
+function findIdItem(node, id) {
+    if (node.id === id) {
+        return node;
+    }
+    else if (node.children) {
+        const findRes = findTree(node.children, id);
+        if (findRes)
+            return findRes;
+    }
+    return false;
+}
+function findTree(tree, id) {
+    if (!id)
+        return false;
+    for (let i = 0; i < tree.length; i++) {
+        const findRes = findIdItem(tree[i], id);
+        if (findRes)
+            return findRes;
+    }
+    return false;
+}
+
+export { getKnowledgeBaseInfo as g, logger as l, main as m };

package/dist/es/index.js

@@ -0,0 +1,8 @@
+import 'node:fs/promises';
+import 'node:path';
+import 'axios';
+export { g as getKnowledgeBaseInfo, m as main } from './index-be42878f.js';
+import 'pull-md-img';
+import 'log4js';
+import 'progress';
+import 'rand-user-agent';

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "yuque-dl",
+  "version": "1.0.0",
+  "description": "yuque 知识库下载",
+  "type": "module",
+  "bin": {
+    "yuque-dl": "bin/index.js"
+  },
+  "scripts": {
+    "dev": "pnpm run build:bundle -w",
+    "build": "run-s build:**",
+    "build:bundle": "rollup -c rollup.config.ts --configPlugin typescript",
+    "build:types": "tsc --emitDeclarationOnly --outDir types -p tsconfig.base.json",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/gxr404/yuque-dl.git"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/gxr404/yuque-dl/issues"
+  },
+  "homepage": "https://github.com/gxr404/yuque-dl#readme",
+  "devDependencies": {
+    "@rollup/plugin-json": "^6.0.0",
+    "@rollup/plugin-typescript": "^11.1.2",
+    "@types/node": "^20.5.0",
+    "@types/progress": "^2.0.5",
+    "@typescript-eslint/eslint-plugin": "^6.3.0",
+    "@typescript-eslint/parser": "^6.3.0",
+    "cac": "^6.7.14",
+    "jest": "^29.6.2",
+    "npm-run-all": "^4.1.5",
+    "rollup": "^3.28.0",
+    "tslib": "^2.6.1"
+  },
+  "main": "dist/es/index.js",
+  "types": "types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./types/index.d.ts",
+      "import": "./es/index.js",
+      "require": "./cjs/index.js"
+    }
+  },
+  "dependencies": {
+    "axios": "^1.4.0",
+    "log4js": "^6.9.1",
+    "progress": "^2.0.3",
+    "pull-md-img": "^0.0.18",
+    "rand-user-agent": "1.0.109"
+  },
+  "engines": {
+    "node": "^14.18.0 || >=16.0.0"
+  }
+}

package/rollup.config.ts

@@ -0,0 +1,26 @@
+import { defineConfig } from 'rollup'
+import typescript from '@rollup/plugin-typescript'
+import json from '@rollup/plugin-json'
+
+export default defineConfig({
+  input: {
+    index: 'src/index.ts',
+    cli: 'src/cli.ts'
+  },
+  output:[
+    {
+      format: "cjs",
+      dir: "dist/cjs"
+    },
+    {
+      format: "es",
+      dir: "dist/es",
+    },
+  ],
+  plugins: [
+    typescript(),
+    json()
+  ]
+})
+
+

package/tsconfig.base.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "strict": true,
+    "declaration": true,
+    "noImplicitOverride": true,
+    "noUnusedLocals": true,
+    "esModuleInterop": true,
+    "useUnknownInCatchVariables": false,
+    "resolveJsonModule": true,
+  },
+  "include": [
+    "src/**/*"
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "declaration": false
+  }
+}