yuque-dl 1.0.3 → 1.0.5

package/.np-config.js

@@ -0,0 +1,3 @@
+export default {
+  tests: false
+}

package/dist/cjs/cli.js

@@ -0,0 +1,46 @@
+'use strict';
+
+var node_fs = require('node:fs');
+var cac = require('cac');
+var index = require('./index-4969d150.js');
+require('node:fs/promises');
+require('node:path');
+require('axios');
+require('log4js');
+require('progress');
+require('rand-user-agent');
+require('pull-md-img');
+
+const cli = cac.cac('yuque-dl');
+// 不能直接使用 import {version} from '../package.json'
+// 否则declaration 生成的d.ts 会多一层src目录
+const { version } = JSON.parse(node_fs.readFileSync(new URL('../../package.json', (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (document.currentScript && document.currentScript.src || new URL('cli.js', document.baseURI).href)))).toString());
+cli
+    .command('<url>', 'request url')
+    .option('-d, --distDir <dir>', '下载的目录 eg: -d download', {
+    default: 'download',
+})
+    .action(async (url, options) => {
+    try {
+        await index.main(url, options);
+    }
+    catch (err) {
+        index.logger.error(err.message || 'unknown exception');
+    }
+});
+cli.help();
+cli.version(version);
+try {
+    cli.parse();
+}
+catch (err) {
+    index.logger.error(err.message || 'unknown exception');
+    process.exit(1);
+}
+// try {
+//   cli.parse(process.argv, { run: false })
+//   await cli.runMatchedCommand()
+// } catch (error) {
+//   logger.error(error.message || 'unknown exception')
+//   process.exit(1)
+// }

package/dist/cjs/index-4969d150.js

@@ -0,0 +1,409 @@
+'use strict';
+
+var fs = require('node:fs/promises');
+var path = require('node:path');
+var axios = require('axios');
+var log4js = require('log4js');
+var progress = require('progress');
+var randUserAgentLib = require('rand-user-agent');
+var mdImg = require('pull-md-img');
+
+var config = {
+    knowledgeBaseReg: /decodeURIComponent\(\"(.+)\"\)\);/m,
+    dist: './download'
+};
+// data = re.findall(r"decodeURIComponent\(\"(.+)\"\)\);", docsdata.content.decode('utf-8'))
+// const config = {
+//   path: '',
+//   suffix: '',
+//   dist: './res/',
+//   imgDir:`./img/${Date.now()}`,
+//   // markdown img 正则 注意多行匹配
+//   mdImgReg: /!\[(.*?)\]\((.*?)\)/gm,
+//   isIgnoreConsole: false
+// }
+
+/**
+ * log 初始化
+ */
+const getLogger = () => {
+    log4js.configure({
+        appenders: {
+            // cheeseLog: { type: 'file', filename: 'cheese.log' },
+            cheese: {
+                type: 'console',
+                layout: {
+                    // type: 'messagePassThrough',
+                    type: 'pattern',
+                    // pattern: '%[%d{yyyy-MM-dd hh:mm:ss} [%p] %c -%] %m%n'
+                    pattern: '%[%c [%p]:%] %m%n'
+                }
+            }
+        },
+        categories: { default: { appenders: ['cheese'], level: 'trace' } }
+    });
+    return log4js.getLogger('yuque-dl');
+};
+var logger = getLogger();
+
+class Progress {
+    constructor(bookPath, total) {
+        this.bookPath = '';
+        this.processFilePath = '';
+        this.processInfo = [];
+        this.curr = 0;
+        this.total = 0;
+        this.isDownloadInterrupted = false;
+        this.bar = null;
+        this.completePromise = null;
+        this.bookPath = bookPath;
+        this.processFilePath = `${bookPath}/process.json`;
+        this.total = total;
+    }
+    async init() {
+        this.processInfo = await this.getProgress();
+        this.curr = this.processInfo.length;
+        let completeResolve;
+        this.completePromise = new Promise(resolve => {
+            completeResolve = resolve;
+        });
+        if (this.curr > 0 && this.curr !== this.total) {
+            this.isDownloadInterrupted = true;
+            logger.info('断点下载');
+        }
+        this.bar = new progress('[:bar] :rate/bps :percent :etas', {
+            // width: 20,
+            total: this.total,
+            curr: this.curr,
+            callback() {
+                completeResolve();
+            }
+        });
+    }
+    async getProgress() {
+        let processInfo = [];
+        try {
+            const processInfoStr = await fs.readFile(this.processFilePath, { encoding: 'utf8' });
+            processInfo = JSON.parse(processInfoStr);
+        }
+        catch (err) {
+            if (err && err.code === 'ENOENT') {
+                await fs.writeFile(this.processFilePath, JSON.stringify(processInfo), { encoding: 'utf8' });
+            }
+        }
+        return processInfo;
+    }
+    async updateProgress(progressItem) {
+        this.processInfo.push(progressItem);
+        await fs.writeFile(this.processFilePath, JSON.stringify(this.processInfo), { encoding: 'utf8' });
+        this.curr = this.curr + 1;
+        if (this.bar) {
+            this.bar?.tick();
+        }
+    }
+}
+
+function randUserAgent({ browser = 'chrome', os = 'mac os', device = 'desktop' }) {
+    device = device.toLowerCase();
+    browser = browser.toLowerCase();
+    os = os.toLowerCase();
+    let UA = randUserAgentLib(device, browser, os);
+    if (browser === 'chrome') {
+        while (UA.includes('Chrome-Lighthouse')
+            || UA.includes('Gener8')
+            || UA.includes('HeadlessChrome')
+            || UA.includes('SMTBot')) {
+            UA = randUserAgentLib(device, browser, os);
+        }
+    }
+    if (browser === 'safari') {
+        while (UA.includes('Applebot')) {
+            UA = randUserAgentLib(device, browser, os);
+        }
+    }
+    return UA;
+}
+
+// SUMMARY.md
+function getKnowledgeBaseInfo(url) {
+    return axios.get(url, {
+        headers: {
+            "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
+        }
+    }).then(({ data = '', status }) => {
+        if (status === 200)
+            return data;
+        return '';
+    }).then(html => {
+        const data = html.match(config.knowledgeBaseReg) || '';
+        if (!data[1])
+            return {};
+        const jsonData = JSON.parse(decodeURIComponent(data[1]));
+        const info = {
+            bookId: jsonData.book.id,
+            tocList: jsonData.book.toc || [],
+            bookName: jsonData.book.name || '',
+            bookDesc: jsonData.book.description || '',
+        };
+        return info;
+    });
+}
+async function main(url, options) {
+    const { bookId, tocList, bookName, bookDesc } = await getKnowledgeBaseInfo(url);
+    if (!bookId)
+        throw new Error('No found book id');
+    if (!tocList || tocList.length === 0)
+        throw new Error('No found toc list');
+    const bookPath = `${options.distDir}/${bookId}`;
+    await fs.mkdir(bookPath, { recursive: true });
+    const total = tocList.length;
+    const progress = new Progress(bookPath, total);
+    await progress.init();
+    if (progress.curr === total) {
+        logger.info('√ 已完成');
+        return;
+    }
+    const uuidMap = new Map();
+    // 下载中断 重新获取下载进度数据
+    if (progress.isDownloadInterrupted) {
+        progress.processInfo.forEach(item => {
+            uuidMap.set(item.toc.uuid, item);
+        });
+    }
+    const articleUrlPrefix = path.parse(url).dir;
+    let errArticleCount = 0;
+    let totalArticleCount = 0;
+    for (let i = 0; i < total; i++) {
+        const item = tocList[i];
+        if (typeof item.type !== 'string')
+            continue;
+        if (uuidMap.get(item.uuid))
+            continue;
+        const itemType = item.type.toLocaleLowerCase();
+        const dirNameReg = /[\\\/:\*\?"<>\|\n\r]/g;
+        // 目录
+        if (itemType === 'title' || item['child_uuid'] !== '') {
+            let tempItem = item;
+            let pathTitleList = [];
+            let pathIdList = [];
+            while (tempItem) {
+                pathTitleList.unshift(tempItem.title.replace(dirNameReg, '_'));
+                pathIdList.unshift(tempItem.uuid);
+                if (uuidMap.get(tempItem['parent_uuid'])) {
+                    tempItem = uuidMap.get(tempItem['parent_uuid']).toc;
+                }
+                else {
+                    tempItem = undefined;
+                }
+            }
+            const progressItem = {
+                path: pathTitleList.join('/'),
+                pathTitleList,
+                pathIdList,
+                toc: item
+            };
+            await fs.mkdir(`${bookPath}/${pathTitleList.join('/')}`, { recursive: true });
+            uuidMap.set(item.uuid, progressItem);
+            progress.updateProgress(progressItem);
+        }
+        else if (item.url) {
+            totalArticleCount += 1;
+            let preItem = {
+                path: '',
+                pathTitleList: [],
+                pathIdList: []
+            };
+            if (uuidMap.get(item['parent_uuid'])) {
+                preItem = uuidMap.get(item['parent_uuid']);
+            }
+            const fileName = item.title.replace(dirNameReg, '_');
+            const pathTitleList = [...preItem.pathTitleList, `${fileName}.md`];
+            const pathIdList = [...preItem.pathIdList, item.uuid];
+            const progressItem = {
+                path: pathTitleList.join('/'),
+                pathTitleList,
+                pathIdList,
+                toc: item
+            };
+            const isSuccess = await downloadArticle({
+                bookId,
+                itemUrl: item.url,
+                savePath: `${bookPath}/${preItem.path}`,
+                saveFilePath: `${bookPath}/${progressItem.path}`,
+                uuid: item.uuid,
+                articleUrl: `${articleUrlPrefix}/${item.url}`,
+                articleTitle: item.title
+            });
+            if (isSuccess) {
+                uuidMap.set(item.uuid, progressItem);
+                progress.updateProgress(progressItem);
+            }
+            else {
+                errArticleCount += 1;
+            }
+        }
+    }
+    await progress.completePromise;
+    logger.info(`总数${totalArticleCount}篇，✕ 失败${errArticleCount}篇`);
+    logger.info('生成目录 SUMMARY.md');
+    await genSummaryFile({
+        bookPath,
+        bookName,
+        bookDesc,
+        uuidMap
+    });
+    if (progress.curr === total) {
+        logger.info('√ 已完成');
+        return;
+    }
+}
+async function downloadArticle(params) {
+    const { bookId, itemUrl, savePath, saveFilePath, uuid, articleUrl, articleTitle } = params;
+    const apiUrl = `https://www.yuque.com/api/docs/${itemUrl}`;
+    const response = await axios.get(apiUrl, {
+        headers: {
+            "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
+        },
+        params: {
+            'book_id': bookId,
+            'merge_dynamic_data': false,
+            mode: 'markdown'
+        }
+    }).then(({ data, status }) => {
+        if (status === 200)
+            return data;
+        logger.error(`download article error: ${apiUrl} http status ${status}`);
+        return null;
+    });
+    if (!response || !response.data || !response.data.sourcecode)
+        return false;
+    let mdData = response.data.sourcecode;
+    try {
+        mdData = await mdImg.run(mdData, {
+            dist: savePath,
+            imgDir: `img/${uuid}`,
+            isIgnoreConsole: true
+        });
+    }
+    catch (err) {
+        logger.error(`download article img Error (api: ${apiUrl})`);
+        return false;
+    }
+    mdData = mdData.replace(/<br(\s?)\/>/gm, '\n');
+    if (articleTitle) {
+        mdData = `# ${articleTitle}\n<!--page header-->\n\n${mdData}\n\n`;
+    }
+    if (articleUrl) {
+        mdData += `<!--page footer-->\n- 原文: <${articleUrl}>`;
+    }
+    try {
+        await fs.writeFile(saveFilePath, mdData);
+        return true;
+    }
+    catch (e) {
+        logger.error(`download article write file Error (api: ${apiUrl})`);
+        return false;
+    }
+}
+async function genSummaryFile(params) {
+    const { bookName, bookDesc, bookPath, uuidMap } = params;
+    const header = `# ${bookName}\n\n > ${bookDesc}\n\n`;
+    let mdContent = header;
+    const summary = [];
+    uuidMap.forEach(processItem => {
+        const toc = processItem.toc;
+        const parentId = toc['parent_uuid'];
+        const findRes = findTree(summary, parentId);
+        if (toc.type.toLocaleLowerCase() === 'title' || toc['child_uuid'] !== '') {
+            if (typeof findRes !== 'boolean') {
+                if (!Array.isArray(findRes.children))
+                    findRes.children = [];
+                findRes.children.push({
+                    id: toc.uuid,
+                    type: 'title',
+                    level: findRes.level + 1,
+                    text: toc.title
+                });
+            }
+            else {
+                summary.push({
+                    id: toc.uuid,
+                    type: 'title',
+                    text: toc.title,
+                    level: 1
+                });
+            }
+        }
+        else {
+            if (typeof findRes !== 'boolean') {
+                if (!Array.isArray(findRes.children))
+                    findRes.children = [];
+                findRes.children.push({
+                    id: toc.uuid,
+                    type: 'link',
+                    level: findRes.level + 1,
+                    link: processItem.path,
+                    text: toc.title
+                });
+            }
+            else {
+                summary.push({
+                    id: toc.uuid,
+                    type: 'link',
+                    text: toc.title,
+                    link: processItem.path,
+                    level: 1
+                });
+            }
+        }
+    });
+    let summaryContent = genSummaryContent(summary, '');
+    mdContent += summaryContent;
+    try {
+        await fs.writeFile(`${bookPath}/SUMMARY.md`, mdContent);
+    }
+    catch (err) {
+        logger.error('Generate Summary Error');
+    }
+}
+function genSummaryContent(summary, summaryContent) {
+    for (let i = 0; i < summary.length; i++) {
+        const item = summary[i];
+        if (item.type === 'title') {
+            summaryContent += `\n${''.padStart(item.level + 1, '#')} ${item.text}\n\n`;
+        }
+        else if (item.type === 'link') {
+            const link = item.link ? item.link.replace(/\s/g, '%20') : item.link;
+            summaryContent += `- [${item.text}](${link})\n`;
+        }
+        if (Array.isArray(item.children)) {
+            summaryContent += genSummaryContent(item.children, '');
+        }
+    }
+    return summaryContent;
+}
+function findIdItem(node, id) {
+    if (node.id === id) {
+        return node;
+    }
+    else if (node.children) {
+        const findRes = findTree(node.children, id);
+        if (findRes)
+            return findRes;
+    }
+    return false;
+}
+function findTree(tree, id) {
+    if (!id)
+        return false;
+    for (let i = 0; i < tree.length; i++) {
+        const findRes = findIdItem(tree[i], id);
+        if (findRes)
+            return findRes;
+    }
+    return false;
+}
+
+exports.getKnowledgeBaseInfo = getKnowledgeBaseInfo;
+exports.logger = logger;
+exports.main = main;

package/dist/cjs/index.js

@@ -0,0 +1,15 @@
+'use strict';
+
+require('node:fs/promises');
+require('node:path');
+require('axios');
+var index = require('./index-4969d150.js');
+require('pull-md-img');
+require('log4js');
+require('progress');
+require('rand-user-agent');
+
+
+
+exports.getKnowledgeBaseInfo = index.getKnowledgeBaseInfo;
+exports.main = index.main;

package/dist/es/cli.js

@@ -0,0 +1,44 @@
+import { readFileSync } from 'node:fs';
+import { cac } from 'cac';
+import { m as main, l as logger } from './index-5cda90ca.js';
+import 'node:fs/promises';
+import 'node:path';
+import 'axios';
+import 'log4js';
+import 'progress';
+import 'rand-user-agent';
+import 'pull-md-img';
+
+const cli = cac('yuque-dl');
+// 不能直接使用 import {version} from '../package.json'
+// 否则declaration 生成的d.ts 会多一层src目录
+const { version } = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url)).toString());
+cli
+    .command('<url>', 'request url')
+    .option('-d, --distDir <dir>', '下载的目录 eg: -d download', {
+    default: 'download',
+})
+    .action(async (url, options) => {
+    try {
+        await main(url, options);
+    }
+    catch (err) {
+        logger.error(err.message || 'unknown exception');
+    }
+});
+cli.help();
+cli.version(version);
+try {
+    cli.parse();
+}
+catch (err) {
+    logger.error(err.message || 'unknown exception');
+    process.exit(1);
+}
+// try {
+//   cli.parse(process.argv, { run: false })
+//   await cli.runMatchedCommand()
+// } catch (error) {
+//   logger.error(error.message || 'unknown exception')
+//   process.exit(1)
+// }