yuque-dl 1.0.3 → 1.0.5

package/dist/es/index-5cda90ca.js

@@ -0,0 +1,405 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import axios from 'axios';
+import log4js from 'log4js';
+import progress from 'progress';
+import randUserAgentLib from 'rand-user-agent';
+import mdImg from 'pull-md-img';
+
+var config = {
+    knowledgeBaseReg: /decodeURIComponent\(\"(.+)\"\)\);/m,
+    dist: './download'
+};
+// data = re.findall(r"decodeURIComponent\(\"(.+)\"\)\);", docsdata.content.decode('utf-8'))
+// const config = {
+//   path: '',
+//   suffix: '',
+//   dist: './res/',
+//   imgDir:`./img/${Date.now()}`,
+//   // markdown img 正则 注意多行匹配
+//   mdImgReg: /!\[(.*?)\]\((.*?)\)/gm,
+//   isIgnoreConsole: false
+// }
+
+/**
+ * log 初始化
+ */
+const getLogger = () => {
+    log4js.configure({
+        appenders: {
+            // cheeseLog: { type: 'file', filename: 'cheese.log' },
+            cheese: {
+                type: 'console',
+                layout: {
+                    // type: 'messagePassThrough',
+                    type: 'pattern',
+                    // pattern: '%[%d{yyyy-MM-dd hh:mm:ss} [%p] %c -%] %m%n'
+                    pattern: '%[%c [%p]:%] %m%n'
+                }
+            }
+        },
+        categories: { default: { appenders: ['cheese'], level: 'trace' } }
+    });
+    return log4js.getLogger('yuque-dl');
+};
+var logger = getLogger();
+
+class Progress {
+    constructor(bookPath, total) {
+        this.bookPath = '';
+        this.processFilePath = '';
+        this.processInfo = [];
+        this.curr = 0;
+        this.total = 0;
+        this.isDownloadInterrupted = false;
+        this.bar = null;
+        this.completePromise = null;
+        this.bookPath = bookPath;
+        this.processFilePath = `${bookPath}/process.json`;
+        this.total = total;
+    }
+    async init() {
+        this.processInfo = await this.getProgress();
+        this.curr = this.processInfo.length;
+        let completeResolve;
+        this.completePromise = new Promise(resolve => {
+            completeResolve = resolve;
+        });
+        if (this.curr > 0 && this.curr !== this.total) {
+            this.isDownloadInterrupted = true;
+            logger.info('断点下载');
+        }
+        this.bar = new progress('[:bar] :rate/bps :percent :etas', {
+            // width: 20,
+            total: this.total,
+            curr: this.curr,
+            callback() {
+                completeResolve();
+            }
+        });
+    }
+    async getProgress() {
+        let processInfo = [];
+        try {
+            const processInfoStr = await fs.readFile(this.processFilePath, { encoding: 'utf8' });
+            processInfo = JSON.parse(processInfoStr);
+        }
+        catch (err) {
+            if (err && err.code === 'ENOENT') {
+                await fs.writeFile(this.processFilePath, JSON.stringify(processInfo), { encoding: 'utf8' });
+            }
+        }
+        return processInfo;
+    }
+    async updateProgress(progressItem) {
+        this.processInfo.push(progressItem);
+        await fs.writeFile(this.processFilePath, JSON.stringify(this.processInfo), { encoding: 'utf8' });
+        this.curr = this.curr + 1;
+        if (this.bar) {
+            this.bar?.tick();
+        }
+    }
+}
+
+function randUserAgent({ browser = 'chrome', os = 'mac os', device = 'desktop' }) {
+    device = device.toLowerCase();
+    browser = browser.toLowerCase();
+    os = os.toLowerCase();
+    let UA = randUserAgentLib(device, browser, os);
+    if (browser === 'chrome') {
+        while (UA.includes('Chrome-Lighthouse')
+            || UA.includes('Gener8')
+            || UA.includes('HeadlessChrome')
+            || UA.includes('SMTBot')) {
+            UA = randUserAgentLib(device, browser, os);
+        }
+    }
+    if (browser === 'safari') {
+        while (UA.includes('Applebot')) {
+            UA = randUserAgentLib(device, browser, os);
+        }
+    }
+    return UA;
+}
+
+// SUMMARY.md
+function getKnowledgeBaseInfo(url) {
+    return axios.get(url, {
+        headers: {
+            "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
+        }
+    }).then(({ data = '', status }) => {
+        if (status === 200)
+            return data;
+        return '';
+    }).then(html => {
+        const data = html.match(config.knowledgeBaseReg) || '';
+        if (!data[1])
+            return {};
+        const jsonData = JSON.parse(decodeURIComponent(data[1]));
+        const info = {
+            bookId: jsonData.book.id,
+            tocList: jsonData.book.toc || [],
+            bookName: jsonData.book.name || '',
+            bookDesc: jsonData.book.description || '',
+        };
+        return info;
+    });
+}
+async function main(url, options) {
+    const { bookId, tocList, bookName, bookDesc } = await getKnowledgeBaseInfo(url);
+    if (!bookId)
+        throw new Error('No found book id');
+    if (!tocList || tocList.length === 0)
+        throw new Error('No found toc list');
+    const bookPath = `${options.distDir}/${bookId}`;
+    await fs.mkdir(bookPath, { recursive: true });
+    const total = tocList.length;
+    const progress = new Progress(bookPath, total);
+    await progress.init();
+    if (progress.curr === total) {
+        logger.info('√ 已完成');
+        return;
+    }
+    const uuidMap = new Map();
+    // 下载中断 重新获取下载进度数据
+    if (progress.isDownloadInterrupted) {
+        progress.processInfo.forEach(item => {
+            uuidMap.set(item.toc.uuid, item);
+        });
+    }
+    const articleUrlPrefix = path.parse(url).dir;
+    let errArticleCount = 0;
+    let totalArticleCount = 0;
+    for (let i = 0; i < total; i++) {
+        const item = tocList[i];
+        if (typeof item.type !== 'string')
+            continue;
+        if (uuidMap.get(item.uuid))
+            continue;
+        const itemType = item.type.toLocaleLowerCase();
+        const dirNameReg = /[\\\/:\*\?"<>\|\n\r]/g;
+        // 目录
+        if (itemType === 'title' || item['child_uuid'] !== '') {
+            let tempItem = item;
+            let pathTitleList = [];
+            let pathIdList = [];
+            while (tempItem) {
+                pathTitleList.unshift(tempItem.title.replace(dirNameReg, '_'));
+                pathIdList.unshift(tempItem.uuid);
+                if (uuidMap.get(tempItem['parent_uuid'])) {
+                    tempItem = uuidMap.get(tempItem['parent_uuid']).toc;
+                }
+                else {
+                    tempItem = undefined;
+                }
+            }
+            const progressItem = {
+                path: pathTitleList.join('/'),
+                pathTitleList,
+                pathIdList,
+                toc: item
+            };
+            await fs.mkdir(`${bookPath}/${pathTitleList.join('/')}`, { recursive: true });
+            uuidMap.set(item.uuid, progressItem);
+            progress.updateProgress(progressItem);
+        }
+        else if (item.url) {
+            totalArticleCount += 1;
+            let preItem = {
+                path: '',
+                pathTitleList: [],
+                pathIdList: []
+            };
+            if (uuidMap.get(item['parent_uuid'])) {
+                preItem = uuidMap.get(item['parent_uuid']);
+            }
+            const fileName = item.title.replace(dirNameReg, '_');
+            const pathTitleList = [...preItem.pathTitleList, `${fileName}.md`];
+            const pathIdList = [...preItem.pathIdList, item.uuid];
+            const progressItem = {
+                path: pathTitleList.join('/'),
+                pathTitleList,
+                pathIdList,
+                toc: item
+            };
+            const isSuccess = await downloadArticle({
+                bookId,
+                itemUrl: item.url,
+                savePath: `${bookPath}/${preItem.path}`,
+                saveFilePath: `${bookPath}/${progressItem.path}`,
+                uuid: item.uuid,
+                articleUrl: `${articleUrlPrefix}/${item.url}`,
+                articleTitle: item.title
+            });
+            if (isSuccess) {
+                uuidMap.set(item.uuid, progressItem);
+                progress.updateProgress(progressItem);
+            }
+            else {
+                errArticleCount += 1;
+            }
+        }
+    }
+    await progress.completePromise;
+    logger.info(`总数${totalArticleCount}篇，✕ 失败${errArticleCount}篇`);
+    logger.info('生成目录 SUMMARY.md');
+    await genSummaryFile({
+        bookPath,
+        bookName,
+        bookDesc,
+        uuidMap
+    });
+    if (progress.curr === total) {
+        logger.info('√ 已完成');
+        return;
+    }
+}
+async function downloadArticle(params) {
+    const { bookId, itemUrl, savePath, saveFilePath, uuid, articleUrl, articleTitle } = params;
+    const apiUrl = `https://www.yuque.com/api/docs/${itemUrl}`;
+    const response = await axios.get(apiUrl, {
+        headers: {
+            "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
+        },
+        params: {
+            'book_id': bookId,
+            'merge_dynamic_data': false,
+            mode: 'markdown'
+        }
+    }).then(({ data, status }) => {
+        if (status === 200)
+            return data;
+        logger.error(`download article error: ${apiUrl} http status ${status}`);
+        return null;
+    });
+    if (!response || !response.data || !response.data.sourcecode)
+        return false;
+    let mdData = response.data.sourcecode;
+    try {
+        mdData = await mdImg.run(mdData, {
+            dist: savePath,
+            imgDir: `img/${uuid}`,
+            isIgnoreConsole: true
+        });
+    }
+    catch (err) {
+        logger.error(`download article img Error (api: ${apiUrl})`);
+        return false;
+    }
+    mdData = mdData.replace(/<br(\s?)\/>/gm, '\n');
+    if (articleTitle) {
+        mdData = `# ${articleTitle}\n<!--page header-->\n\n${mdData}\n\n`;
+    }
+    if (articleUrl) {
+        mdData += `<!--page footer-->\n- 原文: <${articleUrl}>`;
+    }
+    try {
+        await fs.writeFile(saveFilePath, mdData);
+        return true;
+    }
+    catch (e) {
+        logger.error(`download article write file Error (api: ${apiUrl})`);
+        return false;
+    }
+}
+async function genSummaryFile(params) {
+    const { bookName, bookDesc, bookPath, uuidMap } = params;
+    const header = `# ${bookName}\n\n > ${bookDesc}\n\n`;
+    let mdContent = header;
+    const summary = [];
+    uuidMap.forEach(processItem => {
+        const toc = processItem.toc;
+        const parentId = toc['parent_uuid'];
+        const findRes = findTree(summary, parentId);
+        if (toc.type.toLocaleLowerCase() === 'title' || toc['child_uuid'] !== '') {
+            if (typeof findRes !== 'boolean') {
+                if (!Array.isArray(findRes.children))
+                    findRes.children = [];
+                findRes.children.push({
+                    id: toc.uuid,
+                    type: 'title',
+                    level: findRes.level + 1,
+                    text: toc.title
+                });
+            }
+            else {
+                summary.push({
+                    id: toc.uuid,
+                    type: 'title',
+                    text: toc.title,
+                    level: 1
+                });
+            }
+        }
+        else {
+            if (typeof findRes !== 'boolean') {
+                if (!Array.isArray(findRes.children))
+                    findRes.children = [];
+                findRes.children.push({
+                    id: toc.uuid,
+                    type: 'link',
+                    level: findRes.level + 1,
+                    link: processItem.path,
+                    text: toc.title
+                });
+            }
+            else {
+                summary.push({
+                    id: toc.uuid,
+                    type: 'link',
+                    text: toc.title,
+                    link: processItem.path,
+                    level: 1
+                });
+            }
+        }
+    });
+    let summaryContent = genSummaryContent(summary, '');
+    mdContent += summaryContent;
+    try {
+        await fs.writeFile(`${bookPath}/SUMMARY.md`, mdContent);
+    }
+    catch (err) {
+        logger.error('Generate Summary Error');
+    }
+}
+function genSummaryContent(summary, summaryContent) {
+    for (let i = 0; i < summary.length; i++) {
+        const item = summary[i];
+        if (item.type === 'title') {
+            summaryContent += `\n${''.padStart(item.level + 1, '#')} ${item.text}\n\n`;
+        }
+        else if (item.type === 'link') {
+            const link = item.link ? item.link.replace(/\s/g, '%20') : item.link;
+            summaryContent += `- [${item.text}](${link})\n`;
+        }
+        if (Array.isArray(item.children)) {
+            summaryContent += genSummaryContent(item.children, '');
+        }
+    }
+    return summaryContent;
+}
+function findIdItem(node, id) {
+    if (node.id === id) {
+        return node;
+    }
+    else if (node.children) {
+        const findRes = findTree(node.children, id);
+        if (findRes)
+            return findRes;
+    }
+    return false;
+}
+function findTree(tree, id) {
+    if (!id)
+        return false;
+    for (let i = 0; i < tree.length; i++) {
+        const findRes = findIdItem(tree[i], id);
+        if (findRes)
+            return findRes;
+    }
+    return false;
+}
+
+export { getKnowledgeBaseInfo as g, logger as l, main as m };

package/dist/es/index.js

@@ -0,0 +1,8 @@
+import 'node:fs/promises';
+import 'node:path';
+import 'axios';
+export { g as getKnowledgeBaseInfo, m as main } from './index-5cda90ca.js';
+import 'pull-md-img';
+import 'log4js';
+import 'progress';
+import 'rand-user-agent';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yuque-dl",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "yuque 知识库下载",
   "type": "module",
   "bin": {
@@ -12,7 +12,7 @@
     "build:bundle": "rollup -c rollup.config.ts --configPlugin typescript",
     "build:types": "tsc --emitDeclarationOnly --outDir types -p tsconfig.base.json",
     "clean": "rm -rf dist types",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "publish": "np"
   },
   "repository": {
     "type": "git",
@@ -33,6 +33,7 @@
     "@typescript-eslint/eslint-plugin": "^6.3.0",
     "@typescript-eslint/parser": "^6.3.0",
     "jest": "^29.6.2",
+    "np": "^8.0.4",
     "npm-run-all": "^4.1.5",
     "rollup": "^3.28.0",
     "tslib": "^2.6.1"
@@ -48,11 +49,11 @@
   },
   "dependencies": {
     "axios": "^1.4.0",
+    "cac": "^6.7.14",
     "log4js": "^6.9.1",
     "progress": "^2.0.3",
-    "pull-md-img": "^0.0.19",
-    "rand-user-agent": "1.0.109",
-    "cac": "^6.7.14"
+    "pull-md-img": "^0.0.20",
+    "rand-user-agent": "1.0.109"
   },
   "engines": {
     "node": "^14.18.0 || >=16.0.0"

package/tsconfig.base.json

@@ -10,8 +10,12 @@
     "esModuleInterop": true,
     "useUnknownInCatchVariables": false,
     "resolveJsonModule": true,
+    "paths": {
+      "@/*": ["./*"],
+    }
   },
   "include": [
-    "src/**/*"
-  ]
+    "./src/**/*"
+  ],
+
 }

package/types/Process.d.ts

@@ -0,0 +1,23 @@
+import progress from 'progress';
+import type { KnowledgeBase } from './types/KnowledgeBaseResponse';
+export interface IProcessItem {
+    path: string;
+    toc: KnowledgeBase.Toc;
+    pathIdList: string[];
+    pathTitleList: string[];
+}
+export declare type IProcess = IProcessItem[];
+export default class Progress {
+    bookPath: string;
+    processFilePath: string;
+    processInfo: IProcess;
+    curr: number;
+    total: number;
+    isDownloadInterrupted: boolean;
+    bar: progress | null;
+    completePromise: Promise<void> | null;
+    constructor(bookPath: string, total: number);
+    init(): Promise<void>;
+    getProgress(): Promise<IProcess>;
+    updateProgress(progressItem: IProcessItem): Promise<void>;
+}

package/types/cli.d.ts

@@ -0,0 +1,3 @@
+export interface IOptions {
+    distDir: string;
+}

package/types/config.d.ts

@@ -0,0 +1,5 @@
+declare const _default: {
+    knowledgeBaseReg: RegExp;
+    dist: string;
+};
+export default _default;

package/types/index.d.ts

@@ -0,0 +1,11 @@
+import type { IOptions } from './cli';
+import type { KnowledgeBase } from './types/KnowledgeBaseResponse';
+interface IKnowledgeBaseInfo {
+    bookId?: number;
+    tocList?: KnowledgeBase.Toc[];
+    bookName?: string;
+    bookDesc?: string;
+}
+declare function getKnowledgeBaseInfo(url: string): Promise<IKnowledgeBaseInfo>;
+declare function main(url: string, options: IOptions): Promise<void>;
+export { getKnowledgeBaseInfo, main };

package/types/log.d.ts

@@ -0,0 +1,3 @@
+import log4js from 'log4js';
+declare const _default: log4js.Logger;
+export default _default;

package/types/types/ArticleResponse.d.ts

@@ -0,0 +1,42 @@
+export declare namespace ArticleResponse {
+    interface RootObject {
+        meta: Meta;
+        data: Data;
+    }
+    interface Data {
+        id: number;
+        space_id: number;
+        type: string;
+        sub_type?: any;
+        format: string;
+        title: string;
+        slug: string;
+        public: number;
+        status: number;
+        read_status: number;
+        created_at: string;
+        content_updated_at: string;
+        published_at: string;
+        first_published_at: string;
+        sourcecode: string;
+        last_editor?: any;
+        _serializer: string;
+    }
+    interface Meta {
+        abilities: Abilities;
+        latestReviewStatus: number;
+    }
+    interface Abilities {
+        create: boolean;
+        destroy: boolean;
+        update: boolean;
+        read: boolean;
+        export: boolean;
+        manage: boolean;
+        join: boolean;
+        share: boolean;
+        force_delete: boolean;
+        create_collaborator: boolean;
+        destroy_comment: boolean;
+    }
+}