yuque-dl 1.0.9 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierrc.js +1 -1
- package/dist/cjs/cli.js +1 -46
- package/dist/cjs/index-b758b4ba.js +1 -0
- package/dist/cjs/index.js +1 -15
- package/dist/es/cli.js +1 -44
- package/dist/es/index-7580521b.js +1 -0
- package/dist/es/index.js +1 -8
- package/package.json +11 -5
- package/rollup.config.ts +2 -2
- package/types/{Process.d.ts → ProgressBar.d.ts} +7 -7
- package/.editorconfig +0 -16
- package/.eslintignore +0 -5
- package/.eslintrc.js +0 -21
- package/.np-config.js +0 -4
- package/dist/cjs/index-4969d150.js +0 -409
- package/dist/es/index-5cda90ca.js +0 -405
package/.prettierrc.js
CHANGED
package/dist/cjs/cli.js
CHANGED
|
@@ -1,46 +1 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
var node_fs = require('node:fs');
|
|
4
|
-
var cac = require('cac');
|
|
5
|
-
var index = require('./index-4969d150.js');
|
|
6
|
-
require('node:fs/promises');
|
|
7
|
-
require('node:path');
|
|
8
|
-
require('axios');
|
|
9
|
-
require('log4js');
|
|
10
|
-
require('progress');
|
|
11
|
-
require('rand-user-agent');
|
|
12
|
-
require('pull-md-img');
|
|
13
|
-
|
|
14
|
-
const cli = cac.cac('yuque-dl');
|
|
15
|
-
// 不能直接使用 import {version} from '../package.json'
|
|
16
|
-
// 否则declaration 生成的d.ts 会多一层src目录
|
|
17
|
-
const { version } = JSON.parse(node_fs.readFileSync(new URL('../../package.json', (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (document.currentScript && document.currentScript.src || new URL('cli.js', document.baseURI).href)))).toString());
|
|
18
|
-
cli
|
|
19
|
-
.command('<url>', 'request url')
|
|
20
|
-
.option('-d, --distDir <dir>', '下载的目录 eg: -d download', {
|
|
21
|
-
default: 'download',
|
|
22
|
-
})
|
|
23
|
-
.action(async (url, options) => {
|
|
24
|
-
try {
|
|
25
|
-
await index.main(url, options);
|
|
26
|
-
}
|
|
27
|
-
catch (err) {
|
|
28
|
-
index.logger.error(err.message || 'unknown exception');
|
|
29
|
-
}
|
|
30
|
-
});
|
|
31
|
-
cli.help();
|
|
32
|
-
cli.version(version);
|
|
33
|
-
try {
|
|
34
|
-
cli.parse();
|
|
35
|
-
}
|
|
36
|
-
catch (err) {
|
|
37
|
-
index.logger.error(err.message || 'unknown exception');
|
|
38
|
-
process.exit(1);
|
|
39
|
-
}
|
|
40
|
-
// try {
|
|
41
|
-
// cli.parse(process.argv, { run: false })
|
|
42
|
-
// await cli.runMatchedCommand()
|
|
43
|
-
// } catch (error) {
|
|
44
|
-
// logger.error(error.message || 'unknown exception')
|
|
45
|
-
// process.exit(1)
|
|
46
|
-
// }
|
|
1
|
+
"use strict";var e=require("node:fs"),r=require("cac"),n=require("./index-b758b4ba.js");require("node:fs/promises"),require("node:path"),require("axios"),require("log4js"),require("progress"),require("rand-user-agent"),require("pull-md-img");const i=r.cac("yuque-dl"),{version:o}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:document.currentScript&&document.currentScript.src||new URL("cli.js",document.baseURI).href)).toString());i.command("<url>","request url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).action((async(e,r)=>{try{await n.main(e,r)}catch(e){n.logger.error(e.message||"unknown exception")}})),i.help(),i.version(o);try{i.parse()}catch(e){n.logger.error(e.message||"unknown exception"),process.exit(1)}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var e=require("node:fs/promises"),t=require("node:path"),r=require("axios"),i=require("log4js"),o=require("progress"),s=require("rand-user-agent"),n=require("pull-md-img"),a={knowledgeBaseReg:/decodeURIComponent\(\"(.+)\"\)\);/m,dist:"./download"};var l=(i.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),i.getLogger("yuque-dl"));class c{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){let e;this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.completePromise=new Promise((t=>{e=t})),this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,l.info("断点下载")),this.bar=new o("[:bar] :rate/bps :percent :etas",{total:this.total,curr:this.curr,callback(){e()}})}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t){this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"}),this.curr=this.curr+1,this.bar&&this.bar?.tick()}}function u({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let i=s(r,e,t);if("chrome"===e)for(;i.includes("Chrome-Lighthouse")||i.includes("Gener8")||i.includes("HeadlessChrome")||i.includes("SMTBot");)i=s(r,e,t);if("safari"===e)for(;i.includes("Applebot");)i=s(r,e,t);return i}function d(e){return r.get(e,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})}}).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=e.match(a.knowledgeBaseReg)||"";if(!t[1])return{};const r=JSON.parse(decodeURIComponent(t[1]));return{bookId:r.book.id,tocList:r.book.toc||[],bookName:r.book.name||"",bookDesc:r.book.description||""}}))}async function h(t){const{bookId:i,itemUrl:o,savePath:s,saveFilePath:a,uuid:c,articleUrl:d,articleTitle:h}=t,p=`https://www.yuque.com/api/docs/${o}`,g=await r.get(p,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})},params:{book_id:i,merge_dynamic_data:!1,mode:"markdown"}}).then((({data:e,status:t})=>200===t?e:(l.error(`download article error: ${p} http status ${t}`),null)));if(!g||!g.data||!g.data.sourcecode)return!1;let f=g.data.sourcecode;try{f=await n.run(f,{dist:s,imgDir:`img/${c}`,isIgnoreConsole:!0})}catch(e){return l.error(`download article img Error (api: ${p})`),!1}f=f.replace(/<br(\s?)\/>/gm,"\n"),h&&(f=`# ${h}\n\x3c!--page header--\x3e\n\n${f}\n\n`),d&&(f+=`\x3c!--page footer--\x3e\n- 原文: <${d}>`);try{return await e.writeFile(a,f),!0}catch(e){return l.error(`download article write file Error (api: ${p})`),!1}}function p(e,t){for(let r=0;r<e.length;r++){const i=e[r];if("title"===i.type)t+=`\n${"".padStart(i.level+1,"#")} ${i.text}\n\n`;else if("link"===i.type){const e=i.link?i.link.replace(/\s/g,"%20"):i.link;t+=`- [${i.text}](${e})\n`}Array.isArray(i.children)&&(t+=p(i.children,""))}return t}function g(e,t){if(e.id===t)return e;if(e.children){const r=f(e.children,t);if(r)return r}return!1}function f(e,t){if(!t)return!1;for(let r=0;r<e.length;r++){const i=g(e[r],t);if(i)return i}return!1}exports.getKnowledgeBaseInfo=d,exports.logger=l,exports.main=async function(r,i){const{bookId:o,tocList:s,bookName:n,bookDesc:a}=await d(r);if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const u=`${i.distDir}/${o}`;await e.mkdir(u,{recursive:!0});const g=s.length,w=new c(u,g);if(await w.init(),w.curr===g)return void l.info("√ 已完成");const m=new Map;w.isDownloadInterrupted&&w.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const k=t.parse(r).dir;let y=0,b=0;for(let t=0;t<g;t++){const r=s[t];if("string"!=typeof r.type)continue;if(m.get(r.uuid))continue;const i=/[\\\/:\*\?"<>\|\n\r]/g;if("title"===r.type.toLocaleLowerCase()||""!==r.child_uuid){let t=r,o=[],s=[];for(;t;)o.unshift(t.title.replace(i,"_")),s.unshift(t.uuid),t=m.get(t.parent_uuid)?m.get(t.parent_uuid).toc:void 0;const n={path:o.join("/"),pathTitleList:o,pathIdList:s,toc:r};await e.mkdir(`${u}/${o.join("/")}`,{recursive:!0}),m.set(r.uuid,n),w.updateProgress(n)}else if(r.url){b+=1;let e={path:"",pathTitleList:[],pathIdList:[]};m.get(r.parent_uuid)&&(e=m.get(r.parent_uuid));const t=r.title.replace(i,"_"),s=[...e.pathTitleList,`${t}.md`],n=[...e.pathIdList,r.uuid],a={path:s.join("/"),pathTitleList:s,pathIdList:n,toc:r};await h({bookId:o,itemUrl:r.url,savePath:`${u}/${e.path}`,saveFilePath:`${u}/${a.path}`,uuid:r.uuid,articleUrl:`${k}/${r.url}`,articleTitle:r.title})?(m.set(r.uuid,a),w.updateProgress(a)):y+=1}}await w.completePromise,l.info(`本次执行总数${b}篇,✕ 失败${y}篇`),l.info("生成目录 SUMMARY.md"),await async function(t){const{bookName:r,bookDesc:i,bookPath:o,uuidMap:s}=t,n=`# ${r}\n\n > ${i}\n\n`;let a=n;const c=[];s.forEach((e=>{const t=e.toc,r=t.parent_uuid,i=f(c,r);"title"===t.type.toLocaleLowerCase()||""!==t.child_uuid?"boolean"!=typeof i?(Array.isArray(i.children)||(i.children=[]),i.children.push({id:t.uuid,type:"title",level:i.level+1,text:t.title})):c.push({id:t.uuid,type:"title",text:t.title,level:1}):"boolean"!=typeof i?(Array.isArray(i.children)||(i.children=[]),i.children.push({id:t.uuid,type:"link",level:i.level+1,link:e.path,text:t.title})):c.push({id:t.uuid,type:"link",text:t.title,link:e.path,level:1})}));let u=p(c,"");a+=u;try{await e.writeFile(`${o}/SUMMARY.md`,a)}catch(e){l.error("Generate Summary Error")}}({bookPath:u,bookName:n,bookDesc:a,uuidMap:m}),w.curr!==g||l.info("√ 已完成")};
|
package/dist/cjs/index.js
CHANGED
|
@@ -1,15 +1 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require('node:fs/promises');
|
|
4
|
-
require('node:path');
|
|
5
|
-
require('axios');
|
|
6
|
-
var index = require('./index-4969d150.js');
|
|
7
|
-
require('pull-md-img');
|
|
8
|
-
require('log4js');
|
|
9
|
-
require('progress');
|
|
10
|
-
require('rand-user-agent');
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
exports.getKnowledgeBaseInfo = index.getKnowledgeBaseInfo;
|
|
15
|
-
exports.main = index.main;
|
|
1
|
+
"use strict";require("node:fs/promises"),require("node:path"),require("axios");var e=require("./index-b758b4ba.js");require("pull-md-img"),require("log4js"),require("progress"),require("rand-user-agent"),exports.getKnowledgeBaseInfo=e.getKnowledgeBaseInfo,exports.main=e.main;
|
package/dist/es/cli.js
CHANGED
|
@@ -1,44 +1 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { cac } from 'cac';
|
|
3
|
-
import { m as main, l as logger } from './index-5cda90ca.js';
|
|
4
|
-
import 'node:fs/promises';
|
|
5
|
-
import 'node:path';
|
|
6
|
-
import 'axios';
|
|
7
|
-
import 'log4js';
|
|
8
|
-
import 'progress';
|
|
9
|
-
import 'rand-user-agent';
|
|
10
|
-
import 'pull-md-img';
|
|
11
|
-
|
|
12
|
-
const cli = cac('yuque-dl');
|
|
13
|
-
// 不能直接使用 import {version} from '../package.json'
|
|
14
|
-
// 否则declaration 生成的d.ts 会多一层src目录
|
|
15
|
-
const { version } = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url)).toString());
|
|
16
|
-
cli
|
|
17
|
-
.command('<url>', 'request url')
|
|
18
|
-
.option('-d, --distDir <dir>', '下载的目录 eg: -d download', {
|
|
19
|
-
default: 'download',
|
|
20
|
-
})
|
|
21
|
-
.action(async (url, options) => {
|
|
22
|
-
try {
|
|
23
|
-
await main(url, options);
|
|
24
|
-
}
|
|
25
|
-
catch (err) {
|
|
26
|
-
logger.error(err.message || 'unknown exception');
|
|
27
|
-
}
|
|
28
|
-
});
|
|
29
|
-
cli.help();
|
|
30
|
-
cli.version(version);
|
|
31
|
-
try {
|
|
32
|
-
cli.parse();
|
|
33
|
-
}
|
|
34
|
-
catch (err) {
|
|
35
|
-
logger.error(err.message || 'unknown exception');
|
|
36
|
-
process.exit(1);
|
|
37
|
-
}
|
|
38
|
-
// try {
|
|
39
|
-
// cli.parse(process.argv, { run: false })
|
|
40
|
-
// await cli.runMatchedCommand()
|
|
41
|
-
// } catch (error) {
|
|
42
|
-
// logger.error(error.message || 'unknown exception')
|
|
43
|
-
// process.exit(1)
|
|
44
|
-
// }
|
|
1
|
+
import{readFileSync as o}from"node:fs";import{cac as r}from"cac";import{m as e,l as t}from"./index-7580521b.js";import"node:fs/promises";import"node:path";import"axios";import"log4js";import"progress";import"rand-user-agent";import"pull-md-img";const i=r("yuque-dl"),{version:s}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());i.command("<url>","request url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).action((async(o,r)=>{try{await e(o,r)}catch(o){t.error(o.message||"unknown exception")}})),i.help(),i.version(s);try{i.parse()}catch(o){t.error(o.message||"unknown exception"),process.exit(1)}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import t from"node:fs/promises";import e from"node:path";import r from"axios";import i from"log4js";import o from"progress";import s from"rand-user-agent";import a from"pull-md-img";var n={knowledgeBaseReg:/decodeURIComponent\(\"(.+)\"\)\);/m,dist:"./download"};var l=(i.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),i.getLogger("yuque-dl"));class c{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){let t;this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.completePromise=new Promise((e=>{t=e})),this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,l.info("断点下载")),this.bar=new o("[:bar] :rate/bps :percent :etas",{total:this.total,curr:this.curr,callback(){t()}})}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e){this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"}),this.curr=this.curr+1,this.bar&&this.bar?.tick()}}function u({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let i=s(r,t,e);if("chrome"===t)for(;i.includes("Chrome-Lighthouse")||i.includes("Gener8")||i.includes("HeadlessChrome")||i.includes("SMTBot");)i=s(r,t,e);if("safari"===t)for(;i.includes("Applebot");)i=s(r,t,e);return i}function d(t){return r.get(t,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})}}).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=t.match(n.knowledgeBaseReg)||"";if(!e[1])return{};const r=JSON.parse(decodeURIComponent(e[1]));return{bookId:r.book.id,tocList:r.book.toc||[],bookName:r.book.name||"",bookDesc:r.book.description||""}}))}async function h(r,i){const{bookId:o,tocList:s,bookName:a,bookDesc:n}=await d(r);if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const u=`${i.distDir}/${o}`;await t.mkdir(u,{recursive:!0});const h=s.length,g=new c(u,h);if(await g.init(),g.curr===h)return void l.info("√ 已完成");const w=new Map;g.isDownloadInterrupted&&g.progressInfo.forEach((t=>{w.set(t.toc.uuid,t)}));const k=e.parse(r).dir;let y=0,b=0;for(let e=0;e<h;e++){const r=s[e];if("string"!=typeof r.type)continue;if(w.get(r.uuid))continue;const i=/[\\\/:\*\?"<>\|\n\r]/g;if("title"===r.type.toLocaleLowerCase()||""!==r.child_uuid){let e=r,o=[],s=[];for(;e;)o.unshift(e.title.replace(i,"_")),s.unshift(e.uuid),e=w.get(e.parent_uuid)?w.get(e.parent_uuid).toc:void 0;const a={path:o.join("/"),pathTitleList:o,pathIdList:s,toc:r};await t.mkdir(`${u}/${o.join("/")}`,{recursive:!0}),w.set(r.uuid,a),g.updateProgress(a)}else if(r.url){b+=1;let t={path:"",pathTitleList:[],pathIdList:[]};w.get(r.parent_uuid)&&(t=w.get(r.parent_uuid));const e=r.title.replace(i,"_"),s=[...t.pathTitleList,`${e}.md`],a=[...t.pathIdList,r.uuid],n={path:s.join("/"),pathTitleList:s,pathIdList:a,toc:r};await p({bookId:o,itemUrl:r.url,savePath:`${u}/${t.path}`,saveFilePath:`${u}/${n.path}`,uuid:r.uuid,articleUrl:`${k}/${r.url}`,articleTitle:r.title})?(w.set(r.uuid,n),g.updateProgress(n)):y+=1}}await g.completePromise,l.info(`本次执行总数${b}篇,✕ 失败${y}篇`),l.info("生成目录 SUMMARY.md"),await async function(e){const{bookName:r,bookDesc:i,bookPath:o,uuidMap:s}=e,a=`# ${r}\n\n > ${i}\n\n`;let n=a;const c=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,i=m(c,r);"title"===e.type.toLocaleLowerCase()||""!==e.child_uuid?"boolean"!=typeof i?(Array.isArray(i.children)||(i.children=[]),i.children.push({id:e.uuid,type:"title",level:i.level+1,text:e.title})):c.push({id:e.uuid,type:"title",text:e.title,level:1}):"boolean"!=typeof i?(Array.isArray(i.children)||(i.children=[]),i.children.push({id:e.uuid,type:"link",level:i.level+1,link:t.path,text:e.title})):c.push({id:e.uuid,type:"link",text:e.title,link:t.path,level:1})}));let u=f(c,"");n+=u;try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){l.error("Generate Summary Error")}}({bookPath:u,bookName:a,bookDesc:n,uuidMap:w}),g.curr!==h||l.info("√ 已完成")}async function p(e){const{bookId:i,itemUrl:o,savePath:s,saveFilePath:n,uuid:c,articleUrl:d,articleTitle:h}=e,p=`https://www.yuque.com/api/docs/${o}`,f=await r.get(p,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})},params:{book_id:i,merge_dynamic_data:!1,mode:"markdown"}}).then((({data:t,status:e})=>200===e?t:(l.error(`download article error: ${p} http status ${e}`),null)));if(!f||!f.data||!f.data.sourcecode)return!1;let g=f.data.sourcecode;try{g=await a.run(g,{dist:s,imgDir:`img/${c}`,isIgnoreConsole:!0})}catch(t){return l.error(`download article img Error (api: ${p})`),!1}g=g.replace(/<br(\s?)\/>/gm,"\n"),h&&(g=`# ${h}\n\x3c!--page header--\x3e\n\n${g}\n\n`),d&&(g+=`\x3c!--page footer--\x3e\n- 原文: <${d}>`);try{return await t.writeFile(n,g),!0}catch(t){return l.error(`download article write file Error (api: ${p})`),!1}}function f(t,e){for(let r=0;r<t.length;r++){const i=t[r];if("title"===i.type)e+=`\n${"".padStart(i.level+1,"#")} ${i.text}\n\n`;else if("link"===i.type){const t=i.link?i.link.replace(/\s/g,"%20"):i.link;e+=`- [${i.text}](${t})\n`}Array.isArray(i.children)&&(e+=f(i.children,""))}return e}function g(t,e){if(t.id===e)return t;if(t.children){const r=m(t.children,e);if(r)return r}return!1}function m(t,e){if(!e)return!1;for(let r=0;r<t.length;r++){const i=g(t[r],e);if(i)return i}return!1}export{d as g,l,h as m};
|
package/dist/es/index.js
CHANGED
|
@@ -1,8 +1 @@
|
|
|
1
|
-
import
|
|
2
|
-
import 'node:path';
|
|
3
|
-
import 'axios';
|
|
4
|
-
export { g as getKnowledgeBaseInfo, m as main } from './index-5cda90ca.js';
|
|
5
|
-
import 'pull-md-img';
|
|
6
|
-
import 'log4js';
|
|
7
|
-
import 'progress';
|
|
8
|
-
import 'rand-user-agent';
|
|
1
|
+
import"node:fs/promises";import"node:path";import"axios";export{g as getKnowledgeBaseInfo,m as main}from"./index-7580521b.js";import"pull-md-img";import"log4js";import"progress";import"rand-user-agent";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "yuque-dl",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.10",
|
|
4
4
|
"description": "yuque 知识库下载",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -12,21 +12,27 @@
|
|
|
12
12
|
"build:bundle": "rollup -c rollup.config.ts --configPlugin typescript",
|
|
13
13
|
"build:types": "tsc --emitDeclarationOnly --outDir types -p tsconfig.base.json",
|
|
14
14
|
"clean": "rm -rf dist types",
|
|
15
|
-
"
|
|
15
|
+
"np": "np",
|
|
16
|
+
"release": "run-s clean build np"
|
|
16
17
|
},
|
|
17
18
|
"repository": {
|
|
18
19
|
"type": "git",
|
|
19
20
|
"url": "git+https://github.com/gxr404/yuque-dl.git"
|
|
20
21
|
},
|
|
21
|
-
"keywords": [
|
|
22
|
-
|
|
22
|
+
"keywords": [
|
|
23
|
+
"yuque",
|
|
24
|
+
"nodejs",
|
|
25
|
+
"download",
|
|
26
|
+
"yuque-dl"
|
|
27
|
+
],
|
|
28
|
+
"author": "gxr404",
|
|
23
29
|
"license": "ISC",
|
|
24
30
|
"bugs": {
|
|
25
31
|
"url": "https://github.com/gxr404/yuque-dl/issues"
|
|
26
32
|
},
|
|
27
33
|
"homepage": "https://github.com/gxr404/yuque-dl#readme",
|
|
28
34
|
"devDependencies": {
|
|
29
|
-
"@rollup/plugin-
|
|
35
|
+
"@rollup/plugin-terser": "^0.4.3",
|
|
30
36
|
"@rollup/plugin-typescript": "^11.1.2",
|
|
31
37
|
"@types/node": "^20.5.0",
|
|
32
38
|
"@types/progress": "^2.0.5",
|
package/rollup.config.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { defineConfig } from 'rollup'
|
|
2
2
|
import typescript from '@rollup/plugin-typescript'
|
|
3
|
-
import
|
|
3
|
+
import terser from '@rollup/plugin-terser'
|
|
4
4
|
|
|
5
5
|
export default defineConfig({
|
|
6
6
|
input: {
|
|
@@ -19,7 +19,7 @@ export default defineConfig({
|
|
|
19
19
|
],
|
|
20
20
|
plugins: [
|
|
21
21
|
typescript(),
|
|
22
|
-
|
|
22
|
+
terser()
|
|
23
23
|
]
|
|
24
24
|
})
|
|
25
25
|
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import progress from 'progress';
|
|
2
2
|
import type { KnowledgeBase } from './types/KnowledgeBaseResponse';
|
|
3
|
-
export interface
|
|
3
|
+
export interface IProgressItem {
|
|
4
4
|
path: string;
|
|
5
5
|
toc: KnowledgeBase.Toc;
|
|
6
6
|
pathIdList: string[];
|
|
7
7
|
pathTitleList: string[];
|
|
8
8
|
}
|
|
9
|
-
export
|
|
10
|
-
export default class
|
|
9
|
+
export type IProgress = IProgressItem[];
|
|
10
|
+
export default class ProgressBar {
|
|
11
11
|
bookPath: string;
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
progressFilePath: string;
|
|
13
|
+
progressInfo: IProgress;
|
|
14
14
|
curr: number;
|
|
15
15
|
total: number;
|
|
16
16
|
isDownloadInterrupted: boolean;
|
|
@@ -18,6 +18,6 @@ export default class Progress {
|
|
|
18
18
|
completePromise: Promise<void> | null;
|
|
19
19
|
constructor(bookPath: string, total: number);
|
|
20
20
|
init(): Promise<void>;
|
|
21
|
-
getProgress(): Promise<
|
|
22
|
-
updateProgress(progressItem:
|
|
21
|
+
getProgress(): Promise<IProgress>;
|
|
22
|
+
updateProgress(progressItem: IProgressItem): Promise<void>;
|
|
23
23
|
}
|
package/.editorconfig
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
# http://editorconfig.org
|
|
2
|
-
|
|
3
|
-
root = true
|
|
4
|
-
|
|
5
|
-
[*]
|
|
6
|
-
charset = utf-8
|
|
7
|
-
indent_style = space
|
|
8
|
-
indent_size = 2
|
|
9
|
-
end_of_line = lf
|
|
10
|
-
# 最后一行换行 取消掉
|
|
11
|
-
insert_final_newline = false
|
|
12
|
-
trim_trailing_whitespace = true
|
|
13
|
-
|
|
14
|
-
[*.md]
|
|
15
|
-
insert_final_newline = false
|
|
16
|
-
trim_trailing_whitespace = false
|
package/.eslintrc.js
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
module.exports = {
|
|
2
|
-
env: {
|
|
3
|
-
node: true,
|
|
4
|
-
},
|
|
5
|
-
parser: '@typescript-eslint/parser',
|
|
6
|
-
plugins: [
|
|
7
|
-
'@typescript-eslint',
|
|
8
|
-
],
|
|
9
|
-
extends: [
|
|
10
|
-
'eslint:recommended',
|
|
11
|
-
'plugin:@typescript-eslint/recommended',
|
|
12
|
-
],
|
|
13
|
-
rules: {
|
|
14
|
-
semi: ['error', 'never'],
|
|
15
|
-
'no-console': 'off',
|
|
16
|
-
'no-plusplus': ['error', { allowForLoopAfterthoughts: true }],
|
|
17
|
-
// TODO
|
|
18
|
-
'@typescript-eslint/no-explicit-any': 'off',
|
|
19
|
-
'@typescript-eslint/no-non-null-assertion': 'off'
|
|
20
|
-
},
|
|
21
|
-
}
|
package/.np-config.js
DELETED
|
@@ -1,409 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
var fs = require('node:fs/promises');
|
|
4
|
-
var path = require('node:path');
|
|
5
|
-
var axios = require('axios');
|
|
6
|
-
var log4js = require('log4js');
|
|
7
|
-
var progress = require('progress');
|
|
8
|
-
var randUserAgentLib = require('rand-user-agent');
|
|
9
|
-
var mdImg = require('pull-md-img');
|
|
10
|
-
|
|
11
|
-
var config = {
|
|
12
|
-
knowledgeBaseReg: /decodeURIComponent\(\"(.+)\"\)\);/m,
|
|
13
|
-
dist: './download'
|
|
14
|
-
};
|
|
15
|
-
// data = re.findall(r"decodeURIComponent\(\"(.+)\"\)\);", docsdata.content.decode('utf-8'))
|
|
16
|
-
// const config = {
|
|
17
|
-
// path: '',
|
|
18
|
-
// suffix: '',
|
|
19
|
-
// dist: './res/',
|
|
20
|
-
// imgDir:`./img/${Date.now()}`,
|
|
21
|
-
// // markdown img 正则 注意多行匹配
|
|
22
|
-
// mdImgReg: /!\[(.*?)\]\((.*?)\)/gm,
|
|
23
|
-
// isIgnoreConsole: false
|
|
24
|
-
// }
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* log 初始化
|
|
28
|
-
*/
|
|
29
|
-
const getLogger = () => {
|
|
30
|
-
log4js.configure({
|
|
31
|
-
appenders: {
|
|
32
|
-
// cheeseLog: { type: 'file', filename: 'cheese.log' },
|
|
33
|
-
cheese: {
|
|
34
|
-
type: 'console',
|
|
35
|
-
layout: {
|
|
36
|
-
// type: 'messagePassThrough',
|
|
37
|
-
type: 'pattern',
|
|
38
|
-
// pattern: '%[%d{yyyy-MM-dd hh:mm:ss} [%p] %c -%] %m%n'
|
|
39
|
-
pattern: '%[%c [%p]:%] %m%n'
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
},
|
|
43
|
-
categories: { default: { appenders: ['cheese'], level: 'trace' } }
|
|
44
|
-
});
|
|
45
|
-
return log4js.getLogger('yuque-dl');
|
|
46
|
-
};
|
|
47
|
-
var logger = getLogger();
|
|
48
|
-
|
|
49
|
-
class Progress {
|
|
50
|
-
constructor(bookPath, total) {
|
|
51
|
-
this.bookPath = '';
|
|
52
|
-
this.processFilePath = '';
|
|
53
|
-
this.processInfo = [];
|
|
54
|
-
this.curr = 0;
|
|
55
|
-
this.total = 0;
|
|
56
|
-
this.isDownloadInterrupted = false;
|
|
57
|
-
this.bar = null;
|
|
58
|
-
this.completePromise = null;
|
|
59
|
-
this.bookPath = bookPath;
|
|
60
|
-
this.processFilePath = `${bookPath}/process.json`;
|
|
61
|
-
this.total = total;
|
|
62
|
-
}
|
|
63
|
-
async init() {
|
|
64
|
-
this.processInfo = await this.getProgress();
|
|
65
|
-
this.curr = this.processInfo.length;
|
|
66
|
-
let completeResolve;
|
|
67
|
-
this.completePromise = new Promise(resolve => {
|
|
68
|
-
completeResolve = resolve;
|
|
69
|
-
});
|
|
70
|
-
if (this.curr > 0 && this.curr !== this.total) {
|
|
71
|
-
this.isDownloadInterrupted = true;
|
|
72
|
-
logger.info('断点下载');
|
|
73
|
-
}
|
|
74
|
-
this.bar = new progress('[:bar] :rate/bps :percent :etas', {
|
|
75
|
-
// width: 20,
|
|
76
|
-
total: this.total,
|
|
77
|
-
curr: this.curr,
|
|
78
|
-
callback() {
|
|
79
|
-
completeResolve();
|
|
80
|
-
}
|
|
81
|
-
});
|
|
82
|
-
}
|
|
83
|
-
async getProgress() {
|
|
84
|
-
let processInfo = [];
|
|
85
|
-
try {
|
|
86
|
-
const processInfoStr = await fs.readFile(this.processFilePath, { encoding: 'utf8' });
|
|
87
|
-
processInfo = JSON.parse(processInfoStr);
|
|
88
|
-
}
|
|
89
|
-
catch (err) {
|
|
90
|
-
if (err && err.code === 'ENOENT') {
|
|
91
|
-
await fs.writeFile(this.processFilePath, JSON.stringify(processInfo), { encoding: 'utf8' });
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
return processInfo;
|
|
95
|
-
}
|
|
96
|
-
async updateProgress(progressItem) {
|
|
97
|
-
this.processInfo.push(progressItem);
|
|
98
|
-
await fs.writeFile(this.processFilePath, JSON.stringify(this.processInfo), { encoding: 'utf8' });
|
|
99
|
-
this.curr = this.curr + 1;
|
|
100
|
-
if (this.bar) {
|
|
101
|
-
this.bar?.tick();
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
function randUserAgent({ browser = 'chrome', os = 'mac os', device = 'desktop' }) {
|
|
107
|
-
device = device.toLowerCase();
|
|
108
|
-
browser = browser.toLowerCase();
|
|
109
|
-
os = os.toLowerCase();
|
|
110
|
-
let UA = randUserAgentLib(device, browser, os);
|
|
111
|
-
if (browser === 'chrome') {
|
|
112
|
-
while (UA.includes('Chrome-Lighthouse')
|
|
113
|
-
|| UA.includes('Gener8')
|
|
114
|
-
|| UA.includes('HeadlessChrome')
|
|
115
|
-
|| UA.includes('SMTBot')) {
|
|
116
|
-
UA = randUserAgentLib(device, browser, os);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
if (browser === 'safari') {
|
|
120
|
-
while (UA.includes('Applebot')) {
|
|
121
|
-
UA = randUserAgentLib(device, browser, os);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
return UA;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// SUMMARY.md
|
|
128
|
-
function getKnowledgeBaseInfo(url) {
|
|
129
|
-
return axios.get(url, {
|
|
130
|
-
headers: {
|
|
131
|
-
"user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
|
|
132
|
-
}
|
|
133
|
-
}).then(({ data = '', status }) => {
|
|
134
|
-
if (status === 200)
|
|
135
|
-
return data;
|
|
136
|
-
return '';
|
|
137
|
-
}).then(html => {
|
|
138
|
-
const data = html.match(config.knowledgeBaseReg) || '';
|
|
139
|
-
if (!data[1])
|
|
140
|
-
return {};
|
|
141
|
-
const jsonData = JSON.parse(decodeURIComponent(data[1]));
|
|
142
|
-
const info = {
|
|
143
|
-
bookId: jsonData.book.id,
|
|
144
|
-
tocList: jsonData.book.toc || [],
|
|
145
|
-
bookName: jsonData.book.name || '',
|
|
146
|
-
bookDesc: jsonData.book.description || '',
|
|
147
|
-
};
|
|
148
|
-
return info;
|
|
149
|
-
});
|
|
150
|
-
}
|
|
151
|
-
async function main(url, options) {
|
|
152
|
-
const { bookId, tocList, bookName, bookDesc } = await getKnowledgeBaseInfo(url);
|
|
153
|
-
if (!bookId)
|
|
154
|
-
throw new Error('No found book id');
|
|
155
|
-
if (!tocList || tocList.length === 0)
|
|
156
|
-
throw new Error('No found toc list');
|
|
157
|
-
const bookPath = `${options.distDir}/${bookId}`;
|
|
158
|
-
await fs.mkdir(bookPath, { recursive: true });
|
|
159
|
-
const total = tocList.length;
|
|
160
|
-
const progress = new Progress(bookPath, total);
|
|
161
|
-
await progress.init();
|
|
162
|
-
if (progress.curr === total) {
|
|
163
|
-
logger.info('√ 已完成');
|
|
164
|
-
return;
|
|
165
|
-
}
|
|
166
|
-
const uuidMap = new Map();
|
|
167
|
-
// 下载中断 重新获取下载进度数据
|
|
168
|
-
if (progress.isDownloadInterrupted) {
|
|
169
|
-
progress.processInfo.forEach(item => {
|
|
170
|
-
uuidMap.set(item.toc.uuid, item);
|
|
171
|
-
});
|
|
172
|
-
}
|
|
173
|
-
const articleUrlPrefix = path.parse(url).dir;
|
|
174
|
-
let errArticleCount = 0;
|
|
175
|
-
let totalArticleCount = 0;
|
|
176
|
-
for (let i = 0; i < total; i++) {
|
|
177
|
-
const item = tocList[i];
|
|
178
|
-
if (typeof item.type !== 'string')
|
|
179
|
-
continue;
|
|
180
|
-
if (uuidMap.get(item.uuid))
|
|
181
|
-
continue;
|
|
182
|
-
const itemType = item.type.toLocaleLowerCase();
|
|
183
|
-
const dirNameReg = /[\\\/:\*\?"<>\|\n\r]/g;
|
|
184
|
-
// 目录
|
|
185
|
-
if (itemType === 'title' || item['child_uuid'] !== '') {
|
|
186
|
-
let tempItem = item;
|
|
187
|
-
let pathTitleList = [];
|
|
188
|
-
let pathIdList = [];
|
|
189
|
-
while (tempItem) {
|
|
190
|
-
pathTitleList.unshift(tempItem.title.replace(dirNameReg, '_'));
|
|
191
|
-
pathIdList.unshift(tempItem.uuid);
|
|
192
|
-
if (uuidMap.get(tempItem['parent_uuid'])) {
|
|
193
|
-
tempItem = uuidMap.get(tempItem['parent_uuid']).toc;
|
|
194
|
-
}
|
|
195
|
-
else {
|
|
196
|
-
tempItem = undefined;
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
const progressItem = {
|
|
200
|
-
path: pathTitleList.join('/'),
|
|
201
|
-
pathTitleList,
|
|
202
|
-
pathIdList,
|
|
203
|
-
toc: item
|
|
204
|
-
};
|
|
205
|
-
await fs.mkdir(`${bookPath}/${pathTitleList.join('/')}`, { recursive: true });
|
|
206
|
-
uuidMap.set(item.uuid, progressItem);
|
|
207
|
-
progress.updateProgress(progressItem);
|
|
208
|
-
}
|
|
209
|
-
else if (item.url) {
|
|
210
|
-
totalArticleCount += 1;
|
|
211
|
-
let preItem = {
|
|
212
|
-
path: '',
|
|
213
|
-
pathTitleList: [],
|
|
214
|
-
pathIdList: []
|
|
215
|
-
};
|
|
216
|
-
if (uuidMap.get(item['parent_uuid'])) {
|
|
217
|
-
preItem = uuidMap.get(item['parent_uuid']);
|
|
218
|
-
}
|
|
219
|
-
const fileName = item.title.replace(dirNameReg, '_');
|
|
220
|
-
const pathTitleList = [...preItem.pathTitleList, `${fileName}.md`];
|
|
221
|
-
const pathIdList = [...preItem.pathIdList, item.uuid];
|
|
222
|
-
const progressItem = {
|
|
223
|
-
path: pathTitleList.join('/'),
|
|
224
|
-
pathTitleList,
|
|
225
|
-
pathIdList,
|
|
226
|
-
toc: item
|
|
227
|
-
};
|
|
228
|
-
const isSuccess = await downloadArticle({
|
|
229
|
-
bookId,
|
|
230
|
-
itemUrl: item.url,
|
|
231
|
-
savePath: `${bookPath}/${preItem.path}`,
|
|
232
|
-
saveFilePath: `${bookPath}/${progressItem.path}`,
|
|
233
|
-
uuid: item.uuid,
|
|
234
|
-
articleUrl: `${articleUrlPrefix}/${item.url}`,
|
|
235
|
-
articleTitle: item.title
|
|
236
|
-
});
|
|
237
|
-
if (isSuccess) {
|
|
238
|
-
uuidMap.set(item.uuid, progressItem);
|
|
239
|
-
progress.updateProgress(progressItem);
|
|
240
|
-
}
|
|
241
|
-
else {
|
|
242
|
-
errArticleCount += 1;
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
await progress.completePromise;
|
|
247
|
-
logger.info(`总数${totalArticleCount}篇,✕ 失败${errArticleCount}篇`);
|
|
248
|
-
logger.info('生成目录 SUMMARY.md');
|
|
249
|
-
await genSummaryFile({
|
|
250
|
-
bookPath,
|
|
251
|
-
bookName,
|
|
252
|
-
bookDesc,
|
|
253
|
-
uuidMap
|
|
254
|
-
});
|
|
255
|
-
if (progress.curr === total) {
|
|
256
|
-
logger.info('√ 已完成');
|
|
257
|
-
return;
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
async function downloadArticle(params) {
|
|
261
|
-
const { bookId, itemUrl, savePath, saveFilePath, uuid, articleUrl, articleTitle } = params;
|
|
262
|
-
const apiUrl = `https://www.yuque.com/api/docs/${itemUrl}`;
|
|
263
|
-
const response = await axios.get(apiUrl, {
|
|
264
|
-
headers: {
|
|
265
|
-
"user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
|
|
266
|
-
},
|
|
267
|
-
params: {
|
|
268
|
-
'book_id': bookId,
|
|
269
|
-
'merge_dynamic_data': false,
|
|
270
|
-
mode: 'markdown'
|
|
271
|
-
}
|
|
272
|
-
}).then(({ data, status }) => {
|
|
273
|
-
if (status === 200)
|
|
274
|
-
return data;
|
|
275
|
-
logger.error(`download article error: ${apiUrl} http status ${status}`);
|
|
276
|
-
return null;
|
|
277
|
-
});
|
|
278
|
-
if (!response || !response.data || !response.data.sourcecode)
|
|
279
|
-
return false;
|
|
280
|
-
let mdData = response.data.sourcecode;
|
|
281
|
-
try {
|
|
282
|
-
mdData = await mdImg.run(mdData, {
|
|
283
|
-
dist: savePath,
|
|
284
|
-
imgDir: `img/${uuid}`,
|
|
285
|
-
isIgnoreConsole: true
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
catch (err) {
|
|
289
|
-
logger.error(`download article img Error (api: ${apiUrl})`);
|
|
290
|
-
return false;
|
|
291
|
-
}
|
|
292
|
-
mdData = mdData.replace(/<br(\s?)\/>/gm, '\n');
|
|
293
|
-
if (articleTitle) {
|
|
294
|
-
mdData = `# ${articleTitle}\n<!--page header-->\n\n${mdData}\n\n`;
|
|
295
|
-
}
|
|
296
|
-
if (articleUrl) {
|
|
297
|
-
mdData += `<!--page footer-->\n- 原文: <${articleUrl}>`;
|
|
298
|
-
}
|
|
299
|
-
try {
|
|
300
|
-
await fs.writeFile(saveFilePath, mdData);
|
|
301
|
-
return true;
|
|
302
|
-
}
|
|
303
|
-
catch (e) {
|
|
304
|
-
logger.error(`download article write file Error (api: ${apiUrl})`);
|
|
305
|
-
return false;
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
async function genSummaryFile(params) {
|
|
309
|
-
const { bookName, bookDesc, bookPath, uuidMap } = params;
|
|
310
|
-
const header = `# ${bookName}\n\n > ${bookDesc}\n\n`;
|
|
311
|
-
let mdContent = header;
|
|
312
|
-
const summary = [];
|
|
313
|
-
uuidMap.forEach(processItem => {
|
|
314
|
-
const toc = processItem.toc;
|
|
315
|
-
const parentId = toc['parent_uuid'];
|
|
316
|
-
const findRes = findTree(summary, parentId);
|
|
317
|
-
if (toc.type.toLocaleLowerCase() === 'title' || toc['child_uuid'] !== '') {
|
|
318
|
-
if (typeof findRes !== 'boolean') {
|
|
319
|
-
if (!Array.isArray(findRes.children))
|
|
320
|
-
findRes.children = [];
|
|
321
|
-
findRes.children.push({
|
|
322
|
-
id: toc.uuid,
|
|
323
|
-
type: 'title',
|
|
324
|
-
level: findRes.level + 1,
|
|
325
|
-
text: toc.title
|
|
326
|
-
});
|
|
327
|
-
}
|
|
328
|
-
else {
|
|
329
|
-
summary.push({
|
|
330
|
-
id: toc.uuid,
|
|
331
|
-
type: 'title',
|
|
332
|
-
text: toc.title,
|
|
333
|
-
level: 1
|
|
334
|
-
});
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
else {
|
|
338
|
-
if (typeof findRes !== 'boolean') {
|
|
339
|
-
if (!Array.isArray(findRes.children))
|
|
340
|
-
findRes.children = [];
|
|
341
|
-
findRes.children.push({
|
|
342
|
-
id: toc.uuid,
|
|
343
|
-
type: 'link',
|
|
344
|
-
level: findRes.level + 1,
|
|
345
|
-
link: processItem.path,
|
|
346
|
-
text: toc.title
|
|
347
|
-
});
|
|
348
|
-
}
|
|
349
|
-
else {
|
|
350
|
-
summary.push({
|
|
351
|
-
id: toc.uuid,
|
|
352
|
-
type: 'link',
|
|
353
|
-
text: toc.title,
|
|
354
|
-
link: processItem.path,
|
|
355
|
-
level: 1
|
|
356
|
-
});
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
});
|
|
360
|
-
let summaryContent = genSummaryContent(summary, '');
|
|
361
|
-
mdContent += summaryContent;
|
|
362
|
-
try {
|
|
363
|
-
await fs.writeFile(`${bookPath}/SUMMARY.md`, mdContent);
|
|
364
|
-
}
|
|
365
|
-
catch (err) {
|
|
366
|
-
logger.error('Generate Summary Error');
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
function genSummaryContent(summary, summaryContent) {
|
|
370
|
-
for (let i = 0; i < summary.length; i++) {
|
|
371
|
-
const item = summary[i];
|
|
372
|
-
if (item.type === 'title') {
|
|
373
|
-
summaryContent += `\n${''.padStart(item.level + 1, '#')} ${item.text}\n\n`;
|
|
374
|
-
}
|
|
375
|
-
else if (item.type === 'link') {
|
|
376
|
-
const link = item.link ? item.link.replace(/\s/g, '%20') : item.link;
|
|
377
|
-
summaryContent += `- [${item.text}](${link})\n`;
|
|
378
|
-
}
|
|
379
|
-
if (Array.isArray(item.children)) {
|
|
380
|
-
summaryContent += genSummaryContent(item.children, '');
|
|
381
|
-
}
|
|
382
|
-
}
|
|
383
|
-
return summaryContent;
|
|
384
|
-
}
|
|
385
|
-
function findIdItem(node, id) {
|
|
386
|
-
if (node.id === id) {
|
|
387
|
-
return node;
|
|
388
|
-
}
|
|
389
|
-
else if (node.children) {
|
|
390
|
-
const findRes = findTree(node.children, id);
|
|
391
|
-
if (findRes)
|
|
392
|
-
return findRes;
|
|
393
|
-
}
|
|
394
|
-
return false;
|
|
395
|
-
}
|
|
396
|
-
function findTree(tree, id) {
|
|
397
|
-
if (!id)
|
|
398
|
-
return false;
|
|
399
|
-
for (let i = 0; i < tree.length; i++) {
|
|
400
|
-
const findRes = findIdItem(tree[i], id);
|
|
401
|
-
if (findRes)
|
|
402
|
-
return findRes;
|
|
403
|
-
}
|
|
404
|
-
return false;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
exports.getKnowledgeBaseInfo = getKnowledgeBaseInfo;
|
|
408
|
-
exports.logger = logger;
|
|
409
|
-
exports.main = main;
|
|
@@ -1,405 +0,0 @@
|
|
|
1
|
-
import fs from 'node:fs/promises';
|
|
2
|
-
import path from 'node:path';
|
|
3
|
-
import axios from 'axios';
|
|
4
|
-
import log4js from 'log4js';
|
|
5
|
-
import progress from 'progress';
|
|
6
|
-
import randUserAgentLib from 'rand-user-agent';
|
|
7
|
-
import mdImg from 'pull-md-img';
|
|
8
|
-
|
|
9
|
-
var config = {
|
|
10
|
-
knowledgeBaseReg: /decodeURIComponent\(\"(.+)\"\)\);/m,
|
|
11
|
-
dist: './download'
|
|
12
|
-
};
|
|
13
|
-
// data = re.findall(r"decodeURIComponent\(\"(.+)\"\)\);", docsdata.content.decode('utf-8'))
|
|
14
|
-
// const config = {
|
|
15
|
-
// path: '',
|
|
16
|
-
// suffix: '',
|
|
17
|
-
// dist: './res/',
|
|
18
|
-
// imgDir:`./img/${Date.now()}`,
|
|
19
|
-
// // markdown img 正则 注意多行匹配
|
|
20
|
-
// mdImgReg: /!\[(.*?)\]\((.*?)\)/gm,
|
|
21
|
-
// isIgnoreConsole: false
|
|
22
|
-
// }
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* log 初始化
|
|
26
|
-
*/
|
|
27
|
-
const getLogger = () => {
|
|
28
|
-
log4js.configure({
|
|
29
|
-
appenders: {
|
|
30
|
-
// cheeseLog: { type: 'file', filename: 'cheese.log' },
|
|
31
|
-
cheese: {
|
|
32
|
-
type: 'console',
|
|
33
|
-
layout: {
|
|
34
|
-
// type: 'messagePassThrough',
|
|
35
|
-
type: 'pattern',
|
|
36
|
-
// pattern: '%[%d{yyyy-MM-dd hh:mm:ss} [%p] %c -%] %m%n'
|
|
37
|
-
pattern: '%[%c [%p]:%] %m%n'
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
},
|
|
41
|
-
categories: { default: { appenders: ['cheese'], level: 'trace' } }
|
|
42
|
-
});
|
|
43
|
-
return log4js.getLogger('yuque-dl');
|
|
44
|
-
};
|
|
45
|
-
var logger = getLogger();
|
|
46
|
-
|
|
47
|
-
class Progress {
|
|
48
|
-
constructor(bookPath, total) {
|
|
49
|
-
this.bookPath = '';
|
|
50
|
-
this.processFilePath = '';
|
|
51
|
-
this.processInfo = [];
|
|
52
|
-
this.curr = 0;
|
|
53
|
-
this.total = 0;
|
|
54
|
-
this.isDownloadInterrupted = false;
|
|
55
|
-
this.bar = null;
|
|
56
|
-
this.completePromise = null;
|
|
57
|
-
this.bookPath = bookPath;
|
|
58
|
-
this.processFilePath = `${bookPath}/process.json`;
|
|
59
|
-
this.total = total;
|
|
60
|
-
}
|
|
61
|
-
async init() {
|
|
62
|
-
this.processInfo = await this.getProgress();
|
|
63
|
-
this.curr = this.processInfo.length;
|
|
64
|
-
let completeResolve;
|
|
65
|
-
this.completePromise = new Promise(resolve => {
|
|
66
|
-
completeResolve = resolve;
|
|
67
|
-
});
|
|
68
|
-
if (this.curr > 0 && this.curr !== this.total) {
|
|
69
|
-
this.isDownloadInterrupted = true;
|
|
70
|
-
logger.info('断点下载');
|
|
71
|
-
}
|
|
72
|
-
this.bar = new progress('[:bar] :rate/bps :percent :etas', {
|
|
73
|
-
// width: 20,
|
|
74
|
-
total: this.total,
|
|
75
|
-
curr: this.curr,
|
|
76
|
-
callback() {
|
|
77
|
-
completeResolve();
|
|
78
|
-
}
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
async getProgress() {
|
|
82
|
-
let processInfo = [];
|
|
83
|
-
try {
|
|
84
|
-
const processInfoStr = await fs.readFile(this.processFilePath, { encoding: 'utf8' });
|
|
85
|
-
processInfo = JSON.parse(processInfoStr);
|
|
86
|
-
}
|
|
87
|
-
catch (err) {
|
|
88
|
-
if (err && err.code === 'ENOENT') {
|
|
89
|
-
await fs.writeFile(this.processFilePath, JSON.stringify(processInfo), { encoding: 'utf8' });
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
return processInfo;
|
|
93
|
-
}
|
|
94
|
-
async updateProgress(progressItem) {
|
|
95
|
-
this.processInfo.push(progressItem);
|
|
96
|
-
await fs.writeFile(this.processFilePath, JSON.stringify(this.processInfo), { encoding: 'utf8' });
|
|
97
|
-
this.curr = this.curr + 1;
|
|
98
|
-
if (this.bar) {
|
|
99
|
-
this.bar?.tick();
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
function randUserAgent({ browser = 'chrome', os = 'mac os', device = 'desktop' }) {
|
|
105
|
-
device = device.toLowerCase();
|
|
106
|
-
browser = browser.toLowerCase();
|
|
107
|
-
os = os.toLowerCase();
|
|
108
|
-
let UA = randUserAgentLib(device, browser, os);
|
|
109
|
-
if (browser === 'chrome') {
|
|
110
|
-
while (UA.includes('Chrome-Lighthouse')
|
|
111
|
-
|| UA.includes('Gener8')
|
|
112
|
-
|| UA.includes('HeadlessChrome')
|
|
113
|
-
|| UA.includes('SMTBot')) {
|
|
114
|
-
UA = randUserAgentLib(device, browser, os);
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
if (browser === 'safari') {
|
|
118
|
-
while (UA.includes('Applebot')) {
|
|
119
|
-
UA = randUserAgentLib(device, browser, os);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
return UA;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
// SUMMARY.md
|
|
126
|
-
function getKnowledgeBaseInfo(url) {
|
|
127
|
-
return axios.get(url, {
|
|
128
|
-
headers: {
|
|
129
|
-
"user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
|
|
130
|
-
}
|
|
131
|
-
}).then(({ data = '', status }) => {
|
|
132
|
-
if (status === 200)
|
|
133
|
-
return data;
|
|
134
|
-
return '';
|
|
135
|
-
}).then(html => {
|
|
136
|
-
const data = html.match(config.knowledgeBaseReg) || '';
|
|
137
|
-
if (!data[1])
|
|
138
|
-
return {};
|
|
139
|
-
const jsonData = JSON.parse(decodeURIComponent(data[1]));
|
|
140
|
-
const info = {
|
|
141
|
-
bookId: jsonData.book.id,
|
|
142
|
-
tocList: jsonData.book.toc || [],
|
|
143
|
-
bookName: jsonData.book.name || '',
|
|
144
|
-
bookDesc: jsonData.book.description || '',
|
|
145
|
-
};
|
|
146
|
-
return info;
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
async function main(url, options) {
|
|
150
|
-
const { bookId, tocList, bookName, bookDesc } = await getKnowledgeBaseInfo(url);
|
|
151
|
-
if (!bookId)
|
|
152
|
-
throw new Error('No found book id');
|
|
153
|
-
if (!tocList || tocList.length === 0)
|
|
154
|
-
throw new Error('No found toc list');
|
|
155
|
-
const bookPath = `${options.distDir}/${bookId}`;
|
|
156
|
-
await fs.mkdir(bookPath, { recursive: true });
|
|
157
|
-
const total = tocList.length;
|
|
158
|
-
const progress = new Progress(bookPath, total);
|
|
159
|
-
await progress.init();
|
|
160
|
-
if (progress.curr === total) {
|
|
161
|
-
logger.info('√ 已完成');
|
|
162
|
-
return;
|
|
163
|
-
}
|
|
164
|
-
const uuidMap = new Map();
|
|
165
|
-
// 下载中断 重新获取下载进度数据
|
|
166
|
-
if (progress.isDownloadInterrupted) {
|
|
167
|
-
progress.processInfo.forEach(item => {
|
|
168
|
-
uuidMap.set(item.toc.uuid, item);
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
const articleUrlPrefix = path.parse(url).dir;
|
|
172
|
-
let errArticleCount = 0;
|
|
173
|
-
let totalArticleCount = 0;
|
|
174
|
-
for (let i = 0; i < total; i++) {
|
|
175
|
-
const item = tocList[i];
|
|
176
|
-
if (typeof item.type !== 'string')
|
|
177
|
-
continue;
|
|
178
|
-
if (uuidMap.get(item.uuid))
|
|
179
|
-
continue;
|
|
180
|
-
const itemType = item.type.toLocaleLowerCase();
|
|
181
|
-
const dirNameReg = /[\\\/:\*\?"<>\|\n\r]/g;
|
|
182
|
-
// 目录
|
|
183
|
-
if (itemType === 'title' || item['child_uuid'] !== '') {
|
|
184
|
-
let tempItem = item;
|
|
185
|
-
let pathTitleList = [];
|
|
186
|
-
let pathIdList = [];
|
|
187
|
-
while (tempItem) {
|
|
188
|
-
pathTitleList.unshift(tempItem.title.replace(dirNameReg, '_'));
|
|
189
|
-
pathIdList.unshift(tempItem.uuid);
|
|
190
|
-
if (uuidMap.get(tempItem['parent_uuid'])) {
|
|
191
|
-
tempItem = uuidMap.get(tempItem['parent_uuid']).toc;
|
|
192
|
-
}
|
|
193
|
-
else {
|
|
194
|
-
tempItem = undefined;
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
const progressItem = {
|
|
198
|
-
path: pathTitleList.join('/'),
|
|
199
|
-
pathTitleList,
|
|
200
|
-
pathIdList,
|
|
201
|
-
toc: item
|
|
202
|
-
};
|
|
203
|
-
await fs.mkdir(`${bookPath}/${pathTitleList.join('/')}`, { recursive: true });
|
|
204
|
-
uuidMap.set(item.uuid, progressItem);
|
|
205
|
-
progress.updateProgress(progressItem);
|
|
206
|
-
}
|
|
207
|
-
else if (item.url) {
|
|
208
|
-
totalArticleCount += 1;
|
|
209
|
-
let preItem = {
|
|
210
|
-
path: '',
|
|
211
|
-
pathTitleList: [],
|
|
212
|
-
pathIdList: []
|
|
213
|
-
};
|
|
214
|
-
if (uuidMap.get(item['parent_uuid'])) {
|
|
215
|
-
preItem = uuidMap.get(item['parent_uuid']);
|
|
216
|
-
}
|
|
217
|
-
const fileName = item.title.replace(dirNameReg, '_');
|
|
218
|
-
const pathTitleList = [...preItem.pathTitleList, `${fileName}.md`];
|
|
219
|
-
const pathIdList = [...preItem.pathIdList, item.uuid];
|
|
220
|
-
const progressItem = {
|
|
221
|
-
path: pathTitleList.join('/'),
|
|
222
|
-
pathTitleList,
|
|
223
|
-
pathIdList,
|
|
224
|
-
toc: item
|
|
225
|
-
};
|
|
226
|
-
const isSuccess = await downloadArticle({
|
|
227
|
-
bookId,
|
|
228
|
-
itemUrl: item.url,
|
|
229
|
-
savePath: `${bookPath}/${preItem.path}`,
|
|
230
|
-
saveFilePath: `${bookPath}/${progressItem.path}`,
|
|
231
|
-
uuid: item.uuid,
|
|
232
|
-
articleUrl: `${articleUrlPrefix}/${item.url}`,
|
|
233
|
-
articleTitle: item.title
|
|
234
|
-
});
|
|
235
|
-
if (isSuccess) {
|
|
236
|
-
uuidMap.set(item.uuid, progressItem);
|
|
237
|
-
progress.updateProgress(progressItem);
|
|
238
|
-
}
|
|
239
|
-
else {
|
|
240
|
-
errArticleCount += 1;
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
await progress.completePromise;
|
|
245
|
-
logger.info(`总数${totalArticleCount}篇,✕ 失败${errArticleCount}篇`);
|
|
246
|
-
logger.info('生成目录 SUMMARY.md');
|
|
247
|
-
await genSummaryFile({
|
|
248
|
-
bookPath,
|
|
249
|
-
bookName,
|
|
250
|
-
bookDesc,
|
|
251
|
-
uuidMap
|
|
252
|
-
});
|
|
253
|
-
if (progress.curr === total) {
|
|
254
|
-
logger.info('√ 已完成');
|
|
255
|
-
return;
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
async function downloadArticle(params) {
|
|
259
|
-
const { bookId, itemUrl, savePath, saveFilePath, uuid, articleUrl, articleTitle } = params;
|
|
260
|
-
const apiUrl = `https://www.yuque.com/api/docs/${itemUrl}`;
|
|
261
|
-
const response = await axios.get(apiUrl, {
|
|
262
|
-
headers: {
|
|
263
|
-
"user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
|
|
264
|
-
},
|
|
265
|
-
params: {
|
|
266
|
-
'book_id': bookId,
|
|
267
|
-
'merge_dynamic_data': false,
|
|
268
|
-
mode: 'markdown'
|
|
269
|
-
}
|
|
270
|
-
}).then(({ data, status }) => {
|
|
271
|
-
if (status === 200)
|
|
272
|
-
return data;
|
|
273
|
-
logger.error(`download article error: ${apiUrl} http status ${status}`);
|
|
274
|
-
return null;
|
|
275
|
-
});
|
|
276
|
-
if (!response || !response.data || !response.data.sourcecode)
|
|
277
|
-
return false;
|
|
278
|
-
let mdData = response.data.sourcecode;
|
|
279
|
-
try {
|
|
280
|
-
mdData = await mdImg.run(mdData, {
|
|
281
|
-
dist: savePath,
|
|
282
|
-
imgDir: `img/${uuid}`,
|
|
283
|
-
isIgnoreConsole: true
|
|
284
|
-
});
|
|
285
|
-
}
|
|
286
|
-
catch (err) {
|
|
287
|
-
logger.error(`download article img Error (api: ${apiUrl})`);
|
|
288
|
-
return false;
|
|
289
|
-
}
|
|
290
|
-
mdData = mdData.replace(/<br(\s?)\/>/gm, '\n');
|
|
291
|
-
if (articleTitle) {
|
|
292
|
-
mdData = `# ${articleTitle}\n<!--page header-->\n\n${mdData}\n\n`;
|
|
293
|
-
}
|
|
294
|
-
if (articleUrl) {
|
|
295
|
-
mdData += `<!--page footer-->\n- 原文: <${articleUrl}>`;
|
|
296
|
-
}
|
|
297
|
-
try {
|
|
298
|
-
await fs.writeFile(saveFilePath, mdData);
|
|
299
|
-
return true;
|
|
300
|
-
}
|
|
301
|
-
catch (e) {
|
|
302
|
-
logger.error(`download article write file Error (api: ${apiUrl})`);
|
|
303
|
-
return false;
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
async function genSummaryFile(params) {
|
|
307
|
-
const { bookName, bookDesc, bookPath, uuidMap } = params;
|
|
308
|
-
const header = `# ${bookName}\n\n > ${bookDesc}\n\n`;
|
|
309
|
-
let mdContent = header;
|
|
310
|
-
const summary = [];
|
|
311
|
-
uuidMap.forEach(processItem => {
|
|
312
|
-
const toc = processItem.toc;
|
|
313
|
-
const parentId = toc['parent_uuid'];
|
|
314
|
-
const findRes = findTree(summary, parentId);
|
|
315
|
-
if (toc.type.toLocaleLowerCase() === 'title' || toc['child_uuid'] !== '') {
|
|
316
|
-
if (typeof findRes !== 'boolean') {
|
|
317
|
-
if (!Array.isArray(findRes.children))
|
|
318
|
-
findRes.children = [];
|
|
319
|
-
findRes.children.push({
|
|
320
|
-
id: toc.uuid,
|
|
321
|
-
type: 'title',
|
|
322
|
-
level: findRes.level + 1,
|
|
323
|
-
text: toc.title
|
|
324
|
-
});
|
|
325
|
-
}
|
|
326
|
-
else {
|
|
327
|
-
summary.push({
|
|
328
|
-
id: toc.uuid,
|
|
329
|
-
type: 'title',
|
|
330
|
-
text: toc.title,
|
|
331
|
-
level: 1
|
|
332
|
-
});
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
else {
|
|
336
|
-
if (typeof findRes !== 'boolean') {
|
|
337
|
-
if (!Array.isArray(findRes.children))
|
|
338
|
-
findRes.children = [];
|
|
339
|
-
findRes.children.push({
|
|
340
|
-
id: toc.uuid,
|
|
341
|
-
type: 'link',
|
|
342
|
-
level: findRes.level + 1,
|
|
343
|
-
link: processItem.path,
|
|
344
|
-
text: toc.title
|
|
345
|
-
});
|
|
346
|
-
}
|
|
347
|
-
else {
|
|
348
|
-
summary.push({
|
|
349
|
-
id: toc.uuid,
|
|
350
|
-
type: 'link',
|
|
351
|
-
text: toc.title,
|
|
352
|
-
link: processItem.path,
|
|
353
|
-
level: 1
|
|
354
|
-
});
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
});
|
|
358
|
-
let summaryContent = genSummaryContent(summary, '');
|
|
359
|
-
mdContent += summaryContent;
|
|
360
|
-
try {
|
|
361
|
-
await fs.writeFile(`${bookPath}/SUMMARY.md`, mdContent);
|
|
362
|
-
}
|
|
363
|
-
catch (err) {
|
|
364
|
-
logger.error('Generate Summary Error');
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
function genSummaryContent(summary, summaryContent) {
|
|
368
|
-
for (let i = 0; i < summary.length; i++) {
|
|
369
|
-
const item = summary[i];
|
|
370
|
-
if (item.type === 'title') {
|
|
371
|
-
summaryContent += `\n${''.padStart(item.level + 1, '#')} ${item.text}\n\n`;
|
|
372
|
-
}
|
|
373
|
-
else if (item.type === 'link') {
|
|
374
|
-
const link = item.link ? item.link.replace(/\s/g, '%20') : item.link;
|
|
375
|
-
summaryContent += `- [${item.text}](${link})\n`;
|
|
376
|
-
}
|
|
377
|
-
if (Array.isArray(item.children)) {
|
|
378
|
-
summaryContent += genSummaryContent(item.children, '');
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
return summaryContent;
|
|
382
|
-
}
|
|
383
|
-
function findIdItem(node, id) {
|
|
384
|
-
if (node.id === id) {
|
|
385
|
-
return node;
|
|
386
|
-
}
|
|
387
|
-
else if (node.children) {
|
|
388
|
-
const findRes = findTree(node.children, id);
|
|
389
|
-
if (findRes)
|
|
390
|
-
return findRes;
|
|
391
|
-
}
|
|
392
|
-
return false;
|
|
393
|
-
}
|
|
394
|
-
function findTree(tree, id) {
|
|
395
|
-
if (!id)
|
|
396
|
-
return false;
|
|
397
|
-
for (let i = 0; i < tree.length; i++) {
|
|
398
|
-
const findRes = findIdItem(tree[i], id);
|
|
399
|
-
if (findRes)
|
|
400
|
-
return findRes;
|
|
401
|
-
}
|
|
402
|
-
return false;
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
export { getKnowledgeBaseInfo as g, logger as l, main as m };
|