yuque-dl 1.0.8 → 1.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/.prettierrc.js CHANGED
@@ -1,4 +1,4 @@
1
- module.exports = {
1
+ export default {
2
2
  semi: false,
3
3
  endOfLine: 'lf',
4
4
  singleQuote: true,
package/dist/cjs/cli.js CHANGED
@@ -1,46 +1 @@
1
- 'use strict';
2
-
3
- var node_fs = require('node:fs');
4
- var cac = require('cac');
5
- var index = require('./index-4969d150.js');
6
- require('node:fs/promises');
7
- require('node:path');
8
- require('axios');
9
- require('log4js');
10
- require('progress');
11
- require('rand-user-agent');
12
- require('pull-md-img');
13
-
14
- const cli = cac.cac('yuque-dl');
15
- // 不能直接使用 import {version} from '../package.json'
16
- // 否则declaration 生成的d.ts 会多一层src目录
17
- const { version } = JSON.parse(node_fs.readFileSync(new URL('../../package.json', (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (document.currentScript && document.currentScript.src || new URL('cli.js', document.baseURI).href)))).toString());
18
- cli
19
- .command('<url>', 'request url')
20
- .option('-d, --distDir <dir>', '下载的目录 eg: -d download', {
21
- default: 'download',
22
- })
23
- .action(async (url, options) => {
24
- try {
25
- await index.main(url, options);
26
- }
27
- catch (err) {
28
- index.logger.error(err.message || 'unknown exception');
29
- }
30
- });
31
- cli.help();
32
- cli.version(version);
33
- try {
34
- cli.parse();
35
- }
36
- catch (err) {
37
- index.logger.error(err.message || 'unknown exception');
38
- process.exit(1);
39
- }
40
- // try {
41
- // cli.parse(process.argv, { run: false })
42
- // await cli.runMatchedCommand()
43
- // } catch (error) {
44
- // logger.error(error.message || 'unknown exception')
45
- // process.exit(1)
46
- // }
1
+ "use strict";var e=require("node:fs"),r=require("cac"),n=require("./index-b758b4ba.js");require("node:fs/promises"),require("node:path"),require("axios"),require("log4js"),require("progress"),require("rand-user-agent"),require("pull-md-img");const i=r.cac("yuque-dl"),{version:o}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:document.currentScript&&document.currentScript.src||new URL("cli.js",document.baseURI).href)).toString());i.command("<url>","request url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).action((async(e,r)=>{try{await n.main(e,r)}catch(e){n.logger.error(e.message||"unknown exception")}})),i.help(),i.version(o);try{i.parse()}catch(e){n.logger.error(e.message||"unknown exception"),process.exit(1)}
@@ -0,0 +1 @@
1
+ "use strict";var e=require("node:fs/promises"),t=require("node:path"),r=require("axios"),i=require("log4js"),o=require("progress"),s=require("rand-user-agent"),n=require("pull-md-img"),a={knowledgeBaseReg:/decodeURIComponent\(\"(.+)\"\)\);/m,dist:"./download"};var l=(i.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),i.getLogger("yuque-dl"));class c{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){let e;this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.completePromise=new Promise((t=>{e=t})),this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,l.info("断点下载")),this.bar=new o("[:bar] :rate/bps :percent :etas",{total:this.total,curr:this.curr,callback(){e()}})}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t){this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"}),this.curr=this.curr+1,this.bar&&this.bar?.tick()}}function u({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let i=s(r,e,t);if("chrome"===e)for(;i.includes("Chrome-Lighthouse")||i.includes("Gener8")||i.includes("HeadlessChrome")||i.includes("SMTBot");)i=s(r,e,t);if("safari"===e)for(;i.includes("Applebot");)i=s(r,e,t);return i}function d(e){return r.get(e,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})}}).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=e.match(a.knowledgeBaseReg)||"";if(!t[1])return{};const r=JSON.parse(decodeURIComponent(t[1]));return{bookId:r.book.id,tocList:r.book.toc||[],bookName:r.book.name||"",bookDesc:r.book.description||""}}))}async function h(t){const{bookId:i,itemUrl:o,savePath:s,saveFilePath:a,uuid:c,articleUrl:d,articleTitle:h}=t,p=`https://www.yuque.com/api/docs/${o}`,g=await r.get(p,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})},params:{book_id:i,merge_dynamic_data:!1,mode:"markdown"}}).then((({data:e,status:t})=>200===t?e:(l.error(`download article error: ${p} http status ${t}`),null)));if(!g||!g.data||!g.data.sourcecode)return!1;let f=g.data.sourcecode;try{f=await n.run(f,{dist:s,imgDir:`img/${c}`,isIgnoreConsole:!0})}catch(e){return l.error(`download article img Error (api: ${p})`),!1}f=f.replace(/<br(\s?)\/>/gm,"\n"),h&&(f=`# ${h}\n\x3c!--page header--\x3e\n\n${f}\n\n`),d&&(f+=`\x3c!--page footer--\x3e\n- 原文: <${d}>`);try{return await e.writeFile(a,f),!0}catch(e){return l.error(`download article write file Error (api: ${p})`),!1}}function p(e,t){for(let r=0;r<e.length;r++){const i=e[r];if("title"===i.type)t+=`\n${"".padStart(i.level+1,"#")} ${i.text}\n\n`;else if("link"===i.type){const e=i.link?i.link.replace(/\s/g,"%20"):i.link;t+=`- [${i.text}](${e})\n`}Array.isArray(i.children)&&(t+=p(i.children,""))}return t}function g(e,t){if(e.id===t)return e;if(e.children){const r=f(e.children,t);if(r)return r}return!1}function f(e,t){if(!t)return!1;for(let r=0;r<e.length;r++){const i=g(e[r],t);if(i)return i}return!1}exports.getKnowledgeBaseInfo=d,exports.logger=l,exports.main=async function(r,i){const{bookId:o,tocList:s,bookName:n,bookDesc:a}=await d(r);if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const u=`${i.distDir}/${o}`;await e.mkdir(u,{recursive:!0});const g=s.length,w=new c(u,g);if(await w.init(),w.curr===g)return void l.info("√ 已完成");const m=new Map;w.isDownloadInterrupted&&w.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const k=t.parse(r).dir;let y=0,b=0;for(let t=0;t<g;t++){const r=s[t];if("string"!=typeof r.type)continue;if(m.get(r.uuid))continue;const i=/[\\\/:\*\?"<>\|\n\r]/g;if("title"===r.type.toLocaleLowerCase()||""!==r.child_uuid){let t=r,o=[],s=[];for(;t;)o.unshift(t.title.replace(i,"_")),s.unshift(t.uuid),t=m.get(t.parent_uuid)?m.get(t.parent_uuid).toc:void 0;const n={path:o.join("/"),pathTitleList:o,pathIdList:s,toc:r};await e.mkdir(`${u}/${o.join("/")}`,{recursive:!0}),m.set(r.uuid,n),w.updateProgress(n)}else if(r.url){b+=1;let e={path:"",pathTitleList:[],pathIdList:[]};m.get(r.parent_uuid)&&(e=m.get(r.parent_uuid));const t=r.title.replace(i,"_"),s=[...e.pathTitleList,`${t}.md`],n=[...e.pathIdList,r.uuid],a={path:s.join("/"),pathTitleList:s,pathIdList:n,toc:r};await h({bookId:o,itemUrl:r.url,savePath:`${u}/${e.path}`,saveFilePath:`${u}/${a.path}`,uuid:r.uuid,articleUrl:`${k}/${r.url}`,articleTitle:r.title})?(m.set(r.uuid,a),w.updateProgress(a)):y+=1}}await w.completePromise,l.info(`本次执行总数${b}篇,✕ 失败${y}篇`),l.info("生成目录 SUMMARY.md"),await async function(t){const{bookName:r,bookDesc:i,bookPath:o,uuidMap:s}=t,n=`# ${r}\n\n > ${i}\n\n`;let a=n;const c=[];s.forEach((e=>{const t=e.toc,r=t.parent_uuid,i=f(c,r);"title"===t.type.toLocaleLowerCase()||""!==t.child_uuid?"boolean"!=typeof i?(Array.isArray(i.children)||(i.children=[]),i.children.push({id:t.uuid,type:"title",level:i.level+1,text:t.title})):c.push({id:t.uuid,type:"title",text:t.title,level:1}):"boolean"!=typeof i?(Array.isArray(i.children)||(i.children=[]),i.children.push({id:t.uuid,type:"link",level:i.level+1,link:e.path,text:t.title})):c.push({id:t.uuid,type:"link",text:t.title,link:e.path,level:1})}));let u=p(c,"");a+=u;try{await e.writeFile(`${o}/SUMMARY.md`,a)}catch(e){l.error("Generate Summary Error")}}({bookPath:u,bookName:n,bookDesc:a,uuidMap:m}),w.curr!==g||l.info("√ 已完成")};
package/dist/cjs/index.js CHANGED
@@ -1,15 +1 @@
1
- 'use strict';
2
-
3
- require('node:fs/promises');
4
- require('node:path');
5
- require('axios');
6
- var index = require('./index-4969d150.js');
7
- require('pull-md-img');
8
- require('log4js');
9
- require('progress');
10
- require('rand-user-agent');
11
-
12
-
13
-
14
- exports.getKnowledgeBaseInfo = index.getKnowledgeBaseInfo;
15
- exports.main = index.main;
1
+ "use strict";require("node:fs/promises"),require("node:path"),require("axios");var e=require("./index-b758b4ba.js");require("pull-md-img"),require("log4js"),require("progress"),require("rand-user-agent"),exports.getKnowledgeBaseInfo=e.getKnowledgeBaseInfo,exports.main=e.main;
package/dist/es/cli.js CHANGED
@@ -1,44 +1 @@
1
- import { readFileSync } from 'node:fs';
2
- import { cac } from 'cac';
3
- import { m as main, l as logger } from './index-5cda90ca.js';
4
- import 'node:fs/promises';
5
- import 'node:path';
6
- import 'axios';
7
- import 'log4js';
8
- import 'progress';
9
- import 'rand-user-agent';
10
- import 'pull-md-img';
11
-
12
- const cli = cac('yuque-dl');
13
- // 不能直接使用 import {version} from '../package.json'
14
- // 否则declaration 生成的d.ts 会多一层src目录
15
- const { version } = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url)).toString());
16
- cli
17
- .command('<url>', 'request url')
18
- .option('-d, --distDir <dir>', '下载的目录 eg: -d download', {
19
- default: 'download',
20
- })
21
- .action(async (url, options) => {
22
- try {
23
- await main(url, options);
24
- }
25
- catch (err) {
26
- logger.error(err.message || 'unknown exception');
27
- }
28
- });
29
- cli.help();
30
- cli.version(version);
31
- try {
32
- cli.parse();
33
- }
34
- catch (err) {
35
- logger.error(err.message || 'unknown exception');
36
- process.exit(1);
37
- }
38
- // try {
39
- // cli.parse(process.argv, { run: false })
40
- // await cli.runMatchedCommand()
41
- // } catch (error) {
42
- // logger.error(error.message || 'unknown exception')
43
- // process.exit(1)
44
- // }
1
+ import{readFileSync as o}from"node:fs";import{cac as r}from"cac";import{m as e,l as t}from"./index-7580521b.js";import"node:fs/promises";import"node:path";import"axios";import"log4js";import"progress";import"rand-user-agent";import"pull-md-img";const i=r("yuque-dl"),{version:s}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());i.command("<url>","request url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).action((async(o,r)=>{try{await e(o,r)}catch(o){t.error(o.message||"unknown exception")}})),i.help(),i.version(s);try{i.parse()}catch(o){t.error(o.message||"unknown exception"),process.exit(1)}
@@ -0,0 +1 @@
1
+ import t from"node:fs/promises";import e from"node:path";import r from"axios";import i from"log4js";import o from"progress";import s from"rand-user-agent";import a from"pull-md-img";var n={knowledgeBaseReg:/decodeURIComponent\(\"(.+)\"\)\);/m,dist:"./download"};var l=(i.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),i.getLogger("yuque-dl"));class c{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){let t;this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.completePromise=new Promise((e=>{t=e})),this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,l.info("断点下载")),this.bar=new o("[:bar] :rate/bps :percent :etas",{total:this.total,curr:this.curr,callback(){t()}})}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e){this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"}),this.curr=this.curr+1,this.bar&&this.bar?.tick()}}function u({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let i=s(r,t,e);if("chrome"===t)for(;i.includes("Chrome-Lighthouse")||i.includes("Gener8")||i.includes("HeadlessChrome")||i.includes("SMTBot");)i=s(r,t,e);if("safari"===t)for(;i.includes("Applebot");)i=s(r,t,e);return i}function d(t){return r.get(t,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})}}).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=t.match(n.knowledgeBaseReg)||"";if(!e[1])return{};const r=JSON.parse(decodeURIComponent(e[1]));return{bookId:r.book.id,tocList:r.book.toc||[],bookName:r.book.name||"",bookDesc:r.book.description||""}}))}async function h(r,i){const{bookId:o,tocList:s,bookName:a,bookDesc:n}=await d(r);if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const u=`${i.distDir}/${o}`;await t.mkdir(u,{recursive:!0});const h=s.length,g=new c(u,h);if(await g.init(),g.curr===h)return void l.info("√ 已完成");const w=new Map;g.isDownloadInterrupted&&g.progressInfo.forEach((t=>{w.set(t.toc.uuid,t)}));const k=e.parse(r).dir;let y=0,b=0;for(let e=0;e<h;e++){const r=s[e];if("string"!=typeof r.type)continue;if(w.get(r.uuid))continue;const i=/[\\\/:\*\?"<>\|\n\r]/g;if("title"===r.type.toLocaleLowerCase()||""!==r.child_uuid){let e=r,o=[],s=[];for(;e;)o.unshift(e.title.replace(i,"_")),s.unshift(e.uuid),e=w.get(e.parent_uuid)?w.get(e.parent_uuid).toc:void 0;const a={path:o.join("/"),pathTitleList:o,pathIdList:s,toc:r};await t.mkdir(`${u}/${o.join("/")}`,{recursive:!0}),w.set(r.uuid,a),g.updateProgress(a)}else if(r.url){b+=1;let t={path:"",pathTitleList:[],pathIdList:[]};w.get(r.parent_uuid)&&(t=w.get(r.parent_uuid));const e=r.title.replace(i,"_"),s=[...t.pathTitleList,`${e}.md`],a=[...t.pathIdList,r.uuid],n={path:s.join("/"),pathTitleList:s,pathIdList:a,toc:r};await p({bookId:o,itemUrl:r.url,savePath:`${u}/${t.path}`,saveFilePath:`${u}/${n.path}`,uuid:r.uuid,articleUrl:`${k}/${r.url}`,articleTitle:r.title})?(w.set(r.uuid,n),g.updateProgress(n)):y+=1}}await g.completePromise,l.info(`本次执行总数${b}篇,✕ 失败${y}篇`),l.info("生成目录 SUMMARY.md"),await async function(e){const{bookName:r,bookDesc:i,bookPath:o,uuidMap:s}=e,a=`# ${r}\n\n > ${i}\n\n`;let n=a;const c=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,i=m(c,r);"title"===e.type.toLocaleLowerCase()||""!==e.child_uuid?"boolean"!=typeof i?(Array.isArray(i.children)||(i.children=[]),i.children.push({id:e.uuid,type:"title",level:i.level+1,text:e.title})):c.push({id:e.uuid,type:"title",text:e.title,level:1}):"boolean"!=typeof i?(Array.isArray(i.children)||(i.children=[]),i.children.push({id:e.uuid,type:"link",level:i.level+1,link:t.path,text:e.title})):c.push({id:e.uuid,type:"link",text:e.title,link:t.path,level:1})}));let u=f(c,"");n+=u;try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){l.error("Generate Summary Error")}}({bookPath:u,bookName:a,bookDesc:n,uuidMap:w}),g.curr!==h||l.info("√ 已完成")}async function p(e){const{bookId:i,itemUrl:o,savePath:s,saveFilePath:n,uuid:c,articleUrl:d,articleTitle:h}=e,p=`https://www.yuque.com/api/docs/${o}`,f=await r.get(p,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})},params:{book_id:i,merge_dynamic_data:!1,mode:"markdown"}}).then((({data:t,status:e})=>200===e?t:(l.error(`download article error: ${p} http status ${e}`),null)));if(!f||!f.data||!f.data.sourcecode)return!1;let g=f.data.sourcecode;try{g=await a.run(g,{dist:s,imgDir:`img/${c}`,isIgnoreConsole:!0})}catch(t){return l.error(`download article img Error (api: ${p})`),!1}g=g.replace(/<br(\s?)\/>/gm,"\n"),h&&(g=`# ${h}\n\x3c!--page header--\x3e\n\n${g}\n\n`),d&&(g+=`\x3c!--page footer--\x3e\n- 原文: <${d}>`);try{return await t.writeFile(n,g),!0}catch(t){return l.error(`download article write file Error (api: ${p})`),!1}}function f(t,e){for(let r=0;r<t.length;r++){const i=t[r];if("title"===i.type)e+=`\n${"".padStart(i.level+1,"#")} ${i.text}\n\n`;else if("link"===i.type){const t=i.link?i.link.replace(/\s/g,"%20"):i.link;e+=`- [${i.text}](${t})\n`}Array.isArray(i.children)&&(e+=f(i.children,""))}return e}function g(t,e){if(t.id===e)return t;if(t.children){const r=m(t.children,e);if(r)return r}return!1}function m(t,e){if(!e)return!1;for(let r=0;r<t.length;r++){const i=g(t[r],e);if(i)return i}return!1}export{d as g,l,h as m};
package/dist/es/index.js CHANGED
@@ -1,8 +1 @@
1
- import 'node:fs/promises';
2
- import 'node:path';
3
- import 'axios';
4
- export { g as getKnowledgeBaseInfo, m as main } from './index-5cda90ca.js';
5
- import 'pull-md-img';
6
- import 'log4js';
7
- import 'progress';
8
- import 'rand-user-agent';
1
+ import"node:fs/promises";import"node:path";import"axios";export{g as getKnowledgeBaseInfo,m as main}from"./index-7580521b.js";import"pull-md-img";import"log4js";import"progress";import"rand-user-agent";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "yuque-dl",
3
- "version": "1.0.8",
3
+ "version": "1.0.10",
4
4
  "description": "yuque 知识库下载",
5
5
  "type": "module",
6
6
  "bin": {
@@ -12,21 +12,27 @@
12
12
  "build:bundle": "rollup -c rollup.config.ts --configPlugin typescript",
13
13
  "build:types": "tsc --emitDeclarationOnly --outDir types -p tsconfig.base.json",
14
14
  "clean": "rm -rf dist types",
15
- "publish": "np"
15
+ "np": "np",
16
+ "release": "run-s clean build np"
16
17
  },
17
18
  "repository": {
18
19
  "type": "git",
19
20
  "url": "git+https://github.com/gxr404/yuque-dl.git"
20
21
  },
21
- "keywords": [],
22
- "author": "",
22
+ "keywords": [
23
+ "yuque",
24
+ "nodejs",
25
+ "download",
26
+ "yuque-dl"
27
+ ],
28
+ "author": "gxr404",
23
29
  "license": "ISC",
24
30
  "bugs": {
25
31
  "url": "https://github.com/gxr404/yuque-dl/issues"
26
32
  },
27
33
  "homepage": "https://github.com/gxr404/yuque-dl#readme",
28
34
  "devDependencies": {
29
- "@rollup/plugin-json": "^6.0.0",
35
+ "@rollup/plugin-terser": "^0.4.3",
30
36
  "@rollup/plugin-typescript": "^11.1.2",
31
37
  "@types/node": "^20.5.0",
32
38
  "@types/progress": "^2.0.5",
package/rollup.config.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { defineConfig } from 'rollup'
2
2
  import typescript from '@rollup/plugin-typescript'
3
- import json from '@rollup/plugin-json'
3
+ import terser from '@rollup/plugin-terser'
4
4
 
5
5
  export default defineConfig({
6
6
  input: {
@@ -19,7 +19,7 @@ export default defineConfig({
19
19
  ],
20
20
  plugins: [
21
21
  typescript(),
22
- json()
22
+ terser()
23
23
  ]
24
24
  })
25
25
 
@@ -1,16 +1,16 @@
1
1
  import progress from 'progress';
2
2
  import type { KnowledgeBase } from './types/KnowledgeBaseResponse';
3
- export interface IProcessItem {
3
+ export interface IProgressItem {
4
4
  path: string;
5
5
  toc: KnowledgeBase.Toc;
6
6
  pathIdList: string[];
7
7
  pathTitleList: string[];
8
8
  }
9
- export declare type IProcess = IProcessItem[];
10
- export default class Progress {
9
+ export type IProgress = IProgressItem[];
10
+ export default class ProgressBar {
11
11
  bookPath: string;
12
- processFilePath: string;
13
- processInfo: IProcess;
12
+ progressFilePath: string;
13
+ progressInfo: IProgress;
14
14
  curr: number;
15
15
  total: number;
16
16
  isDownloadInterrupted: boolean;
@@ -18,6 +18,6 @@ export default class Progress {
18
18
  completePromise: Promise<void> | null;
19
19
  constructor(bookPath: string, total: number);
20
20
  init(): Promise<void>;
21
- getProgress(): Promise<IProcess>;
22
- updateProgress(progressItem: IProcessItem): Promise<void>;
21
+ getProgress(): Promise<IProgress>;
22
+ updateProgress(progressItem: IProgressItem): Promise<void>;
23
23
  }
package/.editorconfig DELETED
@@ -1,16 +0,0 @@
1
- # http://editorconfig.org
2
-
3
- root = true
4
-
5
- [*]
6
- charset = utf-8
7
- indent_style = space
8
- indent_size = 2
9
- end_of_line = lf
10
- # 最后一行换行 取消掉
11
- insert_final_newline = false
12
- trim_trailing_whitespace = true
13
-
14
- [*.md]
15
- insert_final_newline = false
16
- trim_trailing_whitespace = false
package/.eslintignore DELETED
@@ -1,5 +0,0 @@
1
- dist
2
- types
3
- node_modules
4
- bin
5
- test
package/.eslintrc.js DELETED
@@ -1,21 +0,0 @@
1
- module.exports = {
2
- env: {
3
- node: true,
4
- },
5
- parser: '@typescript-eslint/parser',
6
- plugins: [
7
- '@typescript-eslint',
8
- ],
9
- extends: [
10
- 'eslint:recommended',
11
- 'plugin:@typescript-eslint/recommended',
12
- ],
13
- rules: {
14
- semi: ['error', 'never'],
15
- 'no-console': 'off',
16
- 'no-plusplus': ['error', { allowForLoopAfterthoughts: true }],
17
- // TODO
18
- '@typescript-eslint/no-explicit-any': 'off',
19
- '@typescript-eslint/no-non-null-assertion': 'off'
20
- },
21
- }
package/.np-config.js DELETED
@@ -1,4 +0,0 @@
1
- export default {
2
- tests: false,
3
- '2fa': false
4
- }
@@ -1,409 +0,0 @@
1
- 'use strict';
2
-
3
- var fs = require('node:fs/promises');
4
- var path = require('node:path');
5
- var axios = require('axios');
6
- var log4js = require('log4js');
7
- var progress = require('progress');
8
- var randUserAgentLib = require('rand-user-agent');
9
- var mdImg = require('pull-md-img');
10
-
11
- var config = {
12
- knowledgeBaseReg: /decodeURIComponent\(\"(.+)\"\)\);/m,
13
- dist: './download'
14
- };
15
- // data = re.findall(r"decodeURIComponent\(\"(.+)\"\)\);", docsdata.content.decode('utf-8'))
16
- // const config = {
17
- // path: '',
18
- // suffix: '',
19
- // dist: './res/',
20
- // imgDir:`./img/${Date.now()}`,
21
- // // markdown img 正则 注意多行匹配
22
- // mdImgReg: /!\[(.*?)\]\((.*?)\)/gm,
23
- // isIgnoreConsole: false
24
- // }
25
-
26
- /**
27
- * log 初始化
28
- */
29
- const getLogger = () => {
30
- log4js.configure({
31
- appenders: {
32
- // cheeseLog: { type: 'file', filename: 'cheese.log' },
33
- cheese: {
34
- type: 'console',
35
- layout: {
36
- // type: 'messagePassThrough',
37
- type: 'pattern',
38
- // pattern: '%[%d{yyyy-MM-dd hh:mm:ss} [%p] %c -%] %m%n'
39
- pattern: '%[%c [%p]:%] %m%n'
40
- }
41
- }
42
- },
43
- categories: { default: { appenders: ['cheese'], level: 'trace' } }
44
- });
45
- return log4js.getLogger('yuque-dl');
46
- };
47
- var logger = getLogger();
48
-
49
- class Progress {
50
- constructor(bookPath, total) {
51
- this.bookPath = '';
52
- this.processFilePath = '';
53
- this.processInfo = [];
54
- this.curr = 0;
55
- this.total = 0;
56
- this.isDownloadInterrupted = false;
57
- this.bar = null;
58
- this.completePromise = null;
59
- this.bookPath = bookPath;
60
- this.processFilePath = `${bookPath}/process.json`;
61
- this.total = total;
62
- }
63
- async init() {
64
- this.processInfo = await this.getProgress();
65
- this.curr = this.processInfo.length;
66
- let completeResolve;
67
- this.completePromise = new Promise(resolve => {
68
- completeResolve = resolve;
69
- });
70
- if (this.curr > 0 && this.curr !== this.total) {
71
- this.isDownloadInterrupted = true;
72
- logger.info('断点下载');
73
- }
74
- this.bar = new progress('[:bar] :rate/bps :percent :etas', {
75
- // width: 20,
76
- total: this.total,
77
- curr: this.curr,
78
- callback() {
79
- completeResolve();
80
- }
81
- });
82
- }
83
- async getProgress() {
84
- let processInfo = [];
85
- try {
86
- const processInfoStr = await fs.readFile(this.processFilePath, { encoding: 'utf8' });
87
- processInfo = JSON.parse(processInfoStr);
88
- }
89
- catch (err) {
90
- if (err && err.code === 'ENOENT') {
91
- await fs.writeFile(this.processFilePath, JSON.stringify(processInfo), { encoding: 'utf8' });
92
- }
93
- }
94
- return processInfo;
95
- }
96
- async updateProgress(progressItem) {
97
- this.processInfo.push(progressItem);
98
- await fs.writeFile(this.processFilePath, JSON.stringify(this.processInfo), { encoding: 'utf8' });
99
- this.curr = this.curr + 1;
100
- if (this.bar) {
101
- this.bar?.tick();
102
- }
103
- }
104
- }
105
-
106
- function randUserAgent({ browser = 'chrome', os = 'mac os', device = 'desktop' }) {
107
- device = device.toLowerCase();
108
- browser = browser.toLowerCase();
109
- os = os.toLowerCase();
110
- let UA = randUserAgentLib(device, browser, os);
111
- if (browser === 'chrome') {
112
- while (UA.includes('Chrome-Lighthouse')
113
- || UA.includes('Gener8')
114
- || UA.includes('HeadlessChrome')
115
- || UA.includes('SMTBot')) {
116
- UA = randUserAgentLib(device, browser, os);
117
- }
118
- }
119
- if (browser === 'safari') {
120
- while (UA.includes('Applebot')) {
121
- UA = randUserAgentLib(device, browser, os);
122
- }
123
- }
124
- return UA;
125
- }
126
-
127
- // SUMMARY.md
128
- function getKnowledgeBaseInfo(url) {
129
- return axios.get(url, {
130
- headers: {
131
- "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
132
- }
133
- }).then(({ data = '', status }) => {
134
- if (status === 200)
135
- return data;
136
- return '';
137
- }).then(html => {
138
- const data = html.match(config.knowledgeBaseReg) || '';
139
- if (!data[1])
140
- return {};
141
- const jsonData = JSON.parse(decodeURIComponent(data[1]));
142
- const info = {
143
- bookId: jsonData.book.id,
144
- tocList: jsonData.book.toc || [],
145
- bookName: jsonData.book.name || '',
146
- bookDesc: jsonData.book.description || '',
147
- };
148
- return info;
149
- });
150
- }
151
- async function main(url, options) {
152
- const { bookId, tocList, bookName, bookDesc } = await getKnowledgeBaseInfo(url);
153
- if (!bookId)
154
- throw new Error('No found book id');
155
- if (!tocList || tocList.length === 0)
156
- throw new Error('No found toc list');
157
- const bookPath = `${options.distDir}/${bookId}`;
158
- await fs.mkdir(bookPath, { recursive: true });
159
- const total = tocList.length;
160
- const progress = new Progress(bookPath, total);
161
- await progress.init();
162
- if (progress.curr === total) {
163
- logger.info('√ 已完成');
164
- return;
165
- }
166
- const uuidMap = new Map();
167
- // 下载中断 重新获取下载进度数据
168
- if (progress.isDownloadInterrupted) {
169
- progress.processInfo.forEach(item => {
170
- uuidMap.set(item.toc.uuid, item);
171
- });
172
- }
173
- const articleUrlPrefix = path.parse(url).dir;
174
- let errArticleCount = 0;
175
- let totalArticleCount = 0;
176
- for (let i = 0; i < total; i++) {
177
- const item = tocList[i];
178
- if (typeof item.type !== 'string')
179
- continue;
180
- if (uuidMap.get(item.uuid))
181
- continue;
182
- const itemType = item.type.toLocaleLowerCase();
183
- const dirNameReg = /[\\\/:\*\?"<>\|\n\r]/g;
184
- // 目录
185
- if (itemType === 'title' || item['child_uuid'] !== '') {
186
- let tempItem = item;
187
- let pathTitleList = [];
188
- let pathIdList = [];
189
- while (tempItem) {
190
- pathTitleList.unshift(tempItem.title.replace(dirNameReg, '_'));
191
- pathIdList.unshift(tempItem.uuid);
192
- if (uuidMap.get(tempItem['parent_uuid'])) {
193
- tempItem = uuidMap.get(tempItem['parent_uuid']).toc;
194
- }
195
- else {
196
- tempItem = undefined;
197
- }
198
- }
199
- const progressItem = {
200
- path: pathTitleList.join('/'),
201
- pathTitleList,
202
- pathIdList,
203
- toc: item
204
- };
205
- await fs.mkdir(`${bookPath}/${pathTitleList.join('/')}`, { recursive: true });
206
- uuidMap.set(item.uuid, progressItem);
207
- progress.updateProgress(progressItem);
208
- }
209
- else if (item.url) {
210
- totalArticleCount += 1;
211
- let preItem = {
212
- path: '',
213
- pathTitleList: [],
214
- pathIdList: []
215
- };
216
- if (uuidMap.get(item['parent_uuid'])) {
217
- preItem = uuidMap.get(item['parent_uuid']);
218
- }
219
- const fileName = item.title.replace(dirNameReg, '_');
220
- const pathTitleList = [...preItem.pathTitleList, `${fileName}.md`];
221
- const pathIdList = [...preItem.pathIdList, item.uuid];
222
- const progressItem = {
223
- path: pathTitleList.join('/'),
224
- pathTitleList,
225
- pathIdList,
226
- toc: item
227
- };
228
- const isSuccess = await downloadArticle({
229
- bookId,
230
- itemUrl: item.url,
231
- savePath: `${bookPath}/${preItem.path}`,
232
- saveFilePath: `${bookPath}/${progressItem.path}`,
233
- uuid: item.uuid,
234
- articleUrl: `${articleUrlPrefix}/${item.url}`,
235
- articleTitle: item.title
236
- });
237
- if (isSuccess) {
238
- uuidMap.set(item.uuid, progressItem);
239
- progress.updateProgress(progressItem);
240
- }
241
- else {
242
- errArticleCount += 1;
243
- }
244
- }
245
- }
246
- await progress.completePromise;
247
- logger.info(`总数${totalArticleCount}篇,✕ 失败${errArticleCount}篇`);
248
- logger.info('生成目录 SUMMARY.md');
249
- await genSummaryFile({
250
- bookPath,
251
- bookName,
252
- bookDesc,
253
- uuidMap
254
- });
255
- if (progress.curr === total) {
256
- logger.info('√ 已完成');
257
- return;
258
- }
259
- }
260
- async function downloadArticle(params) {
261
- const { bookId, itemUrl, savePath, saveFilePath, uuid, articleUrl, articleTitle } = params;
262
- const apiUrl = `https://www.yuque.com/api/docs/${itemUrl}`;
263
- const response = await axios.get(apiUrl, {
264
- headers: {
265
- "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
266
- },
267
- params: {
268
- 'book_id': bookId,
269
- 'merge_dynamic_data': false,
270
- mode: 'markdown'
271
- }
272
- }).then(({ data, status }) => {
273
- if (status === 200)
274
- return data;
275
- logger.error(`download article error: ${apiUrl} http status ${status}`);
276
- return null;
277
- });
278
- if (!response || !response.data || !response.data.sourcecode)
279
- return false;
280
- let mdData = response.data.sourcecode;
281
- try {
282
- mdData = await mdImg.run(mdData, {
283
- dist: savePath,
284
- imgDir: `img/${uuid}`,
285
- isIgnoreConsole: true
286
- });
287
- }
288
- catch (err) {
289
- logger.error(`download article img Error (api: ${apiUrl})`);
290
- return false;
291
- }
292
- mdData = mdData.replace(/<br(\s?)\/>/gm, '\n');
293
- if (articleTitle) {
294
- mdData = `# ${articleTitle}\n<!--page header-->\n\n${mdData}\n\n`;
295
- }
296
- if (articleUrl) {
297
- mdData += `<!--page footer-->\n- 原文: <${articleUrl}>`;
298
- }
299
- try {
300
- await fs.writeFile(saveFilePath, mdData);
301
- return true;
302
- }
303
- catch (e) {
304
- logger.error(`download article write file Error (api: ${apiUrl})`);
305
- return false;
306
- }
307
- }
308
- async function genSummaryFile(params) {
309
- const { bookName, bookDesc, bookPath, uuidMap } = params;
310
- const header = `# ${bookName}\n\n > ${bookDesc}\n\n`;
311
- let mdContent = header;
312
- const summary = [];
313
- uuidMap.forEach(processItem => {
314
- const toc = processItem.toc;
315
- const parentId = toc['parent_uuid'];
316
- const findRes = findTree(summary, parentId);
317
- if (toc.type.toLocaleLowerCase() === 'title' || toc['child_uuid'] !== '') {
318
- if (typeof findRes !== 'boolean') {
319
- if (!Array.isArray(findRes.children))
320
- findRes.children = [];
321
- findRes.children.push({
322
- id: toc.uuid,
323
- type: 'title',
324
- level: findRes.level + 1,
325
- text: toc.title
326
- });
327
- }
328
- else {
329
- summary.push({
330
- id: toc.uuid,
331
- type: 'title',
332
- text: toc.title,
333
- level: 1
334
- });
335
- }
336
- }
337
- else {
338
- if (typeof findRes !== 'boolean') {
339
- if (!Array.isArray(findRes.children))
340
- findRes.children = [];
341
- findRes.children.push({
342
- id: toc.uuid,
343
- type: 'link',
344
- level: findRes.level + 1,
345
- link: processItem.path,
346
- text: toc.title
347
- });
348
- }
349
- else {
350
- summary.push({
351
- id: toc.uuid,
352
- type: 'link',
353
- text: toc.title,
354
- link: processItem.path,
355
- level: 1
356
- });
357
- }
358
- }
359
- });
360
- let summaryContent = genSummaryContent(summary, '');
361
- mdContent += summaryContent;
362
- try {
363
- await fs.writeFile(`${bookPath}/SUMMARY.md`, mdContent);
364
- }
365
- catch (err) {
366
- logger.error('Generate Summary Error');
367
- }
368
- }
369
- function genSummaryContent(summary, summaryContent) {
370
- for (let i = 0; i < summary.length; i++) {
371
- const item = summary[i];
372
- if (item.type === 'title') {
373
- summaryContent += `\n${''.padStart(item.level + 1, '#')} ${item.text}\n\n`;
374
- }
375
- else if (item.type === 'link') {
376
- const link = item.link ? item.link.replace(/\s/g, '%20') : item.link;
377
- summaryContent += `- [${item.text}](${link})\n`;
378
- }
379
- if (Array.isArray(item.children)) {
380
- summaryContent += genSummaryContent(item.children, '');
381
- }
382
- }
383
- return summaryContent;
384
- }
385
- function findIdItem(node, id) {
386
- if (node.id === id) {
387
- return node;
388
- }
389
- else if (node.children) {
390
- const findRes = findTree(node.children, id);
391
- if (findRes)
392
- return findRes;
393
- }
394
- return false;
395
- }
396
- function findTree(tree, id) {
397
- if (!id)
398
- return false;
399
- for (let i = 0; i < tree.length; i++) {
400
- const findRes = findIdItem(tree[i], id);
401
- if (findRes)
402
- return findRes;
403
- }
404
- return false;
405
- }
406
-
407
- exports.getKnowledgeBaseInfo = getKnowledgeBaseInfo;
408
- exports.logger = logger;
409
- exports.main = main;
@@ -1,405 +0,0 @@
1
- import fs from 'node:fs/promises';
2
- import path from 'node:path';
3
- import axios from 'axios';
4
- import log4js from 'log4js';
5
- import progress from 'progress';
6
- import randUserAgentLib from 'rand-user-agent';
7
- import mdImg from 'pull-md-img';
8
-
9
- var config = {
10
- knowledgeBaseReg: /decodeURIComponent\(\"(.+)\"\)\);/m,
11
- dist: './download'
12
- };
13
- // data = re.findall(r"decodeURIComponent\(\"(.+)\"\)\);", docsdata.content.decode('utf-8'))
14
- // const config = {
15
- // path: '',
16
- // suffix: '',
17
- // dist: './res/',
18
- // imgDir:`./img/${Date.now()}`,
19
- // // markdown img 正则 注意多行匹配
20
- // mdImgReg: /!\[(.*?)\]\((.*?)\)/gm,
21
- // isIgnoreConsole: false
22
- // }
23
-
24
- /**
25
- * log 初始化
26
- */
27
- const getLogger = () => {
28
- log4js.configure({
29
- appenders: {
30
- // cheeseLog: { type: 'file', filename: 'cheese.log' },
31
- cheese: {
32
- type: 'console',
33
- layout: {
34
- // type: 'messagePassThrough',
35
- type: 'pattern',
36
- // pattern: '%[%d{yyyy-MM-dd hh:mm:ss} [%p] %c -%] %m%n'
37
- pattern: '%[%c [%p]:%] %m%n'
38
- }
39
- }
40
- },
41
- categories: { default: { appenders: ['cheese'], level: 'trace' } }
42
- });
43
- return log4js.getLogger('yuque-dl');
44
- };
45
- var logger = getLogger();
46
-
47
- class Progress {
48
- constructor(bookPath, total) {
49
- this.bookPath = '';
50
- this.processFilePath = '';
51
- this.processInfo = [];
52
- this.curr = 0;
53
- this.total = 0;
54
- this.isDownloadInterrupted = false;
55
- this.bar = null;
56
- this.completePromise = null;
57
- this.bookPath = bookPath;
58
- this.processFilePath = `${bookPath}/process.json`;
59
- this.total = total;
60
- }
61
- async init() {
62
- this.processInfo = await this.getProgress();
63
- this.curr = this.processInfo.length;
64
- let completeResolve;
65
- this.completePromise = new Promise(resolve => {
66
- completeResolve = resolve;
67
- });
68
- if (this.curr > 0 && this.curr !== this.total) {
69
- this.isDownloadInterrupted = true;
70
- logger.info('断点下载');
71
- }
72
- this.bar = new progress('[:bar] :rate/bps :percent :etas', {
73
- // width: 20,
74
- total: this.total,
75
- curr: this.curr,
76
- callback() {
77
- completeResolve();
78
- }
79
- });
80
- }
81
- async getProgress() {
82
- let processInfo = [];
83
- try {
84
- const processInfoStr = await fs.readFile(this.processFilePath, { encoding: 'utf8' });
85
- processInfo = JSON.parse(processInfoStr);
86
- }
87
- catch (err) {
88
- if (err && err.code === 'ENOENT') {
89
- await fs.writeFile(this.processFilePath, JSON.stringify(processInfo), { encoding: 'utf8' });
90
- }
91
- }
92
- return processInfo;
93
- }
94
- async updateProgress(progressItem) {
95
- this.processInfo.push(progressItem);
96
- await fs.writeFile(this.processFilePath, JSON.stringify(this.processInfo), { encoding: 'utf8' });
97
- this.curr = this.curr + 1;
98
- if (this.bar) {
99
- this.bar?.tick();
100
- }
101
- }
102
- }
103
-
104
- function randUserAgent({ browser = 'chrome', os = 'mac os', device = 'desktop' }) {
105
- device = device.toLowerCase();
106
- browser = browser.toLowerCase();
107
- os = os.toLowerCase();
108
- let UA = randUserAgentLib(device, browser, os);
109
- if (browser === 'chrome') {
110
- while (UA.includes('Chrome-Lighthouse')
111
- || UA.includes('Gener8')
112
- || UA.includes('HeadlessChrome')
113
- || UA.includes('SMTBot')) {
114
- UA = randUserAgentLib(device, browser, os);
115
- }
116
- }
117
- if (browser === 'safari') {
118
- while (UA.includes('Applebot')) {
119
- UA = randUserAgentLib(device, browser, os);
120
- }
121
- }
122
- return UA;
123
- }
124
-
125
- // SUMMARY.md
126
- function getKnowledgeBaseInfo(url) {
127
- return axios.get(url, {
128
- headers: {
129
- "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
130
- }
131
- }).then(({ data = '', status }) => {
132
- if (status === 200)
133
- return data;
134
- return '';
135
- }).then(html => {
136
- const data = html.match(config.knowledgeBaseReg) || '';
137
- if (!data[1])
138
- return {};
139
- const jsonData = JSON.parse(decodeURIComponent(data[1]));
140
- const info = {
141
- bookId: jsonData.book.id,
142
- tocList: jsonData.book.toc || [],
143
- bookName: jsonData.book.name || '',
144
- bookDesc: jsonData.book.description || '',
145
- };
146
- return info;
147
- });
148
- }
149
- async function main(url, options) {
150
- const { bookId, tocList, bookName, bookDesc } = await getKnowledgeBaseInfo(url);
151
- if (!bookId)
152
- throw new Error('No found book id');
153
- if (!tocList || tocList.length === 0)
154
- throw new Error('No found toc list');
155
- const bookPath = `${options.distDir}/${bookId}`;
156
- await fs.mkdir(bookPath, { recursive: true });
157
- const total = tocList.length;
158
- const progress = new Progress(bookPath, total);
159
- await progress.init();
160
- if (progress.curr === total) {
161
- logger.info('√ 已完成');
162
- return;
163
- }
164
- const uuidMap = new Map();
165
- // 下载中断 重新获取下载进度数据
166
- if (progress.isDownloadInterrupted) {
167
- progress.processInfo.forEach(item => {
168
- uuidMap.set(item.toc.uuid, item);
169
- });
170
- }
171
- const articleUrlPrefix = path.parse(url).dir;
172
- let errArticleCount = 0;
173
- let totalArticleCount = 0;
174
- for (let i = 0; i < total; i++) {
175
- const item = tocList[i];
176
- if (typeof item.type !== 'string')
177
- continue;
178
- if (uuidMap.get(item.uuid))
179
- continue;
180
- const itemType = item.type.toLocaleLowerCase();
181
- const dirNameReg = /[\\\/:\*\?"<>\|\n\r]/g;
182
- // 目录
183
- if (itemType === 'title' || item['child_uuid'] !== '') {
184
- let tempItem = item;
185
- let pathTitleList = [];
186
- let pathIdList = [];
187
- while (tempItem) {
188
- pathTitleList.unshift(tempItem.title.replace(dirNameReg, '_'));
189
- pathIdList.unshift(tempItem.uuid);
190
- if (uuidMap.get(tempItem['parent_uuid'])) {
191
- tempItem = uuidMap.get(tempItem['parent_uuid']).toc;
192
- }
193
- else {
194
- tempItem = undefined;
195
- }
196
- }
197
- const progressItem = {
198
- path: pathTitleList.join('/'),
199
- pathTitleList,
200
- pathIdList,
201
- toc: item
202
- };
203
- await fs.mkdir(`${bookPath}/${pathTitleList.join('/')}`, { recursive: true });
204
- uuidMap.set(item.uuid, progressItem);
205
- progress.updateProgress(progressItem);
206
- }
207
- else if (item.url) {
208
- totalArticleCount += 1;
209
- let preItem = {
210
- path: '',
211
- pathTitleList: [],
212
- pathIdList: []
213
- };
214
- if (uuidMap.get(item['parent_uuid'])) {
215
- preItem = uuidMap.get(item['parent_uuid']);
216
- }
217
- const fileName = item.title.replace(dirNameReg, '_');
218
- const pathTitleList = [...preItem.pathTitleList, `${fileName}.md`];
219
- const pathIdList = [...preItem.pathIdList, item.uuid];
220
- const progressItem = {
221
- path: pathTitleList.join('/'),
222
- pathTitleList,
223
- pathIdList,
224
- toc: item
225
- };
226
- const isSuccess = await downloadArticle({
227
- bookId,
228
- itemUrl: item.url,
229
- savePath: `${bookPath}/${preItem.path}`,
230
- saveFilePath: `${bookPath}/${progressItem.path}`,
231
- uuid: item.uuid,
232
- articleUrl: `${articleUrlPrefix}/${item.url}`,
233
- articleTitle: item.title
234
- });
235
- if (isSuccess) {
236
- uuidMap.set(item.uuid, progressItem);
237
- progress.updateProgress(progressItem);
238
- }
239
- else {
240
- errArticleCount += 1;
241
- }
242
- }
243
- }
244
- await progress.completePromise;
245
- logger.info(`总数${totalArticleCount}篇,✕ 失败${errArticleCount}篇`);
246
- logger.info('生成目录 SUMMARY.md');
247
- await genSummaryFile({
248
- bookPath,
249
- bookName,
250
- bookDesc,
251
- uuidMap
252
- });
253
- if (progress.curr === total) {
254
- logger.info('√ 已完成');
255
- return;
256
- }
257
- }
258
- async function downloadArticle(params) {
259
- const { bookId, itemUrl, savePath, saveFilePath, uuid, articleUrl, articleTitle } = params;
260
- const apiUrl = `https://www.yuque.com/api/docs/${itemUrl}`;
261
- const response = await axios.get(apiUrl, {
262
- headers: {
263
- "user-agent": randUserAgent({ browser: 'chrome', device: "desktop" })
264
- },
265
- params: {
266
- 'book_id': bookId,
267
- 'merge_dynamic_data': false,
268
- mode: 'markdown'
269
- }
270
- }).then(({ data, status }) => {
271
- if (status === 200)
272
- return data;
273
- logger.error(`download article error: ${apiUrl} http status ${status}`);
274
- return null;
275
- });
276
- if (!response || !response.data || !response.data.sourcecode)
277
- return false;
278
- let mdData = response.data.sourcecode;
279
- try {
280
- mdData = await mdImg.run(mdData, {
281
- dist: savePath,
282
- imgDir: `img/${uuid}`,
283
- isIgnoreConsole: true
284
- });
285
- }
286
- catch (err) {
287
- logger.error(`download article img Error (api: ${apiUrl})`);
288
- return false;
289
- }
290
- mdData = mdData.replace(/<br(\s?)\/>/gm, '\n');
291
- if (articleTitle) {
292
- mdData = `# ${articleTitle}\n<!--page header-->\n\n${mdData}\n\n`;
293
- }
294
- if (articleUrl) {
295
- mdData += `<!--page footer-->\n- 原文: <${articleUrl}>`;
296
- }
297
- try {
298
- await fs.writeFile(saveFilePath, mdData);
299
- return true;
300
- }
301
- catch (e) {
302
- logger.error(`download article write file Error (api: ${apiUrl})`);
303
- return false;
304
- }
305
- }
306
- async function genSummaryFile(params) {
307
- const { bookName, bookDesc, bookPath, uuidMap } = params;
308
- const header = `# ${bookName}\n\n > ${bookDesc}\n\n`;
309
- let mdContent = header;
310
- const summary = [];
311
- uuidMap.forEach(processItem => {
312
- const toc = processItem.toc;
313
- const parentId = toc['parent_uuid'];
314
- const findRes = findTree(summary, parentId);
315
- if (toc.type.toLocaleLowerCase() === 'title' || toc['child_uuid'] !== '') {
316
- if (typeof findRes !== 'boolean') {
317
- if (!Array.isArray(findRes.children))
318
- findRes.children = [];
319
- findRes.children.push({
320
- id: toc.uuid,
321
- type: 'title',
322
- level: findRes.level + 1,
323
- text: toc.title
324
- });
325
- }
326
- else {
327
- summary.push({
328
- id: toc.uuid,
329
- type: 'title',
330
- text: toc.title,
331
- level: 1
332
- });
333
- }
334
- }
335
- else {
336
- if (typeof findRes !== 'boolean') {
337
- if (!Array.isArray(findRes.children))
338
- findRes.children = [];
339
- findRes.children.push({
340
- id: toc.uuid,
341
- type: 'link',
342
- level: findRes.level + 1,
343
- link: processItem.path,
344
- text: toc.title
345
- });
346
- }
347
- else {
348
- summary.push({
349
- id: toc.uuid,
350
- type: 'link',
351
- text: toc.title,
352
- link: processItem.path,
353
- level: 1
354
- });
355
- }
356
- }
357
- });
358
- let summaryContent = genSummaryContent(summary, '');
359
- mdContent += summaryContent;
360
- try {
361
- await fs.writeFile(`${bookPath}/SUMMARY.md`, mdContent);
362
- }
363
- catch (err) {
364
- logger.error('Generate Summary Error');
365
- }
366
- }
367
- function genSummaryContent(summary, summaryContent) {
368
- for (let i = 0; i < summary.length; i++) {
369
- const item = summary[i];
370
- if (item.type === 'title') {
371
- summaryContent += `\n${''.padStart(item.level + 1, '#')} ${item.text}\n\n`;
372
- }
373
- else if (item.type === 'link') {
374
- const link = item.link ? item.link.replace(/\s/g, '%20') : item.link;
375
- summaryContent += `- [${item.text}](${link})\n`;
376
- }
377
- if (Array.isArray(item.children)) {
378
- summaryContent += genSummaryContent(item.children, '');
379
- }
380
- }
381
- return summaryContent;
382
- }
383
- function findIdItem(node, id) {
384
- if (node.id === id) {
385
- return node;
386
- }
387
- else if (node.children) {
388
- const findRes = findTree(node.children, id);
389
- if (findRes)
390
- return findRes;
391
- }
392
- return false;
393
- }
394
- function findTree(tree, id) {
395
- if (!id)
396
- return false;
397
- for (let i = 0; i < tree.length; i++) {
398
- const findRes = findIdItem(tree[i], id);
399
- if (findRes)
400
- return findRes;
401
- }
402
- return false;
403
- }
404
-
405
- export { getKnowledgeBaseInfo as g, logger as l, main as m };