yuque-dl 1.0.11 → 1.0.13

package/README.md

@@ -1,3 +1,41 @@
 # yuque-dl
 
-yuque 知识库下载
+语雀知识库下载为本地markdown
+
+![header](https://socialify.git.ci/gxr404/yuque-dl/image?description=1&descriptionEditable=%E8%AF%AD%E9%9B%80%E7%9F%A5%E8%AF%86%E5%BA%93%E4%B8%8B%E8%BD%BD&issues=1&logo=https%3A%2F%2Fraw.githubusercontent.com%2Fgxr404%2Fyuque-dl%2Fmain%2Fassets%2Flogo.png&name=1&pattern=Circuit%20Board&pulls=1&stargazers=1&theme=Light)
+
+## Install
+
+```bash
+npm i -g yuque-dl
+```
+
+## Usage
+
+```bash
+$ yuque-dl --help
+
+  Usage:
+    $ yuque-dl <url>
+
+  Commands:
+    <url>  request url
+
+  For more info, run any command with the `--help` flag:
+    $ yuque-dl --help
+
+  Options:
+    -d, --distDir <dir>  下载的目录 eg: -d download (default: download)
+    -i, --ignoreImg      忽略图片不下载 (default: false)
+    -h, --help           Display this message
+    -v, --version        Display version number
+```
+
+## Example
+
+![demo](./assets/demo.gif)
+
+## Feature
+
+- [x] 支持下载中断继续
+- [x] 支持图片下载本地

package/dist/cjs/cli.js

@@ -1 +1 @@
-"use strict";var e=require("node:fs"),r=require("cac"),n=require("./index-13483ea3.js");require("node:fs/promises"),require("axios"),require("log4js"),require("progress"),require("rand-user-agent"),require("pull-md-img");const i=r.cac("yuque-dl"),{version:o}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:document.currentScript&&document.currentScript.src||new URL("cli.js",document.baseURI).href)).toString());i.command("<url>","request url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).action((async(e,r)=>{try{await n.main(e,r)}catch(e){n.logger.error(e.message||"unknown exception")}})),i.help(),i.version(o);try{i.parse()}catch(e){n.logger.error(e.message||"unknown exception"),process.exit(1)}
+"use strict";var e=require("node:fs"),r=require("cac"),n=require("./index-f7dedfcf.js");require("node:fs/promises"),require("axios"),require("log4js"),require("progress"),require("rand-user-agent"),require("pull-md-img");const i=r.cac("yuque-dl"),{version:o}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:document.currentScript&&document.currentScript.src||new URL("cli.js",document.baseURI).href)).toString());i.command("<url>","request url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).action((async(e,r)=>{try{await n.main(e,r)}catch(e){n.logger.error(e.message||"unknown exception")}})),i.help(),i.version(o);try{i.parse()}catch(e){n.logger.error(e.message||"unknown exception"),process.exit(1)}

package/dist/cjs/index-f7dedfcf.js

@@ -0,0 +1 @@
+"use strict";var e=require("node:fs/promises"),t=require("axios"),r=require("log4js"),o=require("progress"),i=require("rand-user-agent"),s=require("pull-md-img");var n=(r.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),r.getLogger("yuque-dl"));class a{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){let e;this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.completePromise=new Promise((t=>{e=t})),this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,n.info("断点下载")),this.bar=new o("downloading [:bar] :rate/bps :percent :etas",{width:80,total:this.total,curr:0,callback(){e()}}),this.curr>0&&this.bar.tick(this.curr)}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.curr=this.curr+1,this.bar&&this.bar?.tick()}}function l({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=i(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=i(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=i(r,e,t);return o}class c{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:i}=this.summaryInfo;let s=`# ${t}\n\n`;r&&(s+=`> ${r}\n\n`);let a=s;const l=[];i.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(l,r),i={text:t.title.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""};"title"===t.type.toLocaleLowerCase()||""!==t.child_uuid?(i.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i))):(i.type="link",i.link=e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i)))})),a+=this.genSummaryContent(l,"");try{await e.writeFile(`${o}/SUMMARY.md`,a)}catch(e){n.error("Generate Summary Error")}}genSummaryContent(e,t){for(let r=0;r<e.length;r++){const o=e[r];if("title"===o.type)t+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if("link"===o.type){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`- [${o.text}](${e})\n`}Array.isArray(o.children)&&(t+=this.genSummaryContent(o.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(let r=0;r<e.length;r++){const o=this.findIdItem(e[r],t);if(o)return o}return!1}}function u(e){const r=/decodeURIComponent\(\"(.+)\"\)\);/m;return t.get(e,{headers:{"user-agent":l({browser:"chrome",device:"desktop"})}}).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=e.match(r)||"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));if(!o.book)return{};return{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||""}}))}async function d(r){const{bookId:o,itemUrl:i,savePath:n,saveFilePath:a,uuid:c,articleUrl:u,articleTitle:d,ignoreImg:h}=r;let p=`https://www.yuque.com/api/docs/${i}`;const g={book_id:String(o),merge_dynamic_data:String(!1),mode:"markdown"},f=new URLSearchParams(g).toString();p=`${p}?${f}`;let m=(await t.get(p,{headers:{"user-agent":l({browser:"chrome",device:"desktop"})}}).then((({data:e,status:t})=>{if(200===t){const t=e;if(!t||!t.data||!t.data.sourcecode)throw new Error(`download article Error: ${p}`);return t}throw new Error(`download article Error: ${p} http status ${t}`)})).catch((e=>{throw new Error(`download article Error: ${p} ${e.message}`)}))).data.sourcecode;if(!h)try{m=await s.run(m,{dist:n,imgDir:`img/${c}`,isIgnoreConsole:!0})}catch(e){let t=`download article image Error: ${e.message}`;throw e.error&&e.url&&(t=`download article image Error ${e.url}: ${e.error?.message}`),new Error(t)}m=m.replace(/<br(\s?)\/>/gm,"\n"),d&&(m=`# ${d}\n\x3c!--page header--\x3e\n\n${m}\n\n`),u&&(m+=`\x3c!--page footer--\x3e\n- 原文: <${u}>`);try{return await e.writeFile(a,m),!0}catch(e){throw new Error(`download article Error ${u}: ${e.message}`)}}exports.downloadArticle=d,exports.getKnowledgeBaseInfo=u,exports.logger=n,exports.main=async function(t,r){const{bookId:o,tocList:i,bookName:s,bookDesc:l,bookSlug:h}=await u(t);if(!o)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const p=`${r.distDir}/${o}`;await e.mkdir(p,{recursive:!0});const g=i.length,f=new a(p,g);if(await f.init(),f.curr===g)return void n.info("√ 已完成");const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const w=t.replace(new RegExp(`(.*?/${h}).*`),"$1");let y=0,b=0,$=[];for(let t=0;t<g;t++){const s=i[t];if("string"!=typeof s.type)continue;if(m.get(s.uuid))continue;const n=/[\\\/:\*\?"<>\|\n\r]/g;if("title"===s.type.toLocaleLowerCase()||""!==s.child_uuid){let t=s,r=[],o=[];for(;t;)r.unshift(t.title.replace(n,"_").replace(/\s/,"")),o.unshift(t.uuid),t=m.get(t.parent_uuid)?m.get(t.parent_uuid).toc:void 0;const i={path:r.join("/"),pathTitleList:r,pathIdList:o,toc:s};await e.mkdir(`${p}/${r.join("/")}`,{recursive:!0}),m.set(s.uuid,i),f.updateProgress(i,!0)}else if(s.url){b+=1;let e={path:"",pathTitleList:[],pathIdList:[]};m.get(s.parent_uuid)&&(e=m.get(s.parent_uuid));const t=s.title.replace(n,"_").replace(/\s/,""),i=[...e.pathTitleList,`${t}.md`],a=[...e.pathIdList,s.uuid],l={path:i.join("/"),pathTitleList:i,pathIdList:a,toc:s};let c=!0;try{await d({bookId:o,itemUrl:s.url,savePath:`${p}/${e.path}`,saveFilePath:`${p}/${l.path}`,uuid:s.uuid,articleUrl:`${w}/${s.url}`,articleTitle:s.title,ignoreImg:r.ignoreImg})}catch(e){c=!1,y+=1,$.push(l),f.bar?.interrupt(`✕ ${e.message}`)}m.set(s.uuid,l),f.updateProgress(l,c)}}if(await(f.completePromise?.finally((()=>console.log("")))),y>0){n.info(`本次执行总数${b}篇，✕ 失败${y}篇`);for(let e=0;e<$.length;e++)n.error(`✕ ${$[e].path}`);n.info("o(╥﹏╥)o 由于网络波动以上下载失败，请重新执行命令")}const k=new c({bookPath:p,bookName:s,bookDesc:l,uuidMap:m});await k.genFile(),n.info("√ 生成目录 SUMMARY.md"),f.curr!==g||n.info("√ 已完成")};

package/dist/cjs/index.js

@@ -1 +1 @@
-"use strict";require("node:fs/promises"),require("axios");var e=require("./index-13483ea3.js");require("pull-md-img"),require("log4js"),require("progress"),require("rand-user-agent"),exports.downloadArticle=e.downloadArticle,exports.getKnowledgeBaseInfo=e.getKnowledgeBaseInfo,exports.main=e.main;
+"use strict";require("node:fs/promises"),require("axios");var e=require("./index-f7dedfcf.js");require("pull-md-img"),require("log4js"),require("progress"),require("rand-user-agent"),exports.downloadArticle=e.downloadArticle,exports.getKnowledgeBaseInfo=e.getKnowledgeBaseInfo,exports.main=e.main;

package/dist/es/cli.js

@@ -1 +1 @@
-import{readFileSync as o}from"node:fs";import{cac as r}from"cac";import{m as e,l as t}from"./index-f024e870.js";import"node:fs/promises";import"axios";import"log4js";import"progress";import"rand-user-agent";import"pull-md-img";const s=r("yuque-dl"),{version:i}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());s.command("<url>","request url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).action((async(o,r)=>{try{await e(o,r)}catch(o){t.error(o.message||"unknown exception")}})),s.help(),s.version(i);try{s.parse()}catch(o){t.error(o.message||"unknown exception"),process.exit(1)}
+import{readFileSync as o}from"node:fs";import{cac as r}from"cac";import{m as e,l as t}from"./index-ec0d8de6.js";import"node:fs/promises";import"axios";import"log4js";import"progress";import"rand-user-agent";import"pull-md-img";const i=r("yuque-dl"),{version:n}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());i.command("<url>","request url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).action((async(o,r)=>{try{await e(o,r)}catch(o){t.error(o.message||"unknown exception")}})),i.help(),i.version(n);try{i.parse()}catch(o){t.error(o.message||"unknown exception"),process.exit(1)}

package/dist/es/index-ec0d8de6.js

@@ -0,0 +1 @@
+import e from"node:fs/promises";import t from"axios";import r from"log4js";import o from"progress";import i from"rand-user-agent";import s from"pull-md-img";var n=(r.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),r.getLogger("yuque-dl"));class a{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){let e;this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.completePromise=new Promise((t=>{e=t})),this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,n.info("断点下载")),this.bar=new o("downloading [:bar] :rate/bps :percent :etas",{width:80,total:this.total,curr:0,callback(){e()}}),this.curr>0&&this.bar.tick(this.curr)}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.curr=this.curr+1,this.bar&&this.bar?.tick()}}function l({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=i(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=i(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=i(r,e,t);return o}class c{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:i}=this.summaryInfo;let s=`# ${t}\n\n`;r&&(s+=`> ${r}\n\n`);let a=s;const l=[];i.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(l,r),i={text:t.title.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""};"title"===t.type.toLocaleLowerCase()||""!==t.child_uuid?(i.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i))):(i.type="link",i.link=e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i)))})),a+=this.genSummaryContent(l,"");try{await e.writeFile(`${o}/SUMMARY.md`,a)}catch(e){n.error("Generate Summary Error")}}genSummaryContent(e,t){for(let r=0;r<e.length;r++){const o=e[r];if("title"===o.type)t+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if("link"===o.type){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`- [${o.text}](${e})\n`}Array.isArray(o.children)&&(t+=this.genSummaryContent(o.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(let r=0;r<e.length;r++){const o=this.findIdItem(e[r],t);if(o)return o}return!1}}function u(e){const r=/decodeURIComponent\(\"(.+)\"\)\);/m;return t.get(e,{headers:{"user-agent":l({browser:"chrome",device:"desktop"})}}).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=e.match(r)||"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));if(!o.book)return{};return{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||""}}))}async function d(r){const{bookId:o,itemUrl:i,savePath:n,saveFilePath:a,uuid:c,articleUrl:u,articleTitle:d,ignoreImg:h}=r;let p=`https://www.yuque.com/api/docs/${i}`;const g={book_id:String(o),merge_dynamic_data:String(!1),mode:"markdown"},m=new URLSearchParams(g).toString();p=`${p}?${m}`;let f=(await t.get(p,{headers:{"user-agent":l({browser:"chrome",device:"desktop"})}}).then((({data:e,status:t})=>{if(200===t){const t=e;if(!t||!t.data||!t.data.sourcecode)throw new Error(`download article Error: ${p}`);return t}throw new Error(`download article Error: ${p} http status ${t}`)})).catch((e=>{throw new Error(`download article Error: ${p} ${e.message}`)}))).data.sourcecode;if(!h)try{f=await s.run(f,{dist:n,imgDir:`img/${c}`,isIgnoreConsole:!0})}catch(e){let t=`download article image Error: ${e.message}`;throw e.error&&e.url&&(t=`download article image Error ${e.url}: ${e.error?.message}`),new Error(t)}f=f.replace(/<br(\s?)\/>/gm,"\n"),d&&(f=`# ${d}\n\x3c!--page header--\x3e\n\n${f}\n\n`),u&&(f+=`\x3c!--page footer--\x3e\n- 原文: <${u}>`);try{return await e.writeFile(a,f),!0}catch(e){throw new Error(`download article Error ${u}: ${e.message}`)}}async function h(t,r){const{bookId:o,tocList:i,bookName:s,bookDesc:l,bookSlug:h}=await u(t);if(!o)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const p=`${r.distDir}/${o}`;await e.mkdir(p,{recursive:!0});const g=i.length,m=new a(p,g);if(await m.init(),m.curr===g)return void n.info("√ 已完成");const f=new Map;m.isDownloadInterrupted&&m.progressInfo.forEach((e=>{f.set(e.toc.uuid,e)}));const w=t.replace(new RegExp(`(.*?/${h}).*`),"$1");let y=0,b=0,$=[];for(let t=0;t<g;t++){const s=i[t];if("string"!=typeof s.type)continue;if(f.get(s.uuid))continue;const n=/[\\\/:\*\?"<>\|\n\r]/g;if("title"===s.type.toLocaleLowerCase()||""!==s.child_uuid){let t=s,r=[],o=[];for(;t;)r.unshift(t.title.replace(n,"_").replace(/\s/,"")),o.unshift(t.uuid),t=f.get(t.parent_uuid)?f.get(t.parent_uuid).toc:void 0;const i={path:r.join("/"),pathTitleList:r,pathIdList:o,toc:s};await e.mkdir(`${p}/${r.join("/")}`,{recursive:!0}),f.set(s.uuid,i),m.updateProgress(i,!0)}else if(s.url){b+=1;let e={path:"",pathTitleList:[],pathIdList:[]};f.get(s.parent_uuid)&&(e=f.get(s.parent_uuid));const t=s.title.replace(n,"_").replace(/\s/,""),i=[...e.pathTitleList,`${t}.md`],a=[...e.pathIdList,s.uuid],l={path:i.join("/"),pathTitleList:i,pathIdList:a,toc:s};let c=!0;try{await d({bookId:o,itemUrl:s.url,savePath:`${p}/${e.path}`,saveFilePath:`${p}/${l.path}`,uuid:s.uuid,articleUrl:`${w}/${s.url}`,articleTitle:s.title,ignoreImg:r.ignoreImg})}catch(e){c=!1,y+=1,$.push(l),m.bar?.interrupt(`✕ ${e.message}`)}f.set(s.uuid,l),m.updateProgress(l,c)}}if(await(m.completePromise?.finally((()=>console.log("")))),y>0){n.info(`本次执行总数${b}篇，✕ 失败${y}篇`);for(let e=0;e<$.length;e++)n.error(`✕ ${$[e].path}`);n.info("o(╥﹏╥)o 由于网络波动以上下载失败，请重新执行命令")}const k=new c({bookPath:p,bookName:s,bookDesc:l,uuidMap:f});await k.genFile(),n.info("√ 生成目录 SUMMARY.md"),m.curr!==g||n.info("√ 已完成")}export{d,u as g,n as l,h as m};

package/dist/es/index.js

@@ -1 +1 @@
-import"node:fs/promises";import"axios";export{d as downloadArticle,g as getKnowledgeBaseInfo,m as main}from"./index-f024e870.js";import"pull-md-img";import"log4js";import"progress";import"rand-user-agent";
+import"node:fs/promises";import"axios";export{d as downloadArticle,g as getKnowledgeBaseInfo,m as main}from"./index-ec0d8de6.js";import"pull-md-img";import"log4js";import"progress";import"rand-user-agent";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yuque-dl",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "description": "yuque 知识库下载",
   "type": "module",
   "bin": {

package/types/ProgressBar.d.ts

@@ -6,7 +6,7 @@ export interface IProgressItem {
     pathIdList: string[];
     pathTitleList: string[];
 }
-export declare type IProgress = IProgressItem[];
+export type IProgress = IProgressItem[];
 export default class ProgressBar {
     bookPath: string;
     progressFilePath: string;

package/types/cli.d.ts

@@ -1,3 +1,4 @@
 export interface IOptions {
     distDir: string;
+    ignoreImg: boolean;
 }

package/types/index.d.ts

@@ -15,6 +15,7 @@ interface IDownloadArticleParams {
     uuid: string;
     articleTitle: string;
     articleUrl: string;
+    ignoreImg: boolean;
 }
 /** 获取知识库数据信息 */
 declare function getKnowledgeBaseInfo(url: string): Promise<IKnowledgeBaseInfo>;

package/dist/cjs/index-13483ea3.js

@@ -1 +0,0 @@
-"use strict";var e=require("node:fs/promises"),t=require("axios"),r=require("log4js"),o=require("progress"),i=require("rand-user-agent"),s=require("pull-md-img");var n=(r.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),r.getLogger("yuque-dl"));class a{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){let e;this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.completePromise=new Promise((t=>{e=t})),this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,n.info("断点下载")),this.bar=new o("downloading [:bar] :rate/bps :percent :etas",{width:100,total:this.total,curr:0,callback(){e()}}),this.curr>0&&this.bar.tick(this.curr)}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.curr=this.curr+1,this.bar&&this.bar?.tick()}}function l({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=i(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=i(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=i(r,e,t);return o}class c{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:i}=this.summaryInfo;let s=`# ${t}\n\n`;r&&(s+=`> ${r}\n\n`);let a=s;const l=[];i.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(l,r),i={text:t.title.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""};"title"===t.type.toLocaleLowerCase()||""!==t.child_uuid?(i.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i))):(i.type="link",i.link=e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i)))})),a+=this.genSummaryContent(l,"");try{await e.writeFile(`${o}/SUMMARY.md`,a)}catch(e){n.error("Generate Summary Error")}}genSummaryContent(e,t){for(let r=0;r<e.length;r++){const o=e[r];if("title"===o.type)t+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if("link"===o.type){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`- [${o.text}](${e})\n`}Array.isArray(o.children)&&(t+=this.genSummaryContent(o.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(let r=0;r<e.length;r++){const o=this.findIdItem(e[r],t);if(o)return o}return!1}}function u(e){const r=/decodeURIComponent\(\"(.+)\"\)\);/m;return t.get(e,{headers:{"user-agent":l({browser:"chrome",device:"desktop"})}}).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=e.match(r)||"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));if(!o.book)return{};return{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||""}}))}async function d(r){const{bookId:o,itemUrl:i,savePath:n,saveFilePath:a,uuid:c,articleUrl:u,articleTitle:d}=r;let h=`https://www.yuque.com/api/docs/${i}`;const p={book_id:String(o),merge_dynamic_data:String(!1),mode:"markdown"},g=new URLSearchParams(p).toString();h=`${h}?${g}`;let f=(await t.get(h,{headers:{"user-agent":l({browser:"chrome",device:"desktop"})}}).then((({data:e,status:t})=>{if(200===t){const t=e;if(!t||!t.data||!t.data.sourcecode)throw new Error(`download article Error: ${h}`);return t}throw new Error(`download article Error: ${h} http status ${t}`)})).catch((e=>{throw new Error(`download article Error: ${h} ${e.message}`)}))).data.sourcecode;try{f=await s.run(f,{dist:n,imgDir:`img/${c}`,isIgnoreConsole:!0})}catch(e){let t=`download article image Error: ${e.message}`;throw e.error&&e.url&&(t=`download article image Error ${e.url}: ${e.error?.message}`),new Error(t)}f=f.replace(/<br(\s?)\/>/gm,"\n"),d&&(f=`# ${d}\n\x3c!--page header--\x3e\n\n${f}\n\n`),u&&(f+=`\x3c!--page footer--\x3e\n- 原文: <${u}>`);try{return await e.writeFile(a,f),!0}catch(e){throw new Error(`download article Error ${u}: ${e.message}`)}}exports.downloadArticle=d,exports.getKnowledgeBaseInfo=u,exports.logger=n,exports.main=async function(t,r){const{bookId:o,tocList:i,bookName:s,bookDesc:l,bookSlug:h}=await u(t);if(!o)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const p=`${r.distDir}/${o}`;await e.mkdir(p,{recursive:!0});const g=i.length,f=new a(p,g);if(await f.init(),f.curr===g)return void n.info("√ 已完成");const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const w=t.replace(new RegExp(`(.*?/${h}).*`),"$1");let y=0,b=0,$=[];for(let t=0;t<g;t++){const r=i[t];if("string"!=typeof r.type)continue;if(m.get(r.uuid))continue;const s=/[\\\/:\*\?"<>\|\n\r]/g;if("title"===r.type.toLocaleLowerCase()||""!==r.child_uuid){let t=r,o=[],i=[];for(;t;)o.unshift(t.title.replace(s,"_").replace(/\s/,"")),i.unshift(t.uuid),t=m.get(t.parent_uuid)?m.get(t.parent_uuid).toc:void 0;const n={path:o.join("/"),pathTitleList:o,pathIdList:i,toc:r};await e.mkdir(`${p}/${o.join("/")}`,{recursive:!0}),m.set(r.uuid,n),f.updateProgress(n,!0)}else if(r.url){b+=1;let e={path:"",pathTitleList:[],pathIdList:[]};m.get(r.parent_uuid)&&(e=m.get(r.parent_uuid));const t=r.title.replace(s,"_").replace(/\s/,""),i=[...e.pathTitleList,`${t}.md`],n=[...e.pathIdList,r.uuid],a={path:i.join("/"),pathTitleList:i,pathIdList:n,toc:r};let l=!0;try{await d({bookId:o,itemUrl:r.url,savePath:`${p}/${e.path}`,saveFilePath:`${p}/${a.path}`,uuid:r.uuid,articleUrl:`${w}/${r.url}`,articleTitle:r.title})}catch(e){l=!1,y+=1,$.push(a),f.bar?.interrupt(`✕ ${e.message}`)}m.set(r.uuid,a),f.updateProgress(a,l)}}if(await(f.completePromise?.finally((()=>console.log("")))),y>0){n.info(`本次执行总数${b}篇，✕ 失败${y}篇`);for(let e=0;e<$.length;e++)n.error(`✕ ${$[e].path}`);n.info("o(╥﹏╥)o 由于网络波动以上下载失败，请重新执行命令")}const k=new c({bookPath:p,bookName:s,bookDesc:l,uuidMap:m});await k.genFile(),n.info("√ 生成目录 SUMMARY.md"),f.curr!==g||n.info("√ 已完成")};

package/dist/es/index-f024e870.js

@@ -1 +0,0 @@
-import e from"node:fs/promises";import t from"axios";import r from"log4js";import o from"progress";import i from"rand-user-agent";import s from"pull-md-img";var n=(r.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),r.getLogger("yuque-dl"));class a{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){let e;this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.completePromise=new Promise((t=>{e=t})),this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,n.info("断点下载")),this.bar=new o("downloading [:bar] :rate/bps :percent :etas",{width:100,total:this.total,curr:0,callback(){e()}}),this.curr>0&&this.bar.tick(this.curr)}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.curr=this.curr+1,this.bar&&this.bar?.tick()}}function l({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=i(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=i(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=i(r,e,t);return o}class c{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:i}=this.summaryInfo;let s=`# ${t}\n\n`;r&&(s+=`> ${r}\n\n`);let a=s;const l=[];i.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(l,r),i={text:t.title.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""};"title"===t.type.toLocaleLowerCase()||""!==t.child_uuid?(i.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i))):(i.type="link",i.link=e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i)))})),a+=this.genSummaryContent(l,"");try{await e.writeFile(`${o}/SUMMARY.md`,a)}catch(e){n.error("Generate Summary Error")}}genSummaryContent(e,t){for(let r=0;r<e.length;r++){const o=e[r];if("title"===o.type)t+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if("link"===o.type){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`- [${o.text}](${e})\n`}Array.isArray(o.children)&&(t+=this.genSummaryContent(o.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(let r=0;r<e.length;r++){const o=this.findIdItem(e[r],t);if(o)return o}return!1}}function u(e){const r=/decodeURIComponent\(\"(.+)\"\)\);/m;return t.get(e,{headers:{"user-agent":l({browser:"chrome",device:"desktop"})}}).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=e.match(r)||"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));if(!o.book)return{};return{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||""}}))}async function d(r){const{bookId:o,itemUrl:i,savePath:n,saveFilePath:a,uuid:c,articleUrl:u,articleTitle:d}=r;let h=`https://www.yuque.com/api/docs/${i}`;const p={book_id:String(o),merge_dynamic_data:String(!1),mode:"markdown"},g=new URLSearchParams(p).toString();h=`${h}?${g}`;let f=(await t.get(h,{headers:{"user-agent":l({browser:"chrome",device:"desktop"})}}).then((({data:e,status:t})=>{if(200===t){const t=e;if(!t||!t.data||!t.data.sourcecode)throw new Error(`download article Error: ${h}`);return t}throw new Error(`download article Error: ${h} http status ${t}`)})).catch((e=>{throw new Error(`download article Error: ${h} ${e.message}`)}))).data.sourcecode;try{f=await s.run(f,{dist:n,imgDir:`img/${c}`,isIgnoreConsole:!0})}catch(e){let t=`download article image Error: ${e.message}`;throw e.error&&e.url&&(t=`download article image Error ${e.url}: ${e.error?.message}`),new Error(t)}f=f.replace(/<br(\s?)\/>/gm,"\n"),d&&(f=`# ${d}\n\x3c!--page header--\x3e\n\n${f}\n\n`),u&&(f+=`\x3c!--page footer--\x3e\n- 原文: <${u}>`);try{return await e.writeFile(a,f),!0}catch(e){throw new Error(`download article Error ${u}: ${e.message}`)}}async function h(t,r){const{bookId:o,tocList:i,bookName:s,bookDesc:l,bookSlug:h}=await u(t);if(!o)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const p=`${r.distDir}/${o}`;await e.mkdir(p,{recursive:!0});const g=i.length,f=new a(p,g);if(await f.init(),f.curr===g)return void n.info("√ 已完成");const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const w=t.replace(new RegExp(`(.*?/${h}).*`),"$1");let y=0,b=0,$=[];for(let t=0;t<g;t++){const r=i[t];if("string"!=typeof r.type)continue;if(m.get(r.uuid))continue;const s=/[\\\/:\*\?"<>\|\n\r]/g;if("title"===r.type.toLocaleLowerCase()||""!==r.child_uuid){let t=r,o=[],i=[];for(;t;)o.unshift(t.title.replace(s,"_").replace(/\s/,"")),i.unshift(t.uuid),t=m.get(t.parent_uuid)?m.get(t.parent_uuid).toc:void 0;const n={path:o.join("/"),pathTitleList:o,pathIdList:i,toc:r};await e.mkdir(`${p}/${o.join("/")}`,{recursive:!0}),m.set(r.uuid,n),f.updateProgress(n,!0)}else if(r.url){b+=1;let e={path:"",pathTitleList:[],pathIdList:[]};m.get(r.parent_uuid)&&(e=m.get(r.parent_uuid));const t=r.title.replace(s,"_").replace(/\s/,""),i=[...e.pathTitleList,`${t}.md`],n=[...e.pathIdList,r.uuid],a={path:i.join("/"),pathTitleList:i,pathIdList:n,toc:r};let l=!0;try{await d({bookId:o,itemUrl:r.url,savePath:`${p}/${e.path}`,saveFilePath:`${p}/${a.path}`,uuid:r.uuid,articleUrl:`${w}/${r.url}`,articleTitle:r.title})}catch(e){l=!1,y+=1,$.push(a),f.bar?.interrupt(`✕ ${e.message}`)}m.set(r.uuid,a),f.updateProgress(a,l)}}if(await(f.completePromise?.finally((()=>console.log("")))),y>0){n.info(`本次执行总数${b}篇，✕ 失败${y}篇`);for(let e=0;e<$.length;e++)n.error(`✕ ${$[e].path}`);n.info("o(╥﹏╥)o 由于网络波动以上下载失败，请重新执行命令")}const k=new c({bookPath:p,bookName:s,bookDesc:l,uuidMap:m});await k.genFile(),n.info("√ 生成目录 SUMMARY.md"),f.curr!==g||n.info("√ 已完成")}export{d,u as g,n as l,h as m};