yuque-dl 1.0.33 → 1.0.35

package/README.md

@@ -23,18 +23,21 @@ $ yuque-dl --help
     $ yuque-dl <url>
 
   Commands:
-    <url>  语雀知识库url
+    <url>       语雀知识库url
+    -h, --help  显示帮助
 
   For more info, run any command with the `--help` flag:
     $ yuque-dl --help
+    $ yuque-dl -h, --help --help
 
   Options:
-    -d, --distDir <dir>  下载的目录 eg: -d download (default: download)
-    -i, --ignoreImg      忽略图片不下载 (default: false)
-    -k, --key <key>      指定语雀的 cookie key，默认是 "_yuque_session"
-    -t, --token <token>  语雀的cookie key 对应的值
-    -h, --help           Display this message
-    -v, --version        Display version number
+    -d, --dist-dir <dir>  下载的目录 eg: -d download (default: download)
+    -i, --ignore-img      忽略图片不下载 (default: false)
+    -k, --key <key>       语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样
+    -t, --token <token>   语雀的cookie key 对应的值
+    --ignore-toc          默认输出toc目录,添加此参数则不输出toc目录 (default: false)
+    -h, --help            Display this message
+    -v, --version         Display version number
 ```
 
 ### Start
@@ -72,7 +75,10 @@ yuque-dl "https://www.yuque.com/yuque/thyzgp" -t "abcd..."
 
 ![public_pwd](./docs/assets/public_pwd.png)
 
-公开密码访问的知识库或文档需要使用`verified_books`/`verified_docs`这个cookie
+公开密码访问的知识库两种情况:
+
+- 已经登录语雀，访问需要密码的知识库 输入密码后 使用`_yuque_session`这个cookie
+- 未登录语雀，访问需要密码的知识库 输入密码后需要使用`verified_books`/`verified_docs`这个cookie
 
 ```bash
 yuque-dl "url" -k "verified_books" -t "verified_books的值"
@@ -84,6 +90,7 @@ yuque-dl "url" -k "verified_books" -t "verified_books的值"
 - [x] 支持图片下载本地
 - [x] 支持下载分享私有的知识库
 - [x] 支持转换表格类型的文档 (ps: 表格内插入图表暂不支持)
+- [x] 添加toc目录功能
 - [ ] 支持其他文档类型？🤔
 - [ ] 添加测试 🤔
 

package/dist/cjs/cli.js

@@ -1 +1 @@
-"use strict";var e=require("node:fs"),r=require("cac"),o=require("semver"),n=require("./index-a5e37672.js");require("node:fs/promises"),require("pull-md-img"),require("ora"),require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako");var i="undefined"!=typeof document?document.currentScript:null;const s=r.cac("yuque-dl"),{version:t,engines:u}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:i&&i.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=u.node;o.satisfies(process.version,e)||(n.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),s.command("<url>","语雀知识库url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").action((async(e,r)=>{try{await n.main(e,r)}catch(e){n.logger.error(e.message||"unknown exception")}})),s.help(),s.version(t);try{s.parse()}catch(e){n.logger.error(e.message||"unknown exception"),process.exit(1)}
+"use strict";var e=require("node:fs"),r=require("cac"),o=require("semver"),i=require("./index-327317a7.js");require("node:fs/promises"),require("pull-md-img"),require("ora"),require("markdown-toc"),require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako");var n="undefined"!=typeof document?document.currentScript:null;const s=r.cac("yuque-dl"),{version:t,engines:c}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:n&&n.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=c.node;o.satisfies(process.version,e)||(i.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),s.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(e,r)=>{try{await i.main(e,r)}catch(e){i.logger.error(e.message||"unknown exception")}})),s.help(),s.version(t);try{s.parse()}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}

package/dist/cjs/index-327317a7.js

@@ -0,0 +1 @@
+"use strict";var t=require("node:fs/promises"),e=require("pull-md-img"),r=require("ora"),o=require("markdown-toc"),s=require("cli-progress"),i=require("log4js"),n=require("axios"),a=require("rand-user-agent"),c=require("pako");var l,u,h=(i.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),i.getLogger("yuque-dl"));class d{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,h.info("根据上次数据继续断点下载")),this.bar=new s.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},s.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class p{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${e}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},i=e.type.toLocaleLowerCase();"title"===i||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){h.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if("title"===r.type)e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(l||(l={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(u||(u={}));const f=new Map([[u.BOARD,"画板类型"],[u.TABLE,"数据表类型"],[u.SHEET,"表格类型"],[u.DOC,"文档类型"]]),g="_yuque_session",m="https://www.yuque.com";function $({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=a(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=a(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=a(r,t,e);return o}function w(t){const{key:e=g,token:r}=t,o={"user-agent":$({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function k(t){return{headers:w(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...w(t)}}}}const y=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:i,host:a=m}=t;let c=`${a}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,n.get(c,k({token:s,key:i})).then((({data:t,status:e})=>({apiUrl:c,httpStatus:e,response:t})))},b=t=>{if(!t)return"";const e=JSON.parse(c.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(`![${s?.name}'](${s?.src})`):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return i}(t.data);r=`${r}\n${e}\n${o}`})),r};async function I(s){const{articleInfo:i,progressBar:n,options:a}=s,{token:c,key:l}=a,{bookId:h,itemUrl:d,savePath:p,saveFilePath:g,uuid:m,articleUrl:$,articleTitle:w,ignoreImg:k,host:I}=i,{httpStatus:L,apiUrl:E,response:S}=await y({articleUrl:d,bookId:h,token:c,host:I,key:l}),v=S?.data?.type?.toLocaleLowerCase();let P="";if(v===u.SHEET){const{response:t}=await y({articleUrl:d,bookId:h,token:c,host:I,key:l},!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;P=o?b(o):""}catch(t){const e=f.get(v);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([u.BOARD,u.TABLE].includes(v)){const t=f.get(v);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof S?.data?.sourcecode)throw new Error(`download article Error: ${E}, http status ${L}`);P=S.data.sourcecode}if(!k){n.pause(),console.log("");const t=r({text:`下载 "${w}" 的图片中...`});t.start();try{P=await e.run(P,{dist:p,imgDir:`./img/${m}`,isIgnoreConsole:!0,referer:d||""})}catch(t){let e=`download article image Error: ${t.message}`;throw t.error&&t.url&&(e=`download article image Error ${t.url}: ${t.error?.message}`),new Error(e)}finally{t.stop(),n.continue()}}P=P.replace(/<br(\s?)\/>/gm,"\n"),P=P.replace(/<a.*?>(\s*?)<\/a>/gm,"");const T=w?`# ${w}\n\n`:"";let N=a.ignoreToc?"":o(P).content;N&&(N=`${N}\n\n---\n\n`);P=`${T}${N}${P}${$?`\n\n> 原文: <${$}>`:""}`;try{return await t.writeFile(g,P),!0}catch(t){throw new Error(`download article Error ${$}: ${t.message}`)}}function L(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}exports.downloadArticle=I,exports.logger=h,exports.main=async function(e,r){const{bookId:o,tocList:s,bookName:i,bookDesc:a,bookSlug:c,host:u}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return n.get(t,k(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||m}:{}}))})(e,{token:r.token,key:r.key});if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const f=`${r.distDir}/${i?L(i):o}`;await t.mkdir(f,{recursive:!0});const g=s.length,$=new d(f,g);if(await $.init(),$.curr===g)return $.bar&&$.bar.stop(),void h.info(`√ 已完成: ${process.cwd()}/${f}`);const w=new Map;$.isDownloadInterrupted&&$.progressInfo.forEach((t=>{w.set(t.toc.uuid,t)}));const y=e.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(e){const{articleUrlPrefix:r,total:o,uuidMap:s,tocList:i,bookPath:n,bookId:a,progressBar:c,host:u,options:d}=e;let p=0,f=0,g=0;const m=[],$=[];for(let e=0;e<o;e++){const o=i[e];if("string"!=typeof o.type)continue;if(s.get(o.uuid))continue;const h=o.type.toLocaleLowerCase();if(h===l.TITLE||""!==o.child_uuid||h===l.LINK){let e=o;const r=[],i=[];for(;e;)r.unshift(L(e.title)),i.unshift(e.uuid),e=s.get(e.parent_uuid)?s.get(e.parent_uuid).toc:void 0;const a={path:r.join("/"),pathTitleList:r,pathIdList:i,toc:o};h===l.LINK?(g+=1,$.push(a)):await t.mkdir(`${n}/${r.join("/")}`,{recursive:!0}),s.set(o.uuid,a),await c.updateProgress(a,h!==l.LINK)}else if(o.url){f+=1;let t={path:"",pathTitleList:[],pathIdList:[]};s.get(o.parent_uuid)&&(t=s.get(o.parent_uuid));const e=L(o.title),i=[...t.pathTitleList,`${e}.md`],l=[...t.pathIdList,o.uuid],h={path:i.join("/"),pathTitleList:i,pathIdList:l,toc:o};let g=!0;const $=`${r}/${o.url}`;try{const e={bookId:a,itemUrl:o.url,savePath:`${n}/${t.path}`,saveFilePath:`${n}/${h.path}`,uuid:o.uuid,articleUrl:$,articleTitle:o.title,ignoreImg:d.ignoreImg,host:u};await I({articleInfo:e,progressBar:c,options:d})}catch(t){g=!1,p+=1,m.push({articleUrl:$,errItem:h,errMsg:t.message,err:t})}s.set(o.uuid,h),await c.updateProgress(h,g)}}if(g>0){h.warn("该知识库存在以下外链文章");for(const t of $)h.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(p>0){h.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const t of m)h.error(`${t.errItem.path} ———— ${t.articleUrl}`),h.error(`———— ✕ ${t.errMsg}`);h.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:y,total:g,uuidMap:w,tocList:s,bookPath:f,bookId:o,progressBar:$,host:u,options:r});const b=new p({bookPath:f,bookName:i,bookDesc:a,uuidMap:w});await b.genFile();const E=process.cwd();h.info(`√ 生成目录 ${E}/${f}/SUMMARY.md`),$.curr===g&&h.info(`√ 已完成: ${E}/${f}`),process.exit(0)};

package/dist/cjs/index.js

@@ -1 +1 @@
-"use strict";require("node:fs/promises"),require("pull-md-img"),require("ora");var r=require("./index-a5e37672.js");require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),exports.downloadArticle=r.downloadArticle,exports.main=r.main;
+"use strict";require("node:fs/promises"),require("pull-md-img"),require("ora"),require("markdown-toc");var r=require("./index-327317a7.js");require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),exports.downloadArticle=r.downloadArticle,exports.main=r.main;

package/dist/es/cli.js

@@ -1 +1 @@
-import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import r from"semver";import{m as i,l as s}from"./index-0a71dec6.js";import"node:fs/promises";import"pull-md-img";import"ora";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";const n=e("yuque-dl"),{version:t,engines:p}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=p.node;r.satisfies(process.version,o)||(s.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),n.command("<url>","语雀知识库url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").action((async(o,e)=>{try{await i(o,e)}catch(o){s.error(o.message||"unknown exception")}})),n.help(),n.version(t);try{n.parse()}catch(o){s.error(o.message||"unknown exception"),process.exit(1)}
+import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import r from"semver";import{m as i,l as t}from"./index-6d8832a0.js";import"node:fs/promises";import"pull-md-img";import"ora";import"markdown-toc";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";const n=e("yuque-dl"),{version:s,engines:p}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=p.node;r.satisfies(process.version,o)||(t.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),n.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(o,e)=>{try{await i(o,e)}catch(o){t.error(o.message||"unknown exception")}})),n.help(),n.version(s);try{n.parse()}catch(o){t.error(o.message||"unknown exception"),process.exit(1)}

package/dist/es/index-6d8832a0.js

@@ -0,0 +1 @@
+import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"pull-md-img";import s from"ora";import i from"markdown-toc";import n from"cli-progress";import a from"log4js";import c from"axios";import l from"rand-user-agent";import u from"pako";var h,p,d=(a.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),a.getLogger("yuque-dl"));class f{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,d.info("根据上次数据继续断点下载")),this.bar=new n.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},n.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class g{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${e}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},i=e.type.toLocaleLowerCase();"title"===i||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){d.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if("title"===r.type)e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(h||(h={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(p||(p={}));const m=new Map([[p.BOARD,"画板类型"],[p.TABLE,"数据表类型"],[p.SHEET,"表格类型"],[p.DOC,"文档类型"]]),$="_yuque_session",w="https://www.yuque.com";function k({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=l(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=l(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=l(r,t,e);return o}function y(t){const{key:e=$,token:r}=t,o={"user-agent":k({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function b(t){return{headers:y(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...y(t)}}}}const I=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:i,host:n=w}=t;let a=`${n}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return a=`${a}?${u}`,c.get(a,b({token:s,key:i})).then((({data:t,status:e})=>({apiUrl:a,httpStatus:e,response:t})))},L=t=>{if(!t)return"";const e=JSON.parse(u.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(`![${s?.name}'](${s?.src})`):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return i}(t.data);r=`${r}\n${e}\n${o}`})),r};async function E(t){const{articleInfo:r,progressBar:n,options:a}=t,{token:c,key:l}=a,{bookId:u,itemUrl:h,savePath:d,saveFilePath:f,uuid:g,articleUrl:$,articleTitle:w,ignoreImg:k,host:y}=r,{httpStatus:b,apiUrl:E,response:S}=await I({articleUrl:h,bookId:u,token:c,host:y,key:l}),P=S?.data?.type?.toLocaleLowerCase();let v="";if(P===p.SHEET){const{response:t}=await I({articleUrl:h,bookId:u,token:c,host:y,key:l},!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;v=o?L(o):""}catch(t){const e=m.get(P);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([p.BOARD,p.TABLE].includes(P)){const t=m.get(P);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof S?.data?.sourcecode)throw new Error(`download article Error: ${E}, http status ${b}`);v=S.data.sourcecode}if(!k){n.pause(),console.log("");const t=s({text:`下载 "${w}" 的图片中...`});t.start();try{v=await o.run(v,{dist:d,imgDir:`./img/${g}`,isIgnoreConsole:!0,referer:h||""})}catch(t){let e=`download article image Error: ${t.message}`;throw t.error&&t.url&&(e=`download article image Error ${t.url}: ${t.error?.message}`),new Error(e)}finally{t.stop(),n.continue()}}v=v.replace(/<br(\s?)\/>/gm,"\n"),v=v.replace(/<a.*?>(\s*?)<\/a>/gm,"");const T=w?`# ${w}\n\n`:"";let N=a.ignoreToc?"":i(v).content;N&&(N=`${N}\n\n---\n\n`);v=`${T}${N}${v}${$?`\n\n> 原文: <${$}>`:""}`;try{return await e(f,v),!0}catch(t){throw new Error(`download article Error ${$}: ${t.message}`)}}function S(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}async function P(t,e){const{bookId:o,tocList:s,bookName:i,bookDesc:n,bookSlug:a,host:l}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return c.get(t,b(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||w}:{}}))})(t,{token:e.token,key:e.key});if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const u=`${e.distDir}/${i?S(i):o}`;await r(u,{recursive:!0});const p=s.length,m=new f(u,p);if(await m.init(),m.curr===p)return m.bar&&m.bar.stop(),void d.info(`√ 已完成: ${process.cwd()}/${u}`);const $=new Map;m.isDownloadInterrupted&&m.progressInfo.forEach((t=>{$.set(t.toc.uuid,t)}));const k=t.replace(new RegExp(`(.*?/${a}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:o,uuidMap:s,tocList:i,bookPath:n,bookId:a,progressBar:c,host:l,options:u}=t;let p=0,f=0,g=0;const m=[],$=[];for(let t=0;t<o;t++){const o=i[t];if("string"!=typeof o.type)continue;if(s.get(o.uuid))continue;const d=o.type.toLocaleLowerCase();if(d===h.TITLE||""!==o.child_uuid||d===h.LINK){let t=o;const e=[],i=[];for(;t;)e.unshift(S(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const a={path:e.join("/"),pathTitleList:e,pathIdList:i,toc:o};d===h.LINK?(g+=1,$.push(a)):await r(`${n}/${e.join("/")}`,{recursive:!0}),s.set(o.uuid,a),await c.updateProgress(a,d!==h.LINK)}else if(o.url){f+=1;let t={path:"",pathTitleList:[],pathIdList:[]};s.get(o.parent_uuid)&&(t=s.get(o.parent_uuid));const r=S(o.title),i=[...t.pathTitleList,`${r}.md`],h=[...t.pathIdList,o.uuid],d={path:i.join("/"),pathTitleList:i,pathIdList:h,toc:o};let g=!0;const $=`${e}/${o.url}`;try{const e={bookId:a,itemUrl:o.url,savePath:`${n}/${t.path}`,saveFilePath:`${n}/${d.path}`,uuid:o.uuid,articleUrl:$,articleTitle:o.title,ignoreImg:u.ignoreImg,host:l};await E({articleInfo:e,progressBar:c,options:u})}catch(t){g=!1,p+=1,m.push({articleUrl:$,errItem:d,errMsg:t.message,err:t})}s.set(o.uuid,d),await c.updateProgress(d,g)}}if(g>0){d.warn("该知识库存在以下外链文章");for(const t of $)d.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(p>0){d.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const t of m)d.error(`${t.errItem.path} ———— ${t.articleUrl}`),d.error(`———— ✕ ${t.errMsg}`);d.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:k,total:p,uuidMap:$,tocList:s,bookPath:u,bookId:o,progressBar:m,host:l,options:e});const y=new g({bookPath:u,bookName:i,bookDesc:n,uuidMap:$});await y.genFile();const I=process.cwd();d.info(`√ 生成目录 ${I}/${u}/SUMMARY.md`),m.curr===p&&d.info(`√ 已完成: ${I}/${u}`),process.exit(0)}export{E as d,d as l,P as m};

package/dist/es/index.js

@@ -1 +1 @@
-import"node:fs/promises";import"pull-md-img";import"ora";export{d as downloadArticle,m as main}from"./index-0a71dec6.js";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";
+import"node:fs/promises";import"pull-md-img";import"ora";import"markdown-toc";export{d as downloadArticle,m as main}from"./index-6d8832a0.js";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yuque-dl",
-  "version": "1.0.33",
+  "version": "1.0.35",
   "description": "yuque 知识库下载",
   "type": "module",
   "bin": {
@@ -61,9 +61,10 @@
     "cac": "^6.7.14",
     "cli-progress": "^3.12.0",
     "log4js": "^6.9.1",
+    "markdown-toc": "^1.2.0",
     "ora": "^7.0.1",
     "pako": "1.0.11",
-    "pull-md-img": "^0.0.26",
+    "pull-md-img": "^0.0.41",
     "rand-user-agent": "1.0.109",
     "semver": "^7.6.0"
   },

package/types/ProgressBar.d.ts

@@ -6,7 +6,7 @@ export interface IProgressItem {
     pathIdList: string[];
     pathTitleList: string[];
 }
-export type IProgress = IProgressItem[];
+export declare type IProgress = IProgressItem[];
 export default class ProgressBar {
     bookPath: string;
     progressFilePath: string;

package/types/api.d.ts

@@ -12,7 +12,7 @@ interface GetHeaderParams {
     key?: string;
     token?: string;
 }
-type TGetKnowledgeBaseInfo = (url: string, headerParams: GetHeaderParams) => Promise<IKnowledgeBaseInfo>;
+declare type TGetKnowledgeBaseInfo = (url: string, headerParams: GetHeaderParams) => Promise<IKnowledgeBaseInfo>;
 /** 获取知识库数据信息 */
 export declare const getKnowledgeBaseInfo: TGetKnowledgeBaseInfo;
 interface GetMdDataParams {
@@ -27,6 +27,6 @@ interface IGetDocsMdDataRes {
     httpStatus: number;
     response?: ArticleResponse.RootObject;
 }
-type TGetMdData = (params: GetMdDataParams, isMd?: boolean) => Promise<IGetDocsMdDataRes>;
+declare type TGetMdData = (params: GetMdDataParams, isMd?: boolean) => Promise<IGetDocsMdDataRes>;
 export declare const getDocsMdData: TGetMdData;
 export {};

package/types/cli.d.ts

@@ -3,4 +3,5 @@ export interface IOptions {
     ignoreImg: boolean;
     token?: string;
     key?: string;
+    ignoreToc: boolean;
 }

package/types/index.d.ts

@@ -16,10 +16,8 @@ interface DownloadArticleParams {
     articleInfo: ArticleInfo;
     /** 进度条实例 */
     progressBar: ProgressBar;
-    /** 语雀cookie token的值 */
-    token?: string;
-    /** 语雀cookie token的key */
-    key?: string;
+    /** cli options */
+    options: IOptions;
 }
 /** 下载单篇文章 */
 declare function downloadArticle(params: DownloadArticleParams): Promise<boolean>;

package/dist/cjs/index-a5e37672.js

@@ -1 +0,0 @@
-"use strict";var t=require("node:fs/promises"),e=require("pull-md-img"),r=require("ora"),o=require("cli-progress"),s=require("log4js"),i=require("axios"),n=require("rand-user-agent"),a=require("pako");var c,l,u=(s.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),s.getLogger("yuque-dl"));class h{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,u.info("根据上次数据继续断点下载")),this.bar=new o.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},o.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class d{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${e}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},i=e.type.toLocaleLowerCase();"title"===i||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){u.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if("title"===r.type)e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(c||(c={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(l||(l={}));const p=new Map([[l.BOARD,"画板类型"],[l.TABLE,"数据表类型"],[l.SHEET,"表格类型"],[l.DOC,"文档类型"]]),f="_yuque_session",g="https://www.yuque.com";function m({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=n(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=n(r,t,e);return o}function k(t){const{key:e=f,token:r}=t,o={"user-agent":m({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function $(t){return{headers:k(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...k(t)}}}}const w=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:n,host:a=g}=t;let c=`${a}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,i.get(c,$({token:s,key:n})).then((({data:t,status:e})=>({apiUrl:c,httpStatus:e,response:t})))},y=t=>{if(!t)return"";const e=JSON.parse(a.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(`![${s?.name}'](${s?.src})`):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return i}(t.data);r=`${r}\n${e}\n${o}`})),r};async function b(o){const{articleInfo:s,progressBar:i,token:n,key:a}=o,{bookId:c,itemUrl:u,savePath:h,saveFilePath:d,uuid:f,articleUrl:g,articleTitle:m,ignoreImg:k,host:$}=s,{httpStatus:b,apiUrl:I,response:L}=await w({articleUrl:u,bookId:c,token:n,host:$,key:a}),E=L?.data?.type?.toLocaleLowerCase();let S="";if(E===l.SHEET){const{response:t}=await w({articleUrl:u,bookId:c,token:n,host:$,key:a},!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;S=o?y(o):""}catch(t){const e=p.get(E);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([l.BOARD,l.TABLE].includes(E)){const t=p.get(E);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof L?.data?.sourcecode)throw new Error(`download article Error: ${I}, http status ${b}`);S=L.data.sourcecode}if(!k){i.pause(),console.log("");const t=r({text:`下载 "${m}" 的图片中...`});t.start();try{S=await e.run(S,{dist:h,imgDir:`./img/${f}`,isIgnoreConsole:!0})}catch(t){let e=`download article image Error: ${t.message}`;throw t.error&&t.url&&(e=`download article image Error ${t.url}: ${t.error?.message}`),new Error(e)}finally{t.stop(),i.continue()}}S=S.replace(/<br(\s?)\/>/gm,"\n"),S=S.replace(/<a.*?>(\s*?)<\/a>/gm,""),m&&(S=`# ${m}\n\n${S}\n\n`),g&&(S+=`> 原文: <${g}>`);try{return await t.writeFile(d,S),!0}catch(t){throw new Error(`download article Error ${g}: ${t.message}`)}}function I(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}exports.downloadArticle=b,exports.logger=u,exports.main=async function(e,r){const{bookId:o,tocList:s,bookName:n,bookDesc:a,bookSlug:l,host:p}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return i.get(t,$(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||g}:{}}))})(e,{token:r.token,key:r.key});if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const f=`${r.distDir}/${n?I(n):o}`;await t.mkdir(f,{recursive:!0});const m=s.length,k=new h(f,m);if(await k.init(),k.curr===m)return k.bar&&k.bar.stop(),void u.info(`√ 已完成: ${process.cwd()}/${f}`);const w=new Map;k.isDownloadInterrupted&&k.progressInfo.forEach((t=>{w.set(t.toc.uuid,t)}));const y=e.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(e){const{articleUrlPrefix:r,total:o,uuidMap:s,tocList:i,bookPath:n,bookId:a,progressBar:l,host:h,options:d}=e;let p=0,f=0,g=0;const m=[],k=[];for(let e=0;e<o;e++){const o=i[e];if("string"!=typeof o.type)continue;if(s.get(o.uuid))continue;const u=o.type.toLocaleLowerCase();if(u===c.TITLE||""!==o.child_uuid||u===c.LINK){let e=o;const r=[],i=[];for(;e;)r.unshift(I(e.title)),i.unshift(e.uuid),e=s.get(e.parent_uuid)?s.get(e.parent_uuid).toc:void 0;const a={path:r.join("/"),pathTitleList:r,pathIdList:i,toc:o};u===c.LINK?(g+=1,k.push(a)):await t.mkdir(`${n}/${r.join("/")}`,{recursive:!0}),s.set(o.uuid,a),await l.updateProgress(a,u!==c.LINK)}else if(o.url){f+=1;let t={path:"",pathTitleList:[],pathIdList:[]};s.get(o.parent_uuid)&&(t=s.get(o.parent_uuid));const e=I(o.title),i=[...t.pathTitleList,`${e}.md`],c=[...t.pathIdList,o.uuid],u={path:i.join("/"),pathTitleList:i,pathIdList:c,toc:o};let g=!0;const k=`${r}/${o.url}`;try{const e={bookId:a,itemUrl:o.url,savePath:`${n}/${t.path}`,saveFilePath:`${n}/${u.path}`,uuid:o.uuid,articleUrl:k,articleTitle:o.title,ignoreImg:d.ignoreImg,host:h};await b({articleInfo:e,progressBar:l,token:d.token,key:d.key})}catch(t){g=!1,p+=1,m.push({articleUrl:k,errItem:u,errMsg:t.message,err:t})}s.set(o.uuid,u),await l.updateProgress(u,g)}}if(g>0){u.warn("该知识库存在以下外链文章");for(const t of k)u.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(p>0){u.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const t of m)u.error(`${t.errItem.path} ———— ${t.articleUrl}`),u.error(`———— ✕ ${t.errMsg}`);u.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:y,total:m,uuidMap:w,tocList:s,bookPath:f,bookId:o,progressBar:k,host:p,options:r});const L=new d({bookPath:f,bookName:n,bookDesc:a,uuidMap:w});await L.genFile();const E=process.cwd();u.info(`√ 生成目录 ${E}/${f}/SUMMARY.md`),k.curr===m&&u.info(`√ 已完成: ${E}/${f}`),process.exit(0)};

package/dist/es/index-0a71dec6.js

@@ -1 +0,0 @@
-import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"pull-md-img";import s from"ora";import i from"cli-progress";import n from"log4js";import a from"axios";import c from"rand-user-agent";import l from"pako";var u,h,p=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class d{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,p.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class f{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${e}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},i=e.type.toLocaleLowerCase();"title"===i||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){p.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if("title"===r.type)e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(u||(u={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(h||(h={}));const g=new Map([[h.BOARD,"画板类型"],[h.TABLE,"数据表类型"],[h.SHEET,"表格类型"],[h.DOC,"文档类型"]]),m="_yuque_session",k="https://www.yuque.com";function $({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=c(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=c(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=c(r,t,e);return o}function w(t){const{key:e=m,token:r}=t,o={"user-agent":$({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function y(t){return{headers:w(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...w(t)}}}}const b=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:i,host:n=k}=t;let c=`${n}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,y({token:s,key:i})).then((({data:t,status:e})=>({apiUrl:c,httpStatus:e,response:t})))},I=t=>{if(!t)return"";const e=JSON.parse(l.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(`![${s?.name}'](${s?.src})`):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return i}(t.data);r=`${r}\n${e}\n${o}`})),r};async function L(t){const{articleInfo:r,progressBar:i,token:n,key:a}=t,{bookId:c,itemUrl:l,savePath:u,saveFilePath:p,uuid:d,articleUrl:f,articleTitle:m,ignoreImg:k,host:$}=r,{httpStatus:w,apiUrl:y,response:L}=await b({articleUrl:l,bookId:c,token:n,host:$,key:a}),E=L?.data?.type?.toLocaleLowerCase();let S="";if(E===h.SHEET){const{response:t}=await b({articleUrl:l,bookId:c,token:n,host:$,key:a},!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;S=o?I(o):""}catch(t){const e=g.get(E);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([h.BOARD,h.TABLE].includes(E)){const t=g.get(E);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof L?.data?.sourcecode)throw new Error(`download article Error: ${y}, http status ${w}`);S=L.data.sourcecode}if(!k){i.pause(),console.log("");const t=s({text:`下载 "${m}" 的图片中...`});t.start();try{S=await o.run(S,{dist:u,imgDir:`./img/${d}`,isIgnoreConsole:!0})}catch(t){let e=`download article image Error: ${t.message}`;throw t.error&&t.url&&(e=`download article image Error ${t.url}: ${t.error?.message}`),new Error(e)}finally{t.stop(),i.continue()}}S=S.replace(/<br(\s?)\/>/gm,"\n"),S=S.replace(/<a.*?>(\s*?)<\/a>/gm,""),m&&(S=`# ${m}\n\n${S}\n\n`),f&&(S+=`> 原文: <${f}>`);try{return await e(p,S),!0}catch(t){throw new Error(`download article Error ${f}: ${t.message}`)}}function E(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}async function S(t,e){const{bookId:o,tocList:s,bookName:i,bookDesc:n,bookSlug:c,host:l}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(t,y(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||k}:{}}))})(t,{token:e.token,key:e.key});if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const h=`${e.distDir}/${i?E(i):o}`;await r(h,{recursive:!0});const g=s.length,m=new d(h,g);if(await m.init(),m.curr===g)return m.bar&&m.bar.stop(),void p.info(`√ 已完成: ${process.cwd()}/${h}`);const $=new Map;m.isDownloadInterrupted&&m.progressInfo.forEach((t=>{$.set(t.toc.uuid,t)}));const w=t.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:o,uuidMap:s,tocList:i,bookPath:n,bookId:a,progressBar:c,host:l,options:h}=t;let d=0,f=0,g=0;const m=[],k=[];for(let t=0;t<o;t++){const o=i[t];if("string"!=typeof o.type)continue;if(s.get(o.uuid))continue;const p=o.type.toLocaleLowerCase();if(p===u.TITLE||""!==o.child_uuid||p===u.LINK){let t=o;const e=[],i=[];for(;t;)e.unshift(E(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const a={path:e.join("/"),pathTitleList:e,pathIdList:i,toc:o};p===u.LINK?(g+=1,k.push(a)):await r(`${n}/${e.join("/")}`,{recursive:!0}),s.set(o.uuid,a),await c.updateProgress(a,p!==u.LINK)}else if(o.url){f+=1;let t={path:"",pathTitleList:[],pathIdList:[]};s.get(o.parent_uuid)&&(t=s.get(o.parent_uuid));const r=E(o.title),i=[...t.pathTitleList,`${r}.md`],u=[...t.pathIdList,o.uuid],p={path:i.join("/"),pathTitleList:i,pathIdList:u,toc:o};let g=!0;const k=`${e}/${o.url}`;try{const e={bookId:a,itemUrl:o.url,savePath:`${n}/${t.path}`,saveFilePath:`${n}/${p.path}`,uuid:o.uuid,articleUrl:k,articleTitle:o.title,ignoreImg:h.ignoreImg,host:l};await L({articleInfo:e,progressBar:c,token:h.token,key:h.key})}catch(t){g=!1,d+=1,m.push({articleUrl:k,errItem:p,errMsg:t.message,err:t})}s.set(o.uuid,p),await c.updateProgress(p,g)}}if(g>0){p.warn("该知识库存在以下外链文章");for(const t of k)p.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(d>0){p.error(`本次执行总数${f}篇，✕ 失败${d}篇`);for(const t of m)p.error(`${t.errItem.path} ———— ${t.articleUrl}`),p.error(`———— ✕ ${t.errMsg}`);p.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:w,total:g,uuidMap:$,tocList:s,bookPath:h,bookId:o,progressBar:m,host:l,options:e});const b=new f({bookPath:h,bookName:i,bookDesc:n,uuidMap:$});await b.genFile();const I=process.cwd();p.info(`√ 生成目录 ${I}/${h}/SUMMARY.md`),m.curr===g&&p.info(`√ 已完成: ${I}/${h}`),process.exit(0)}export{L as d,p as l,S as m};