yuque-dl 1.0.15 → 1.0.17

package/GET_TOEKN.md

@@ -0,0 +1,18 @@
+# 如何获取语雀的token
+
+> 以chrome为例其他浏览器也类似
+
+1. 登录语雀，浏览器右击菜单"检查"或点击快捷键 F12(Mac是Option+Command+J)
+2. 退出控制台后点击 `Application`
+3. 点击左侧`Cookies` 下的 `https://www.yuque.com`
+4. 右侧列表中找到 `Name`为 `_yuque_session` 双击`Value`列复制 **Value的值**(也就是下面图片中绿色部分)
+
+![geToken](./assets/getoken.png)
+
+之后就可以在终端执行
+
+```bash
+yuque-dl "知识库的url" -t "复制的token"
+```
+
+> 🚨Tips: cookie为个人登录的信息，请勿泄露自己的cookie给其他人

package/README.md

@@ -19,7 +19,7 @@ $ yuque-dl --help
     $ yuque-dl <url>
 
   Commands:
-    <url> 语雀知识库url
+    <url>  语雀知识库url
 
   For more info, run any command with the `--help` flag:
     $ yuque-dl --help
@@ -27,6 +27,7 @@ $ yuque-dl --help
   Options:
     -d, --distDir <dir>  下载的目录 eg: -d download (default: download)
     -i, --ignoreImg      忽略图片不下载 (default: false)
+    -t, --token <token>  语雀的cookie "_yuque_session (default: )
     -h, --help           Display this message
     -v, --version        Display version number
 ```
@@ -42,10 +43,21 @@ yuque-dl "https://www.yuque.com/yuque/thyzgp"
 
 ![demo](./assets/demo.gif)
 
+## 私有知识库
+
+通过别人私有知识库 分享的链接，需使用`-t`添加token才能下载
+
+```bash
+yuque-dl "https://www.yuque.com/yuque/thyzgp" -t "abcd..."
+```
+
+[token的获取请看](./GET_TOEKN.md)
+
 ## Feature
 
 - [x] 支持下载中断继续
 - [x] 支持图片下载本地
+- [x] 支持下载分享私有的知识库
 
 ## Tips
 

package/dist/cjs/cli.js

@@ -1 +1 @@
-"use strict";var e=require("node:fs"),r=require("cac"),i=require("./index-d6956d87.js");require("node:fs/promises"),require("axios"),require("pull-md-img"),require("ora"),require("cli-progress"),require("log4js"),require("rand-user-agent");const n=r.cac("yuque-dl"),{version:o}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:document.currentScript&&document.currentScript.src||new URL("cli.js",document.baseURI).href)).toString());n.command("<url>","语雀知识库url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).action((async(e,r)=>{try{await i.main(e,r)}catch(e){i.logger.error(e.message||"unknown exception")}})),n.help(),n.version(o);try{n.parse()}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}
+"use strict";var e=require("node:fs"),r=require("cac"),o=require("./index-8cc00c20.js");require("node:fs/promises"),require("axios"),require("pull-md-img"),require("ora"),require("cli-progress"),require("log4js"),require("rand-user-agent");const i=r.cac("yuque-dl"),{version:n}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:document.currentScript&&document.currentScript.src||new URL("cli.js",document.baseURI).href)).toString());i.command("<url>","语雀知识库url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).option("-t, --token <token>",'语雀的cookie "_yuque_session"').action((async(e,r)=>{try{await o.main(e,r)}catch(e){o.logger.error(e.message||"unknown exception")}})),i.help(),i.version(n);try{i.parse()}catch(e){o.logger.error(e.message||"unknown exception"),process.exit(1)}

package/dist/cjs/index-8cc00c20.js

@@ -0,0 +1 @@
+"use strict";var e=require("node:fs/promises"),t=require("axios"),r=require("pull-md-img"),o=require("ora"),i=require("cli-progress"),s=require("log4js"),n=require("rand-user-agent");var a=(s.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),s.getLogger("yuque-dl"));class l{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,a.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class c{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:i}=this.summaryInfo;let s=`# ${t}\n\n`;r&&(s+=`> ${r}\n\n`);let n=s;const l=[];i.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(l,r),i={text:t.title.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""},s=t.type.toLocaleLowerCase();"title"===s||""!==t.child_uuid?(i.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i))):(i.type="link",i.link="link"===s?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i)))})),n+=this.genSummaryContent(l,"");try{await e.writeFile(`${o}/SUMMARY.md`,n)}catch(e){a.error("Generate Summary Error")}}genSummaryContent(e,t){for(let r=0;r<e.length;r++){const o=e[r];if("title"===o.type)t+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if("link"===o.type){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`- [${o.text}](${e})\n`}Array.isArray(o.children)&&(t+=this.genSummaryContent(o.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(let r=0;r<e.length;r++){const o=this.findIdItem(e[r],t);if(o)return o}return!1}}function u({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=n(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=n(r,e,t);return o}function h(e){const t={"user-agent":u({browser:"chrome",device:"desktop"})};return e&&(t.cookie=`_yuque_session=${e}`),t}function d(e,r){const o=/decodeURIComponent\(\"(.+)\"\)\);/m;return t.get(e,{headers:h(r)}).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=e.match(o)||"";if(!t[1])return{};const r=JSON.parse(decodeURIComponent(t[1]));if(!r.book)return{};return{bookId:r.book.id,bookSlug:r.book.slug,tocList:r.book.toc||[],bookName:r.book.name||"",bookDesc:r.book.description||""}}))}async function p(i,s,n){const{bookId:a,itemUrl:l,savePath:c,saveFilePath:u,uuid:d,articleUrl:p,articleTitle:g,ignoreImg:f}=i;let w=`https://www.yuque.com/api/docs/${l}`;const m={book_id:String(a),merge_dynamic_data:String(!1),mode:"markdown"},$=new URLSearchParams(m).toString();w=`${w}?${$}`;let y=(await t.get(w,{headers:h(n)}).then((({data:e,status:t})=>{if(200===t){const t=e;if(!t||!t.data||!t.data.sourcecode)throw new Error(`download article Error: ${w}`);return t}throw new Error(`download article Error: ${w} http status ${t}`)})).catch((e=>{throw new Error(`download article Error: ${w} ${e.message}`)}))).data.sourcecode;if(!f){s.pause(),console.log("");const e=o({text:`下载 "${g}" 的图片中...`});e.start();try{y=await r.run(y,{dist:c,imgDir:`img/${d}`,isIgnoreConsole:!0})}catch(e){let t=`download article image Error: ${e.message}`;throw e.error&&e.url&&(t=`download article image Error ${e.url}: ${e.error?.message}`),new Error(t)}finally{e.stop(),s.continue()}}y=y.replace(/<br(\s?)\/>/gm,"\n"),g&&(y=`# ${g}\n\x3c!--page header--\x3e\n\n${y}\n\n`),p&&(y+=`\x3c!--page footer--\x3e\n- 原文: <${p}>`);try{return await e.writeFile(u,y),!0}catch(e){throw new Error(`download article Error ${p}: ${e.message}`)}}function g(e){if(!e)return"";return e.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,"")}exports.downloadArticle=p,exports.getKnowledgeBaseInfo=d,exports.logger=a,exports.main=async function(t,r){const{bookId:o,tocList:i,bookName:s,bookDesc:n,bookSlug:u}=await d(t,r.token);if(!o)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const h=`${r.distDir}/${s?g(s):o}`;await e.mkdir(h,{recursive:!0});const f=i.length,w=new l(h,f);if(await w.init(),w.curr===f)return w.bar&&w.bar.stop(),void a.info(`√ 已完成: ${process.cwd()}/${h}`);const m=new Map;w.isDownloadInterrupted&&w.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const $=t.replace(new RegExp(`(.*?/${u}).*`),"$1");let y=0,b=0,k=0,I=[],v=[];for(let t=0;t<f;t++){const s=i[t];if("string"!=typeof s.type)continue;if(m.get(s.uuid))continue;const n=s.type.toLocaleLowerCase();if("title"===n||""!==s.child_uuid||"link"===n){let t=s,r=[],o=[];for(;t;)r.unshift(g(t.title)),o.unshift(t.uuid),t=m.get(t.parent_uuid)?m.get(t.parent_uuid).toc:void 0;const i={path:r.join("/"),pathTitleList:r,pathIdList:o,toc:s};"link"===n?(k+=1,v.push(i)):await e.mkdir(`${h}/${r.join("/")}`,{recursive:!0}),m.set(s.uuid,i),await w.updateProgress(i,"link"!==n)}else if(s.url){b+=1;let e={path:"",pathTitleList:[],pathIdList:[]};m.get(s.parent_uuid)&&(e=m.get(s.parent_uuid));const t=g(s.title),i=[...e.pathTitleList,`${t}.md`],n=[...e.pathIdList,s.uuid],a={path:i.join("/"),pathTitleList:i,pathIdList:n,toc:s};let l=!0;const c=`${$}/${s.url}`;try{await p({bookId:o,itemUrl:s.url,savePath:`${h}/${e.path}`,saveFilePath:`${h}/${a.path}`,uuid:s.uuid,articleUrl:c,articleTitle:s.title,ignoreImg:r.ignoreImg},w,r.token)}catch(e){l=!1,y+=1,I.push({articleUrl:c,errItem:a,errMsg:e.message,err:e})}m.set(s.uuid,a),await w.updateProgress(a,l)}}if(k>0){a.warn("该知识库存在以下外链文章");for(let e=0;e<v.length;e++)a.warn(`———— ✕ ${v[e].path} ${v[e].toc.url}`)}if(y>0){a.error(`本次执行总数${b}篇，✕ 失败${y}篇`);for(let e=0;e<I.length;e++){const t=I[e];a.error(`${t.errItem.path} ———— ${t.articleUrl}`),a.error(`———— ✕ ${t.errMsg}`)}a.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(不会影响已下载成功的数据)~~")}const L=new c({bookPath:h,bookName:s,bookDesc:n,uuidMap:m});await L.genFile();const P=process.cwd();a.info(`√ 生成目录 ${P}/${h}/SUMMARY.md`),w.curr!==f||a.info(`√ 已完成: ${P}/${h}`)};

package/dist/cjs/index.js

@@ -1 +1 @@
-"use strict";require("node:fs/promises"),require("axios"),require("pull-md-img"),require("ora");var e=require("./index-d6956d87.js");require("cli-progress"),require("log4js"),require("rand-user-agent"),exports.downloadArticle=e.downloadArticle,exports.getKnowledgeBaseInfo=e.getKnowledgeBaseInfo,exports.main=e.main;
+"use strict";require("node:fs/promises"),require("axios"),require("pull-md-img"),require("ora");var e=require("./index-8cc00c20.js");require("cli-progress"),require("log4js"),require("rand-user-agent"),exports.downloadArticle=e.downloadArticle,exports.getKnowledgeBaseInfo=e.getKnowledgeBaseInfo,exports.main=e.main;

package/dist/es/cli.js

@@ -1 +1 @@
-import{readFileSync as o}from"node:fs";import{cac as r}from"cac";import{m as e,l as i}from"./index-36090493.js";import"node:fs/promises";import"axios";import"pull-md-img";import"ora";import"cli-progress";import"log4js";import"rand-user-agent";const t=r("yuque-dl"),{version:n}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());t.command("<url>","语雀知识库url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).action((async(o,r)=>{try{await e(o,r)}catch(o){i.error(o.message||"unknown exception")}})),t.help(),t.version(n);try{t.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}
+import{readFileSync as o}from"node:fs";import{cac as r}from"cac";import{m as e,l as i}from"./index-996c9ed5.js";import"node:fs/promises";import"axios";import"pull-md-img";import"ora";import"cli-progress";import"log4js";import"rand-user-agent";const t=r("yuque-dl"),{version:n}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());t.command("<url>","语雀知识库url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).option("-t, --token <token>",'语雀的cookie "_yuque_session"').action((async(o,r)=>{try{await e(o,r)}catch(o){i.error(o.message||"unknown exception")}})),t.help(),t.version(n);try{t.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}

package/dist/es/index-996c9ed5.js

@@ -0,0 +1 @@
+import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"axios";import i from"pull-md-img";import s from"ora";import n from"cli-progress";import a from"log4js";import l from"rand-user-agent";var c=(a.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),a.getLogger("yuque-dl"));class u{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,c.info("根据上次数据继续断点下载")),this.bar=new n.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},n.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class h{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:i}=this.summaryInfo;let s=`# ${e}\n\n`;r&&(s+=`> ${r}\n\n`);let n=s;const a=[];i.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),i={text:e.title.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},s=e.type.toLocaleLowerCase();"title"===s||""!==e.child_uuid?(i.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,a.push(i))):(i.type="link",i.link="link"===s?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,a.push(i)))})),n+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){c.error("Generate Summary Error")}}genSummaryContent(t,e){for(let r=0;r<t.length;r++){const o=t[r];if("title"===o.type)e+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if("link"===o.type){const t=o.link?o.link.replace(/\s/g,"%20"):o.link;e+=`- [${o.text}](${t})\n`}Array.isArray(o.children)&&(e+=this.genSummaryContent(o.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(let r=0;r<t.length;r++){const o=this.findIdItem(t[r],e);if(o)return o}return!1}}function d({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=l(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=l(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=l(r,t,e);return o}function p(t){const e={"user-agent":d({browser:"chrome",device:"desktop"})};return t&&(e.cookie=`_yuque_session=${t}`),e}function g(t,e){const r=/decodeURIComponent\(\"(.+)\"\)\);/m;return o.get(t,{headers:p(e)}).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=t.match(r)||"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));if(!o.book)return{};return{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||""}}))}async function f(t,r,n){const{bookId:a,itemUrl:l,savePath:c,saveFilePath:u,uuid:h,articleUrl:d,articleTitle:g,ignoreImg:f}=t;let m=`https://www.yuque.com/api/docs/${l}`;const w={book_id:String(a),merge_dynamic_data:String(!1),mode:"markdown"},$=new URLSearchParams(w).toString();m=`${m}?${$}`;let y=(await o.get(m,{headers:p(n)}).then((({data:t,status:e})=>{if(200===e){const e=t;if(!e||!e.data||!e.data.sourcecode)throw new Error(`download article Error: ${m}`);return e}throw new Error(`download article Error: ${m} http status ${e}`)})).catch((t=>{throw new Error(`download article Error: ${m} ${t.message}`)}))).data.sourcecode;if(!f){r.pause(),console.log("");const t=s({text:`下载 "${g}" 的图片中...`});t.start();try{y=await i.run(y,{dist:c,imgDir:`img/${h}`,isIgnoreConsole:!0})}catch(t){let e=`download article image Error: ${t.message}`;throw t.error&&t.url&&(e=`download article image Error ${t.url}: ${t.error?.message}`),new Error(e)}finally{t.stop(),r.continue()}}y=y.replace(/<br(\s?)\/>/gm,"\n"),g&&(y=`# ${g}\n\x3c!--page header--\x3e\n\n${y}\n\n`),d&&(y+=`\x3c!--page footer--\x3e\n- 原文: <${d}>`);try{return await e(u,y),!0}catch(t){throw new Error(`download article Error ${d}: ${t.message}`)}}function m(t){if(!t)return"";return t.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,"")}async function w(t,e){const{bookId:o,tocList:i,bookName:s,bookDesc:n,bookSlug:a}=await g(t,e.token);if(!o)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const l=`${e.distDir}/${s?m(s):o}`;await r(l,{recursive:!0});const d=i.length,p=new u(l,d);if(await p.init(),p.curr===d)return p.bar&&p.bar.stop(),void c.info(`√ 已完成: ${process.cwd()}/${l}`);const w=new Map;p.isDownloadInterrupted&&p.progressInfo.forEach((t=>{w.set(t.toc.uuid,t)}));const $=t.replace(new RegExp(`(.*?/${a}).*`),"$1");let y=0,b=0,k=0,I=[],L=[];for(let t=0;t<d;t++){const s=i[t];if("string"!=typeof s.type)continue;if(w.get(s.uuid))continue;const n=s.type.toLocaleLowerCase();if("title"===n||""!==s.child_uuid||"link"===n){let t=s,e=[],o=[];for(;t;)e.unshift(m(t.title)),o.unshift(t.uuid),t=w.get(t.parent_uuid)?w.get(t.parent_uuid).toc:void 0;const i={path:e.join("/"),pathTitleList:e,pathIdList:o,toc:s};"link"===n?(k+=1,L.push(i)):await r(`${l}/${e.join("/")}`,{recursive:!0}),w.set(s.uuid,i),await p.updateProgress(i,"link"!==n)}else if(s.url){b+=1;let t={path:"",pathTitleList:[],pathIdList:[]};w.get(s.parent_uuid)&&(t=w.get(s.parent_uuid));const r=m(s.title),i=[...t.pathTitleList,`${r}.md`],n=[...t.pathIdList,s.uuid],a={path:i.join("/"),pathTitleList:i,pathIdList:n,toc:s};let c=!0;const u=`${$}/${s.url}`;try{await f({bookId:o,itemUrl:s.url,savePath:`${l}/${t.path}`,saveFilePath:`${l}/${a.path}`,uuid:s.uuid,articleUrl:u,articleTitle:s.title,ignoreImg:e.ignoreImg},p,e.token)}catch(t){c=!1,y+=1,I.push({articleUrl:u,errItem:a,errMsg:t.message,err:t})}w.set(s.uuid,a),await p.updateProgress(a,c)}}if(k>0){c.warn("该知识库存在以下外链文章");for(let t=0;t<L.length;t++)c.warn(`———— ✕ ${L[t].path} ${L[t].toc.url}`)}if(y>0){c.error(`本次执行总数${b}篇，✕ 失败${y}篇`);for(let t=0;t<I.length;t++){const e=I[t];c.error(`${e.errItem.path} ———— ${e.articleUrl}`),c.error(`———— ✕ ${e.errMsg}`)}c.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(不会影响已下载成功的数据)~~")}const v=new h({bookPath:l,bookName:s,bookDesc:n,uuidMap:w});await v.genFile();const P=process.cwd();c.info(`√ 生成目录 ${P}/${l}/SUMMARY.md`),p.curr!==d||c.info(`√ 已完成: ${P}/${l}`)}export{f as d,g,c as l,w as m};

package/dist/es/index.js

@@ -1 +1 @@
-import"node:fs/promises";import"axios";import"pull-md-img";import"ora";export{d as downloadArticle,g as getKnowledgeBaseInfo,m as main}from"./index-36090493.js";import"cli-progress";import"log4js";import"rand-user-agent";
+import"node:fs/promises";import"axios";import"pull-md-img";import"ora";export{d as downloadArticle,g as getKnowledgeBaseInfo,m as main}from"./index-996c9ed5.js";import"cli-progress";import"log4js";import"rand-user-agent";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yuque-dl",
-  "version": "1.0.15",
+  "version": "1.0.17",
   "description": "yuque 知识库下载",
   "type": "module",
   "bin": {

package/types/cli.d.ts

@@ -1,4 +1,5 @@
 export interface IOptions {
     distDir: string;
     ignoreImg: boolean;
+    token: string;
 }

package/types/index.d.ts

@@ -19,8 +19,8 @@ interface IDownloadArticleParams {
     ignoreImg: boolean;
 }
 /** 获取知识库数据信息 */
-declare function getKnowledgeBaseInfo(url: string): Promise<IKnowledgeBaseInfo>;
+declare function getKnowledgeBaseInfo(url: string, token?: string): Promise<IKnowledgeBaseInfo>;
 /** 下载单篇文章 */
-declare function downloadArticle(params: IDownloadArticleParams, progressBar: ProgressBar): Promise<boolean>;
+declare function downloadArticle(params: IDownloadArticleParams, progressBar: ProgressBar, token?: string): Promise<boolean>;
 declare function main(url: string, options: IOptions): Promise<void>;
 export { getKnowledgeBaseInfo, downloadArticle, main };

package/dist/cjs/index-d6956d87.js

@@ -1 +0,0 @@
-"use strict";var e=require("node:fs/promises"),t=require("axios"),r=require("pull-md-img"),o=require("ora"),i=require("cli-progress"),s=require("log4js"),n=require("rand-user-agent");var a=(s.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),s.getLogger("yuque-dl"));class l{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,a.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class c{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:i}=this.summaryInfo;let s=`# ${t}\n\n`;r&&(s+=`> ${r}\n\n`);let n=s;const l=[];i.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(l,r),i={text:t.title.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""},s=t.type.toLocaleLowerCase();"title"===s||""!==t.child_uuid?(i.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i))):(i.type="link",i.link="link"===s?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,l.push(i)))})),n+=this.genSummaryContent(l,"");try{await e.writeFile(`${o}/SUMMARY.md`,n)}catch(e){a.error("Generate Summary Error")}}genSummaryContent(e,t){for(let r=0;r<e.length;r++){const o=e[r];if("title"===o.type)t+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if("link"===o.type){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`- [${o.text}](${e})\n`}Array.isArray(o.children)&&(t+=this.genSummaryContent(o.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(let r=0;r<e.length;r++){const o=this.findIdItem(e[r],t);if(o)return o}return!1}}function u({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=n(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=n(r,e,t);return o}function h(e){const r=/decodeURIComponent\(\"(.+)\"\)\);/m;return t.get(e,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})}}).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=e.match(r)||"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));if(!o.book)return{};return{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||""}}))}async function d(i,s){const{bookId:n,itemUrl:a,savePath:l,saveFilePath:c,uuid:h,articleUrl:d,articleTitle:p,ignoreImg:g}=i;let f=`https://www.yuque.com/api/docs/${a}`;const w={book_id:String(n),merge_dynamic_data:String(!1),mode:"markdown"},m=new URLSearchParams(w).toString();f=`${f}?${m}`;let $=(await t.get(f,{headers:{"user-agent":u({browser:"chrome",device:"desktop"})}}).then((({data:e,status:t})=>{if(200===t){const t=e;if(!t||!t.data||!t.data.sourcecode)throw new Error(`download article Error: ${f}`);return t}throw new Error(`download article Error: ${f} http status ${t}`)})).catch((e=>{throw new Error(`download article Error: ${f} ${e.message}`)}))).data.sourcecode;if(!g){s.pause(),console.log("");const e=o({text:`下载 "${p}" 的图片中...`});e.start();try{$=await r.run($,{dist:l,imgDir:`img/${h}`,isIgnoreConsole:!0})}catch(e){let t=`download article image Error: ${e.message}`;throw e.error&&e.url&&(t=`download article image Error ${e.url}: ${e.error?.message}`),new Error(t)}finally{e.stop(),s.continue()}}$=$.replace(/<br(\s?)\/>/gm,"\n"),p&&($=`# ${p}\n\x3c!--page header--\x3e\n\n${$}\n\n`),d&&($+=`\x3c!--page footer--\x3e\n- 原文: <${d}>`);try{return await e.writeFile(c,$),!0}catch(e){throw new Error(`download article Error ${d}: ${e.message}`)}}function p(e){if(!e)return"";return e.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,"")}exports.downloadArticle=d,exports.getKnowledgeBaseInfo=h,exports.logger=a,exports.main=async function(t,r){const{bookId:o,tocList:i,bookName:s,bookDesc:n,bookSlug:u}=await h(t);if(!o)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const g=`${r.distDir}/${s?p(s):o}`;await e.mkdir(g,{recursive:!0});const f=i.length,w=new l(g,f);if(await w.init(),w.curr===f)return w.bar&&w.bar.stop(),void a.info(`√ 已完成: ${process.cwd()}/${g}`);const m=new Map;w.isDownloadInterrupted&&w.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const $=t.replace(new RegExp(`(.*?/${u}).*`),"$1");let b=0,y=0,k=0,I=[],v=[];for(let t=0;t<f;t++){const s=i[t];if("string"!=typeof s.type)continue;if(m.get(s.uuid))continue;const n=s.type.toLocaleLowerCase();if("title"===n||""!==s.child_uuid||"link"===n){let t=s,r=[],o=[];for(;t;)r.unshift(p(t.title)),o.unshift(t.uuid),t=m.get(t.parent_uuid)?m.get(t.parent_uuid).toc:void 0;const i={path:r.join("/"),pathTitleList:r,pathIdList:o,toc:s};"link"===n?(k+=1,v.push(i)):await e.mkdir(`${g}/${r.join("/")}`,{recursive:!0}),m.set(s.uuid,i),await w.updateProgress(i,"link"!==n)}else if(s.url){y+=1;let e={path:"",pathTitleList:[],pathIdList:[]};m.get(s.parent_uuid)&&(e=m.get(s.parent_uuid));const t=p(s.title),i=[...e.pathTitleList,`${t}.md`],n=[...e.pathIdList,s.uuid],a={path:i.join("/"),pathTitleList:i,pathIdList:n,toc:s};let l=!0;try{await d({bookId:o,itemUrl:s.url,savePath:`${g}/${e.path}`,saveFilePath:`${g}/${a.path}`,uuid:s.uuid,articleUrl:`${$}/${s.url}`,articleTitle:s.title,ignoreImg:r.ignoreImg},w)}catch(e){l=!1,b+=1,I.push({errItem:a,errMsg:e.message,err:e})}m.set(s.uuid,a),await w.updateProgress(a,l)}}if(k>0){a.warn("该知识库存在以下外链文章");for(let e=0;e<v.length;e++)a.warn(`———— ✕ ${v[e].path} ${v[e].toc.url}`)}if(b>0){a.error(`本次执行总数${y}篇，✕ 失败${b}篇`);for(let e=0;e<I.length;e++)a.error(`${I[e].errItem.path}`),a.error(`———— ✕ ${I[e].errMsg}`);a.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(不会影响已下载成功的数据)~~")}const L=new c({bookPath:g,bookName:s,bookDesc:n,uuidMap:m});await L.genFile();const P=process.cwd();a.info(`√ 生成目录 ${P}/${g}/SUMMARY.md`),w.curr!==f||a.info(`√ 已完成: ${P}/${g}`)};

package/dist/es/index-36090493.js

@@ -1 +0,0 @@
-import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"axios";import i from"pull-md-img";import s from"ora";import a from"cli-progress";import n from"log4js";import l from"rand-user-agent";var c=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class u{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,c.info("根据上次数据继续断点下载")),this.bar=new a.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},a.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class h{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:i}=this.summaryInfo;let s=`# ${e}\n\n`;r&&(s+=`> ${r}\n\n`);let a=s;const n=[];i.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(n,r),i={text:e.title.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},s=e.type.toLocaleLowerCase();"title"===s||""!==e.child_uuid?(i.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,n.push(i))):(i.type="link",i.link="link"===s?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),i.level=o.level+1,o.children.push(i)):(i.level=1,n.push(i)))})),a+=this.genSummaryContent(n,"");try{await t.writeFile(`${o}/SUMMARY.md`,a)}catch(t){c.error("Generate Summary Error")}}genSummaryContent(t,e){for(let r=0;r<t.length;r++){const o=t[r];if("title"===o.type)e+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if("link"===o.type){const t=o.link?o.link.replace(/\s/g,"%20"):o.link;e+=`- [${o.text}](${t})\n`}Array.isArray(o.children)&&(e+=this.genSummaryContent(o.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(let r=0;r<t.length;r++){const o=this.findIdItem(t[r],e);if(o)return o}return!1}}function d({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=l(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=l(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=l(r,t,e);return o}function p(t){const e=/decodeURIComponent\(\"(.+)\"\)\);/m;return o.get(t,{headers:{"user-agent":d({browser:"chrome",device:"desktop"})}}).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const r=t.match(e)||"";if(!r[1])return{};const o=JSON.parse(decodeURIComponent(r[1]));if(!o.book)return{};return{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||""}}))}async function g(t,r){const{bookId:a,itemUrl:n,savePath:l,saveFilePath:c,uuid:u,articleUrl:h,articleTitle:p,ignoreImg:g}=t;let f=`https://www.yuque.com/api/docs/${n}`;const m={book_id:String(a),merge_dynamic_data:String(!1),mode:"markdown"},w=new URLSearchParams(m).toString();f=`${f}?${w}`;let $=(await o.get(f,{headers:{"user-agent":d({browser:"chrome",device:"desktop"})}}).then((({data:t,status:e})=>{if(200===e){const e=t;if(!e||!e.data||!e.data.sourcecode)throw new Error(`download article Error: ${f}`);return e}throw new Error(`download article Error: ${f} http status ${e}`)})).catch((t=>{throw new Error(`download article Error: ${f} ${t.message}`)}))).data.sourcecode;if(!g){r.pause(),console.log("");const t=s({text:`下载 "${p}" 的图片中...`});t.start();try{$=await i.run($,{dist:l,imgDir:`img/${u}`,isIgnoreConsole:!0})}catch(t){let e=`download article image Error: ${t.message}`;throw t.error&&t.url&&(e=`download article image Error ${t.url}: ${t.error?.message}`),new Error(e)}finally{t.stop(),r.continue()}}$=$.replace(/<br(\s?)\/>/gm,"\n"),p&&($=`# ${p}\n\x3c!--page header--\x3e\n\n${$}\n\n`),h&&($+=`\x3c!--page footer--\x3e\n- 原文: <${h}>`);try{return await e(c,$),!0}catch(t){throw new Error(`download article Error ${h}: ${t.message}`)}}function f(t){if(!t)return"";return t.replace(/[\\\/:\*\?"<>\|\n\r]/g,"_").replace(/\s/,"")}async function m(t,e){const{bookId:o,tocList:i,bookName:s,bookDesc:a,bookSlug:n}=await p(t);if(!o)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const l=`${e.distDir}/${s?f(s):o}`;await r(l,{recursive:!0});const d=i.length,m=new u(l,d);if(await m.init(),m.curr===d)return m.bar&&m.bar.stop(),void c.info(`√ 已完成: ${process.cwd()}/${l}`);const w=new Map;m.isDownloadInterrupted&&m.progressInfo.forEach((t=>{w.set(t.toc.uuid,t)}));const $=t.replace(new RegExp(`(.*?/${n}).*`),"$1");let b=0,y=0,k=0,I=[],v=[];for(let t=0;t<d;t++){const s=i[t];if("string"!=typeof s.type)continue;if(w.get(s.uuid))continue;const a=s.type.toLocaleLowerCase();if("title"===a||""!==s.child_uuid||"link"===a){let t=s,e=[],o=[];for(;t;)e.unshift(f(t.title)),o.unshift(t.uuid),t=w.get(t.parent_uuid)?w.get(t.parent_uuid).toc:void 0;const i={path:e.join("/"),pathTitleList:e,pathIdList:o,toc:s};"link"===a?(k+=1,v.push(i)):await r(`${l}/${e.join("/")}`,{recursive:!0}),w.set(s.uuid,i),await m.updateProgress(i,"link"!==a)}else if(s.url){y+=1;let t={path:"",pathTitleList:[],pathIdList:[]};w.get(s.parent_uuid)&&(t=w.get(s.parent_uuid));const r=f(s.title),i=[...t.pathTitleList,`${r}.md`],a=[...t.pathIdList,s.uuid],n={path:i.join("/"),pathTitleList:i,pathIdList:a,toc:s};let c=!0;try{await g({bookId:o,itemUrl:s.url,savePath:`${l}/${t.path}`,saveFilePath:`${l}/${n.path}`,uuid:s.uuid,articleUrl:`${$}/${s.url}`,articleTitle:s.title,ignoreImg:e.ignoreImg},m)}catch(t){c=!1,b+=1,I.push({errItem:n,errMsg:t.message,err:t})}w.set(s.uuid,n),await m.updateProgress(n,c)}}if(k>0){c.warn("该知识库存在以下外链文章");for(let t=0;t<v.length;t++)c.warn(`———— ✕ ${v[t].path} ${v[t].toc.url}`)}if(b>0){c.error(`本次执行总数${y}篇，✕ 失败${b}篇`);for(let t=0;t<I.length;t++)c.error(`${I[t].errItem.path}`),c.error(`———— ✕ ${I[t].errMsg}`);c.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(不会影响已下载成功的数据)~~")}const L=new h({bookPath:l,bookName:s,bookDesc:a,uuidMap:w});await L.genFile();const P=process.cwd();c.info(`√ 生成目录 ${P}/${l}/SUMMARY.md`),m.curr!==d||c.info(`√ 已完成: ${P}/${l}`)}export{g as d,p as g,c as l,m};