yuque-dl 1.0.31 → 1.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -4,6 +4,10 @@
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
|
|
7
|
+
## Prerequisite
|
|
8
|
+
|
|
9
|
+
- Node.js 18.4 or later
|
|
10
|
+
|
|
7
11
|
## Install
|
|
8
12
|
|
|
9
13
|
```bash
|
|
@@ -44,7 +48,9 @@ yuque-dl "https://www.yuque.com/yuque/thyzgp"
|
|
|
44
48
|
|
|
45
49
|
|
|
46
50
|
|
|
47
|
-
##
|
|
51
|
+
## 其他场景
|
|
52
|
+
|
|
53
|
+
### 私有知识库
|
|
48
54
|
|
|
49
55
|
通过别人私有知识库 分享的链接，需使用`-t`添加token才能下载
|
|
50
56
|
|
|
@@ -54,14 +60,24 @@ yuque-dl "https://www.yuque.com/yuque/thyzgp" -t "abcd..."
|
|
|
54
60
|
|
|
55
61
|
[token的获取请看](./docs/GET_TOEKN.md)
|
|
56
62
|
|
|
57
|
-
|
|
63
|
+
### 企业私有服务
|
|
58
64
|
|
|
59
65
|
企业服务有自己的域名(黄色语雀logo)，非`yuque.com`结尾, 如`https://yuque.antfin.com/r/zone`
|
|
60
66
|
|
|
61
|
-
这种情况 token的key不唯一, 不一定是为`_yuque_session` 需用户使用 `-
|
|
67
|
+
这种情况 token的key不唯一, 不一定是为`_yuque_session` 需用户使用 `-k` 指定 token的key,`-t` 指定 token的值。
|
|
62
68
|
|
|
63
69
|
至于`key`具体是什么只能靠用户自己在 `浏览器Devtools-> Application -> Cookies` 里找了🤔
|
|
64
70
|
|
|
71
|
+
### 公开密码访问的知识库
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
公开密码访问的知识库或文档需要使用`verified_books`/`verified_docs`这个cookie
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
yuque-dl "url" -k "verified_books" -t "verified_books的值"
|
|
79
|
+
```
|
|
80
|
+
|
|
65
81
|
## Feature
|
|
66
82
|
|
|
67
83
|
- [x] 支持下载中断继续
|
|
@@ -71,6 +87,21 @@ yuque-dl "https://www.yuque.com/yuque/thyzgp" -t "abcd..."
|
|
|
71
87
|
- [ ] 支持其他文档类型？🤔
|
|
72
88
|
- [ ] 添加测试 🤔
|
|
73
89
|
|
|
90
|
+
## 常见错误
|
|
91
|
+
|
|
92
|
+
1. 由于token可能含有 特殊字符导致参数识别错误
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
yuque-dl "https://www.yuque.com/yuque/thyzgp" -t "-a123"
|
|
96
|
+
yuque-dl [ERROR]: Unknown option `-1`
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
解决方案
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
yuque-dl "https://www.yuque.com/yuque/thyzgp" -t="-a123"
|
|
103
|
+
```
|
|
104
|
+
|
|
74
105
|
## Tips
|
|
75
106
|
|
|
76
107
|
由于网络波动下载失败的，重新运行即可，已下载的进度不会受到影响
|
package/dist/cjs/cli.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var e=require("node:fs"),r=require("cac"),o=require("semver"),n=require("./index-
|
|
1
|
+
"use strict";var e=require("node:fs"),r=require("cac"),o=require("semver"),n=require("./index-c9bc9b75.js");require("node:fs/promises"),require("pull-md-img"),require("ora"),require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako");var i="undefined"!=typeof document?document.currentScript:null;const s=r.cac("yuque-dl"),{version:t,engines:u}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:i&&i.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=u.node;o.satisfies(process.version,e)||(n.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),s.command("<url>","语雀知识库url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").action((async(e,r)=>{try{await n.main(e,r)}catch(e){n.logger.error(e.message||"unknown exception")}})),s.help(),s.version(t);try{s.parse()}catch(e){n.logger.error(e.message||"unknown exception"),process.exit(1)}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var e=require("node:fs/promises"),t=require("pull-md-img"),r=require("ora"),o=require("cli-progress"),s=require("log4js"),i=require("axios"),n=require("rand-user-agent"),a=require("pako");var c,l,u=(s.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),s.getLogger("yuque-dl"));class h{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,u.info("根据上次数据继续断点下载")),this.bar=new o.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},o.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class d{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${t}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(a,r),s={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""},i=t.type.toLocaleLowerCase();"title"===i||""!==t.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await e.writeFile(`${o}/SUMMARY.md`,n)}catch(e){u.error("Generate Summary Error")}}genSummaryContent(e,t){for(const r of e){if("title"===r.type)t+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`- [${r.text}](${e})\n`}Array.isArray(r.children)&&(t+=this.genSummaryContent(r.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(c||(c={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(l||(l={}));const p=new Map([[l.BOARD,"画板类型"],[l.TABLE,"数据表类型"],[l.SHEET,"表格类型"],[l.DOC,"文档类型"]]),f="_yuque_session",g="https://www.yuque.com";function m({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=n(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=n(r,e,t);return o}function k(e){const{key:t=f,token:r}=e,o={"user-agent":m({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function $(e){return{headers:k(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...k(e)}}}}const w=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:s,key:n,host:a=g}=e;let c=`${a}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,i.get(c,$({token:s,key:n})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))},y=e=>{if(!e)return"";const t=JSON.parse(a.inflate(e,{to:"string"}));let r="";return t.forEach((e=>{const t=`## ${e.name}\n`,o=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),s=Math.max(...r.map((e=>Number(e))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((e,t)=>{const r=t%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<s+1;o++){const s=e?.[t]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(``):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return i}(e.data);r=`${r}\n${t}\n${o}`})),r};async function b(o){const{articleInfo:s,progressBar:i,token:n,key:a}=o,{bookId:c,itemUrl:u,savePath:h,saveFilePath:d,uuid:f,articleUrl:g,articleTitle:m,ignoreImg:k,host:$}=s,{httpStatus:b,apiUrl:I,response:L}=await w({articleUrl:u,bookId:c,token:n,host:$,key:a}),E=L?.data?.type?.toLocaleLowerCase();let S="";if(E===l.SHEET){const{response:e}=await w({articleUrl:u,bookId:c,token:n,host:$,key:a},!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;S=o?y(o):""}catch(e){const t=p.get(E);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([l.BOARD,l.TABLE].includes(E)){const e=p.get(E);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof L?.data?.sourcecode)throw new Error(`download article Error: ${I}, http status ${b}`);S=L.data.sourcecode}if(!k){i.pause(),console.log("");const e=r({text:`下载 "${m}" 的图片中...`});e.start();try{S=await t.run(S,{dist:h,imgDir:`./img/${f}`,isIgnoreConsole:!0})}catch(e){let t=`download article image Error: ${e.message}`;throw e.error&&e.url&&(t=`download article image Error ${e.url}: ${e.error?.message}`),new Error(t)}finally{e.stop(),i.continue()}}S=S.replace(/<br(\s?)\/>/gm,"\n"),m&&(S=`# ${m}\n\x3c!--page header--\x3e\n\n${S}\n\n`),g&&(S+=`\x3c!--page footer--\x3e\n- 原文: <${g}>`);try{return await e.writeFile(d,S),!0}catch(e){throw new Error(`download article Error ${g}: ${e.message}`)}}function I(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"")}exports.downloadArticle=b,exports.logger=u,exports.main=async function(t,r){const{bookId:o,tocList:s,bookName:n,bookDesc:a,bookSlug:l,host:p}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return i.get(e,$(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||g}:{}}))})(t,{token:r.token,key:r.key});if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const f=`${r.distDir}/${n?I(n):o}`;await e.mkdir(f,{recursive:!0});const m=s.length,k=new h(f,m);if(await k.init(),k.curr===m)return k.bar&&k.bar.stop(),void u.info(`√ 已完成: ${process.cwd()}/${f}`);const w=new Map;k.isDownloadInterrupted&&k.progressInfo.forEach((e=>{w.set(e.toc.uuid,e)}));const y=t.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(t){const{articleUrlPrefix:r,total:o,uuidMap:s,tocList:i,bookPath:n,bookId:a,progressBar:l,host:h,options:d}=t;let p=0,f=0,g=0;const m=[],k=[];for(let t=0;t<o;t++){const o=i[t];if("string"!=typeof o.type)continue;if(s.get(o.uuid))continue;const u=o.type.toLocaleLowerCase();if(u===c.TITLE||""!==o.child_uuid||u===c.LINK){let t=o;const r=[],i=[];for(;t;)r.unshift(I(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const a={path:r.join("/"),pathTitleList:r,pathIdList:i,toc:o};u===c.LINK?(g+=1,k.push(a)):await e.mkdir(`${n}/${r.join("/")}`,{recursive:!0}),s.set(o.uuid,a),await l.updateProgress(a,u!==c.LINK)}else if(o.url){f+=1;let e={path:"",pathTitleList:[],pathIdList:[]};s.get(o.parent_uuid)&&(e=s.get(o.parent_uuid));const t=I(o.title),i=[...e.pathTitleList,`${t}.md`],c=[...e.pathIdList,o.uuid],u={path:i.join("/"),pathTitleList:i,pathIdList:c,toc:o};let g=!0;const k=`${r}/${o.url}`;try{const t={bookId:a,itemUrl:o.url,savePath:`${n}/${e.path}`,saveFilePath:`${n}/${u.path}`,uuid:o.uuid,articleUrl:k,articleTitle:o.title,ignoreImg:d.ignoreImg,host:h};await b({articleInfo:t,progressBar:l,token:d.token,key:d.key})}catch(e){g=!1,p+=1,m.push({articleUrl:k,errItem:u,errMsg:e.message,err:e})}s.set(o.uuid,u),await l.updateProgress(u,g)}}if(g>0){u.warn("该知识库存在以下外链文章");for(const e of k)u.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(p>0){u.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const e of m)u.error(`${e.errItem.path} ———— ${e.articleUrl}`),u.error(`———— ✕ ${e.errMsg}`);u.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:y,total:m,uuidMap:w,tocList:s,bookPath:f,bookId:o,progressBar:k,host:p,options:r});const L=new d({bookPath:f,bookName:n,bookDesc:a,uuidMap:w});await L.genFile();const E=process.cwd();u.info(`√ 生成目录 ${E}/${f}/SUMMARY.md`),k.curr===m&&u.info(`√ 已完成: ${E}/${f}`),process.exit(0)};
|
|
1
|
+
"use strict";var e=require("node:fs/promises"),t=require("pull-md-img"),r=require("ora"),o=require("cli-progress"),s=require("log4js"),i=require("axios"),n=require("rand-user-agent"),a=require("pako");var c,l,u=(s.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),s.getLogger("yuque-dl"));class h{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,u.info("根据上次数据继续断点下载")),this.bar=new o.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},o.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class d{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${t}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(a,r),s={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""},i=t.type.toLocaleLowerCase();"title"===i||""!==t.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await e.writeFile(`${o}/SUMMARY.md`,n)}catch(e){u.error("Generate Summary Error")}}genSummaryContent(e,t){for(const r of e){if("title"===r.type)t+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`- [${r.text}](${e})\n`}Array.isArray(r.children)&&(t+=this.genSummaryContent(r.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(c||(c={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(l||(l={}));const p=new Map([[l.BOARD,"画板类型"],[l.TABLE,"数据表类型"],[l.SHEET,"表格类型"],[l.DOC,"文档类型"]]),f="_yuque_session",g="https://www.yuque.com";function m({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=n(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=n(r,e,t);return o}function k(e){const{key:t=f,token:r}=e,o={"user-agent":m({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function $(e){return{headers:k(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...k(e)}}}}const w=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:s,key:n,host:a=g}=e;let c=`${a}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,i.get(c,$({token:s,key:n})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))},y=e=>{if(!e)return"";const t=JSON.parse(a.inflate(e,{to:"string"}));let r="";return t.forEach((e=>{const t=`## ${e.name}\n`,o=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),s=Math.max(...r.map((e=>Number(e))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((e,t)=>{const r=t%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<s+1;o++){const s=e?.[t]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(``):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return i}(e.data);r=`${r}\n${t}\n${o}`})),r};async function b(o){const{articleInfo:s,progressBar:i,token:n,key:a}=o,{bookId:c,itemUrl:u,savePath:h,saveFilePath:d,uuid:f,articleUrl:g,articleTitle:m,ignoreImg:k,host:$}=s,{httpStatus:b,apiUrl:I,response:L}=await w({articleUrl:u,bookId:c,token:n,host:$,key:a}),E=L?.data?.type?.toLocaleLowerCase();let S="";if(E===l.SHEET){const{response:e}=await w({articleUrl:u,bookId:c,token:n,host:$,key:a},!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;S=o?y(o):""}catch(e){const t=p.get(E);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([l.BOARD,l.TABLE].includes(E)){const e=p.get(E);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof L?.data?.sourcecode)throw new Error(`download article Error: ${I}, http status ${b}`);S=L.data.sourcecode}if(!k){i.pause(),console.log("");const e=r({text:`下载 "${m}" 的图片中...`});e.start();try{S=await t.run(S,{dist:h,imgDir:`./img/${f}`,isIgnoreConsole:!0})}catch(e){let t=`download article image Error: ${e.message}`;throw e.error&&e.url&&(t=`download article image Error ${e.url}: ${e.error?.message}`),new Error(t)}finally{e.stop(),i.continue()}}S=S.replace(/<br(\s?)\/>/gm,"\n"),m&&(S=`# ${m}\n\x3c!--page header--\x3e\n\n${S}\n\n`),g&&(S+=`\x3c!--page footer--\x3e\n- 原文: <${g}>`);try{return await e.writeFile(d,S),!0}catch(e){throw new Error(`download article Error ${g}: ${e.message}`)}}function I(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}exports.downloadArticle=b,exports.logger=u,exports.main=async function(t,r){const{bookId:o,tocList:s,bookName:n,bookDesc:a,bookSlug:l,host:p}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return i.get(e,$(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||g}:{}}))})(t,{token:r.token,key:r.key});if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const f=`${r.distDir}/${n?I(n):o}`;await e.mkdir(f,{recursive:!0});const m=s.length,k=new h(f,m);if(await k.init(),k.curr===m)return k.bar&&k.bar.stop(),void u.info(`√ 已完成: ${process.cwd()}/${f}`);const w=new Map;k.isDownloadInterrupted&&k.progressInfo.forEach((e=>{w.set(e.toc.uuid,e)}));const y=t.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(t){const{articleUrlPrefix:r,total:o,uuidMap:s,tocList:i,bookPath:n,bookId:a,progressBar:l,host:h,options:d}=t;let p=0,f=0,g=0;const m=[],k=[];for(let t=0;t<o;t++){const o=i[t];if("string"!=typeof o.type)continue;if(s.get(o.uuid))continue;const u=o.type.toLocaleLowerCase();if(u===c.TITLE||""!==o.child_uuid||u===c.LINK){let t=o;const r=[],i=[];for(;t;)r.unshift(I(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const a={path:r.join("/"),pathTitleList:r,pathIdList:i,toc:o};u===c.LINK?(g+=1,k.push(a)):await e.mkdir(`${n}/${r.join("/")}`,{recursive:!0}),s.set(o.uuid,a),await l.updateProgress(a,u!==c.LINK)}else if(o.url){f+=1;let e={path:"",pathTitleList:[],pathIdList:[]};s.get(o.parent_uuid)&&(e=s.get(o.parent_uuid));const t=I(o.title),i=[...e.pathTitleList,`${t}.md`],c=[...e.pathIdList,o.uuid],u={path:i.join("/"),pathTitleList:i,pathIdList:c,toc:o};let g=!0;const k=`${r}/${o.url}`;try{const t={bookId:a,itemUrl:o.url,savePath:`${n}/${e.path}`,saveFilePath:`${n}/${u.path}`,uuid:o.uuid,articleUrl:k,articleTitle:o.title,ignoreImg:d.ignoreImg,host:h};await b({articleInfo:t,progressBar:l,token:d.token,key:d.key})}catch(e){g=!1,p+=1,m.push({articleUrl:k,errItem:u,errMsg:e.message,err:e})}s.set(o.uuid,u),await l.updateProgress(u,g)}}if(g>0){u.warn("该知识库存在以下外链文章");for(const e of k)u.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(p>0){u.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const e of m)u.error(`${e.errItem.path} ———— ${e.articleUrl}`),u.error(`———— ✕ ${e.errMsg}`);u.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:y,total:m,uuidMap:w,tocList:s,bookPath:f,bookId:o,progressBar:k,host:p,options:r});const L=new d({bookPath:f,bookName:n,bookDesc:a,uuidMap:w});await L.genFile();const E=process.cwd();u.info(`√ 生成目录 ${E}/${f}/SUMMARY.md`),k.curr===m&&u.info(`√ 已完成: ${E}/${f}`),process.exit(0)};
|
package/dist/cjs/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";require("node:fs/promises"),require("pull-md-img"),require("ora");var r=require("./index-
|
|
1
|
+
"use strict";require("node:fs/promises"),require("pull-md-img"),require("ora");var r=require("./index-c9bc9b75.js");require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),exports.downloadArticle=r.downloadArticle,exports.main=r.main;
|
package/dist/es/cli.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import r from"semver";import{m as i,l as s}from"./index-
|
|
1
|
+
import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import r from"semver";import{m as i,l as s}from"./index-5461d7d6.js";import"node:fs/promises";import"pull-md-img";import"ora";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";const n=e("yuque-dl"),{version:t,engines:p}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=p.node;r.satisfies(process.version,o)||(s.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),n.command("<url>","语雀知识库url").option("-d, --distDir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").action((async(o,e)=>{try{await i(o,e)}catch(o){s.error(o.message||"unknown exception")}})),n.help(),n.version(t);try{n.parse()}catch(o){s.error(o.message||"unknown exception"),process.exit(1)}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"pull-md-img";import s from"ora";import i from"cli-progress";import n from"log4js";import a from"axios";import c from"rand-user-agent";import l from"pako";var u,h,p=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class d{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,p.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class f{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${e}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},i=e.type.toLocaleLowerCase();"title"===i||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){p.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if("title"===r.type)e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(u||(u={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(h||(h={}));const g=new Map([[h.BOARD,"画板类型"],[h.TABLE,"数据表类型"],[h.SHEET,"表格类型"],[h.DOC,"文档类型"]]),m="_yuque_session",k="https://www.yuque.com";function $({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=c(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=c(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=c(r,t,e);return o}function w(t){const{key:e=m,token:r}=t,o={"user-agent":$({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function y(t){return{headers:w(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...w(t)}}}}const b=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:i,host:n=k}=t;let c=`${n}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,y({token:s,key:i})).then((({data:t,status:e})=>({apiUrl:c,httpStatus:e,response:t})))},I=t=>{if(!t)return"";const e=JSON.parse(l.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(``):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return i}(t.data);r=`${r}\n${e}\n${o}`})),r};async function L(t){const{articleInfo:r,progressBar:i,token:n,key:a}=t,{bookId:c,itemUrl:l,savePath:u,saveFilePath:p,uuid:d,articleUrl:f,articleTitle:m,ignoreImg:k,host:$}=r,{httpStatus:w,apiUrl:y,response:L}=await b({articleUrl:l,bookId:c,token:n,host:$,key:a}),E=L?.data?.type?.toLocaleLowerCase();let S="";if(E===h.SHEET){const{response:t}=await b({articleUrl:l,bookId:c,token:n,host:$,key:a},!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;S=o?I(o):""}catch(t){const e=g.get(E);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([h.BOARD,h.TABLE].includes(E)){const t=g.get(E);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof L?.data?.sourcecode)throw new Error(`download article Error: ${y}, http status ${w}`);S=L.data.sourcecode}if(!k){i.pause(),console.log("");const t=s({text:`下载 "${m}" 的图片中...`});t.start();try{S=await o.run(S,{dist:u,imgDir:`./img/${d}`,isIgnoreConsole:!0})}catch(t){let e=`download article image Error: ${t.message}`;throw t.error&&t.url&&(e=`download article image Error ${t.url}: ${t.error?.message}`),new Error(e)}finally{t.stop(),i.continue()}}S=S.replace(/<br(\s?)\/>/gm,"\n"),m&&(S=`# ${m}\n\x3c!--page header--\x3e\n\n${S}\n\n`),f&&(S+=`\x3c!--page footer--\x3e\n- 原文: <${f}>`);try{return await e(p,S),!0}catch(t){throw new Error(`download article Error ${f}: ${t.message}`)}}function E(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"")}async function S(t,e){const{bookId:o,tocList:s,bookName:i,bookDesc:n,bookSlug:c,host:l}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(t,y(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||k}:{}}))})(t,{token:e.token,key:e.key});if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const h=`${e.distDir}/${i?E(i):o}`;await r(h,{recursive:!0});const g=s.length,m=new d(h,g);if(await m.init(),m.curr===g)return m.bar&&m.bar.stop(),void p.info(`√ 已完成: ${process.cwd()}/${h}`);const $=new Map;m.isDownloadInterrupted&&m.progressInfo.forEach((t=>{$.set(t.toc.uuid,t)}));const w=t.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:o,uuidMap:s,tocList:i,bookPath:n,bookId:a,progressBar:c,host:l,options:h}=t;let d=0,f=0,g=0;const m=[],k=[];for(let t=0;t<o;t++){const o=i[t];if("string"!=typeof o.type)continue;if(s.get(o.uuid))continue;const p=o.type.toLocaleLowerCase();if(p===u.TITLE||""!==o.child_uuid||p===u.LINK){let t=o;const e=[],i=[];for(;t;)e.unshift(E(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const a={path:e.join("/"),pathTitleList:e,pathIdList:i,toc:o};p===u.LINK?(g+=1,k.push(a)):await r(`${n}/${e.join("/")}`,{recursive:!0}),s.set(o.uuid,a),await c.updateProgress(a,p!==u.LINK)}else if(o.url){f+=1;let t={path:"",pathTitleList:[],pathIdList:[]};s.get(o.parent_uuid)&&(t=s.get(o.parent_uuid));const r=E(o.title),i=[...t.pathTitleList,`${r}.md`],u=[...t.pathIdList,o.uuid],p={path:i.join("/"),pathTitleList:i,pathIdList:u,toc:o};let g=!0;const k=`${e}/${o.url}`;try{const e={bookId:a,itemUrl:o.url,savePath:`${n}/${t.path}`,saveFilePath:`${n}/${p.path}`,uuid:o.uuid,articleUrl:k,articleTitle:o.title,ignoreImg:h.ignoreImg,host:l};await L({articleInfo:e,progressBar:c,token:h.token,key:h.key})}catch(t){g=!1,d+=1,m.push({articleUrl:k,errItem:p,errMsg:t.message,err:t})}s.set(o.uuid,p),await c.updateProgress(p,g)}}if(g>0){p.warn("该知识库存在以下外链文章");for(const t of k)p.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(d>0){p.error(`本次执行总数${f}篇，✕ 失败${d}篇`);for(const t of m)p.error(`${t.errItem.path} ———— ${t.articleUrl}`),p.error(`———— ✕ ${t.errMsg}`);p.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:w,total:g,uuidMap:$,tocList:s,bookPath:h,bookId:o,progressBar:m,host:l,options:e});const b=new f({bookPath:h,bookName:i,bookDesc:n,uuidMap:$});await b.genFile();const I=process.cwd();p.info(`√ 生成目录 ${I}/${h}/SUMMARY.md`),m.curr===g&&p.info(`√ 已完成: ${I}/${h}`),process.exit(0)}export{L as d,p as l,S as m};
|
|
1
|
+
import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"pull-md-img";import s from"ora";import i from"cli-progress";import n from"log4js";import a from"axios";import c from"rand-user-agent";import l from"pako";var u,h,p=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class d{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,p.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class f{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${e}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},i=e.type.toLocaleLowerCase();"title"===i||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,n)}catch(t){p.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if("title"===r.type)e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(u||(u={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(h||(h={}));const g=new Map([[h.BOARD,"画板类型"],[h.TABLE,"数据表类型"],[h.SHEET,"表格类型"],[h.DOC,"文档类型"]]),m="_yuque_session",k="https://www.yuque.com";function $({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=c(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=c(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=c(r,t,e);return o}function w(t){const{key:e=m,token:r}=t,o={"user-agent":$({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function y(t){return{headers:w(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...w(t)}}}}const b=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:i,host:n=k}=t;let c=`${n}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,y({token:s,key:i})).then((({data:t,status:e})=>({apiUrl:c,httpStatus:e,response:t})))},I=t=>{if(!t)return"";const e=JSON.parse(l.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(``):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return i}(t.data);r=`${r}\n${e}\n${o}`})),r};async function L(t){const{articleInfo:r,progressBar:i,token:n,key:a}=t,{bookId:c,itemUrl:l,savePath:u,saveFilePath:p,uuid:d,articleUrl:f,articleTitle:m,ignoreImg:k,host:$}=r,{httpStatus:w,apiUrl:y,response:L}=await b({articleUrl:l,bookId:c,token:n,host:$,key:a}),E=L?.data?.type?.toLocaleLowerCase();let S="";if(E===h.SHEET){const{response:t}=await b({articleUrl:l,bookId:c,token:n,host:$,key:a},!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;S=o?I(o):""}catch(t){const e=g.get(E);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([h.BOARD,h.TABLE].includes(E)){const t=g.get(E);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof L?.data?.sourcecode)throw new Error(`download article Error: ${y}, http status ${w}`);S=L.data.sourcecode}if(!k){i.pause(),console.log("");const t=s({text:`下载 "${m}" 的图片中...`});t.start();try{S=await o.run(S,{dist:u,imgDir:`./img/${d}`,isIgnoreConsole:!0})}catch(t){let e=`download article image Error: ${t.message}`;throw t.error&&t.url&&(e=`download article image Error ${t.url}: ${t.error?.message}`),new Error(e)}finally{t.stop(),i.continue()}}S=S.replace(/<br(\s?)\/>/gm,"\n"),m&&(S=`# ${m}\n\x3c!--page header--\x3e\n\n${S}\n\n`),f&&(S+=`\x3c!--page footer--\x3e\n- 原文: <${f}>`);try{return await e(p,S),!0}catch(t){throw new Error(`download article Error ${f}: ${t.message}`)}}function E(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}async function S(t,e){const{bookId:o,tocList:s,bookName:i,bookDesc:n,bookSlug:c,host:l}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(t,y(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||k}:{}}))})(t,{token:e.token,key:e.key});if(!o)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const h=`${e.distDir}/${i?E(i):o}`;await r(h,{recursive:!0});const g=s.length,m=new d(h,g);if(await m.init(),m.curr===g)return m.bar&&m.bar.stop(),void p.info(`√ 已完成: ${process.cwd()}/${h}`);const $=new Map;m.isDownloadInterrupted&&m.progressInfo.forEach((t=>{$.set(t.toc.uuid,t)}));const w=t.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:o,uuidMap:s,tocList:i,bookPath:n,bookId:a,progressBar:c,host:l,options:h}=t;let d=0,f=0,g=0;const m=[],k=[];for(let t=0;t<o;t++){const o=i[t];if("string"!=typeof o.type)continue;if(s.get(o.uuid))continue;const p=o.type.toLocaleLowerCase();if(p===u.TITLE||""!==o.child_uuid||p===u.LINK){let t=o;const e=[],i=[];for(;t;)e.unshift(E(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const a={path:e.join("/"),pathTitleList:e,pathIdList:i,toc:o};p===u.LINK?(g+=1,k.push(a)):await r(`${n}/${e.join("/")}`,{recursive:!0}),s.set(o.uuid,a),await c.updateProgress(a,p!==u.LINK)}else if(o.url){f+=1;let t={path:"",pathTitleList:[],pathIdList:[]};s.get(o.parent_uuid)&&(t=s.get(o.parent_uuid));const r=E(o.title),i=[...t.pathTitleList,`${r}.md`],u=[...t.pathIdList,o.uuid],p={path:i.join("/"),pathTitleList:i,pathIdList:u,toc:o};let g=!0;const k=`${e}/${o.url}`;try{const e={bookId:a,itemUrl:o.url,savePath:`${n}/${t.path}`,saveFilePath:`${n}/${p.path}`,uuid:o.uuid,articleUrl:k,articleTitle:o.title,ignoreImg:h.ignoreImg,host:l};await L({articleInfo:e,progressBar:c,token:h.token,key:h.key})}catch(t){g=!1,d+=1,m.push({articleUrl:k,errItem:p,errMsg:t.message,err:t})}s.set(o.uuid,p),await c.updateProgress(p,g)}}if(g>0){p.warn("该知识库存在以下外链文章");for(const t of k)p.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(d>0){p.error(`本次执行总数${f}篇，✕ 失败${d}篇`);for(const t of m)p.error(`${t.errItem.path} ———— ${t.articleUrl}`),p.error(`———— ✕ ${t.errMsg}`);p.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:w,total:g,uuidMap:$,tocList:s,bookPath:h,bookId:o,progressBar:m,host:l,options:e});const b=new f({bookPath:h,bookName:i,bookDesc:n,uuidMap:$});await b.genFile();const I=process.cwd();p.info(`√ 生成目录 ${I}/${h}/SUMMARY.md`),m.curr===g&&p.info(`√ 已完成: ${I}/${h}`),process.exit(0)}export{L as d,p as l,S as m};
|
package/dist/es/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import"node:fs/promises";import"pull-md-img";import"ora";export{d as downloadArticle,m as main}from"./index-
|
|
1
|
+
import"node:fs/promises";import"pull-md-img";import"ora";export{d as downloadArticle,m as main}from"./index-5461d7d6.js";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";
|