yuque-dl 1.0.48 → 1.0.50

package/README.md

@@ -92,7 +92,7 @@ yuque-dl "url" -k "verified_books" -t "verified_books的值"
 - [x] 支持转换表格类型的文档 (ps: 表格内插入图表暂不支持)
 - [x] 添加toc目录功能
 - [ ] 支持其他文档类型？🤔
-- [ ] 添加测试 🤔
+- [x] 添加测试 🤔
 
 ## 常见错误
 

package/dist/cjs/cli.js

@@ -1 +1 @@
-"use strict";var e=require("node:fs"),r=require("cac"),o=require("semver"),i=require("./index-42f6f944.js");require("node:fs/promises"),require("node:path"),require("pull-md-img"),require("ora"),require("markdown-toc"),require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),require("node:crypto");var n="undefined"!=typeof document?document.currentScript:null;const s=r.cac("yuque-dl"),{version:t,engines:u}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:n&&n.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=u.node;o.satisfies(process.version,e)||(i.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),s.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(e,r)=>{try{await i.main(e,r)}catch(e){i.logger.error(e.message||"unknown exception")}})),s.help(),s.version(t);try{s.parse()}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}
+"use strict";var e=require("node:fs"),r=require("cac"),o=require("semver"),i=require("./index-73995800.js");require("rand-user-agent"),require("node:fs/promises"),require("cli-progress"),require("node:path"),require("log4js"),require("axios"),require("node:process"),require("ora"),require("pull-md-img"),require("markdown-toc"),require("pako"),require("node:crypto");var n="undefined"!=typeof document?document.currentScript:null;const s=r.cac("yuque-dl"),{version:t,engines:c}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:n&&n.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=c.node;o.satisfies(process.version,e)||(i.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),s.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(e,r)=>{try{await i.main(e,r),process.exit(0)}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}})),s.help(),s.version(t);try{s.parse()}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}

package/dist/cjs/index-73995800.js

@@ -0,0 +1 @@
+"use strict";var e,t,r=require("node:fs/promises"),o=require("node:path"),n=require("rand-user-agent"),s=require("log4js"),i=require("cli-progress"),a=require("axios"),c=require("node:process"),l=require("ora"),u=require("pull-md-img"),h=require("markdown-toc"),p=require("pako"),d=require("node:crypto");!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(e||(e={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(t||(t={}));const f=new Map([[t.BOARD,"画板类型"],[t.TABLE,"数据表类型"],[t.SHEET,"表格类型"],[t.DOC,"文档类型"]]),g="_yuque_session",m="https://www.yuque.com",w="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",$=(s.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),s.getLogger("yuque-dl"));class k{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,$.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const t=await r.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(t)}catch(t){t&&"ENOENT"===t.code&&await r.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,t){this.curr!==this.total&&(this.curr=this.curr+1,t&&(this.progressInfo.push(e),await r.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function y({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=n(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=n(r,e,t);return o}class b{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:o,bookPath:n,uuidMap:s}=this.summaryInfo;let i=`# ${t}\n\n`;o&&(i+=`> ${o}\n\n`);let a=i;const c=[];s.forEach((t=>{const r=t.toc,o=r.parent_uuid,n=this.findTree(c,o),s={text:r.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:r.uuid,level:1,type:"link"},i=r.type.toLocaleLowerCase();i===e.TITLE||""!==r.child_uuid?(s.type="title","boolean"!=typeof n?(Array.isArray(n.children)||(n.children=[]),s.level=n.level+1,n.children.push(s)):(s.level=1,c.push(s)),i===e.DOC&&(s.link=t.path)):(s.type="link",s.link=i===e.LINK?t.toc.url:t.path,"boolean"!=typeof n?(Array.isArray(n.children)||(n.children=[]),s.level=n.level+1,n.children.push(s)):(s.level=1,c.push(s)))}));a+=this.genSummaryContent(c,"");try{await r.writeFile(`${n}/SUMMARY.md`,a)}catch(e){$.error("Generate Summary Error")}return a}genSummaryContent(t,r){for(const o of t){if(o.type===e.TITLE)if(o.link){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;r+=`\n${"".padStart(o.level+1,"#")} [${o.text}](${e})\n\n`}else r+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if(o.type===e.LINK){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;r+=`- [${o.text}](${e})\n`}Array.isArray(o.children)&&(r+=this.genSummaryContent(o.children,""))}return r}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}function I(e){const{key:t=g,token:r}=e,o={"user-agent":y({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function L(e){return{headers:I(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...I(e)}}}}const v=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:n,key:s,host:i=m}=e;let c=`${i}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,L({token:n,key:s})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))};function S(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const T=e=>{if(!e)return"";const t=p.inflate(e,{to:"string"}),r=JSON.parse(t);let o="";return r.forEach((e=>{const t=`## ${e.name}\n`,r=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),n=Math.max(...r.map((e=>Number(e))));if(o<0||n<0)return"";let s="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(n+1).fill(" ").map(((e,t)=>{const r=t%i.length;return i[r]})).join(" | ");a=`| |${a}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,s=`${a}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<n+1;o++){const n=e?.[t]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(`![${n?.name}'](${n?.src})`):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}s=`${s}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return s}(e.data);o=`${o}\n${t}\n${r}`})),o};function E(e,t=[]){return function(e,t){try{const{host:r,pathname:o}=new URL(e);return!t.includes(r)&&Boolean(o)}catch(e){return!1}}(e,t)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(e)}&sign=${function(e){const t=d.createHash("sha256");return t.update(`${w}${e}`),t.digest("hex")}(e)}`:e}async function P(e){const{articleInfo:o,progressBar:n,options:s}=e,{token:i,key:a}=s,{bookId:h,itemUrl:p,savePath:d,saveFilePath:g,uuid:m,articleUrl:w,articleTitle:$,ignoreImg:k,host:y,imageServiceDomains:b}=o,I={articleUrl:p,bookId:h,token:i,host:y,key:a},{httpStatus:L,apiUrl:S,response:P}=await v(I),U=P?.data?.type?.toLocaleLowerCase();let C="";if(U===t.SHEET){const{response:e}=await v(I,!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;C=o?T(o):""}catch(e){const t=f.get(U);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([t.BOARD,t.TABLE].includes(U)){const e=f.get(U);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof P?.data?.sourcecode)throw new Error(`download article Error: ${S}, http status ${L}`);C=P.data.sourcecode,C=function(e){const t=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=e.match(t);let o=e;const n=e;try{r?.forEach((e=>{t.lastIndex=0;const r=t.exec(e)?.[2]??"",{pathname:n,search:s}=new URL(r);if(!n.endsWith(".svg")&&s){const t=decodeURIComponent(s);o=o.replace(e,t.slice(1))}}))}catch(e){return n}return o}(C)}const x=function(e){if(!e)return[];const t=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(e.match(t)||[]);return r=r.map((e=>(e=e.replace(t,"$2"),/^http.*/g.test(e)?e:""))).filter((e=>Boolean(e))),r}(C);if(x.length&&!k){const e=await v(I,!1);C=function(e,t,r){if(!r)return t;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let s,i=!0;for(;i||Boolean(s);)if(i=!1,s=o.exec(r),s?.[1])try{const e=decodeURIComponent(s[1]),t=JSON.parse(e);n.push(t?.src||"")}catch{n.push("")}const a=new Map;return e.forEach((e=>{const{origin:r,pathname:o}=new URL(e),s=`${r}${o}`,i=n.find(((e,t)=>{const r=new RegExp(`${s}.*?`).test(e);return r&&n.splice(t,1),r}));if(i){const r=new RegExp(e,"g"),o=a.get(e)||0;let n=0;t=t.replace(r,(e=>{let t=e;return n===o&&(t=i),n+=1,t})),a.set(e,o+1)}})),t}(x,C,e.response?.data?.content??"")}const N={ignoreToc:s.ignoreToc,articleTitle:$,articleUrl:w};if(x.length&&!k){let e;n.pause(),console.log(""),"test"!==c.env.NODE_ENV&&(e=l({text:`下载 "${$}" 的图片中...`,stream:c.stdout}),e.start());const{data:t,errorInfo:o}=await u.run(C,{dist:d,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:w||"",transform:e=>E(e,b)});C=t;const s=()=>{e&&e.stop(),n.continue()};if(o.length>0){const e=`图片下载失败(失败的以远程链接保存): ${o[0].url}`;throw await r.writeFile(g,D(C,N)),s(),new Error(`${e}`)}s()}try{return await r.writeFile(g,D(C,N)),!0}catch(e){throw new Error(`download article Error ${w}: ${e.message}`)}}function D(e,t){const{articleTitle:r,articleUrl:o,ignoreToc:n}=t;let s=e;s=s.replace(/<br(\s?)\/>/gm,"\n"),s=s.replace(/<a.*?>(\s*?)<\/a>/gm,"");const i=r?`# ${r}\n\n`:"";let a=n?"":h(s).content;a&&(a=`${a}\n\n---\n\n`);return s=`${i}${a}${s}${o?`\n\n> 原文: <${o}>`:""}`,s}exports.logger=$,exports.main=async function(n,s){const{bookId:i,tocList:c,bookName:l,bookDesc:u,bookSlug:h,host:p,imageServiceDomains:d}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(e,L(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||m,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((()=>({})))})(n,{token:s.token,key:s.key});if(!i)throw new Error("No found book id");if(!c||0===c.length)throw new Error("No found toc list");const f=o.resolve(s.distDir,l?S(l):String(i));await r.mkdir(f,{recursive:!0});const g=c.length,w=new k(f,g);if(await w.init(),w.curr===g)return w.bar&&w.bar.stop(),void $.info(`√ 已完成: ${f}`);const y=new Map;w.isDownloadInterrupted&&w.progressInfo.forEach((e=>{y.set(e.toc.uuid,e)}));const I=n.replace(new RegExp(`(.*?/${h}).*`),"$1");await async function(n){const{articleUrlPrefix:s,total:i,uuidMap:a,tocList:c,bookPath:l,bookId:u,progressBar:h,host:p,options:d,imageServiceDomains:f=[]}=n;let g=0,m=0,w=0;const k=[],y=[];for(let o=0;o<i;o++){const n=c[o];if("string"!=typeof n.type)continue;if(a.get(n.uuid))continue;const s=n.type.toLocaleLowerCase();if(s===e.TITLE||""!==n.child_uuid||s===e.LINK){let o=n;const i=[],c=[];for(;o;)i.unshift(S(o.title)),c.unshift(o.uuid),o=a.get(o.parent_uuid)?a.get(o.parent_uuid).toc:void 0;const u={path:i.map(S).join("/"),pathTitleList:i,pathIdList:c,toc:n};s===e.LINK?(w+=1,y.push(u)):await r.mkdir(`${l}/${i.map(S).join("/")}`,{recursive:!0}),a.set(n.uuid,u),s===t.DOC?await b(n):await h.updateProgress(u,s!==e.LINK)}else n.url&&await b(n)}async function b(e){m+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=e.type.toLocaleLowerCase();a.get(e.parent_uuid)&&(r=a.get(e.parent_uuid));const i=S(e.title),c=[...r.pathTitleList,i],w=[...r.pathIdList,e.uuid];let $=[...r.pathTitleList,`${i}.md`].map(S).join("/"),y=r.pathTitleList.map(S).join("/");n===t.DOC&&e.child_uuid&&($=[...r.pathTitleList,i,"index.md"].map(S).join("/"),y=c.map(S).join("/"));const b={path:$,savePath:y,pathTitleList:c,pathIdList:w,toc:e};let I=!0;const L=`${s}/${e.url}`;try{const t={bookId:u,itemUrl:e.url,savePath:o.resolve(l,b.savePath),saveFilePath:o.resolve(l,b.path),uuid:e.uuid,articleUrl:L,articleTitle:e.title,ignoreImg:d.ignoreImg,host:p,imageServiceDomains:f};await P({articleInfo:t,progressBar:h,options:d})}catch(e){I=!1,g+=1,k.push({articleUrl:L,errItem:b,errMsg:e.message,err:e})}a.set(e.uuid,b),await h.updateProgress(b,I)}if(w>0){$.warn("该知识库存在以下外链文章");for(const e of y)$.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(g>0){$.error(`本次执行总数${m}篇，✕ 失败${g}篇`);for(const e of k)$.error(`《${e.errItem.path}》: ${e.articleUrl}`),$.error(`———— ✕ ${e.errMsg}`);$.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:I,total:g,uuidMap:y,tocList:c,bookPath:f,bookId:i,progressBar:w,host:p,options:s,imageServiceDomains:d});const v=new b({bookPath:f,bookName:l,bookDesc:u,uuidMap:y});await v.genFile(),$.info(`√ 生成目录 ${o.resolve(f,"SUMMARY.md")}`),w.curr===g&&$.info(`√ 已完成: ${f}`)};

package/dist/cjs/index.js

@@ -1 +1 @@
-"use strict";require("node:fs/promises"),require("node:path"),require("pull-md-img"),require("ora"),require("markdown-toc");var r=require("./index-42f6f944.js");require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),require("node:crypto"),exports.downloadArticle=r.downloadArticle,exports.main=r.main;
+"use strict";require("node:fs/promises"),require("node:path");var e=require("./index-73995800.js");require("rand-user-agent"),require("log4js"),require("cli-progress"),require("axios"),require("node:process"),require("ora"),require("pull-md-img"),require("markdown-toc"),require("pako"),require("node:crypto"),exports.main=e.main;

package/dist/es/cli.js

@@ -1 +1 @@
-import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import r from"semver";import{m as t,l as i}from"./index-2974a20a.js";import"node:fs/promises";import"node:path";import"pull-md-img";import"ora";import"markdown-toc";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";import"node:crypto";const n=e("yuque-dl"),{version:s,engines:p}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=p.node;r.satisfies(process.version,o)||(i.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),n.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(o,e)=>{try{await t(o,e)}catch(o){i.error(o.message||"unknown exception")}})),n.help(),n.version(s);try{n.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}
+import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import r from"semver";import{m as t,l as i}from"./index-7a4efc0f.js";import"rand-user-agent";import"node:fs/promises";import"cli-progress";import"node:path";import"log4js";import"axios";import"node:process";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:crypto";const s=e("yuque-dl"),{version:n,engines:p}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=p.node;r.satisfies(process.version,o)||(i.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),s.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(o,e)=>{try{await t(o,e),process.exit(0)}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}})),s.help(),s.version(n);try{s.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}

package/dist/es/index-7a4efc0f.js

@@ -0,0 +1 @@
+import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"node:path";import n from"rand-user-agent";import s from"log4js";import i from"cli-progress";import a from"axios";import{env as c,stdout as l}from"node:process";import u from"ora";import p from"pull-md-img";import h from"markdown-toc";import d from"pako";import f from"node:crypto";var m,g;!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(m||(m={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(g||(g={}));const w=new Map([[g.BOARD,"画板类型"],[g.TABLE,"数据表类型"],[g.SHEET,"表格类型"],[g.DOC,"文档类型"]]),$="_yuque_session",k="https://www.yuque.com",y="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",b=(s.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),s.getLogger("yuque-dl"));class I{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,b.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr!==this.total&&(this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function L({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=n(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=n(r,t,e);return o}class v{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:n}=this.summaryInfo;let s=`# ${e}\n\n`;r&&(s+=`> ${r}\n\n`);let i=s;const a=[];n.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),n={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:"link"},s=e.type.toLocaleLowerCase();s===m.TITLE||""!==e.child_uuid?(n.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,a.push(n)),s===m.DOC&&(n.link=t.path)):(n.type="link",n.link=s===m.LINK?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,a.push(n)))}));i+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,i)}catch(t){b.error("Generate Summary Error")}return i}genSummaryContent(t,e){for(const r of t){if(r.type===m.TITLE)if(r.link){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${t})\n\n`}else e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===m.LINK){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}function S(t){const{key:e=$,token:r}=t,o={"user-agent":L({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function T(t){return{headers:S(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...S(t)}}}}const E=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:n,key:s,host:i=k}=t;let c=`${i}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,T({token:n,key:s})).then((({data:t,status:e})=>({apiUrl:c,httpStatus:e,response:t})))};function P(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const D=t=>{if(!t)return"";const e=d.inflate(t,{to:"string"}),r=JSON.parse(e);let o="";return r.forEach((t=>{const e=`## ${t.name}\n`,r=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),n=Math.max(...r.map((t=>Number(t))));if(o<0||n<0)return"";let s="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(n+1).fill(" ").map(((t,e)=>{const r=e%i.length;return i[r]})).join(" | ");a=`| |${a}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,s=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<n+1;o++){const n=t?.[e]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(`![${n?.name}'](${n?.src})`):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}s=`${s}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return s}(t.data);o=`${o}\n${e}\n${r}`})),o};function U(t,e=[]){return function(t,e){try{const{host:r,pathname:o}=new URL(t);return!e.includes(r)&&Boolean(o)}catch(t){return!1}}(t,e)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(t)}&sign=${function(t){const e=f.createHash("sha256");return e.update(`${y}${t}`),e.digest("hex")}(t)}`:t}async function C(t){const{articleInfo:r,progressBar:o,options:n}=t,{token:s,key:i}=n,{bookId:a,itemUrl:h,savePath:d,saveFilePath:f,uuid:m,articleUrl:$,articleTitle:k,ignoreImg:y,host:b,imageServiceDomains:I}=r,L={articleUrl:h,bookId:a,token:s,host:b,key:i},{httpStatus:v,apiUrl:S,response:T}=await E(L),P=T?.data?.type?.toLocaleLowerCase();let C="";if(P===g.SHEET){const{response:t}=await E(L,!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;C=o?D(o):""}catch(t){const e=w.get(P);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([g.BOARD,g.TABLE].includes(P)){const t=w.get(P);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof T?.data?.sourcecode)throw new Error(`download article Error: ${S}, http status ${v}`);C=T.data.sourcecode,C=function(t){const e=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=t.match(e);let o=t;const n=t;try{r?.forEach((t=>{e.lastIndex=0;const r=e.exec(t)?.[2]??"",{pathname:n,search:s}=new URL(r);if(!n.endsWith(".svg")&&s){const e=decodeURIComponent(s);o=o.replace(t,e.slice(1))}}))}catch(t){return n}return o}(C)}const x=function(t){if(!t)return[];const e=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(t.match(e)||[]);return r=r.map((t=>(t=t.replace(e,"$2"),/^http.*/g.test(t)?t:""))).filter((t=>Boolean(t))),r}(C);if(x.length&&!y){const t=await E(L,!1);C=function(t,e,r){if(!r)return e;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let s,i=!0;for(;i||Boolean(s);)if(i=!1,s=o.exec(r),s?.[1])try{const t=decodeURIComponent(s[1]),e=JSON.parse(t);n.push(e?.src||"")}catch{n.push("")}const a=new Map;return t.forEach((t=>{const{origin:r,pathname:o}=new URL(t),s=`${r}${o}`,i=n.find(((t,e)=>{const r=new RegExp(`${s}.*?`).test(t);return r&&n.splice(e,1),r}));if(i){const r=new RegExp(t,"g"),o=a.get(t)||0;let n=0;e=e.replace(r,(t=>{let e=t;return n===o&&(e=i),n+=1,e})),a.set(t,o+1)}})),e}(x,C,t.response?.data?.content??"")}const O={ignoreToc:n.ignoreToc,articleTitle:k,articleUrl:$};if(x.length&&!y){let t;o.pause(),console.log(""),"test"!==c.NODE_ENV&&(t=u({text:`下载 "${k}" 的图片中...`,stream:l}),t.start());const{data:r,errorInfo:n}=await p.run(C,{dist:d,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:$||"",transform:t=>U(t,I)});C=r;const s=()=>{t&&t.stop(),o.continue()};if(n.length>0){const t=`图片下载失败(失败的以远程链接保存): ${n[0].url}`;throw await e(f,N(C,O)),s(),new Error(`${t}`)}s()}try{return await e(f,N(C,O)),!0}catch(t){throw new Error(`download article Error ${$}: ${t.message}`)}}function N(t,e){const{articleTitle:r,articleUrl:o,ignoreToc:n}=e;let s=t;s=s.replace(/<br(\s?)\/>/gm,"\n"),s=s.replace(/<a.*?>(\s*?)<\/a>/gm,"");const i=r?`# ${r}\n\n`:"";let a=n?"":h(s).content;a&&(a=`${a}\n\n---\n\n`);return s=`${i}${a}${s}${o?`\n\n> 原文: <${o}>`:""}`,s}async function x(t,e){const{bookId:n,tocList:s,bookName:i,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:p}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(t,T(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||k,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((()=>({})))})(t,{token:e.token,key:e.key});if(!n)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const h=o.resolve(e.distDir,i?P(i):String(n));await r(h,{recursive:!0});const d=s.length,f=new I(h,d);if(await f.init(),f.curr===d)return f.bar&&f.bar.stop(),void b.info(`√ 已完成: ${h}`);const w=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((t=>{w.set(t.toc.uuid,t)}));const $=t.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:n,uuidMap:s,tocList:i,bookPath:a,bookId:c,progressBar:l,host:u,options:p,imageServiceDomains:h=[]}=t;let d=0,f=0,w=0;const $=[],k=[];for(let t=0;t<n;t++){const e=i[t];if("string"!=typeof e.type)continue;if(s.get(e.uuid))continue;const o=e.type.toLocaleLowerCase();if(o===m.TITLE||""!==e.child_uuid||o===m.LINK){let t=e;const n=[],i=[];for(;t;)n.unshift(P(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const c={path:n.map(P).join("/"),pathTitleList:n,pathIdList:i,toc:e};o===m.LINK?(w+=1,k.push(c)):await r(`${a}/${n.map(P).join("/")}`,{recursive:!0}),s.set(e.uuid,c),o===g.DOC?await y(e):await l.updateProgress(c,o!==m.LINK)}else e.url&&await y(e)}async function y(t){f+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=t.type.toLocaleLowerCase();s.get(t.parent_uuid)&&(r=s.get(t.parent_uuid));const i=P(t.title),m=[...r.pathTitleList,i],w=[...r.pathIdList,t.uuid];let k=[...r.pathTitleList,`${i}.md`].map(P).join("/"),y=r.pathTitleList.map(P).join("/");n===g.DOC&&t.child_uuid&&(k=[...r.pathTitleList,i,"index.md"].map(P).join("/"),y=m.map(P).join("/"));const b={path:k,savePath:y,pathTitleList:m,pathIdList:w,toc:t};let I=!0;const L=`${e}/${t.url}`;try{const e={bookId:c,itemUrl:t.url,savePath:o.resolve(a,b.savePath),saveFilePath:o.resolve(a,b.path),uuid:t.uuid,articleUrl:L,articleTitle:t.title,ignoreImg:p.ignoreImg,host:u,imageServiceDomains:h};await C({articleInfo:e,progressBar:l,options:p})}catch(t){I=!1,d+=1,$.push({articleUrl:L,errItem:b,errMsg:t.message,err:t})}s.set(t.uuid,b),await l.updateProgress(b,I)}if(w>0){b.warn("该知识库存在以下外链文章");for(const t of k)b.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(d>0){b.error(`本次执行总数${f}篇，✕ 失败${d}篇`);for(const t of $)b.error(`《${t.errItem.path}》: ${t.articleUrl}`),b.error(`———— ✕ ${t.errMsg}`);b.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:$,total:d,uuidMap:w,tocList:s,bookPath:h,bookId:n,progressBar:f,host:u,options:e,imageServiceDomains:p});const y=new v({bookPath:h,bookName:i,bookDesc:c,uuidMap:w});await y.genFile(),b.info(`√ 生成目录 ${o.resolve(h,"SUMMARY.md")}`),f.curr===d&&b.info(`√ 已完成: ${h}`)}export{b as l,x as m};

package/dist/es/index.js

@@ -1 +1 @@
-import"node:fs/promises";import"node:path";import"pull-md-img";import"ora";import"markdown-toc";export{d as downloadArticle,m as main}from"./index-2974a20a.js";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";import"node:crypto";
+import"node:fs/promises";import"node:path";export{m as main}from"./index-7a4efc0f.js";import"rand-user-agent";import"log4js";import"cli-progress";import"axios";import"node:process";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:crypto";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yuque-dl",
-  "version": "1.0.48",
+  "version": "1.0.50",
   "description": "yuque 知识库下载",
   "type": "module",
   "bin": {
@@ -16,7 +16,8 @@
     "eslintLog": "eslint . > eslint.log",
     "changelog:gen": "conventional-changelog -p angular -i CHANGELOG.md -s -r 2",
     "changelog:commit": "git add CHANGELOG.md && git commit CHANGELOG.md -m 'chore: update changelog' && git push",
-    "release": "run-s clean build np changelog:gen changelog:commit"
+    "release": "run-s clean build np changelog:gen changelog:commit",
+    "test": "vitest --run"
   },
   "repository": {
     "type": "git",
@@ -44,11 +45,12 @@
     "@typescript-eslint/eslint-plugin": "^6.3.0",
     "@typescript-eslint/parser": "^6.3.0",
     "conventional-changelog-cli": "^5.0.0",
-    "jest": "^29.6.2",
+    "msw": "^2.3.0",
     "np": "^8.0.4",
     "npm-run-all": "^4.1.5",
     "rollup": "^3.28.0",
-    "tslib": "^2.6.1"
+    "tslib": "^2.6.1",
+    "vitest": "^1.6.0"
   },
   "main": "dist/es/index.js",
   "types": "types/index.d.ts",
@@ -60,7 +62,7 @@
     "markdown-toc": "^1.2.0",
     "ora": "^7.0.1",
     "pako": "1.0.11",
-    "pull-md-img": "^0.0.54",
+    "pull-md-img": "^0.0.58",
     "rand-user-agent": "1.0.109",
     "semver": "^7.6.0"
   },

package/types/api.d.ts

@@ -1,33 +1,4 @@
-import type { ArticleResponse } from './types/ArticleResponse';
-import type { KnowledgeBase } from './types/KnowledgeBaseResponse';
-interface IKnowledgeBaseInfo {
-    bookId?: number;
-    bookSlug?: string;
-    tocList?: KnowledgeBase.Toc[];
-    bookName?: string;
-    bookDesc?: string;
-    host?: string;
-    imageServiceDomains?: string[];
-}
-interface GetHeaderParams {
-    key?: string;
-    token?: string;
-}
-type TGetKnowledgeBaseInfo = (url: string, headerParams: GetHeaderParams) => Promise<IKnowledgeBaseInfo>;
+import type { TGetKnowledgeBaseInfo, TGetMdData } from './types';
 /** 获取知识库数据信息 */
 export declare const getKnowledgeBaseInfo: TGetKnowledgeBaseInfo;
-interface GetMdDataParams {
-    articleUrl: string;
-    bookId: number;
-    host?: string;
-    token?: string;
-    key?: string;
-}
-interface IGetDocsMdDataRes {
-    apiUrl: string;
-    httpStatus: number;
-    response?: ArticleResponse.RootObject;
-}
-type TGetMdData = (params: GetMdDataParams, isMd?: boolean) => Promise<IGetDocsMdDataRes>;
 export declare const getDocsMdData: TGetMdData;
-export {};

package/types/cli.d.ts

@@ -1,7 +1 @@
-export interface IOptions {
-    distDir: string;
-    ignoreImg: boolean;
-    token?: string;
-    key?: string;
-    ignoreToc: boolean;
-}
+export {};

package/types/constant.d.ts

@@ -15,10 +15,10 @@ declare enum ARTICLE_CONTENT_TYPE {
     /** 文档 */
     DOC = "doc"
 }
-declare const articleContentMap: Map<ARTICLE_CONTENT_TYPE, string>;
+declare const ARTICLE_CONTENT_MAP: Map<ARTICLE_CONTENT_TYPE, string>;
 /** 默认语雀cookie KEY */
 declare const DEFAULT_COOKIE_KEY = "_yuque_session";
 /** 默认语雀域名 */
 declare const DEFAULT_DOMAIN = "https://www.yuque.com";
 declare const IMAGE_SING_KEY = "UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD";
-export { ARTICLE_TOC_TYPE, ARTICLE_CONTENT_TYPE, articleContentMap, DEFAULT_COOKIE_KEY, DEFAULT_DOMAIN, IMAGE_SING_KEY };
+export { ARTICLE_TOC_TYPE, ARTICLE_CONTENT_TYPE, ARTICLE_CONTENT_MAP, DEFAULT_COOKIE_KEY, DEFAULT_DOMAIN, IMAGE_SING_KEY };

package/types/crypto/index.d.ts

@@ -1 +1,2 @@
+export declare function genSign(url: string): string;
 export declare function captureImageURL(url: string, imageServiceDomains?: string[]): string;

package/types/index.d.ts

@@ -1,26 +1,2 @@
-import ProgressBar from './ProgressBar';
-import type { IOptions } from './cli';
-interface ArticleInfo {
-    bookId: number;
-    itemUrl: string;
-    savePath: string;
-    saveFilePath: string;
-    uuid: string;
-    articleTitle: string;
-    articleUrl: string;
-    ignoreImg: boolean;
-    host?: string;
-    imageServiceDomains: string[];
-}
-interface DownloadArticleParams {
-    /** 文章信息 */
-    articleInfo: ArticleInfo;
-    /** 进度条实例 */
-    progressBar: ProgressBar;
-    /** cli options */
-    options: IOptions;
-}
-/** 下载单篇文章 */
-declare function downloadArticle(params: DownloadArticleParams): Promise<boolean>;
-declare function main(url: string, options: IOptions): Promise<void>;
-export { downloadArticle, main };
+import type { ICliOptions } from './types';
+export declare function main(url: string, options: ICliOptions): Promise<void>;

package/types/parse/Summary.d.ts

@@ -0,0 +1,9 @@
+import type { IGenSummaryFile, SummaryItem } from '../types';
+export default class Summary {
+    summaryInfo: IGenSummaryFile;
+    constructor(summaryInfo: IGenSummaryFile);
+    genFile(): Promise<string>;
+    genSummaryContent(summary: SummaryItem[], summaryContent: string): string;
+    findIdItem(node: SummaryItem, id: string): boolean | SummaryItem;
+    findTree(tree: SummaryItem[], id: string): SummaryItem | boolean;
+}

package/types/parse/fix.d.ts

@@ -0,0 +1,3 @@
+export declare function fixLatex(mdData: string): string;
+export declare function fixMarkdownImage(imgList: string[], mdData: string, htmlData: string): string;
+export declare function fixPath(dirPath: string): string;

package/types/parse/sheet.d.ts

@@ -0,0 +1,3 @@
+import type { SheetItemData } from '../types';
+export declare const parseSheet: (sheetStr: string) => string;
+export declare function genMarkdownTable(data: SheetItemData): string;

package/types/types/index.d.ts

@@ -0,0 +1,141 @@
+import { ArticleResponse } from './ArticleResponse';
+import { KnowledgeBase } from './KnowledgeBaseResponse';
+import { ProgressBar } from '../utils/ProgressBar';
+export interface ICliOptions {
+    /** 目标目录 */
+    distDir: string;
+    /** 是否忽略图片 */
+    ignoreImg: boolean;
+    /** 私有知识库 token */
+    token?: string;
+    /** 自定义token key(企业所有部署) */
+    key?: string;
+    /** 是否忽略markdown中toc的生成 */
+    ignoreToc: boolean;
+}
+export interface ArticleInfo {
+    bookId: number;
+    itemUrl: string;
+    savePath: string;
+    saveFilePath: string;
+    uuid: string;
+    articleTitle: string;
+    articleUrl: string;
+    ignoreImg: boolean;
+    host?: string;
+    imageServiceDomains: string[];
+}
+export interface DownloadArticleParams {
+    /** 文章信息 */
+    articleInfo: ArticleInfo;
+    /** 进度条实例 */
+    progressBar: ProgressBar;
+    /** cli options */
+    options: ICliOptions;
+}
+export interface IHandleMdDataOptions {
+    articleUrl: string;
+    articleTitle: string;
+    ignoreToc: boolean;
+}
+export interface IErrArticleInfo {
+    articleUrl: string;
+    errItem: IProgressItem;
+    errMsg: string;
+    err: any;
+}
+export interface IDownloadArticleListParams {
+    articleUrlPrefix: string;
+    total: number;
+    uuidMap: Map<string, IProgressItem>;
+    tocList: KnowledgeBase.Toc[];
+    bookPath: string;
+    bookId: number;
+    progressBar: ProgressBar;
+    host?: string;
+    options: ICliOptions;
+    imageServiceDomains?: string[];
+}
+export interface IProgressItem {
+    path: string;
+    toc: KnowledgeBase.Toc;
+    pathIdList: string[];
+    pathTitleList: string[];
+}
+export type IProgress = IProgressItem[];
+export interface IGenSummaryFile {
+    bookPath: string;
+    bookName?: string;
+    bookDesc?: string;
+    uuidMap: Map<string, IProgressItem>;
+}
+export interface SummaryItem {
+    id: string;
+    children?: SummaryItem[];
+    type: 'link' | 'title';
+    text: string;
+    level: number;
+    link?: string;
+}
+export interface SheetItemData {
+    [key: string]: {
+        [key: string]: {
+            v: string;
+        };
+    };
+}
+export interface SheetItem {
+    name: string;
+    rowCount: number;
+    selections: {
+        row: number;
+        col: number;
+        rowCount: number;
+        colCount: number;
+        activeCol: number;
+        activeRow: number;
+    };
+    rows: any;
+    columns: any;
+    filter: any;
+    index: 0;
+    colCount: 26;
+    mergeCells: any;
+    id: string;
+    data: SheetItemData;
+    vStore: any;
+}
+export interface IKnowledgeBaseInfo {
+    bookId?: number;
+    bookSlug?: string;
+    tocList?: KnowledgeBase.Toc[];
+    bookName?: string;
+    bookDesc?: string;
+    host?: string;
+    imageServiceDomains?: string[];
+}
+export interface IReqHeader {
+    [key: string]: string;
+}
+export interface GetHeaderParams {
+    /** token key */
+    key?: string;
+    /** token value */
+    token?: string;
+}
+export type TGetKnowledgeBaseInfo = (url: string, headerParams: GetHeaderParams) => Promise<IKnowledgeBaseInfo>;
+export interface GetMdDataParams {
+    articleUrl: string;
+    bookId: number;
+    host?: string;
+    token?: string;
+    key?: string;
+}
+export interface IGetDocsMdDataRes {
+    apiUrl: string;
+    httpStatus: number;
+    response?: ArticleResponse.RootObject;
+}
+export type TGetMdData = (params: GetMdDataParams, isMd?: boolean) => Promise<IGetDocsMdDataRes>;
+export * from './ArticleResponse';
+export * from './KnowledgeBaseResponse';

package/types/{ProgressBar.d.ts → utils/ProgressBar.d.ts}

@@ -1,13 +1,6 @@
 import cliProgress from 'cli-progress';
-import type { KnowledgeBase } from './types/KnowledgeBaseResponse';
-export interface IProgressItem {
-    path: string;
-    toc: KnowledgeBase.Toc;
-    pathIdList: string[];
-    pathTitleList: string[];
-}
-export type IProgress = IProgressItem[];
-export default class ProgressBar {
+import type { IProgress, IProgressItem } from '../types';
+export declare class ProgressBar {
     bookPath: string;
     progressFilePath: string;
     progressInfo: IProgress;

package/types/{utils.d.ts → utils/index.d.ts}

@@ -1,3 +1,4 @@
+/** 随机生成UA */
 declare function randUserAgent({ browser, os, device }: {
     browser?: string | undefined;
     os?: string | undefined;
@@ -7,4 +8,7 @@ declare function randUserAgent({ browser, os, device }: {
  * 获取md中的img url
  */
 declare function getMarkdownImageList(mdStr: string): string[];
-export { randUserAgent, getMarkdownImageList };
+declare function removeEmojis(dirName: string): string;
+export { randUserAgent, getMarkdownImageList, removeEmojis };
+export * from './log';
+export * from './ProgressBar';

package/types/utils/log.d.ts

@@ -0,0 +1,2 @@
+import log4js from 'log4js';
+export declare const logger: log4js.Logger;

package/dist/cjs/index-42f6f944.js

@@ -1 +0,0 @@
-"use strict";var e=require("node:fs/promises"),t=require("node:path"),r=require("pull-md-img"),o=require("ora"),n=require("markdown-toc"),s=require("cli-progress"),i=require("log4js"),a=require("axios"),c=require("rand-user-agent"),l=require("pako"),u=require("node:crypto");var h,p,d=(i.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),i.getLogger("yuque-dl"));class f{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,d.info("根据上次数据继续断点下载")),this.bar=new s.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},s.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(h||(h={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(p||(p={}));const g=new Map([[p.BOARD,"画板类型"],[p.TABLE,"数据表类型"],[p.SHEET,"表格类型"],[p.DOC,"文档类型"]]),m="_yuque_session",w="https://www.yuque.com",$="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD";class k{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:n}=this.summaryInfo;let s=`# ${t}\n\n`;r&&(s+=`> ${r}\n\n`);let i=s;const a=[];n.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(a,r),n={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:"link"},s=t.type.toLocaleLowerCase();s===h.TITLE||""!==t.child_uuid?(n.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,a.push(n)),s===h.DOC&&(n.link=e.path)):(n.type="link",n.link=s===h.LINK?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,a.push(n)))}));i+=this.genSummaryContent(a,"");try{await e.writeFile(`${o}/SUMMARY.md`,i)}catch(e){d.error("Generate Summary Error")}}genSummaryContent(e,t){for(const r of e){if(r.type===h.TITLE)if(r.link){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${e})\n\n`}else t+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===h.LINK){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`- [${r.text}](${e})\n`}Array.isArray(r.children)&&(t+=this.genSummaryContent(r.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}function y({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=c(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=c(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=c(r,e,t);return o}function b(e){const{key:t=m,token:r}=e,o={"user-agent":y({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function I(e){return{headers:b(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...b(e)}}}}const L=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:n,key:s,host:i=w}=e;let c=`${i}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,I({token:n,key:s})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))},v=e=>{if(!e)return"";const t=JSON.parse(l.inflate(e,{to:"string"}));let r="";return t.forEach((e=>{const t=`## ${e.name}\n`,o=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),n=Math.max(...r.map((e=>Number(e))));if(o<0||n<0)return"";let s="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(n+1).fill(" ").map(((e,t)=>{const r=t%i.length;return i[r]})).join(" | ");a=`| |${a}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,s=`${a}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<n+1;o++){const n=e?.[t]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(`![${n?.name}'](${n?.src})`):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}s=`${s}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return s}(e.data);r=`${r}\n${t}\n${o}`})),r};function S(e,t=[]){return function(e,t){try{const{host:r,pathname:o}=new URL(e);return!t.includes(r)&&Boolean(o)}catch(e){return!1}}(e,t)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(e)}&sign=${function(e){const t=u.createHash("sha256");return t.update(`${$}${e}`),t.digest("hex")}(e)}`:e}async function E(t){const{articleInfo:s,progressBar:i,options:a}=t,{token:c,key:l}=a,{bookId:u,itemUrl:h,savePath:d,saveFilePath:f,uuid:m,articleUrl:w,articleTitle:$,ignoreImg:k,host:y,imageServiceDomains:b}=s,I={articleUrl:h,bookId:u,token:c,host:y,key:l},{httpStatus:E,apiUrl:T,response:P}=await L(I),D=P?.data?.type?.toLocaleLowerCase();let U="";if(D===p.SHEET){const{response:e}=await L(I,!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;U=o?v(o):""}catch(e){const t=g.get(D);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([p.BOARD,p.TABLE].includes(D)){const e=g.get(D);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof P?.data?.sourcecode)throw new Error(`download article Error: ${T}, http status ${E}`);U=P.data.sourcecode,U=function(e){const t=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=e.match(t);let o=e;const n=e;try{r?.forEach((e=>{t.lastIndex=0;const r=t.exec(e)?.[2]||"",{pathname:n,search:s}=new URL(r);if(!n.endsWith(".svg")&&s){const t=decodeURIComponent(s);o=o.replace(e,t.slice(1))}}))}catch(e){return n}return o}(U)}const x=function(e){if(!e)return[];const t=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(e.match(t)||[]);return r=r.map((e=>(e=e.replace(t,"$2"),/^http.*/g.test(e)?e:""))).filter((e=>Boolean(e))),r}(U);if(x.length&&!k){const e=await L(I,!1);U=function(e,t,r){if(!r)return t;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let s,i=!0;for(;i||Boolean(s);)if(i=!1,s=o.exec(r),s?.[1])try{const e=decodeURIComponent(s[1]),t=JSON.parse(e);n.push(t?.src||"")}catch{n.push("")}return e.forEach((e=>{const{origin:r,pathname:o}=new URL(e),s=`${r}${o}`,i=n.find((e=>new RegExp(`${s}.*?`).test(e)));i&&(t=t.replace(e,i))})),t}(x,U,e.response?.data?.content??"")}if(x.length&&!k){i.pause(),console.log("");const t=o({text:`下载 "${$}" 的图片中...`});t.start();const{data:n,errorInfo:s}=await r.run(U,{dist:d,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:w||"",transform:e=>S(e,b)});U=n;const a=()=>{t.stop(),i.continue()};if(s.length>0){const t="图片下载失败(失败的以远程链接保存): \n";let r="";throw s.forEach(((e,t)=>{r=`${r} ———————— ${t+1}. ${e.error?.message}: ${e.url} \n`})),await e.writeFile(f,C(U)),a(),new Error(`${t}\n${r}`)}a()}function C(e){let t=e;t=t.replace(/<br(\s?)\/>/gm,"\n"),t=t.replace(/<a.*?>(\s*?)<\/a>/gm,"");const r=$?`# ${$}\n\n`:"";let o=a.ignoreToc?"":n(t).content;o&&(o=`${o}\n\n---\n\n`);return t=`${r}${o}${t}${w?`\n\n> 原文: <${w}>`:""}`,t}try{return await e.writeFile(f,C(U)),!0}catch(e){throw new Error(`download article Error ${w}: ${e.message}`)}}function T(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}exports.downloadArticle=E,exports.logger=d,exports.main=async function(r,o){const{bookId:n,tocList:s,bookName:i,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:g}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(e,I(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||w,imageServiceDomains:o.imageServiceDomains||[]}:{}}))})(r,{token:o.token,key:o.key});if(!n)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const m=t.resolve(o.distDir,i?T(i):String(n));await e.mkdir(m,{recursive:!0});const $=s.length,y=new f(m,$);if(await y.init(),y.curr===$)return y.bar&&y.bar.stop(),void d.info(`√ 已完成: ${m}`);const b=new Map;y.isDownloadInterrupted&&y.progressInfo.forEach((e=>{b.set(e.toc.uuid,e)}));const L=r.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(r){const{articleUrlPrefix:o,total:n,uuidMap:s,tocList:i,bookPath:a,bookId:c,progressBar:l,host:u,options:f,imageServiceDomains:g=[]}=r;let m=0,w=0,$=0;const k=[],y=[];for(let t=0;t<n;t++){const r=i[t];if("string"!=typeof r.type)continue;if(s.get(r.uuid))continue;const o=r.type.toLocaleLowerCase();if(o===h.TITLE||""!==r.child_uuid||o===h.LINK){let t=r;const n=[],i=[];for(;t;)n.unshift(T(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const c={path:n.map(T).join("/"),pathTitleList:n,pathIdList:i,toc:r};o===h.LINK?($+=1,y.push(c)):await e.mkdir(`${a}/${n.map(T).join("/")}`,{recursive:!0}),s.set(r.uuid,c),o===p.DOC?await b(r):await l.updateProgress(c,o!==h.LINK)}else r.url&&await b(r)}async function b(e){w+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=e.type.toLocaleLowerCase();s.get(e.parent_uuid)&&(r=s.get(e.parent_uuid));const i=T(e.title),h=[...r.pathTitleList,i],d=[...r.pathIdList,e.uuid];let $=[...r.pathTitleList,`${i}.md`].map(T).join("/"),y=r.pathTitleList.map(T).join("/");n===p.DOC&&e.child_uuid&&($=[...r.pathTitleList,i,"index.md"].map(T).join("/"),y=h.map(T).join("/"));const b={path:$,savePath:y,pathTitleList:h,pathIdList:d,toc:e};let I=!0;const L=`${o}/${e.url}`;try{const r={bookId:c,itemUrl:e.url,savePath:t.resolve(a,b.savePath),saveFilePath:t.resolve(a,b.path),uuid:e.uuid,articleUrl:L,articleTitle:e.title,ignoreImg:f.ignoreImg,host:u,imageServiceDomains:g};await E({articleInfo:r,progressBar:l,options:f})}catch(e){I=!1,m+=1,k.push({articleUrl:L,errItem:b,errMsg:e.message,err:e})}s.set(e.uuid,b),await l.updateProgress(b,I)}if($>0){d.warn("该知识库存在以下外链文章");for(const e of y)d.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(m>0){d.error(`本次执行总数${w}篇，✕ 失败${m}篇`);for(const e of k)d.error(`《${e.errItem.path}》: ${e.articleUrl}`),d.error(`———— ✕ ${e.errMsg}`);d.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:L,total:$,uuidMap:b,tocList:s,bookPath:m,bookId:n,progressBar:y,host:u,options:o,imageServiceDomains:g});const v=new k({bookPath:m,bookName:i,bookDesc:c,uuidMap:b});await v.genFile(),d.info(`√ 生成目录 ${t.resolve(m,"SUMMARY.md")}`),y.curr===$&&d.info(`√ 已完成: ${m}`),process.exit(0)};

package/dist/es/index-2974a20a.js

@@ -1 +0,0 @@
-import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"node:path";import n from"pull-md-img";import s from"ora";import i from"markdown-toc";import a from"cli-progress";import c from"log4js";import l from"axios";import u from"rand-user-agent";import h from"pako";import p from"node:crypto";var d,f,m=(c.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),c.getLogger("yuque-dl"));class g{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,m.info("根据上次数据继续断点下载")),this.bar=new a.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},a.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(d||(d={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(f||(f={}));const $=new Map([[f.BOARD,"画板类型"],[f.TABLE,"数据表类型"],[f.SHEET,"表格类型"],[f.DOC,"文档类型"]]),w="_yuque_session",k="https://www.yuque.com",y="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD";class b{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:n}=this.summaryInfo;let s=`# ${e}\n\n`;r&&(s+=`> ${r}\n\n`);let i=s;const a=[];n.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),n={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:"link"},s=e.type.toLocaleLowerCase();s===d.TITLE||""!==e.child_uuid?(n.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,a.push(n)),s===d.DOC&&(n.link=t.path)):(n.type="link",n.link=s===d.LINK?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,a.push(n)))}));i+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,i)}catch(t){m.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if(r.type===d.TITLE)if(r.link){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${t})\n\n`}else e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===d.LINK){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}function I({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=u(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=u(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=u(r,t,e);return o}function L(t){const{key:e=w,token:r}=t,o={"user-agent":I({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function v(t){return{headers:L(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...L(t)}}}}const S=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:n,key:s,host:i=k}=t;let a=`${i}/api/docs/${r}`;const c={book_id:String(o),merge_dynamic_data:String(!1)};e&&(c.mode="markdown");const u=new URLSearchParams(c).toString();return a=`${a}?${u}`,l.get(a,v({token:n,key:s})).then((({data:t,status:e})=>({apiUrl:a,httpStatus:e,response:t})))},E=t=>{if(!t)return"";const e=JSON.parse(h.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),n=Math.max(...r.map((t=>Number(t))));if(o<0||n<0)return"";let s="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(n+1).fill(" ").map(((t,e)=>{const r=e%i.length;return i[r]})).join(" | ");a=`| |${a}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,s=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<n+1;o++){const n=t?.[e]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(`![${n?.name}'](${n?.src})`):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}s=`${s}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return s}(t.data);r=`${r}\n${e}\n${o}`})),r};function T(t,e=[]){return function(t,e){try{const{host:r,pathname:o}=new URL(t);return!e.includes(r)&&Boolean(o)}catch(t){return!1}}(t,e)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(t)}&sign=${function(t){const e=p.createHash("sha256");return e.update(`${y}${t}`),e.digest("hex")}(t)}`:t}async function P(t){const{articleInfo:r,progressBar:o,options:a}=t,{token:c,key:l}=a,{bookId:u,itemUrl:h,savePath:p,saveFilePath:d,uuid:m,articleUrl:g,articleTitle:w,ignoreImg:k,host:y,imageServiceDomains:b}=r,I={articleUrl:h,bookId:u,token:c,host:y,key:l},{httpStatus:L,apiUrl:v,response:P}=await S(I),D=P?.data?.type?.toLocaleLowerCase();let U="";if(D===f.SHEET){const{response:t}=await S(I,!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;U=o?E(o):""}catch(t){const e=$.get(D);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([f.BOARD,f.TABLE].includes(D)){const t=$.get(D);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof P?.data?.sourcecode)throw new Error(`download article Error: ${v}, http status ${L}`);U=P.data.sourcecode,U=function(t){const e=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=t.match(e);let o=t;const n=t;try{r?.forEach((t=>{e.lastIndex=0;const r=e.exec(t)?.[2]||"",{pathname:n,search:s}=new URL(r);if(!n.endsWith(".svg")&&s){const e=decodeURIComponent(s);o=o.replace(t,e.slice(1))}}))}catch(t){return n}return o}(U)}const C=function(t){if(!t)return[];const e=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(t.match(e)||[]);return r=r.map((t=>(t=t.replace(e,"$2"),/^http.*/g.test(t)?t:""))).filter((t=>Boolean(t))),r}(U);if(C.length&&!k){const t=await S(I,!1);U=function(t,e,r){if(!r)return e;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let s,i=!0;for(;i||Boolean(s);)if(i=!1,s=o.exec(r),s?.[1])try{const t=decodeURIComponent(s[1]),e=JSON.parse(t);n.push(e?.src||"")}catch{n.push("")}return t.forEach((t=>{const{origin:r,pathname:o}=new URL(t),s=`${r}${o}`,i=n.find((t=>new RegExp(`${s}.*?`).test(t)));i&&(e=e.replace(t,i))})),e}(C,U,t.response?.data?.content??"")}if(C.length&&!k){o.pause(),console.log("");const t=s({text:`下载 "${w}" 的图片中...`});t.start();const{data:r,errorInfo:i}=await n.run(U,{dist:p,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:g||"",transform:t=>T(t,b)});U=r;const a=()=>{t.stop(),o.continue()};if(i.length>0){const t="图片下载失败(失败的以远程链接保存): \n";let r="";throw i.forEach(((t,e)=>{r=`${r} ———————— ${e+1}. ${t.error?.message}: ${t.url} \n`})),await e(d,x(U)),a(),new Error(`${t}\n${r}`)}a()}function x(t){let e=t;e=e.replace(/<br(\s?)\/>/gm,"\n"),e=e.replace(/<a.*?>(\s*?)<\/a>/gm,"");const r=w?`# ${w}\n\n`:"";let o=a.ignoreToc?"":i(e).content;o&&(o=`${o}\n\n---\n\n`);return e=`${r}${o}${e}${g?`\n\n> 原文: <${g}>`:""}`,e}try{return await e(d,x(U)),!0}catch(t){throw new Error(`download article Error ${g}: ${t.message}`)}}function D(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}async function U(t,e){const{bookId:n,tocList:s,bookName:i,bookDesc:a,bookSlug:c,host:u,imageServiceDomains:h}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return l.get(t,v(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||k,imageServiceDomains:o.imageServiceDomains||[]}:{}}))})(t,{token:e.token,key:e.key});if(!n)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const p=o.resolve(e.distDir,i?D(i):String(n));await r(p,{recursive:!0});const $=s.length,w=new g(p,$);if(await w.init(),w.curr===$)return w.bar&&w.bar.stop(),void m.info(`√ 已完成: ${p}`);const y=new Map;w.isDownloadInterrupted&&w.progressInfo.forEach((t=>{y.set(t.toc.uuid,t)}));const I=t.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:n,uuidMap:s,tocList:i,bookPath:a,bookId:c,progressBar:l,host:u,options:h,imageServiceDomains:p=[]}=t;let g=0,$=0,w=0;const k=[],y=[];for(let t=0;t<n;t++){const e=i[t];if("string"!=typeof e.type)continue;if(s.get(e.uuid))continue;const o=e.type.toLocaleLowerCase();if(o===d.TITLE||""!==e.child_uuid||o===d.LINK){let t=e;const n=[],i=[];for(;t;)n.unshift(D(t.title)),i.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const c={path:n.map(D).join("/"),pathTitleList:n,pathIdList:i,toc:e};o===d.LINK?(w+=1,y.push(c)):await r(`${a}/${n.map(D).join("/")}`,{recursive:!0}),s.set(e.uuid,c),o===f.DOC?await b(e):await l.updateProgress(c,o!==d.LINK)}else e.url&&await b(e)}async function b(t){$+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=t.type.toLocaleLowerCase();s.get(t.parent_uuid)&&(r=s.get(t.parent_uuid));const i=D(t.title),d=[...r.pathTitleList,i],m=[...r.pathIdList,t.uuid];let w=[...r.pathTitleList,`${i}.md`].map(D).join("/"),y=r.pathTitleList.map(D).join("/");n===f.DOC&&t.child_uuid&&(w=[...r.pathTitleList,i,"index.md"].map(D).join("/"),y=d.map(D).join("/"));const b={path:w,savePath:y,pathTitleList:d,pathIdList:m,toc:t};let I=!0;const L=`${e}/${t.url}`;try{const e={bookId:c,itemUrl:t.url,savePath:o.resolve(a,b.savePath),saveFilePath:o.resolve(a,b.path),uuid:t.uuid,articleUrl:L,articleTitle:t.title,ignoreImg:h.ignoreImg,host:u,imageServiceDomains:p};await P({articleInfo:e,progressBar:l,options:h})}catch(t){I=!1,g+=1,k.push({articleUrl:L,errItem:b,errMsg:t.message,err:t})}s.set(t.uuid,b),await l.updateProgress(b,I)}if(w>0){m.warn("该知识库存在以下外链文章");for(const t of y)m.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(g>0){m.error(`本次执行总数${$}篇，✕ 失败${g}篇`);for(const t of k)m.error(`《${t.errItem.path}》: ${t.articleUrl}`),m.error(`———— ✕ ${t.errMsg}`);m.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:I,total:$,uuidMap:y,tocList:s,bookPath:p,bookId:n,progressBar:w,host:u,options:e,imageServiceDomains:h});const L=new b({bookPath:p,bookName:i,bookDesc:a,uuidMap:y});await L.genFile(),m.info(`√ 生成目录 ${o.resolve(p,"SUMMARY.md")}`),w.curr===$&&m.info(`√ 已完成: ${p}`),process.exit(0)}export{P as d,m as l,U as m};

package/types/Summary.d.ts

@@ -1,24 +0,0 @@
-import type { IProgressItem } from './ProgressBar';
-interface IGenSummaryFile {
-    bookPath: string;
-    bookName?: string;
-    bookDesc?: string;
-    uuidMap: Map<string, IProgressItem>;
-}
-interface SummaryItem {
-    id: string;
-    children?: SummaryItem[];
-    type: 'link' | 'title';
-    text: string;
-    level: number;
-    link?: string;
-}
-export default class Summary {
-    summaryInfo: IGenSummaryFile;
-    constructor(summaryInfo: IGenSummaryFile);
-    genFile(): Promise<void>;
-    genSummaryContent(summary: SummaryItem[], summaryContent: string): string;
-    findIdItem(node: SummaryItem, id: string): boolean | SummaryItem;
-    findTree(tree: SummaryItem[], id: string): SummaryItem | boolean;
-}
-export {};

package/types/log.d.ts

@@ -1,3 +0,0 @@
-import log4js from 'log4js';
-declare const _default: log4js.Logger;
-export default _default;

package/types/parseSheet.d.ts

@@ -1 +0,0 @@
-export declare const parseSheet: (sheetStr: string) => string;