yuque-dl 1.0.41 → 1.0.42

package/dist/cjs/cli.js

@@ -1 +1 @@
-"use strict";var e=require("node:fs"),r=require("cac"),o=require("semver"),i=require("./index-f22a3e66.js");require("node:fs/promises"),require("node:path"),require("pull-md-img"),require("ora"),require("markdown-toc"),require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),require("node:crypto");var n="undefined"!=typeof document?document.currentScript:null;const s=r.cac("yuque-dl"),{version:t,engines:u}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:n&&n.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=u.node;o.satisfies(process.version,e)||(i.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),s.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(e,r)=>{try{await i.main(e,r)}catch(e){i.logger.error(e.message||"unknown exception")}})),s.help(),s.version(t);try{s.parse()}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}
+"use strict";var e=require("node:fs"),r=require("cac"),o=require("semver"),i=require("./index-23bb041a.js");require("node:fs/promises"),require("node:path"),require("pull-md-img"),require("ora"),require("markdown-toc"),require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),require("node:crypto");var n="undefined"!=typeof document?document.currentScript:null;const s=r.cac("yuque-dl"),{version:t,engines:u}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:n&&n.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=u.node;o.satisfies(process.version,e)||(i.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),s.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(e,r)=>{try{await i.main(e,r)}catch(e){i.logger.error(e.message||"unknown exception")}})),s.help(),s.version(t);try{s.parse()}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}

package/dist/cjs/index-23bb041a.js

@@ -0,0 +1 @@
+"use strict";var e=require("node:fs/promises"),t=require("node:path"),r=require("pull-md-img"),o=require("ora"),s=require("markdown-toc"),i=require("cli-progress"),n=require("log4js"),a=require("axios"),c=require("rand-user-agent"),l=require("pako"),u=require("node:crypto");var h,p,d=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class f{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,d.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(h||(h={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(p||(p={}));const g=new Map([[p.BOARD,"画板类型"],[p.TABLE,"数据表类型"],[p.SHEET,"表格类型"],[p.DOC,"文档类型"]]),m="_yuque_session",w="https://www.yuque.com",k="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD";class ${constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${t}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(a,r),s={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:"link"},i=t.type.toLocaleLowerCase();i===h.TITLE||""!==t.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)),i===h.DOC&&(s.link=e.path)):(s.type="link",s.link=i===h.LINK?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await e.writeFile(`${o}/SUMMARY.md`,n)}catch(e){d.error("Generate Summary Error")}}genSummaryContent(e,t){for(const r of e){if(r.type===h.TITLE)if(r.link){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${e})\n\n`}else t+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===h.LINK){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`- [${r.text}](${e})\n`}Array.isArray(r.children)&&(t+=this.genSummaryContent(r.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}function y({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=c(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=c(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=c(r,e,t);return o}function b(e){const{key:t=m,token:r}=e,o={"user-agent":y({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function I(e){return{headers:b(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...b(e)}}}}const L=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:s,key:i,host:n=w}=e;let c=`${n}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,I({token:s,key:i})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))},v=e=>{if(!e)return"";const t=JSON.parse(l.inflate(e,{to:"string"}));let r="";return t.forEach((e=>{const t=`## ${e.name}\n`,o=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),s=Math.max(...r.map((e=>Number(e))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((e,t)=>{const r=t%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<s+1;o++){const s=e?.[t]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(`![${s?.name}'](${s?.src})`):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return i}(e.data);r=`${r}\n${t}\n${o}`})),r};function S(e,t=[]){if(!function(e,t){try{const{host:r,pathname:o}=new URL(e);return!t.includes(r)&&Boolean(o)}catch(e){return!1}}(e,t))return e;let r=e;try{const{origin:e,pathname:t,hash:o}=new URL(r),s=o.split("#");s.splice(s.length-1,1),r=`${e}${t}${s.join("#")}`}catch(t){return e}return`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(r)}&sign=${function(e){const t=u.createHash("sha256");return t.update(`${k}${e}`),t.digest("hex")}(r)}`}async function T(t){const{articleInfo:i,progressBar:n,options:a}=t,{token:c,key:l}=a,{bookId:u,itemUrl:h,savePath:d,saveFilePath:f,uuid:m,articleUrl:w,articleTitle:k,ignoreImg:$,host:y,imageServiceDomains:b}=i,{httpStatus:I,apiUrl:T,response:E}=await L({articleUrl:h,bookId:u,token:c,host:y,key:l}),D=E?.data?.type?.toLocaleLowerCase();let P="";if(D===p.SHEET){const{response:e}=await L({articleUrl:h,bookId:u,token:c,host:y,key:l},!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;P=o?v(o):""}catch(e){const t=g.get(D);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([p.BOARD,p.TABLE].includes(D)){const e=g.get(D);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof E?.data?.sourcecode)throw new Error(`download article Error: ${T}, http status ${I}`);P=E.data.sourcecode}if(!$){n.pause(),console.log("");const t=o({text:`下载 "${k}" 的图片中...`});t.start();try{P=await r.run(P,{dist:d,imgDir:`./img/${m}`,isIgnoreConsole:!0,referer:w||"",transform:e=>S(e,b)})}catch(t){let r=`图片下载失败(md的图片按远程url保存): ${t.message}`;throw t.error&&t.url&&(r=`图片下载失败(md的图片按远程url保存) ${t.url}: ${t.error?.message}`),await e.writeFile(f,U(P)),new Error(r)}finally{t.stop(),n.continue()}}function U(e){let t=e;t=t.replace(/<br(\s?)\/>/gm,"\n"),t=t.replace(/<a.*?>(\s*?)<\/a>/gm,"");const r=k?`# ${k}\n\n`:"";let o=a.ignoreToc?"":s(t).content;o&&(o=`${o}\n\n---\n\n`);return t=`${r}${o}${t}${w?`\n\n> 原文: <${w}>`:""}`,t}try{return await e.writeFile(f,U(P)),!0}catch(e){throw new Error(`download article Error ${w}: ${e.message}`)}}function E(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}exports.downloadArticle=T,exports.logger=d,exports.main=async function(r,o){const{bookId:s,tocList:i,bookName:n,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:g}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(e,I(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||w,imageServiceDomains:o.imageServiceDomains||[]}:{}}))})(r,{token:o.token,key:o.key});if(!s)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const m=t.resolve(o.distDir,n?E(n):String(s));await e.mkdir(m,{recursive:!0});const k=i.length,y=new f(m,k);if(await y.init(),y.curr===k)return y.bar&&y.bar.stop(),void d.info(`√ 已完成: ${m}`);const b=new Map;y.isDownloadInterrupted&&y.progressInfo.forEach((e=>{b.set(e.toc.uuid,e)}));const L=r.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(r){const{articleUrlPrefix:o,total:s,uuidMap:i,tocList:n,bookPath:a,bookId:c,progressBar:l,host:u,options:f,imageServiceDomains:g=[]}=r;let m=0,w=0,k=0;const $=[],y=[];for(let t=0;t<s;t++){const r=n[t];if("string"!=typeof r.type)continue;if(i.get(r.uuid))continue;const o=r.type.toLocaleLowerCase();if(o===h.TITLE||""!==r.child_uuid||o===h.LINK){let t=r;const s=[],n=[];for(;t;)s.unshift(E(t.title)),n.unshift(t.uuid),t=i.get(t.parent_uuid)?i.get(t.parent_uuid).toc:void 0;const c={path:s.map(E).join("/"),pathTitleList:s,pathIdList:n,toc:r};o===h.LINK?(k+=1,y.push(c)):await e.mkdir(`${a}/${s.map(E).join("/")}`,{recursive:!0}),i.set(r.uuid,c),o===p.DOC?await b(r):await l.updateProgress(c,o!==h.LINK)}else r.url&&await b(r)}async function b(e){w+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const s=e.type.toLocaleLowerCase();i.get(e.parent_uuid)&&(r=i.get(e.parent_uuid));const n=E(e.title),h=[...r.pathTitleList,n],d=[...r.pathIdList,e.uuid];let k=[...r.pathTitleList,`${n}.md`].map(E).join("/");s===p.DOC&&e.child_uuid&&(k=[...r.pathTitleList,n,"index.md"].map(E).join("/"));const y={path:k,pathTitleList:h,pathIdList:d,toc:e};let b=!0;const I=`${o}/${e.url}`;try{const o={bookId:c,itemUrl:e.url,savePath:t.resolve(a,r.pathTitleList.map(E).join("/")),saveFilePath:t.resolve(a,y.path),uuid:e.uuid,articleUrl:I,articleTitle:e.title,ignoreImg:f.ignoreImg,host:u,imageServiceDomains:g};await T({articleInfo:o,progressBar:l,options:f})}catch(e){b=!1,m+=1,$.push({articleUrl:I,errItem:y,errMsg:e.message,err:e})}i.set(e.uuid,y),await l.updateProgress(y,b)}if(k>0){d.warn("该知识库存在以下外链文章");for(const e of y)d.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(m>0){d.error(`本次执行总数${w}篇，✕ 失败${m}篇`);for(const e of $)d.error(`${e.errItem.path} ———— ${e.articleUrl}`),d.error(`———— ✕ ${e.errMsg}`);d.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:L,total:k,uuidMap:b,tocList:i,bookPath:m,bookId:s,progressBar:y,host:u,options:o,imageServiceDomains:g});const v=new $({bookPath:m,bookName:n,bookDesc:c,uuidMap:b});await v.genFile(),d.info(`√ 生成目录 ${t.resolve(m,"SUMMARY.md")}`),y.curr===k&&d.info(`√ 已完成: ${m}`),process.exit(0)};

package/dist/cjs/index.js

@@ -1 +1 @@
-"use strict";require("node:fs/promises"),require("node:path"),require("pull-md-img"),require("ora"),require("markdown-toc");var r=require("./index-f22a3e66.js");require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),require("node:crypto"),exports.downloadArticle=r.downloadArticle,exports.main=r.main;
+"use strict";require("node:fs/promises"),require("node:path"),require("pull-md-img"),require("ora"),require("markdown-toc");var r=require("./index-23bb041a.js");require("cli-progress"),require("log4js"),require("axios"),require("rand-user-agent"),require("pako"),require("node:crypto"),exports.downloadArticle=r.downloadArticle,exports.main=r.main;

package/dist/es/cli.js

@@ -1 +1 @@
-import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import r from"semver";import{m as t,l as i}from"./index-8f94b4d7.js";import"node:fs/promises";import"node:path";import"pull-md-img";import"ora";import"markdown-toc";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";import"node:crypto";const n=e("yuque-dl"),{version:s,engines:p}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=p.node;r.satisfies(process.version,o)||(i.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),n.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(o,e)=>{try{await t(o,e)}catch(o){i.error(o.message||"unknown exception")}})),n.help(),n.version(s);try{n.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}
+import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import r from"semver";import{m as t,l as i}from"./index-e9f4eef4.js";import"node:fs/promises";import"node:path";import"pull-md-img";import"ora";import"markdown-toc";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";import"node:crypto";const n=e("yuque-dl"),{version:s,engines:p}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=p.node;r.satisfies(process.version,o)||(i.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),n.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--ignore-toc","默认输出toc目录,添加此参数则不输出toc目录",{default:!1}).action((async(o,e)=>{try{await t(o,e)}catch(o){i.error(o.message||"unknown exception")}})),n.help(),n.version(s);try{n.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}

package/dist/es/index-e9f4eef4.js

@@ -0,0 +1 @@
+import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"node:path";import s from"pull-md-img";import n from"ora";import i from"markdown-toc";import a from"cli-progress";import c from"log4js";import l from"axios";import u from"rand-user-agent";import h from"pako";import p from"node:crypto";var d,f,m=(c.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),c.getLogger("yuque-dl"));class g{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,m.info("根据上次数据继续断点下载")),this.bar=new a.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},a.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(d||(d={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(f||(f={}));const w=new Map([[f.BOARD,"画板类型"],[f.TABLE,"数据表类型"],[f.SHEET,"表格类型"],[f.DOC,"文档类型"]]),$="_yuque_session",k="https://www.yuque.com",y="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD";class b{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let n=`# ${e}\n\n`;r&&(n+=`> ${r}\n\n`);let i=n;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:"link"},n=e.type.toLocaleLowerCase();n===d.TITLE||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)),n===d.DOC&&(s.link=t.path)):(s.type="link",s.link=n===d.LINK?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));i+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,i)}catch(t){m.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if(r.type===d.TITLE)if(r.link){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${t})\n\n`}else e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===d.LINK){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}function I({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=u(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=u(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=u(r,t,e);return o}function L(t){const{key:e=$,token:r}=t,o={"user-agent":I({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function v(t){return{headers:L(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...L(t)}}}}const S=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:n,host:i=k}=t;let a=`${i}/api/docs/${r}`;const c={book_id:String(o),merge_dynamic_data:String(!1)};e&&(c.mode="markdown");const u=new URLSearchParams(c).toString();return a=`${a}?${u}`,l.get(a,v({token:s,key:n})).then((({data:t,status:e})=>({apiUrl:a,httpStatus:e,response:t})))},T=t=>{if(!t)return"";const e=JSON.parse(h.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let n="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%i.length;return i[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,n=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(`![${s?.name}'](${s?.src})`):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}n=`${n}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return n}(t.data);r=`${r}\n${e}\n${o}`})),r};function E(t,e=[]){if(!function(t,e){try{const{host:r,pathname:o}=new URL(t);return!e.includes(r)&&Boolean(o)}catch(t){return!1}}(t,e))return t;let r=t;try{const{origin:t,pathname:e,hash:o}=new URL(r),s=o.split("#");s.splice(s.length-1,1),r=`${t}${e}${s.join("#")}`}catch(e){return t}return`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(r)}&sign=${function(t){const e=p.createHash("sha256");return e.update(`${y}${t}`),e.digest("hex")}(r)}`}async function D(t){const{articleInfo:r,progressBar:o,options:a}=t,{token:c,key:l}=a,{bookId:u,itemUrl:h,savePath:p,saveFilePath:d,uuid:m,articleUrl:g,articleTitle:$,ignoreImg:k,host:y,imageServiceDomains:b}=r,{httpStatus:I,apiUrl:L,response:v}=await S({articleUrl:h,bookId:u,token:c,host:y,key:l}),D=v?.data?.type?.toLocaleLowerCase();let P="";if(D===f.SHEET){const{response:t}=await S({articleUrl:h,bookId:u,token:c,host:y,key:l},!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;P=o?T(o):""}catch(t){const e=w.get(D);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([f.BOARD,f.TABLE].includes(D)){const t=w.get(D);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof v?.data?.sourcecode)throw new Error(`download article Error: ${L}, http status ${I}`);P=v.data.sourcecode}if(!k){o.pause(),console.log("");const t=n({text:`下载 "${$}" 的图片中...`});t.start();try{P=await s.run(P,{dist:p,imgDir:`./img/${m}`,isIgnoreConsole:!0,referer:g||"",transform:t=>E(t,b)})}catch(t){let r=`图片下载失败(md的图片按远程url保存): ${t.message}`;throw t.error&&t.url&&(r=`图片下载失败(md的图片按远程url保存) ${t.url}: ${t.error?.message}`),await e(d,U(P)),new Error(r)}finally{t.stop(),o.continue()}}function U(t){let e=t;e=e.replace(/<br(\s?)\/>/gm,"\n"),e=e.replace(/<a.*?>(\s*?)<\/a>/gm,"");const r=$?`# ${$}\n\n`:"";let o=a.ignoreToc?"":i(e).content;o&&(o=`${o}\n\n---\n\n`);return e=`${r}${o}${e}${g?`\n\n> 原文: <${g}>`:""}`,e}try{return await e(d,U(P)),!0}catch(t){throw new Error(`download article Error ${g}: ${t.message}`)}}function P(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}async function U(t,e){const{bookId:s,tocList:n,bookName:i,bookDesc:a,bookSlug:c,host:u,imageServiceDomains:h}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return l.get(t,v(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||k,imageServiceDomains:o.imageServiceDomains||[]}:{}}))})(t,{token:e.token,key:e.key});if(!s)throw new Error("No found book id");if(!n||0===n.length)throw new Error("No found toc list");const p=o.resolve(e.distDir,i?P(i):String(s));await r(p,{recursive:!0});const w=n.length,$=new g(p,w);if(await $.init(),$.curr===w)return $.bar&&$.bar.stop(),void m.info(`√ 已完成: ${p}`);const y=new Map;$.isDownloadInterrupted&&$.progressInfo.forEach((t=>{y.set(t.toc.uuid,t)}));const I=t.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:s,uuidMap:n,tocList:i,bookPath:a,bookId:c,progressBar:l,host:u,options:h,imageServiceDomains:p=[]}=t;let g=0,w=0,$=0;const k=[],y=[];for(let t=0;t<s;t++){const e=i[t];if("string"!=typeof e.type)continue;if(n.get(e.uuid))continue;const o=e.type.toLocaleLowerCase();if(o===d.TITLE||""!==e.child_uuid||o===d.LINK){let t=e;const s=[],i=[];for(;t;)s.unshift(P(t.title)),i.unshift(t.uuid),t=n.get(t.parent_uuid)?n.get(t.parent_uuid).toc:void 0;const c={path:s.map(P).join("/"),pathTitleList:s,pathIdList:i,toc:e};o===d.LINK?($+=1,y.push(c)):await r(`${a}/${s.map(P).join("/")}`,{recursive:!0}),n.set(e.uuid,c),o===f.DOC?await b(e):await l.updateProgress(c,o!==d.LINK)}else e.url&&await b(e)}async function b(t){w+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const s=t.type.toLocaleLowerCase();n.get(t.parent_uuid)&&(r=n.get(t.parent_uuid));const i=P(t.title),d=[...r.pathTitleList,i],m=[...r.pathIdList,t.uuid];let $=[...r.pathTitleList,`${i}.md`].map(P).join("/");s===f.DOC&&t.child_uuid&&($=[...r.pathTitleList,i,"index.md"].map(P).join("/"));const y={path:$,pathTitleList:d,pathIdList:m,toc:t};let b=!0;const I=`${e}/${t.url}`;try{const e={bookId:c,itemUrl:t.url,savePath:o.resolve(a,r.pathTitleList.map(P).join("/")),saveFilePath:o.resolve(a,y.path),uuid:t.uuid,articleUrl:I,articleTitle:t.title,ignoreImg:h.ignoreImg,host:u,imageServiceDomains:p};await D({articleInfo:e,progressBar:l,options:h})}catch(t){b=!1,g+=1,k.push({articleUrl:I,errItem:y,errMsg:t.message,err:t})}n.set(t.uuid,y),await l.updateProgress(y,b)}if($>0){m.warn("该知识库存在以下外链文章");for(const t of y)m.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(g>0){m.error(`本次执行总数${w}篇，✕ 失败${g}篇`);for(const t of k)m.error(`${t.errItem.path} ———— ${t.articleUrl}`),m.error(`———— ✕ ${t.errMsg}`);m.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:I,total:w,uuidMap:y,tocList:n,bookPath:p,bookId:s,progressBar:$,host:u,options:e,imageServiceDomains:h});const L=new b({bookPath:p,bookName:i,bookDesc:a,uuidMap:y});await L.genFile(),m.info(`√ 生成目录 ${o.resolve(p,"SUMMARY.md")}`),$.curr===w&&m.info(`√ 已完成: ${p}`),process.exit(0)}export{D as d,m as l,U as m};

package/dist/es/index.js

@@ -1 +1 @@
-import"node:fs/promises";import"node:path";import"pull-md-img";import"ora";import"markdown-toc";export{d as downloadArticle,m as main}from"./index-8f94b4d7.js";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";import"node:crypto";
+import"node:fs/promises";import"node:path";import"pull-md-img";import"ora";import"markdown-toc";export{d as downloadArticle,m as main}from"./index-e9f4eef4.js";import"cli-progress";import"log4js";import"axios";import"rand-user-agent";import"pako";import"node:crypto";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yuque-dl",
-  "version": "1.0.41",
+  "version": "1.0.42",
   "description": "yuque 知识库下载",
   "type": "module",
   "bin": {
@@ -13,8 +13,10 @@
     "build:types": "tsc --emitDeclarationOnly --outDir types -p tsconfig.base.json",
     "clean": "rm -rf dist types",
     "np": "np",
-    "release": "run-s clean build np",
-    "eslintLog": "eslint . > eslint.log"
+    "eslintLog": "eslint . > eslint.log",
+    "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s",
+    "changelog:commit": "git commit CHANGELOG.md -m 'chore: update changelog' && git push",
+    "release": "run-s clean build np changelog && changelog:commit"
   },
   "repository": {
     "type": "git",
@@ -41,6 +43,7 @@
     "@types/semver": "^7.5.8",
     "@typescript-eslint/eslint-plugin": "^6.3.0",
     "@typescript-eslint/parser": "^6.3.0",
+    "conventional-changelog-cli": "^5.0.0",
     "jest": "^29.6.2",
     "np": "^8.0.4",
     "npm-run-all": "^4.1.5",

package/types/Summary.d.ts

@@ -8,7 +8,7 @@ interface IGenSummaryFile {
 interface SummaryItem {
     id: string;
     children?: SummaryItem[];
-    type: string;
+    type: 'link' | 'title';
     text: string;
     level: number;
     link?: string;

package/dist/cjs/index-f22a3e66.js

@@ -1 +0,0 @@
-"use strict";var e=require("node:fs/promises"),t=require("node:path"),r=require("pull-md-img"),o=require("ora"),s=require("markdown-toc"),i=require("cli-progress"),n=require("log4js"),a=require("axios"),c=require("rand-user-agent"),l=require("pako"),u=require("node:crypto");var h,p,d=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class f{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,d.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class g{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let i=`# ${t}\n\n`;r&&(i+=`> ${r}\n\n`);let n=i;const a=[];s.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(a,r),s={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:""},i=t.type.toLocaleLowerCase();"title"===i||""!==t.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===i?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));n+=this.genSummaryContent(a,"");try{await e.writeFile(`${o}/SUMMARY.md`,n)}catch(e){d.error("Generate Summary Error")}}genSummaryContent(e,t){for(const r of e){if("title"===r.type)t+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`- [${r.text}](${e})\n`}Array.isArray(r.children)&&(t+=this.genSummaryContent(r.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(h||(h={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(p||(p={}));const m=new Map([[p.BOARD,"画板类型"],[p.TABLE,"数据表类型"],[p.SHEET,"表格类型"],[p.DOC,"文档类型"]]),w="_yuque_session",k="https://www.yuque.com",$="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD";function y({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=c(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=c(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=c(r,e,t);return o}function b(e){const{key:t=w,token:r}=e,o={"user-agent":y({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function I(e){return{headers:b(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...b(e)}}}}const v=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:s,key:i,host:n=k}=e;let c=`${n}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,I({token:s,key:i})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))},L=e=>{if(!e)return"";const t=JSON.parse(l.inflate(e,{to:"string"}));let r="";return t.forEach((e=>{const t=`## ${e.name}\n`,o=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),s=Math.max(...r.map((e=>Number(e))));if(o<0||s<0)return"";let i="";const n="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((e,t)=>{const r=t%n.length;return n[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,i=`${a}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<s+1;o++){const s=e?.[t]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(`![${s?.name}'](${s?.src})`):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}i=`${i}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return i}(e.data);r=`${r}\n${t}\n${o}`})),r};function S(e,t=[]){if(!function(e,t){try{const{host:r,pathname:o}=new URL(e);return!t.includes(r)&&Boolean(o)}catch(e){return!1}}(e,t))return e;let r=e;try{const{origin:e,pathname:t,hash:o}=new URL(r),s=o.split("#");s.splice(s.length-1,1),r=`${e}${t}${s.join("#")}`}catch(t){return e}return`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(r)}&sign=${function(e){const t=u.createHash("sha256");return t.update(`${$}${e}`),t.digest("hex")}(r)}`}async function E(t){const{articleInfo:i,progressBar:n,options:a}=t,{token:c,key:l}=a,{bookId:u,itemUrl:h,savePath:d,saveFilePath:f,uuid:g,articleUrl:w,articleTitle:k,ignoreImg:$,host:y,imageServiceDomains:b}=i,{httpStatus:I,apiUrl:E,response:P}=await v({articleUrl:h,bookId:u,token:c,host:y,key:l}),T=P?.data?.type?.toLocaleLowerCase();let D="";if(T===p.SHEET){const{response:e}=await v({articleUrl:h,bookId:u,token:c,host:y,key:l},!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;D=o?L(o):""}catch(e){const t=m.get(T);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([p.BOARD,p.TABLE].includes(T)){const e=m.get(T);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof P?.data?.sourcecode)throw new Error(`download article Error: ${E}, http status ${I}`);D=P.data.sourcecode}if(!$){n.pause(),console.log("");const t=o({text:`下载 "${k}" 的图片中...`});t.start();try{D=await r.run(D,{dist:d,imgDir:`./img/${g}`,isIgnoreConsole:!0,referer:w||"",transform:e=>S(e,b)})}catch(t){let r=`图片下载失败(md的图片按远程url保存): ${t.message}`;throw t.error&&t.url&&(r=`图片下载失败(md的图片按远程url保存) ${t.url}: ${t.error?.message}`),await e.writeFile(f,U(D)),new Error(r)}finally{t.stop(),n.continue()}}function U(e){let t=e;t=t.replace(/<br(\s?)\/>/gm,"\n"),t=t.replace(/<a.*?>(\s*?)<\/a>/gm,"");const r=k?`# ${k}\n\n`:"";let o=a.ignoreToc?"":s(t).content;o&&(o=`${o}\n\n---\n\n`);return t=`${r}${o}${t}${w?`\n\n> 原文: <${w}>`:""}`,t}try{return await e.writeFile(f,U(D)),!0}catch(e){throw new Error(`download article Error ${w}: ${e.message}`)}}function P(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}exports.downloadArticle=E,exports.logger=d,exports.main=async function(r,o){const{bookId:s,tocList:i,bookName:n,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:p}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(e,I(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||k,imageServiceDomains:o.imageServiceDomains||[]}:{}}))})(r,{token:o.token,key:o.key});if(!s)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const m=t.resolve(o.distDir,n?P(n):String(s));await e.mkdir(m,{recursive:!0});const w=i.length,$=new f(m,w);if(await $.init(),$.curr===w)return $.bar&&$.bar.stop(),void d.info(`√ 已完成: ${m}`);const y=new Map;$.isDownloadInterrupted&&$.progressInfo.forEach((e=>{y.set(e.toc.uuid,e)}));const b=r.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(r){const{articleUrlPrefix:o,total:s,uuidMap:i,tocList:n,bookPath:a,bookId:c,progressBar:l,host:u,options:p,imageServiceDomains:f=[]}=r;let g=0,m=0,w=0;const k=[],$=[];for(let r=0;r<s;r++){const s=n[r];if("string"!=typeof s.type)continue;if(i.get(s.uuid))continue;const d=s.type.toLocaleLowerCase();if(d===h.TITLE||""!==s.child_uuid||d===h.LINK){let t=s;const r=[],o=[];for(;t;)r.unshift(P(t.title)),o.unshift(t.uuid),t=i.get(t.parent_uuid)?i.get(t.parent_uuid).toc:void 0;const n={path:r.map(P).join("/"),pathTitleList:r,pathIdList:o,toc:s};d===h.LINK?(w+=1,$.push(n)):await e.mkdir(`${a}/${r.map(P).join("/")}`,{recursive:!0}),i.set(s.uuid,n),await l.updateProgress(n,d!==h.LINK)}else if(s.url){m+=1;let e={path:"",pathTitleList:[],pathIdList:[]};i.get(s.parent_uuid)&&(e=i.get(s.parent_uuid));const r=P(s.title),n=[...e.pathTitleList,`${r}.md`],h=[...e.pathIdList,s.uuid],d={path:n.map(P).join("/"),pathTitleList:n,pathIdList:h,toc:s};let w=!0;const $=`${o}/${s.url}`;try{const r={bookId:c,itemUrl:s.url,savePath:t.resolve(a,e.path),saveFilePath:t.resolve(a,d.path),uuid:s.uuid,articleUrl:$,articleTitle:s.title,ignoreImg:p.ignoreImg,host:u,imageServiceDomains:f};await E({articleInfo:r,progressBar:l,options:p})}catch(e){w=!1,g+=1,k.push({articleUrl:$,errItem:d,errMsg:e.message,err:e})}i.set(s.uuid,d),await l.updateProgress(d,w)}}if(w>0){d.warn("该知识库存在以下外链文章");for(const e of $)d.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(g>0){d.error(`本次执行总数${m}篇，✕ 失败${g}篇`);for(const e of k)d.error(`${e.errItem.path} ———— ${e.articleUrl}`),d.error(`———— ✕ ${e.errMsg}`);d.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:b,total:w,uuidMap:y,tocList:i,bookPath:m,bookId:s,progressBar:$,host:u,options:o,imageServiceDomains:p});const v=new g({bookPath:m,bookName:n,bookDesc:c,uuidMap:y});await v.genFile(),d.info(`√ 生成目录 ${t.resolve(m,"SUMMARY.md")}`),$.curr===w&&d.info(`√ 已完成: ${m}`),process.exit(0)};

package/dist/es/index-8f94b4d7.js

@@ -1 +0,0 @@
-import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"node:path";import s from"pull-md-img";import n from"ora";import i from"markdown-toc";import a from"cli-progress";import c from"log4js";import l from"axios";import u from"rand-user-agent";import h from"pako";import p from"node:crypto";var d,f,m=(c.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),c.getLogger("yuque-dl"));class g{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,m.info("根据上次数据继续断点下载")),this.bar=new a.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},a.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}class w{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let n=`# ${e}\n\n`;r&&(n+=`> ${r}\n\n`);let i=n;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:""},n=e.type.toLocaleLowerCase();"title"===n||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s))):(s.type="link",s.link="link"===n?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));i+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/SUMMARY.md`,i)}catch(t){m.error("Generate Summary Error")}}genSummaryContent(t,e){for(const r of t){if("title"===r.type)e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if("link"===r.type){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(d||(d={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(f||(f={}));const $=new Map([[f.BOARD,"画板类型"],[f.TABLE,"数据表类型"],[f.SHEET,"表格类型"],[f.DOC,"文档类型"]]),k="_yuque_session",y="https://www.yuque.com",b="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD";function I({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=u(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=u(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=u(r,t,e);return o}function v(t){const{key:e=k,token:r}=t,o={"user-agent":I({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function L(t){return{headers:v(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...v(t)}}}}const S=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:n,host:i=y}=t;let a=`${i}/api/docs/${r}`;const c={book_id:String(o),merge_dynamic_data:String(!1)};e&&(c.mode="markdown");const u=new URLSearchParams(c).toString();return a=`${a}?${u}`,l.get(a,L({token:s,key:n})).then((({data:t,status:e})=>({apiUrl:a,httpStatus:e,response:t})))},E=t=>{if(!t)return"";const e=JSON.parse(h.inflate(t,{to:"string"}));let r="";return e.forEach((t=>{const e=`## ${t.name}\n`,o=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let n="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%i.length;return i[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,n=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(`![${s?.name}'](${s?.src})`):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}n=`${n}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return n}(t.data);r=`${r}\n${e}\n${o}`})),r};function P(t,e=[]){if(!function(t,e){try{const{host:r,pathname:o}=new URL(t);return!e.includes(r)&&Boolean(o)}catch(t){return!1}}(t,e))return t;let r=t;try{const{origin:t,pathname:e,hash:o}=new URL(r),s=o.split("#");s.splice(s.length-1,1),r=`${t}${e}${s.join("#")}`}catch(e){return t}return`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(r)}&sign=${function(t){const e=p.createHash("sha256");return e.update(`${b}${t}`),e.digest("hex")}(r)}`}async function T(t){const{articleInfo:r,progressBar:o,options:a}=t,{token:c,key:l}=a,{bookId:u,itemUrl:h,savePath:p,saveFilePath:d,uuid:m,articleUrl:g,articleTitle:w,ignoreImg:k,host:y,imageServiceDomains:b}=r,{httpStatus:I,apiUrl:v,response:L}=await S({articleUrl:h,bookId:u,token:c,host:y,key:l}),T=L?.data?.type?.toLocaleLowerCase();let D="";if(T===f.SHEET){const{response:t}=await S({articleUrl:h,bookId:u,token:c,host:y,key:l},!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;D=o?E(o):""}catch(t){const e=$.get(T);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([f.BOARD,f.TABLE].includes(T)){const t=$.get(T);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof L?.data?.sourcecode)throw new Error(`download article Error: ${v}, http status ${I}`);D=L.data.sourcecode}if(!k){o.pause(),console.log("");const t=n({text:`下载 "${w}" 的图片中...`});t.start();try{D=await s.run(D,{dist:p,imgDir:`./img/${m}`,isIgnoreConsole:!0,referer:g||"",transform:t=>P(t,b)})}catch(t){let r=`图片下载失败(md的图片按远程url保存): ${t.message}`;throw t.error&&t.url&&(r=`图片下载失败(md的图片按远程url保存) ${t.url}: ${t.error?.message}`),await e(d,U(D)),new Error(r)}finally{t.stop(),o.continue()}}function U(t){let e=t;e=e.replace(/<br(\s?)\/>/gm,"\n"),e=e.replace(/<a.*?>(\s*?)<\/a>/gm,"");const r=w?`# ${w}\n\n`:"";let o=a.ignoreToc?"":i(e).content;o&&(o=`${o}\n\n---\n\n`);return e=`${r}${o}${e}${g?`\n\n> 原文: <${g}>`:""}`,e}try{return await e(d,U(D)),!0}catch(t){throw new Error(`download article Error ${g}: ${t.message}`)}}function D(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}async function U(t,e){const{bookId:s,tocList:n,bookName:i,bookDesc:a,bookSlug:c,host:u,imageServiceDomains:h}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return l.get(t,L(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||y,imageServiceDomains:o.imageServiceDomains||[]}:{}}))})(t,{token:e.token,key:e.key});if(!s)throw new Error("No found book id");if(!n||0===n.length)throw new Error("No found toc list");const p=o.resolve(e.distDir,i?D(i):String(s));await r(p,{recursive:!0});const f=n.length,$=new g(p,f);if(await $.init(),$.curr===f)return $.bar&&$.bar.stop(),void m.info(`√ 已完成: ${p}`);const k=new Map;$.isDownloadInterrupted&&$.progressInfo.forEach((t=>{k.set(t.toc.uuid,t)}));const b=t.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:s,uuidMap:n,tocList:i,bookPath:a,bookId:c,progressBar:l,host:u,options:h,imageServiceDomains:p=[]}=t;let f=0,g=0,w=0;const $=[],k=[];for(let t=0;t<s;t++){const s=i[t];if("string"!=typeof s.type)continue;if(n.get(s.uuid))continue;const m=s.type.toLocaleLowerCase();if(m===d.TITLE||""!==s.child_uuid||m===d.LINK){let t=s;const e=[],o=[];for(;t;)e.unshift(D(t.title)),o.unshift(t.uuid),t=n.get(t.parent_uuid)?n.get(t.parent_uuid).toc:void 0;const i={path:e.map(D).join("/"),pathTitleList:e,pathIdList:o,toc:s};m===d.LINK?(w+=1,k.push(i)):await r(`${a}/${e.map(D).join("/")}`,{recursive:!0}),n.set(s.uuid,i),await l.updateProgress(i,m!==d.LINK)}else if(s.url){g+=1;let t={path:"",pathTitleList:[],pathIdList:[]};n.get(s.parent_uuid)&&(t=n.get(s.parent_uuid));const r=D(s.title),i=[...t.pathTitleList,`${r}.md`],d=[...t.pathIdList,s.uuid],m={path:i.map(D).join("/"),pathTitleList:i,pathIdList:d,toc:s};let w=!0;const k=`${e}/${s.url}`;try{const e={bookId:c,itemUrl:s.url,savePath:o.resolve(a,t.path),saveFilePath:o.resolve(a,m.path),uuid:s.uuid,articleUrl:k,articleTitle:s.title,ignoreImg:h.ignoreImg,host:u,imageServiceDomains:p};await T({articleInfo:e,progressBar:l,options:h})}catch(t){w=!1,f+=1,$.push({articleUrl:k,errItem:m,errMsg:t.message,err:t})}n.set(s.uuid,m),await l.updateProgress(m,w)}}if(w>0){m.warn("该知识库存在以下外链文章");for(const t of k)m.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(f>0){m.error(`本次执行总数${g}篇，✕ 失败${f}篇`);for(const t of $)m.error(`${t.errItem.path} ———— ${t.articleUrl}`),m.error(`———— ✕ ${t.errMsg}`);m.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:b,total:f,uuidMap:k,tocList:n,bookPath:p,bookId:s,progressBar:$,host:u,options:e,imageServiceDomains:h});const I=new w({bookPath:p,bookName:i,bookDesc:a,uuidMap:k});await I.genFile(),m.info(`√ 生成目录 ${o.resolve(p,"SUMMARY.md")}`),$.curr===f&&m.info(`√ 已完成: ${p}`),process.exit(0)}export{T as d,m as l,U as m};