yuque-dl 1.0.66 → 1.0.67

package/dist/cjs/cli.js

@@ -1 +1 @@
-"use strict";var e=require("node:fs"),n=require("cac"),r=require("semver"),t=require("./index-KK9Csh23.js");require("rand-user-agent");var i=require("node:fs/promises");require("cli-progress");var o=require("vitepress"),s=require("node:path");require("log4js"),require("node:process"),require("axios"),require("ora"),require("pull-md-img"),require("markdown-toc"),require("pako"),require("node:crypto"),require("node:stream"),require("node:util"),require("mdast-util-from-markdown"),require("mdast-util-to-markdown");var a="undefined"!=typeof document?document.currentScript:null;let c;async function d(e){const n=s.resolve(e);if(!await u(n))throw new Error("server root not found");await async function(e){const n=e.split(s.sep).filter(Boolean).at(-1)||"yuque-dl",r=s.join(e,"/.vitepress");await i.mkdir(r,{recursive:!0});const t=s.join(r,"config.ts"),o=await async function(e){const n=[],r=[".vitepress","img","index.md","progress.json","attachments"];let t=await i.readdir(e);t=t.filter((e=>!r.includes(e)));for(const r of t){const t=s.join(e,r);if((await i.stat(t)).isDirectory()){const i=await u(s.join(t,"index.md")),o={text:r,collapsed:!0,items:await l(e,r)};i&&(o.link=`/${r}/`),n.push(o)}else n.push({text:r.slice(0,r.lastIndexOf(".")),link:`/${r}`})}return n}(e),a=`\n  export default {\n    title: "${n}",\n    themeConfig: {\n      search: {\n        provider: 'local'\n      },\n      sidebar: ${JSON.stringify(o)}\n    },\n    vite: {\n      optimizeDeps: {\n        include: []\n      }\n    },\n    markdown: {\n      html: false\n    }\n  }\n  `;await i.writeFile(t,a)}(n);const r=async()=>{const n=await o.createServer(e,{},(async()=>(c||(c=(async()=>{await n.close(),await r()})().finally((()=>{c=void 0}))),c)));await n.listen(),n.printUrls()};await r().catch((e=>{console.error(e),process.exit(1)}))}async function u(e){try{return await i.access(e),!0}catch(e){if("ENOENT"===e.code)return!1;throw e}}async function l(e,...n){const r=await i.readdir(s.join(e,...n)),t=[];for(const o of r){const r=s.join(e,...n,o);if((await i.stat(r)).isDirectory()){const i=await l(s.join(e),...n,o);if(i.length>0){const e={text:o,items:i,collapsed:false};await u(s.join(r,"index.md"))&&(e.link="/"+[...n,o].map(decodeURIComponent).join("/")+"/"),t.push(e)}}else{if("index.md"===o||/^-.*\.(md|MD)$/.test(o)||!o.endsWith(".md"))continue;const e=o.replace(/\.md$/,""),r={text:e,link:"/"+[...n.map(decodeURIComponent),`${encodeURI(e)}.html`].join("/")};t.push(r)}}return t}const p=n.cac("yuque-dl"),{version:m,engines:f}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:a&&a.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=f.node;r.satisfies(process.version,e)||(t.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),p.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--toc","是否输出文档toc目录",{default:!1}).action((async(e,n)=>{try{await t.main(e,n),process.exit(0)}catch(e){t.logger.error(e.message||"unknown exception"),process.exit(1)}})),p.command("server <serverPath>","启动web服务").action((async e=>{try{await d(e)}catch(e){t.logger.error(e.message||"unknown exception"),process.exit(1)}})),p.help(),p.version(m);try{p.parse()}catch(e){t.logger.error(e.message||"unknown exception"),process.exit(1)}
+"use strict";var e=require("node:fs"),n=require("cac"),r=require("semver"),t=require("./index-Q-HocOjx.js");require("rand-user-agent");var i=require("node:fs/promises");require("cli-progress");var o=require("vitepress"),s=require("node:path");require("log4js"),require("node:process"),require("axios"),require("ora"),require("pull-md-img"),require("markdown-toc"),require("pako"),require("node:crypto"),require("node:stream"),require("node:util"),require("mdast-util-from-markdown"),require("mdast-util-to-markdown");var a="undefined"!=typeof document?document.currentScript:null;let c;async function d(e){const n=s.resolve(e);if(!await u(n))throw new Error("server root not found");await async function(e){const n=e.split(s.sep).filter(Boolean).at(-1)||"yuque-dl",r=s.join(e,"/.vitepress");await i.mkdir(r,{recursive:!0});const t=s.join(r,"config.ts"),o=await async function(e){const n=[],r=[".vitepress","img","index.md","progress.json","attachments"];let t=await i.readdir(e);t=t.filter((e=>!r.includes(e)));for(const r of t){const t=s.join(e,r);if((await i.stat(t)).isDirectory()){const i=await u(s.join(t,"index.md")),o={text:r,collapsed:!0,items:await l(e,r)};i&&(o.link=`/${r}/`),n.push(o)}else n.push({text:r.slice(0,r.lastIndexOf(".")),link:`/${r}`})}return n}(e),a=`\n  export default {\n    title: "${n}",\n    themeConfig: {\n      search: {\n        provider: 'local'\n      },\n      sidebar: ${JSON.stringify(o)}\n    },\n    vite: {\n      optimizeDeps: {\n        include: []\n      }\n    },\n    markdown: {\n      html: false\n    }\n  }\n  `;await i.writeFile(t,a)}(n);const r=async()=>{const n=await o.createServer(e,{},(async()=>(c||(c=(async()=>{await n.close(),await r()})().finally((()=>{c=void 0}))),c)));await n.listen(),n.printUrls()};await r().catch((e=>{console.error(e),process.exit(1)}))}async function u(e){try{return await i.access(e),!0}catch(e){if("ENOENT"===e.code)return!1;throw e}}async function l(e,...n){const r=await i.readdir(s.join(e,...n)),t=[];for(const o of r){const r=s.join(e,...n,o);if((await i.stat(r)).isDirectory()){const i=await l(s.join(e),...n,o);if(i.length>0){const e={text:o,items:i,collapsed:false};await u(s.join(r,"index.md"))&&(e.link="/"+[...n,o].map(decodeURIComponent).join("/")+"/"),t.push(e)}}else{if("index.md"===o||/^-.*\.(md|MD)$/.test(o)||!o.endsWith(".md"))continue;const e=o.replace(/\.md$/,""),r={text:e,link:"/"+[...n.map(decodeURIComponent),`${encodeURI(e)}.html`].join("/")};t.push(r)}}return t}const p=n.cac("yuque-dl"),{version:m,engines:f}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:a&&a.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=f.node;r.satisfies(process.version,e)||(t.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),p.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--toc","是否输出文档toc目录",{default:!1}).action((async(e,n)=>{try{await t.main(e,n),process.exit(0)}catch(e){t.logger.error(e.message||"unknown exception"),process.exit(1)}})),p.command("server <serverPath>","启动web服务").action((async e=>{try{await d(e)}catch(e){t.logger.error(e.message||"unknown exception"),process.exit(1)}})),p.help(),p.version(m);try{p.parse()}catch(e){t.logger.error(e.message||"unknown exception"),process.exit(1)}

package/dist/cjs/index-Q-HocOjx.js

@@ -0,0 +1 @@
+"use strict";var e=require("node:fs/promises"),t=require("node:path"),r=require("rand-user-agent"),n=require("log4js"),o=require("cli-progress"),a=require("node:process"),i=require("axios"),s=require("ora"),c=require("pull-md-img"),l=require("markdown-toc"),u=require("pako"),d=require("node:crypto"),h=require("node:fs"),p=require("node:stream"),f=require("node:util"),m=require("mdast-util-from-markdown"),g=require("mdast-util-to-markdown");function w(e){var t=Object.create(null);return e&&Object.keys(e).forEach((function(r){if("default"!==r){var n=Object.getOwnPropertyDescriptor(e,r);Object.defineProperty(t,r,n.get?n:{enumerable:!0,get:function(){return e[r]}})}})),t.default=e,Object.freeze(t)}var k,y,$=w(p);!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(k||(k={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(y||(y={}));const v=new Map([[y.BOARD,"画板类型"],[y.TABLE,"数据表类型"],[y.SHEET,"表格类型"],[y.DOC,"文档类型"]]),I="_yuque_session",b="https://www.yuque.com",E="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",N=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class P{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,N.info("根据上次数据继续断点下载")),this.bar=new o.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},o.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr!==this.total&&(this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function L({browser:e="chrome",os:t="mac os",device:n="desktop"}){n=n.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=r(n,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=r(n,e,t);if("safari"===e)for(;o.includes("Applebot");)o=r(n,e,t);return o}function D(e){return e.toString().padStart(2,"0")}function S(e){const t=new Date(e);if(isNaN(t.getTime()))return"";const r=t.getFullYear(),n=t.getMonth()+1,o=t.getDate(),a=t.getHours(),i=t.getMinutes(),s=t.getSeconds();return`${r}-${D(n)}-${D(o)} ${D(a)}:${D(i)}:${D(s)}`}class T{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:n,uuidMap:o}=this.summaryInfo;let a=`# ${t}\n\n`;r&&(a+=`> ${r}\n\n`);let i=a;const s=[];o.forEach((e=>{const t=e.toc,r=t.parent_uuid,n=this.findTree(s,r),o={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:"link"},a=t.type.toLocaleLowerCase();a===k.TITLE||""!==t.child_uuid?(o.type="title","boolean"!=typeof n?(Array.isArray(n.children)||(n.children=[]),o.level=n.level+1,n.children.push(o)):(o.level=1,s.push(o)),a===k.DOC&&(o.link=e.path)):(o.type="link",o.link=a===k.LINK?e.toc.url:e.path,"boolean"!=typeof n?(Array.isArray(n.children)||(n.children=[]),o.level=n.level+1,n.children.push(o)):(o.level=1,s.push(o)))}));i+=this.genSummaryContent(s,"");try{await e.writeFile(`${n}/index.md`,i)}catch(e){N.error("Generate Summary Error")}return i}genSummaryContent(e,t){for(const r of e){if(r.type===k.TITLE)if(r.link){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${e})\n\n`}else t+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===k.LINK){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`- [${r.text}](${e})\n`}Array.isArray(r.children)&&(t+=this.genSummaryContent(r.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}function U(e){const{key:t=I,token:r}=e,n={"user-agent":L({browser:"chrome",device:"desktop"})};return r&&(n.cookie=`${t}=${r};`),n}function O(e){const t={headers:U(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...U(e)}}};return"test"===a.env.NODE_ENV&&(t.proxy=!1),t}const x=(e,t=!0)=>{const{articleUrl:r,bookId:n,token:o,key:a,host:s=b}=e;let c=`${s}/api/docs/${r}`;const l={book_id:String(n),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,i.get(c,O({token:o,key:a})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))};function C(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const R=e=>{if(!e)return"";const t=u.inflate(e,{to:"string"}),r=JSON.parse(t);let n="";return r.forEach((e=>{const t=`## ${e.name}\n`,r=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const n=e[t];n&&(r=r.concat(Object.keys(n)))}));const n=Math.max(...t.map((e=>Number(e)))),o=Math.max(...r.map((e=>Number(e))));if(n<0||o<0)return"";let a="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let s=Array(o+1).fill(" ").map(((e,t)=>{const r=t%i.length;return i[r]})).join(" | ");s=`| |${s}|`;let c=Array(o+2).fill("---").join(" |");c=`|${c}|`,a=`${s}\n${c}\n`;for(let t=0;t<n+1;t++){const r=[];for(let n=0;n<o+1;n++){const o=e?.[t]?.[n]?.v||null;o&&"string"==typeof o?r.push(o):o&&"object"==typeof o?"image"===o?.class&&o?.src?r.push(`![${o?.name}'](${o?.src})`):"checkbox"===o?.class?r.push(o?.value?"[x] ":"[ ] "):"link"===o?.class?r.push(`[${o?.text}](${o?.url})`):"select"===o?.class&&r.push(o?.value?.join(",")):r.push(null)}a=`${a}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return a}(e.data);n=`${n}\n${t}\n${r}`})),n};function q(e,t=[]){return function(e,t){try{const{host:r,pathname:n}=new URL(e);return!t.includes(r)&&Boolean(n)}catch(e){return!1}}(e,t)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(e)}&sign=${function(e){const t=d.createHash("sha256");return t.update(`${E}${e}`),t.digest("hex")}(e)}`:e}const F=f.promisify($.finished);async function _(e){const{fileUrl:t,savePath:r,token:n,key:o,fileName:a}=e;return i.get(t,{...O({token:n,key:o}),responseType:"stream"}).then((async e=>{if(e.request?.path?.startsWith("/login"))throw new Error(`"${a}" need token`);if(200===e.status){const n=h.createWriteStream(r);return e.data?.pipe(n),F(n).then((()=>({fileUrl:t,savePath:r})))}throw new Error(`response status ${e.status}`)}))}const j=/\[(.*?)\]\((.*?)\)/g,A=/\[(.*?)\]\((.*?\.yuque\.com\/attachments.*?)\)/;async function M(e){const{mdData:r,savePath:n,attachmentsDir:o,articleTitle:i,token:c,key:l}=e,u=(r.match(j)||[]).filter((e=>A.test(e)));if(0===u.length)return{mdData:r};const d=s({text:`下载 "${i}" 的附件中...`,stream:a.stdout});"test"!==a.env.NODE_ENV&&d.start();const p=t.resolve(n,o),f=u.map((e=>function(e,r){const[,n,o]=A.exec(e)||[];if(!o)return!1;const a=n||o.split("/").at(-1);if(!a)return!1;const i=t.join(r,a);return{fileName:a,url:o,rawMd:e,currentFilePath:i}}(e,p))).filter((e=>!1!==e));h.mkdirSync(p,{recursive:!0});const m=f.map((e=>_({fileUrl:e.url,savePath:e.currentFilePath,token:c,key:l,fileName:e.fileName}))),g=await Promise.all(m).finally((function(){d&&d.stop()}));let w=r;return g.forEach((e=>{const t=f.find((t=>t.url===e.fileUrl));if(t){const e=`[附件: ${t.fileName}](${o}/${t.fileName})`;w=w.replace(t.rawMd,e)}})),{mdData:w}}function B(e){const t=[];return J(e,(function(e,r){"link"===e.type&&t.push({node:e,keyChain:r})})),t}function J(e,t,r=[]){t(e,r),Array.isArray(e.children)&&(r.push("children"),e.children.forEach(((e,n)=>{J(e,t,[...r,String(n)])})))}const H=/name="audio" value="data:(.*?audioId.*?)".*?><\/card>/gm;async function K(e){const{mdData:r,htmlData:n,savePath:o,attachmentsDir:i,articleTitle:c,token:l,key:u}=e,d=function(e){return m.fromMarkdown(e)}(r),p=B(d).filter((e=>/_lake_card.*?videoId/.test(e.node.url))),f=function(e){const t=e.match(H)||[];try{return t.map((e=>e.replace(H,"$1"))).map((e=>JSON.parse(decodeURIComponent(e))))}catch(e){return[]}}(n);if(0===p.length&&0===f.length)return{mdData:r};const w=s({text:`下载 "${c}" 的音视频中...`,stream:a.stdout});"test"!==a.env.NODE_ENV&&w.start();const k=t.resolve(o,i);h.mkdirSync(k,{recursive:!0});let y=r;try{if(p.length>0){const r=await async function(e,r,n){const{key:o,token:a}=r,i=e.map((async e=>{const r=function(e){try{const t=new URL(e).searchParams.get("_lake_card")??"",r=decodeURIComponent(t),n=JSON.parse(r);return{name:n?.name||"",videoId:n?.videoId||""}}catch(e){return!1}}(e.node.url);if(!r)return!1;const i=await W({videoId:r.videoId,key:o,token:a});if(!i)return!1;const s=r.name??r.videoId.split("/").at(-1)??r.videoId;return{videoInfo:{...r,...i},astNode:e,fileName:s,currentFilePath:t.join(n,s)}})),s=await Promise.all(i);return s.filter(V)}(p,e,k),n=r.map((e=>_({fileUrl:e.videoInfo.video,savePath:e.currentFilePath,token:l,key:u,fileName:e.videoInfo.name})));(await Promise.all(n)).forEach((e=>{const n=r.find((t=>t.videoInfo.video===e.fileUrl));n&&(n.astNode.node.url=`${i}${t.sep}${n.fileName}`,n.astNode.node.children=[{type:"text",value:`音视频附件: ${n.videoInfo.name}`}])})),y=function(e){return g.toMarkdown(e)}(d)}if(f.length>0){const r=await async function(e,r,n){const{key:o,token:a}=r,i=e.map((async e=>{const r=await W({videoId:e.audioId,key:o,token:a});if(!r)return!1;const i=e?.fileName??e.id;return{audioInfo:{...e,...r},fileName:i,currentFilePath:t.join(n,i)}})),s=await Promise.all(i);return s.filter(V)}(f,e,k),n=r.map((e=>_({fileUrl:e.audioInfo.audio,savePath:e.currentFilePath,token:l,key:u,fileName:e.audioInfo.fileName}))),o=await Promise.all(n);let a="\n\n> [yuque-dl warn]: 由于语雀markdown接口限制, 无法准确定位音频文件在文档中所在位置, 所以统一所有音频放到一起\n";o.forEach((e=>{const n=r.find((t=>t.audioInfo.audio===e.fileUrl));n&&(a+=`> - [音视频附件: ${n.audioInfo.fileName}](${i}${t.sep}${n.fileName})\n`)})),y+=a}}finally{w&&w.stop()}return{mdData:y}}function W(e){let t="https://www.yuque.com/api/video";const{videoId:r,token:n,key:o}=e,a=new URLSearchParams;return a.set("video_id",r),t=`${t}?${a.toString()}`,i.get(t,O({token:n,key:o})).then((({data:e,status:t})=>{const r=e.data;return 200===t&&"success"===r.status&&r.info})).catch((()=>!1))}function V(e){return!!e}async function z(t){const{articleInfo:r,progressBar:n,options:o}=t,{token:i,key:l}=o,{bookId:u,itemUrl:d,savePath:h,saveFilePath:p,uuid:f,articleUrl:m,articleTitle:g,ignoreImg:w,host:k,imageServiceDomains:$}=r,I={articleUrl:d,bookId:u,token:i,host:k,key:l},{httpStatus:b,apiUrl:E,response:N}=await x(I),P=N?.data?.type?.toLocaleLowerCase();let L="";if(P===y.SHEET){const{response:e}=await x(I,!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},n=r?.sheet;L=n?R(n):""}catch(e){const t=v.get(P);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([y.BOARD,y.TABLE].includes(P)){const e=v.get(P);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof N?.data?.sourcecode)throw new Error(`download article Error: ${E}, http status ${b}`);L=N.data.sourcecode,L=function(e){const t=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=e.match(t);let n=e;const o=e;try{r?.forEach((e=>{t.lastIndex=0;const r=t.exec(e)?.[2]??"",{pathname:o,search:a}=new URL(r);if(!o.endsWith(".svg")&&a){const t=decodeURIComponent(a);n=n.replace(e,t.slice(1))}}))}catch(e){return o}return n}(L)}const D=function(e){if(!e)return[];const t=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(e.match(t)||[]);return r=r.map((e=>(e=e.replace(t,"$2"),/^http.*/g.test(e)?e:""))).filter((e=>Boolean(e))),r}(L),T=await x(I,!1),U=T.response?.data?.content??"";D.length&&!w&&(L=function(e,t,r){if(!r)return t;const n=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,o=[];let a,i=!0;for(;i||Boolean(a);)if(i=!1,a=n.exec(r),a?.[1])try{const e=decodeURIComponent(a[1]),t=JSON.parse(e);o.push(t?.src||"")}catch(e){o.push("")}const s=new Map;return e.forEach((e=>{const{origin:r,pathname:n}=new URL(e),a=`${r}${n}`,i=o.find(((e,t)=>{const r=new RegExp(`${a}.*?`).test(e);return r&&o.splice(t,1),r}));if(i){const r=new RegExp(e,"g"),n=s.get(e)||0;let o=0;t=t.replace(r,(e=>{let t=e;return o===n&&(t=i),o+=1,t})),s.set(e,n+1)}})),t}(D,L,U));const O={toc:o.toc,articleTitle:g,articleUrl:m,articleUpdateTime:S(N?.data?.content_updated_at??"")},C=[];try{n.pause(),console.log("");L=(await M({mdData:L,savePath:h,attachmentsDir:`./attachments/${f}`,articleTitle:g,token:i,key:l})).mdData}catch(e){C.push(`附件下载失败: ${e.message||"unknown error"}`)}finally{n.continue()}try{n.pause(),console.log("");L=(await K({mdData:L,htmlData:U,savePath:h,attachmentsDir:`./attachments/${f}`,articleTitle:g,token:i,key:l})).mdData}catch(e){C.push(`音视频下载失败: ${e.message||"unknown error"}`)}finally{n.continue()}if(D.length&&!w){let t;n.pause(),console.log(""),"test"!==a.env.NODE_ENV&&(t=s({text:`下载 "${g}" 的图片中...`,stream:a.stdout}),t.start());let r=[],o=L;try{const e=await c.run(L,{dist:h,imgDir:`./img/${f}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:m||"",transform:e=>q(e=e.replace("x-oss-process=image%2Fwatermark%2C",""),$)});r=e.errorInfo,o=e.data}catch(e){r=[e]}L=o;const i=()=>{t&&t.stop(),n.continue()};if(r.length>0){const t=r[0];let n="图片下载失败(失败的以远程链接保存): ";throw n=t.url?`${n}${t.error?.message} ${t.url.slice(0,20)}...`:`${n}${t.message}`,await e.writeFile(p,G(L,O)),i(),new Error(`${n}`)}i()}try{if(await e.writeFile(p,G(L,O)),C.length>0)throw new Error(C[0]);return!0}catch(e){throw new Error(`${e.message}`)}}function G(e,t){const{articleTitle:r,articleUrl:n,toc:o}=t;let a=e;a=a.replace(/\|\s?<br \/>- \[(x|\s)\]([\s\S\n]*?)\|/gm,((e,t,r)=>`| <br />- [${t}]${r}|`.replace(/\n/gm,"").replace(/<br(\s?)\/>/gm,""))),a=a.replace(/(\|?)(.*?)\|/gm,((e,t,r)=>`${t}${r.replace(/<br(\s?)\/>/gm,"")}|`)),a=a.replace(/<br(\s?)\/>/gm,"\n"),a=a.replace(/<a.*?>(\s*?)<\/a>/gm,"");const i=r?`# ${r}\n\n`:"";let s=o?l(a).content:"";s&&(s=`${s}\n\n---\n\n`);let c="\n\n";return t.articleUpdateTime&&(c+=`> 更新: ${t.articleUpdateTime}  \n`),n&&(c+=`> 原文: <${n}>`),a=`${i}${s}${a}${c}`,a}exports.logger=N,exports.main=async function(r,n){if(!function(e){if("function"==typeof URL.canParse)return URL.canParse(e);try{return new URL(e),!0}catch(e){return!1}}(r))throw new Error("Please enter a valid URL");const{bookId:o,tocList:a,bookName:s,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:d}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return i.get(e,O(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const n=JSON.parse(decodeURIComponent(t[1]));return n.book?{bookId:n.book.id,bookSlug:n.book.slug,tocList:n.book.toc||[],bookName:n.book.name||"",bookDesc:n.book.description||"",host:n.space?.host||b,imageServiceDomains:n.imageServiceDomains||[]}:{}})).catch((e=>{const t=e?.message??"";if(!t)throw new Error("unknown error");if(["getaddrinfo ENOTFOUND","read ECONNRESET","Client network socket disconnected before secure TLS connection was established"].some((e=>t.startsWith(e))))throw new Error("请检查网络(是否正常联网/是否开启了代理软件)");throw new Error(t)}))})(r,{token:n.token,key:n.key});if(!o)throw new Error("No found book id");if(!a||0===a.length)throw new Error("No found toc list");const h=t.resolve(n.distDir,s?C(s):String(o));await e.mkdir(h,{recursive:!0});const p=a.length,f=new P(h,p);if(await f.init(),f.curr===p)return f.bar&&f.bar.stop(),void N.info(`√ 已完成: ${h}`);const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const g=r.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(r){const{articleUrlPrefix:n,total:o,uuidMap:a,tocList:i,bookPath:s,bookId:c,progressBar:l,host:u,options:d,imageServiceDomains:h=[]}=r;let p=0,f=0,m=0;const g=[],w=[];for(let t=0;t<o;t++){const r=i[t];if("string"!=typeof r.type)continue;if(a.get(r.uuid))continue;const n=r.type.toLocaleLowerCase();if(n===k.TITLE||""!==r.child_uuid||n===k.LINK){let t=r;const o=[],i=[];for(;t;)o.unshift(C(t.title)),i.unshift(t.uuid),t=a.get(t.parent_uuid)?a.get(t.parent_uuid).toc:void 0;const c={path:o.map(C).join("/"),pathTitleList:o,pathIdList:i,toc:r};n===k.LINK?(m+=1,w.push(c)):await e.mkdir(`${s}/${o.map(C).join("/")}`,{recursive:!0}),a.set(r.uuid,c),n===y.DOC?await $(r):await l.updateProgress(c,n!==k.LINK)}else r.url&&await $(r)}async function $(e){f+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const o=e.type.toLocaleLowerCase();a.get(e.parent_uuid)&&(r=a.get(e.parent_uuid));const i=C(e.title),m=[...r.pathTitleList,i],w=[...r.pathIdList,e.uuid];let k=[...r.pathTitleList,`${i}.md`].map(C).join("/"),$=r.pathTitleList.map(C).join("/");o===y.DOC&&e.child_uuid&&(k=[...r.pathTitleList,i,"index.md"].map(C).join("/"),$=m.map(C).join("/"));const v={path:k,savePath:$,pathTitleList:m,pathIdList:w,toc:e};let I=!0;const b=`${n}/${e.url}`;try{const r={bookId:c,itemUrl:e.url,savePath:t.resolve(s,v.savePath),saveFilePath:t.resolve(s,v.path),uuid:e.uuid,articleUrl:b,articleTitle:e.title,ignoreImg:d.ignoreImg,host:u,imageServiceDomains:h};await z({articleInfo:r,progressBar:l,options:d})}catch(e){I=!1,p+=1,g.push({articleUrl:b,errItem:v,errMsg:e.message,err:e})}a.set(e.uuid,v),await l.updateProgress(v,I)}if(m>0){N.warn("该知识库存在以下外链文章");for(const e of w)N.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(p>0){N.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const e of g)N.error(`《${e.errItem.path}》: ${e.articleUrl}`),e.errMsg.split("\n").forEach((e=>{N.error(`———— ✕ ${e}`)}));N.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:g,total:p,uuidMap:m,tocList:a,bookPath:h,bookId:o,progressBar:f,host:u,options:n,imageServiceDomains:d});const w=new T({bookPath:h,bookName:s,bookDesc:c,uuidMap:m});await w.genFile(),N.info(`√ 生成目录 ${t.resolve(h,"index.md")}`),f.curr===p&&N.info(`√ 已完成: ${h}`)};

package/dist/cjs/index.js

@@ -1 +1 @@
-"use strict";require("node:fs/promises"),require("node:path");var e=require("./index-KK9Csh23.js");require("rand-user-agent"),require("log4js"),require("cli-progress"),require("node:process"),require("axios"),require("ora"),require("pull-md-img"),require("markdown-toc"),require("pako"),require("node:crypto"),require("node:fs"),require("node:stream"),require("node:util"),require("mdast-util-from-markdown"),require("mdast-util-to-markdown"),exports.main=e.main;
+"use strict";require("node:fs/promises"),require("node:path");var e=require("./index-Q-HocOjx.js");require("rand-user-agent"),require("log4js"),require("cli-progress"),require("node:process"),require("axios"),require("ora"),require("pull-md-img"),require("markdown-toc"),require("pako"),require("node:crypto"),require("node:fs"),require("node:stream"),require("node:util"),require("mdast-util-from-markdown"),require("mdast-util-to-markdown"),exports.main=e.main;

package/dist/es/cli.js

@@ -1 +1 @@
-import{readFileSync as o}from"node:fs";import{cac as t}from"cac";import e from"semver";import{m as n,l as i}from"./index-DH0mvJUN.js";import"rand-user-agent";import{mkdir as r,writeFile as s,access as a,readdir as c,stat as m}from"node:fs/promises";import"cli-progress";import{createServer as p}from"vitepress";import{resolve as d,sep as l,join as f}from"node:path";import"log4js";import"node:process";import"axios";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:crypto";import"node:stream";import"node:util";import"mdast-util-from-markdown";import"mdast-util-to-markdown";let u;async function w(o){const t=d(o);if(!await y(t))throw new Error("server root not found");await async function(o){const t=o.split(l).filter(Boolean).at(-1)||"yuque-dl",e=f(o,"/.vitepress");await r(e,{recursive:!0});const n=f(e,"config.ts"),i=await async function(o){const t=[],e=[".vitepress","img","index.md","progress.json","attachments"];let n=await c(o);n=n.filter((o=>!e.includes(o)));for(const e of n){const n=f(o,e);if((await m(n)).isDirectory()){const i=await y(f(n,"index.md")),r={text:e,collapsed:!0,items:await k(o,e)};i&&(r.link=`/${e}/`),t.push(r)}else t.push({text:e.slice(0,e.lastIndexOf(".")),link:`/${e}`})}return t}(o),a=`\n  export default {\n    title: "${t}",\n    themeConfig: {\n      search: {\n        provider: 'local'\n      },\n      sidebar: ${JSON.stringify(i)}\n    },\n    vite: {\n      optimizeDeps: {\n        include: []\n      }\n    },\n    markdown: {\n      html: false\n    }\n  }\n  `;await s(n,a)}(t);const e=async()=>{const t=await p(o,{},(async()=>(u||(u=(async()=>{await t.close(),await e()})().finally((()=>{u=void 0}))),u)));await t.listen(),t.printUrls()};await e().catch((o=>{console.error(o),process.exit(1)}))}async function y(o){try{return await a(o),!0}catch(o){if("ENOENT"===o.code)return!1;throw o}}async function k(o,...t){const e=await c(f(o,...t)),n=[];for(const i of e){const e=f(o,...t,i);if((await m(e)).isDirectory()){const r=await k(f(o),...t,i);if(r.length>0){const o={text:i,items:r,collapsed:false};await y(f(e,"index.md"))&&(o.link="/"+[...t,i].map(decodeURIComponent).join("/")+"/"),n.push(o)}}else{if("index.md"===i||/^-.*\.(md|MD)$/.test(i)||!i.endsWith(".md"))continue;const o=i.replace(/\.md$/,""),e={text:o,link:"/"+[...t.map(decodeURIComponent),`${encodeURI(o)}.html`].join("/")};n.push(e)}}return n}const h=t("yuque-dl"),{version:x,engines:g}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=g.node;e.satisfies(process.version,o)||(i.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),h.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--toc","是否输出文档toc目录",{default:!1}).action((async(o,t)=>{try{await n(o,t),process.exit(0)}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}})),h.command("server <serverPath>","启动web服务").action((async o=>{try{await w(o)}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}})),h.help(),h.version(x);try{h.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}
+import{readFileSync as o}from"node:fs";import{cac as t}from"cac";import e from"semver";import{m as n,l as i}from"./index-Bw1wFGlC.js";import"rand-user-agent";import{mkdir as r,writeFile as s,access as a,readdir as c,stat as m}from"node:fs/promises";import"cli-progress";import{createServer as p}from"vitepress";import{resolve as d,sep as l,join as f}from"node:path";import"log4js";import"node:process";import"axios";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:crypto";import"node:stream";import"node:util";import"mdast-util-from-markdown";import"mdast-util-to-markdown";let u;async function w(o){const t=d(o);if(!await y(t))throw new Error("server root not found");await async function(o){const t=o.split(l).filter(Boolean).at(-1)||"yuque-dl",e=f(o,"/.vitepress");await r(e,{recursive:!0});const n=f(e,"config.ts"),i=await async function(o){const t=[],e=[".vitepress","img","index.md","progress.json","attachments"];let n=await c(o);n=n.filter((o=>!e.includes(o)));for(const e of n){const n=f(o,e);if((await m(n)).isDirectory()){const i=await y(f(n,"index.md")),r={text:e,collapsed:!0,items:await k(o,e)};i&&(r.link=`/${e}/`),t.push(r)}else t.push({text:e.slice(0,e.lastIndexOf(".")),link:`/${e}`})}return t}(o),a=`\n  export default {\n    title: "${t}",\n    themeConfig: {\n      search: {\n        provider: 'local'\n      },\n      sidebar: ${JSON.stringify(i)}\n    },\n    vite: {\n      optimizeDeps: {\n        include: []\n      }\n    },\n    markdown: {\n      html: false\n    }\n  }\n  `;await s(n,a)}(t);const e=async()=>{const t=await p(o,{},(async()=>(u||(u=(async()=>{await t.close(),await e()})().finally((()=>{u=void 0}))),u)));await t.listen(),t.printUrls()};await e().catch((o=>{console.error(o),process.exit(1)}))}async function y(o){try{return await a(o),!0}catch(o){if("ENOENT"===o.code)return!1;throw o}}async function k(o,...t){const e=await c(f(o,...t)),n=[];for(const i of e){const e=f(o,...t,i);if((await m(e)).isDirectory()){const r=await k(f(o),...t,i);if(r.length>0){const o={text:i,items:r,collapsed:false};await y(f(e,"index.md"))&&(o.link="/"+[...t,i].map(decodeURIComponent).join("/")+"/"),n.push(o)}}else{if("index.md"===i||/^-.*\.(md|MD)$/.test(i)||!i.endsWith(".md"))continue;const o=i.replace(/\.md$/,""),e={text:o,link:"/"+[...t.map(decodeURIComponent),`${encodeURI(o)}.html`].join("/")};n.push(e)}}return n}const h=t("yuque-dl"),{version:x,engines:g}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=g.node;e.satisfies(process.version,o)||(i.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),h.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--toc","是否输出文档toc目录",{default:!1}).action((async(o,t)=>{try{await n(o,t),process.exit(0)}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}})),h.command("server <serverPath>","启动web服务").action((async o=>{try{await w(o)}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}})),h.help(),h.version(x);try{h.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}

package/dist/es/index-Bw1wFGlC.js

@@ -0,0 +1 @@
+import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"node:path";import n from"rand-user-agent";import a from"log4js";import s from"cli-progress";import{env as i,stdout as c}from"node:process";import l from"axios";import u from"ora";import d from"pull-md-img";import h from"markdown-toc";import p from"pako";import f from"node:crypto";import{createWriteStream as m,mkdirSync as g}from"node:fs";import*as w from"node:stream";import{promisify as k}from"node:util";import{fromMarkdown as y}from"mdast-util-from-markdown";import{toMarkdown as $}from"mdast-util-to-markdown";var v,I;!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(v||(v={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(I||(I={}));const b=new Map([[I.BOARD,"画板类型"],[I.TABLE,"数据表类型"],[I.SHEET,"表格类型"],[I.DOC,"文档类型"]]),E="_yuque_session",N="https://www.yuque.com",L="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",P=(a.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),a.getLogger("yuque-dl"));class D{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,P.info("根据上次数据继续断点下载")),this.bar=new s.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},s.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr!==this.total&&(this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function S({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=n(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=n(r,t,e);return o}function T(t){return t.toString().padStart(2,"0")}function U(t){const e=new Date(t);if(isNaN(e.getTime()))return"";const r=e.getFullYear(),o=e.getMonth()+1,n=e.getDate(),a=e.getHours(),s=e.getMinutes(),i=e.getSeconds();return`${r}-${T(o)}-${T(n)} ${T(a)}:${T(s)}:${T(i)}`}class x{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:n}=this.summaryInfo;let a=`# ${e}\n\n`;r&&(a+=`> ${r}\n\n`);let s=a;const i=[];n.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(i,r),n={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:"link"},a=e.type.toLocaleLowerCase();a===v.TITLE||""!==e.child_uuid?(n.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)),a===v.DOC&&(n.link=t.path)):(n.type="link",n.link=a===v.LINK?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)))}));s+=this.genSummaryContent(i,"");try{await t.writeFile(`${o}/index.md`,s)}catch(t){P.error("Generate Summary Error")}return s}genSummaryContent(t,e){for(const r of t){if(r.type===v.TITLE)if(r.link){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${t})\n\n`}else e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===v.LINK){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}function C(t){const{key:e=E,token:r}=t,o={"user-agent":S({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function O(t){const e={headers:C(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...C(t)}}};return"test"===i.NODE_ENV&&(e.proxy=!1),e}const R=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:n,key:a,host:s=N}=t;let i=`${s}/api/docs/${r}`;const c={book_id:String(o),merge_dynamic_data:String(!1)};e&&(c.mode="markdown");const u=new URLSearchParams(c).toString();return i=`${i}?${u}`,l.get(i,O({token:n,key:a})).then((({data:t,status:e})=>({apiUrl:i,httpStatus:e,response:t})))};function _(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const F=t=>{if(!t)return"";const e=p.inflate(t,{to:"string"}),r=JSON.parse(e);let o="";return r.forEach((t=>{const e=`## ${t.name}\n`,r=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),n=Math.max(...r.map((t=>Number(t))));if(o<0||n<0)return"";let a="";const s="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let i=Array(n+1).fill(" ").map(((t,e)=>{const r=e%s.length;return s[r]})).join(" | ");i=`| |${i}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,a=`${i}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<n+1;o++){const n=t?.[e]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(`![${n?.name}'](${n?.src})`):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}a=`${a}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return a}(t.data);o=`${o}\n${e}\n${r}`})),o};function A(t,e=[]){return function(t,e){try{const{host:r,pathname:o}=new URL(t);return!e.includes(r)&&Boolean(o)}catch(t){return!1}}(t,e)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(t)}&sign=${function(t){const e=f.createHash("sha256");return e.update(`${L}${t}`),e.digest("hex")}(t)}`:t}const j=k(w.finished);async function M(t){const{fileUrl:e,savePath:r,token:o,key:n,fileName:a}=t;return l.get(e,{...O({token:o,key:n}),responseType:"stream"}).then((async t=>{if(t.request?.path?.startsWith("/login"))throw new Error(`"${a}" need token`);if(200===t.status){const o=m(r);return t.data?.pipe(o),j(o).then((()=>({fileUrl:e,savePath:r})))}throw new Error(`response status ${t.status}`)}))}const B=/\[(.*?)\]\((.*?)\)/g,J=/\[(.*?)\]\((.*?\.yuque\.com\/attachments.*?)\)/;async function q(t){const{mdData:e,savePath:r,attachmentsDir:n,articleTitle:a,token:s,key:l}=t,d=(e.match(B)||[]).filter((t=>J.test(t)));if(0===d.length)return{mdData:e};const h=u({text:`下载 "${a}" 的附件中...`,stream:c});"test"!==i.NODE_ENV&&h.start();const p=o.resolve(r,n),f=d.map((t=>function(t,e){const[,r,n]=J.exec(t)||[];if(!n)return!1;const a=r||n.split("/").at(-1);if(!a)return!1;const s=o.join(e,a);return{fileName:a,url:n,rawMd:t,currentFilePath:s}}(t,p))).filter((t=>!1!==t));g(p,{recursive:!0});const m=f.map((t=>M({fileUrl:t.url,savePath:t.currentFilePath,token:s,key:l,fileName:t.fileName}))),w=await Promise.all(m).finally((function(){h&&h.stop()}));let k=e;return w.forEach((t=>{const e=f.find((e=>e.url===t.fileUrl));if(e){const t=`[附件: ${e.fileName}](${n}/${e.fileName})`;k=k.replace(e.rawMd,t)}})),{mdData:k}}function H(t){const e=[];return K(t,(function(t,r){"link"===t.type&&e.push({node:t,keyChain:r})})),e}function K(t,e,r=[]){e(t,r),Array.isArray(t.children)&&(r.push("children"),t.children.forEach(((t,o)=>{K(t,e,[...r,String(o)])})))}const V=/name="audio" value="data:(.*?audioId.*?)".*?><\/card>/gm;async function W(t){const{mdData:e,htmlData:r,savePath:n,attachmentsDir:a,articleTitle:s,token:l,key:d}=t,h=function(t){return y(t)}(e),p=H(h).filter((t=>/_lake_card.*?videoId/.test(t.node.url))),f=function(t){const e=t.match(V)||[];try{return e.map((t=>t.replace(V,"$1"))).map((t=>JSON.parse(decodeURIComponent(t))))}catch(t){return[]}}(r);if(0===p.length&&0===f.length)return{mdData:e};const m=u({text:`下载 "${s}" 的音视频中...`,stream:c});"test"!==i.NODE_ENV&&m.start();const w=o.resolve(n,a);g(w,{recursive:!0});let k=e;try{if(p.length>0){const e=await async function(t,e,r){const{key:n,token:a}=e,s=t.map((async t=>{const e=function(t){try{const e=new URL(t).searchParams.get("_lake_card")??"",r=decodeURIComponent(e),o=JSON.parse(r);return{name:o?.name||"",videoId:o?.videoId||""}}catch(t){return!1}}(t.node.url);if(!e)return!1;const s=await G({videoId:e.videoId,key:n,token:a});if(!s)return!1;const i=e.name??e.videoId.split("/").at(-1)??e.videoId;return{videoInfo:{...e,...s},astNode:t,fileName:i,currentFilePath:o.join(r,i)}})),i=await Promise.all(s);return i.filter(Q)}(p,t,w),r=e.map((t=>M({fileUrl:t.videoInfo.video,savePath:t.currentFilePath,token:l,key:d,fileName:t.videoInfo.name})));(await Promise.all(r)).forEach((t=>{const r=e.find((e=>e.videoInfo.video===t.fileUrl));r&&(r.astNode.node.url=`${a}${o.sep}${r.fileName}`,r.astNode.node.children=[{type:"text",value:`音视频附件: ${r.videoInfo.name}`}])})),k=function(t){return $(t)}(h)}if(f.length>0){const e=await async function(t,e,r){const{key:n,token:a}=e,s=t.map((async t=>{const e=await G({videoId:t.audioId,key:n,token:a});if(!e)return!1;const s=t?.fileName??t.id;return{audioInfo:{...t,...e},fileName:s,currentFilePath:o.join(r,s)}})),i=await Promise.all(s);return i.filter(Q)}(f,t,w),r=e.map((t=>M({fileUrl:t.audioInfo.audio,savePath:t.currentFilePath,token:l,key:d,fileName:t.audioInfo.fileName}))),n=await Promise.all(r);let s="\n\n> [yuque-dl warn]: 由于语雀markdown接口限制, 无法准确定位音频文件在文档中所在位置, 所以统一所有音频放到一起\n";n.forEach((t=>{const r=e.find((e=>e.audioInfo.audio===t.fileUrl));r&&(s+=`> - [音视频附件: ${r.audioInfo.fileName}](${a}${o.sep}${r.fileName})\n`)})),k+=s}}finally{m&&m.stop()}return{mdData:k}}function G(t){let e="https://www.yuque.com/api/video";const{videoId:r,token:o,key:n}=t,a=new URLSearchParams;return a.set("video_id",r),e=`${e}?${a.toString()}`,l.get(e,O({token:o,key:n})).then((({data:t,status:e})=>{const r=t.data;return 200===e&&"success"===r.status&&r.info})).catch((()=>!1))}function Q(t){return!!t}async function Y(t){const{articleInfo:r,progressBar:o,options:n}=t,{token:a,key:s}=n,{bookId:l,itemUrl:h,savePath:p,saveFilePath:f,uuid:m,articleUrl:g,articleTitle:w,ignoreImg:k,host:y,imageServiceDomains:$}=r,v={articleUrl:h,bookId:l,token:a,host:y,key:s},{httpStatus:E,apiUrl:N,response:L}=await R(v),P=L?.data?.type?.toLocaleLowerCase();let D="";if(P===I.SHEET){const{response:t}=await R(v,!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;D=o?F(o):""}catch(t){const e=b.get(P);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([I.BOARD,I.TABLE].includes(P)){const t=b.get(P);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof L?.data?.sourcecode)throw new Error(`download article Error: ${N}, http status ${E}`);D=L.data.sourcecode,D=function(t){const e=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=t.match(e);let o=t;const n=t;try{r?.forEach((t=>{e.lastIndex=0;const r=e.exec(t)?.[2]??"",{pathname:n,search:a}=new URL(r);if(!n.endsWith(".svg")&&a){const e=decodeURIComponent(a);o=o.replace(t,e.slice(1))}}))}catch(t){return n}return o}(D)}const S=function(t){if(!t)return[];const e=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(t.match(e)||[]);return r=r.map((t=>(t=t.replace(e,"$2"),/^http.*/g.test(t)?t:""))).filter((t=>Boolean(t))),r}(D),T=await R(v,!1),x=T.response?.data?.content??"";S.length&&!k&&(D=function(t,e,r){if(!r)return e;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let a,s=!0;for(;s||Boolean(a);)if(s=!1,a=o.exec(r),a?.[1])try{const t=decodeURIComponent(a[1]),e=JSON.parse(t);n.push(e?.src||"")}catch(t){n.push("")}const i=new Map;return t.forEach((t=>{const{origin:r,pathname:o}=new URL(t),a=`${r}${o}`,s=n.find(((t,e)=>{const r=new RegExp(`${a}.*?`).test(t);return r&&n.splice(e,1),r}));if(s){const r=new RegExp(t,"g"),o=i.get(t)||0;let n=0;e=e.replace(r,(t=>{let e=t;return n===o&&(e=s),n+=1,e})),i.set(t,o+1)}})),e}(S,D,x));const C={toc:n.toc,articleTitle:w,articleUrl:g,articleUpdateTime:U(L?.data?.content_updated_at??"")},O=[];try{o.pause(),console.log("");D=(await q({mdData:D,savePath:p,attachmentsDir:`./attachments/${m}`,articleTitle:w,token:a,key:s})).mdData}catch(t){O.push(`附件下载失败: ${t.message||"unknown error"}`)}finally{o.continue()}try{o.pause(),console.log("");D=(await W({mdData:D,htmlData:x,savePath:p,attachmentsDir:`./attachments/${m}`,articleTitle:w,token:a,key:s})).mdData}catch(t){O.push(`音视频下载失败: ${t.message||"unknown error"}`)}finally{o.continue()}if(S.length&&!k){let t;o.pause(),console.log(""),"test"!==i.NODE_ENV&&(t=u({text:`下载 "${w}" 的图片中...`,stream:c}),t.start());let r=[],n=D;try{const t=await d.run(D,{dist:p,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:g||"",transform:t=>A(t=t.replace("x-oss-process=image%2Fwatermark%2C",""),$)});r=t.errorInfo,n=t.data}catch(t){r=[t]}D=n;const a=()=>{t&&t.stop(),o.continue()};if(r.length>0){const t=r[0];let o="图片下载失败(失败的以远程链接保存): ";throw o=t.url?`${o}${t.error?.message} ${t.url.slice(0,20)}...`:`${o}${t.message}`,await e(f,z(D,C)),a(),new Error(`${o}`)}a()}try{if(await e(f,z(D,C)),O.length>0)throw new Error(O[0]);return!0}catch(t){throw new Error(`${t.message}`)}}function z(t,e){const{articleTitle:r,articleUrl:o,toc:n}=e;let a=t;a=a.replace(/\|\s?<br \/>- \[(x|\s)\]([\s\S\n]*?)\|/gm,((t,e,r)=>`| <br />- [${e}]${r}|`.replace(/\n/gm,"").replace(/<br(\s?)\/>/gm,""))),a=a.replace(/(\|?)(.*?)\|/gm,((t,e,r)=>`${e}${r.replace(/<br(\s?)\/>/gm,"")}|`)),a=a.replace(/<br(\s?)\/>/gm,"\n"),a=a.replace(/<a.*?>(\s*?)<\/a>/gm,"");const s=r?`# ${r}\n\n`:"";let i=n?h(a).content:"";i&&(i=`${i}\n\n---\n\n`);let c="\n\n";return e.articleUpdateTime&&(c+=`> 更新: ${e.articleUpdateTime}  \n`),o&&(c+=`> 原文: <${o}>`),a=`${s}${i}${a}${c}`,a}async function X(t,e){if(!function(t){if("function"==typeof URL.canParse)return URL.canParse(t);try{return new URL(t),!0}catch(t){return!1}}(t))throw new Error("Please enter a valid URL");const{bookId:n,tocList:a,bookName:s,bookDesc:i,bookSlug:c,host:u,imageServiceDomains:d}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return l.get(t,O(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||N,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((t=>{const e=t?.message??"";if(!e)throw new Error("unknown error");if(["getaddrinfo ENOTFOUND","read ECONNRESET","Client network socket disconnected before secure TLS connection was established"].some((t=>e.startsWith(t))))throw new Error("请检查网络(是否正常联网/是否开启了代理软件)");throw new Error(e)}))})(t,{token:e.token,key:e.key});if(!n)throw new Error("No found book id");if(!a||0===a.length)throw new Error("No found toc list");const h=o.resolve(e.distDir,s?_(s):String(n));await r(h,{recursive:!0});const p=a.length,f=new D(h,p);if(await f.init(),f.curr===p)return f.bar&&f.bar.stop(),void P.info(`√ 已完成: ${h}`);const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((t=>{m.set(t.toc.uuid,t)}));const g=t.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:n,uuidMap:a,tocList:s,bookPath:i,bookId:c,progressBar:l,host:u,options:d,imageServiceDomains:h=[]}=t;let p=0,f=0,m=0;const g=[],w=[];for(let t=0;t<n;t++){const e=s[t];if("string"!=typeof e.type)continue;if(a.get(e.uuid))continue;const o=e.type.toLocaleLowerCase();if(o===v.TITLE||""!==e.child_uuid||o===v.LINK){let t=e;const n=[],s=[];for(;t;)n.unshift(_(t.title)),s.unshift(t.uuid),t=a.get(t.parent_uuid)?a.get(t.parent_uuid).toc:void 0;const c={path:n.map(_).join("/"),pathTitleList:n,pathIdList:s,toc:e};o===v.LINK?(m+=1,w.push(c)):await r(`${i}/${n.map(_).join("/")}`,{recursive:!0}),a.set(e.uuid,c),o===I.DOC?await k(e):await l.updateProgress(c,o!==v.LINK)}else e.url&&await k(e)}async function k(t){f+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=t.type.toLocaleLowerCase();a.get(t.parent_uuid)&&(r=a.get(t.parent_uuid));const s=_(t.title),m=[...r.pathTitleList,s],w=[...r.pathIdList,t.uuid];let k=[...r.pathTitleList,`${s}.md`].map(_).join("/"),y=r.pathTitleList.map(_).join("/");n===I.DOC&&t.child_uuid&&(k=[...r.pathTitleList,s,"index.md"].map(_).join("/"),y=m.map(_).join("/"));const $={path:k,savePath:y,pathTitleList:m,pathIdList:w,toc:t};let v=!0;const b=`${e}/${t.url}`;try{const e={bookId:c,itemUrl:t.url,savePath:o.resolve(i,$.savePath),saveFilePath:o.resolve(i,$.path),uuid:t.uuid,articleUrl:b,articleTitle:t.title,ignoreImg:d.ignoreImg,host:u,imageServiceDomains:h};await Y({articleInfo:e,progressBar:l,options:d})}catch(t){v=!1,p+=1,g.push({articleUrl:b,errItem:$,errMsg:t.message,err:t})}a.set(t.uuid,$),await l.updateProgress($,v)}if(m>0){P.warn("该知识库存在以下外链文章");for(const t of w)P.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(p>0){P.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const t of g)P.error(`《${t.errItem.path}》: ${t.articleUrl}`),t.errMsg.split("\n").forEach((t=>{P.error(`———— ✕ ${t}`)}));P.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:g,total:p,uuidMap:m,tocList:a,bookPath:h,bookId:n,progressBar:f,host:u,options:e,imageServiceDomains:d});const w=new x({bookPath:h,bookName:s,bookDesc:i,uuidMap:m});await w.genFile(),P.info(`√ 生成目录 ${o.resolve(h,"index.md")}`),f.curr===p&&P.info(`√ 已完成: ${h}`)}export{P as l,X as m};

package/dist/es/index.js

@@ -1 +1 @@
-import"node:fs/promises";import"node:path";export{m as main}from"./index-DH0mvJUN.js";import"rand-user-agent";import"log4js";import"cli-progress";import"node:process";import"axios";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:crypto";import"node:fs";import"node:stream";import"node:util";import"mdast-util-from-markdown";import"mdast-util-to-markdown";
+import"node:fs/promises";import"node:path";export{m as main}from"./index-Bw1wFGlC.js";import"rand-user-agent";import"log4js";import"cli-progress";import"node:process";import"axios";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:crypto";import"node:fs";import"node:stream";import"node:util";import"mdast-util-from-markdown";import"mdast-util-to-markdown";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yuque-dl",
-  "version": "1.0.66",
+  "version": "1.0.67",
   "description": "yuque 知识库下载",
   "type": "module",
   "bin": {

package/types/types/index.d.ts

@@ -37,6 +37,7 @@ export interface IHandleMdDataOptions {
     articleUrl: string;
     articleTitle: string;
     toc: boolean;
+    articleUpdateTime: string;
 }
 export interface IErrArticleInfo {
     articleUrl: string;

package/types/utils/index.d.ts

@@ -1,5 +1,5 @@
 /** 随机生成UA */
-declare function randUserAgent({ browser, os, device }: {
+export declare function randUserAgent({ browser, os, device }: {
     browser?: string | undefined;
     os?: string | undefined;
     device?: string | undefined;
@@ -7,9 +7,9 @@ declare function randUserAgent({ browser, os, device }: {
 /**
  * 获取md中的img url
  */
-declare function getMarkdownImageList(mdStr: string): string[];
-declare function removeEmojis(dirName: string): string;
-declare function isValidUrl(url: string): boolean;
-export { randUserAgent, getMarkdownImageList, removeEmojis, isValidUrl };
+export declare function getMarkdownImageList(mdStr: string): string[];
+export declare function removeEmojis(dirName: string): string;
+export declare function isValidUrl(url: string): boolean;
+export declare function formateDate(d: string): string;
 export * from './log';
 export * from './ProgressBar';

package/dist/cjs/index-KK9Csh23.js

@@ -1 +0,0 @@
-"use strict";var e=require("node:fs/promises"),t=require("node:path"),r=require("rand-user-agent"),o=require("log4js"),n=require("cli-progress"),a=require("node:process"),s=require("axios"),i=require("ora"),c=require("pull-md-img"),l=require("markdown-toc"),u=require("pako"),d=require("node:crypto"),h=require("node:fs"),p=require("node:stream"),f=require("node:util"),m=require("mdast-util-from-markdown"),g=require("mdast-util-to-markdown");function w(e){var t=Object.create(null);return e&&Object.keys(e).forEach((function(r){if("default"!==r){var o=Object.getOwnPropertyDescriptor(e,r);Object.defineProperty(t,r,o.get?o:{enumerable:!0,get:function(){return e[r]}})}})),t.default=e,Object.freeze(t)}var k,y,$=w(p);!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(k||(k={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(y||(y={}));const v=new Map([[y.BOARD,"画板类型"],[y.TABLE,"数据表类型"],[y.SHEET,"表格类型"],[y.DOC,"文档类型"]]),I="_yuque_session",b="https://www.yuque.com",E="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",P=(o.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),o.getLogger("yuque-dl"));class L{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,P.info("根据上次数据继续断点下载")),this.bar=new n.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},n.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr!==this.total&&(this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function N({browser:e="chrome",os:t="mac os",device:o="desktop"}){o=o.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let n=r(o,e,t);if("chrome"===e)for(;n.includes("Chrome-Lighthouse")||n.includes("Gener8")||n.includes("HeadlessChrome")||n.includes("SMTBot");)n=r(o,e,t);if("safari"===e)for(;n.includes("Applebot");)n=r(o,e,t);return n}class D{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:n}=this.summaryInfo;let a=`# ${t}\n\n`;r&&(a+=`> ${r}\n\n`);let s=a;const i=[];n.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(i,r),n={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:"link"},a=t.type.toLocaleLowerCase();a===k.TITLE||""!==t.child_uuid?(n.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)),a===k.DOC&&(n.link=e.path)):(n.type="link",n.link=a===k.LINK?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)))}));s+=this.genSummaryContent(i,"");try{await e.writeFile(`${o}/index.md`,s)}catch(e){P.error("Generate Summary Error")}return s}genSummaryContent(e,t){for(const r of e){if(r.type===k.TITLE)if(r.link){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${e})\n\n`}else t+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===k.LINK){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`- [${r.text}](${e})\n`}Array.isArray(r.children)&&(t+=this.genSummaryContent(r.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}function S(e){const{key:t=I,token:r}=e,o={"user-agent":N({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function T(e){const t={headers:S(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...S(e)}}};return"test"===a.env.NODE_ENV&&(t.proxy=!1),t}const U=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:n,key:a,host:i=b}=e;let c=`${i}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,s.get(c,T({token:n,key:a})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))};function O(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const x=e=>{if(!e)return"";const t=u.inflate(e,{to:"string"}),r=JSON.parse(t);let o="";return r.forEach((e=>{const t=`## ${e.name}\n`,r=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),n=Math.max(...r.map((e=>Number(e))));if(o<0||n<0)return"";let a="";const s="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let i=Array(n+1).fill(" ").map(((e,t)=>{const r=t%s.length;return s[r]})).join(" | ");i=`| |${i}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,a=`${i}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<n+1;o++){const n=e?.[t]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(`![${n?.name}'](${n?.src})`):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}a=`${a}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return a}(e.data);o=`${o}\n${t}\n${r}`})),o};function C(e,t=[]){return function(e,t){try{const{host:r,pathname:o}=new URL(e);return!t.includes(r)&&Boolean(o)}catch(e){return!1}}(e,t)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(e)}&sign=${function(e){const t=d.createHash("sha256");return t.update(`${E}${e}`),t.digest("hex")}(e)}`:e}const R=f.promisify($.finished);async function q(e){const{fileUrl:t,savePath:r,token:o,key:n,fileName:a}=e;return s.get(t,{...T({token:o,key:n}),responseType:"stream"}).then((async e=>{if(e.request?.path?.startsWith("/login"))throw new Error(`"${a}" need token`);if(200===e.status){const o=h.createWriteStream(r);return e.data?.pipe(o),R(o).then((()=>({fileUrl:t,savePath:r})))}throw new Error(`response status ${e.status}`)}))}const F=/\[(.*?)\]\((.*?)\)/g,j=/\[(.*?)\]\((.*?\.yuque\.com\/attachments.*?)\)/;async function _(e){const{mdData:r,savePath:o,attachmentsDir:n,articleTitle:s,token:c,key:l}=e,u=(r.match(F)||[]).filter((e=>j.test(e)));if(0===u.length)return{mdData:r};const d=i({text:`下载 "${s}" 的附件中...`,stream:a.stdout});"test"!==a.env.NODE_ENV&&d.start();const p=t.resolve(o,n),f=u.map((e=>function(e,r){const[,o,n]=j.exec(e)||[];if(!n)return!1;const a=o||n.split("/").at(-1);if(!a)return!1;const s=t.join(r,a);return{fileName:a,url:n,rawMd:e,currentFilePath:s}}(e,p))).filter((e=>!1!==e));h.mkdirSync(p,{recursive:!0});const m=f.map((e=>q({fileUrl:e.url,savePath:e.currentFilePath,token:c,key:l,fileName:e.fileName}))),g=await Promise.all(m).finally((function(){d&&d.stop()}));let w=r;return g.forEach((e=>{const t=f.find((t=>t.url===e.fileUrl));if(t){const e=`[附件: ${t.fileName}](${n}/${t.fileName})`;w=w.replace(t.rawMd,e)}})),{mdData:w}}function A(e){const t=[];return M(e,(function(e,r){"link"===e.type&&t.push({node:e,keyChain:r})})),t}function M(e,t,r=[]){t(e,r),Array.isArray(e.children)&&(r.push("children"),e.children.forEach(((e,o)=>{M(e,t,[...r,String(o)])})))}const B=/name="audio" value="data:(.*?audioId.*?)".*?><\/card>/gm;async function J(e){const{mdData:r,htmlData:o,savePath:n,attachmentsDir:s,articleTitle:c,token:l,key:u}=e,d=function(e){return m.fromMarkdown(e)}(r),p=A(d).filter((e=>/_lake_card.*?videoId/.test(e.node.url))),f=function(e){const t=e.match(B)||[];try{return t.map((e=>e.replace(B,"$1"))).map((e=>JSON.parse(decodeURIComponent(e))))}catch(e){return[]}}(o);if(0===p.length&&0===f.length)return{mdData:r};const w=i({text:`下载 "${c}" 的音视频中...`,stream:a.stdout});"test"!==a.env.NODE_ENV&&w.start();const k=t.resolve(n,s);h.mkdirSync(k,{recursive:!0});let y=r;try{if(p.length>0){const r=await async function(e,r,o){const{key:n,token:a}=r,s=e.map((async e=>{const r=function(e){try{const t=new URL(e).searchParams.get("_lake_card")??"",r=decodeURIComponent(t),o=JSON.parse(r);return{name:o?.name||"",videoId:o?.videoId||""}}catch(e){return!1}}(e.node.url);if(!r)return!1;const s=await K({videoId:r.videoId,key:n,token:a});if(!s)return!1;const i=r.name??r.videoId.split("/").at(-1)??r.videoId;return{videoInfo:{...r,...s},astNode:e,fileName:i,currentFilePath:t.join(o,i)}})),i=await Promise.all(s);return i.filter(H)}(p,e,k),o=r.map((e=>q({fileUrl:e.videoInfo.video,savePath:e.currentFilePath,token:l,key:u,fileName:e.videoInfo.name})));(await Promise.all(o)).forEach((e=>{const o=r.find((t=>t.videoInfo.video===e.fileUrl));o&&(o.astNode.node.url=`${s}${t.sep}${o.fileName}`,o.astNode.node.children=[{type:"text",value:`音视频附件: ${o.videoInfo.name}`}])})),y=function(e){return g.toMarkdown(e)}(d)}if(f.length>0){const r=await async function(e,r,o){const{key:n,token:a}=r,s=e.map((async e=>{const r=await K({videoId:e.audioId,key:n,token:a});if(!r)return!1;const s=e?.fileName??e.id;return{audioInfo:{...e,...r},fileName:s,currentFilePath:t.join(o,s)}})),i=await Promise.all(s);return i.filter(H)}(f,e,k),o=r.map((e=>q({fileUrl:e.audioInfo.audio,savePath:e.currentFilePath,token:l,key:u,fileName:e.audioInfo.fileName}))),n=await Promise.all(o);let a="\n\n> [yuque-dl warn]: 由于语雀markdown接口限制, 无法准确定位音频文件在文档中所在位置, 所以统一所有音频放到一起\n";n.forEach((e=>{const o=r.find((t=>t.audioInfo.audio===e.fileUrl));o&&(a+=`> - [音视频附件: ${o.audioInfo.fileName}](${s}${t.sep}${o.fileName})\n`)})),y+=a}}finally{w&&w.stop()}return{mdData:y}}function K(e){let t="https://www.yuque.com/api/video";const{videoId:r,token:o,key:n}=e,a=new URLSearchParams;return a.set("video_id",r),t=`${t}?${a.toString()}`,s.get(t,T({token:o,key:n})).then((({data:e,status:t})=>{const r=e.data;return 200===t&&"success"===r.status&&r.info})).catch((()=>!1))}function H(e){return!!e}async function W(t){const{articleInfo:r,progressBar:o,options:n}=t,{token:s,key:l}=n,{bookId:u,itemUrl:d,savePath:h,saveFilePath:p,uuid:f,articleUrl:m,articleTitle:g,ignoreImg:w,host:k,imageServiceDomains:$}=r,I={articleUrl:d,bookId:u,token:s,host:k,key:l},{httpStatus:b,apiUrl:E,response:P}=await U(I),L=P?.data?.type?.toLocaleLowerCase();let N="";if(L===y.SHEET){const{response:e}=await U(I,!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;N=o?x(o):""}catch(e){const t=v.get(L);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([y.BOARD,y.TABLE].includes(L)){const e=v.get(L);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof P?.data?.sourcecode)throw new Error(`download article Error: ${E}, http status ${b}`);N=P.data.sourcecode,N=function(e){const t=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=e.match(t);let o=e;const n=e;try{r?.forEach((e=>{t.lastIndex=0;const r=t.exec(e)?.[2]??"",{pathname:n,search:a}=new URL(r);if(!n.endsWith(".svg")&&a){const t=decodeURIComponent(a);o=o.replace(e,t.slice(1))}}))}catch(e){return n}return o}(N)}const D=function(e){if(!e)return[];const t=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(e.match(t)||[]);return r=r.map((e=>(e=e.replace(t,"$2"),/^http.*/g.test(e)?e:""))).filter((e=>Boolean(e))),r}(N),S=await U(I,!1),T=S.response?.data?.content??"";D.length&&!w&&(N=function(e,t,r){if(!r)return t;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let a,s=!0;for(;s||Boolean(a);)if(s=!1,a=o.exec(r),a?.[1])try{const e=decodeURIComponent(a[1]),t=JSON.parse(e);n.push(t?.src||"")}catch(e){n.push("")}const i=new Map;return e.forEach((e=>{const{origin:r,pathname:o}=new URL(e),a=`${r}${o}`,s=n.find(((e,t)=>{const r=new RegExp(`${a}.*?`).test(e);return r&&n.splice(t,1),r}));if(s){const r=new RegExp(e,"g"),o=i.get(e)||0;let n=0;t=t.replace(r,(e=>{let t=e;return n===o&&(t=s),n+=1,t})),i.set(e,o+1)}})),t}(D,N,T));const O={toc:n.toc,articleTitle:g,articleUrl:m},R=[];try{o.pause(),console.log("");N=(await _({mdData:N,savePath:h,attachmentsDir:`./attachments/${f}`,articleTitle:g,token:s,key:l})).mdData}catch(e){R.push(`附件下载失败: ${e.message||"unknown error"}`)}finally{o.continue()}try{o.pause(),console.log("");N=(await J({mdData:N,htmlData:T,savePath:h,attachmentsDir:`./attachments/${f}`,articleTitle:g,token:s,key:l})).mdData}catch(e){R.push(`音视频下载失败: ${e.message||"unknown error"}`)}finally{o.continue()}if(D.length&&!w){let t;o.pause(),console.log(""),"test"!==a.env.NODE_ENV&&(t=i({text:`下载 "${g}" 的图片中...`,stream:a.stdout}),t.start());let r=[],n=N;try{const e=await c.run(N,{dist:h,imgDir:`./img/${f}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:m||"",transform:e=>C(e=e.replace("x-oss-process=image%2Fwatermark%2C",""),$)});r=e.errorInfo,n=e.data}catch(e){r=[e]}N=n;const s=()=>{t&&t.stop(),o.continue()};if(r.length>0){const t=r[0];let o="图片下载失败(失败的以远程链接保存): ";throw o=t.url?`${o}${t.error?.message} ${t.url.slice(0,20)}...`:`${o}${t.message}`,await e.writeFile(p,V(N,O)),s(),new Error(`${o}`)}s()}try{if(await e.writeFile(p,V(N,O)),R.length>0)throw new Error(R[0]);return!0}catch(e){throw new Error(`${e.message}`)}}function V(e,t){const{articleTitle:r,articleUrl:o,toc:n}=t;let a=e;a=a.replace(/\|\s?<br \/>- \[(x|\s)\]([\s\S\n]*?)\|/gm,((e,t,r)=>`| <br />- [${t}]${r}|`.replace(/\n/gm,"").replace(/<br(\s?)\/>/gm,""))),a=a.replace(/(\|?)(.*?)\|/gm,((e,t,r)=>`${t}${r.replace(/<br(\s?)\/>/gm,"")}|`)),a=a.replace(/<br(\s?)\/>/gm,"\n"),a=a.replace(/<a.*?>(\s*?)<\/a>/gm,"");const s=r?`# ${r}\n\n`:"";let i=n?l(a).content:"";i&&(i=`${i}\n\n---\n\n`);return a=`${s}${i}${a}${o?`\n\n> 原文: <${o}>`:""}`,a}exports.logger=P,exports.main=async function(r,o){if(!function(e){if("function"==typeof URL.canParse)return URL.canParse(e);try{return new URL(e),!0}catch(e){return!1}}(r))throw new Error("Please enter a valid URL");const{bookId:n,tocList:a,bookName:i,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:d}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return s.get(e,T(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||b,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((e=>{const t=e?.message??"";if(!t)throw new Error("unknown error");if(["getaddrinfo ENOTFOUND","read ECONNRESET","Client network socket disconnected before secure TLS connection was established"].some((e=>t.startsWith(e))))throw new Error("请检查网络(是否正常联网/是否开启了代理软件)");throw new Error(t)}))})(r,{token:o.token,key:o.key});if(!n)throw new Error("No found book id");if(!a||0===a.length)throw new Error("No found toc list");const h=t.resolve(o.distDir,i?O(i):String(n));await e.mkdir(h,{recursive:!0});const p=a.length,f=new L(h,p);if(await f.init(),f.curr===p)return f.bar&&f.bar.stop(),void P.info(`√ 已完成: ${h}`);const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const g=r.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(r){const{articleUrlPrefix:o,total:n,uuidMap:a,tocList:s,bookPath:i,bookId:c,progressBar:l,host:u,options:d,imageServiceDomains:h=[]}=r;let p=0,f=0,m=0;const g=[],w=[];for(let t=0;t<n;t++){const r=s[t];if("string"!=typeof r.type)continue;if(a.get(r.uuid))continue;const o=r.type.toLocaleLowerCase();if(o===k.TITLE||""!==r.child_uuid||o===k.LINK){let t=r;const n=[],s=[];for(;t;)n.unshift(O(t.title)),s.unshift(t.uuid),t=a.get(t.parent_uuid)?a.get(t.parent_uuid).toc:void 0;const c={path:n.map(O).join("/"),pathTitleList:n,pathIdList:s,toc:r};o===k.LINK?(m+=1,w.push(c)):await e.mkdir(`${i}/${n.map(O).join("/")}`,{recursive:!0}),a.set(r.uuid,c),o===y.DOC?await $(r):await l.updateProgress(c,o!==k.LINK)}else r.url&&await $(r)}async function $(e){f+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=e.type.toLocaleLowerCase();a.get(e.parent_uuid)&&(r=a.get(e.parent_uuid));const s=O(e.title),m=[...r.pathTitleList,s],w=[...r.pathIdList,e.uuid];let k=[...r.pathTitleList,`${s}.md`].map(O).join("/"),$=r.pathTitleList.map(O).join("/");n===y.DOC&&e.child_uuid&&(k=[...r.pathTitleList,s,"index.md"].map(O).join("/"),$=m.map(O).join("/"));const v={path:k,savePath:$,pathTitleList:m,pathIdList:w,toc:e};let I=!0;const b=`${o}/${e.url}`;try{const r={bookId:c,itemUrl:e.url,savePath:t.resolve(i,v.savePath),saveFilePath:t.resolve(i,v.path),uuid:e.uuid,articleUrl:b,articleTitle:e.title,ignoreImg:d.ignoreImg,host:u,imageServiceDomains:h};await W({articleInfo:r,progressBar:l,options:d})}catch(e){I=!1,p+=1,g.push({articleUrl:b,errItem:v,errMsg:e.message,err:e})}a.set(e.uuid,v),await l.updateProgress(v,I)}if(m>0){P.warn("该知识库存在以下外链文章");for(const e of w)P.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(p>0){P.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const e of g)P.error(`《${e.errItem.path}》: ${e.articleUrl}`),e.errMsg.split("\n").forEach((e=>{P.error(`———— ✕ ${e}`)}));P.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:g,total:p,uuidMap:m,tocList:a,bookPath:h,bookId:n,progressBar:f,host:u,options:o,imageServiceDomains:d});const w=new D({bookPath:h,bookName:i,bookDesc:c,uuidMap:m});await w.genFile(),P.info(`√ 生成目录 ${t.resolve(h,"index.md")}`),f.curr===p&&P.info(`√ 已完成: ${h}`)};

package/dist/es/index-DH0mvJUN.js

@@ -1 +0,0 @@
-import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"node:path";import n from"rand-user-agent";import a from"log4js";import s from"cli-progress";import{env as i,stdout as c}from"node:process";import l from"axios";import u from"ora";import h from"pull-md-img";import d from"markdown-toc";import p from"pako";import f from"node:crypto";import{createWriteStream as m,mkdirSync as g}from"node:fs";import*as w from"node:stream";import{promisify as k}from"node:util";import{fromMarkdown as y}from"mdast-util-from-markdown";import{toMarkdown as $}from"mdast-util-to-markdown";var v,I;!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(v||(v={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(I||(I={}));const b=new Map([[I.BOARD,"画板类型"],[I.TABLE,"数据表类型"],[I.SHEET,"表格类型"],[I.DOC,"文档类型"]]),E="_yuque_session",L="https://www.yuque.com",N="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",P=(a.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),a.getLogger("yuque-dl"));class D{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,P.info("根据上次数据继续断点下载")),this.bar=new s.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},s.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr!==this.total&&(this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function S({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=n(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=n(r,t,e);return o}class T{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:n}=this.summaryInfo;let a=`# ${e}\n\n`;r&&(a+=`> ${r}\n\n`);let s=a;const i=[];n.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(i,r),n={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:"link"},a=e.type.toLocaleLowerCase();a===v.TITLE||""!==e.child_uuid?(n.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)),a===v.DOC&&(n.link=t.path)):(n.type="link",n.link=a===v.LINK?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)))}));s+=this.genSummaryContent(i,"");try{await t.writeFile(`${o}/index.md`,s)}catch(t){P.error("Generate Summary Error")}return s}genSummaryContent(t,e){for(const r of t){if(r.type===v.TITLE)if(r.link){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${t})\n\n`}else e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===v.LINK){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}function U(t){const{key:e=E,token:r}=t,o={"user-agent":S({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function x(t){const e={headers:U(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...U(t)}}};return"test"===i.NODE_ENV&&(e.proxy=!1),e}const C=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:n,key:a,host:s=L}=t;let i=`${s}/api/docs/${r}`;const c={book_id:String(o),merge_dynamic_data:String(!1)};e&&(c.mode="markdown");const u=new URLSearchParams(c).toString();return i=`${i}?${u}`,l.get(i,x({token:n,key:a})).then((({data:t,status:e})=>({apiUrl:i,httpStatus:e,response:t})))};function O(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const R=t=>{if(!t)return"";const e=p.inflate(t,{to:"string"}),r=JSON.parse(e);let o="";return r.forEach((t=>{const e=`## ${t.name}\n`,r=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),n=Math.max(...r.map((t=>Number(t))));if(o<0||n<0)return"";let a="";const s="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let i=Array(n+1).fill(" ").map(((t,e)=>{const r=e%s.length;return s[r]})).join(" | ");i=`| |${i}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,a=`${i}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<n+1;o++){const n=t?.[e]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(`![${n?.name}'](${n?.src})`):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}a=`${a}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return a}(t.data);o=`${o}\n${e}\n${r}`})),o};function _(t,e=[]){return function(t,e){try{const{host:r,pathname:o}=new URL(t);return!e.includes(r)&&Boolean(o)}catch(t){return!1}}(t,e)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(t)}&sign=${function(t){const e=f.createHash("sha256");return e.update(`${N}${t}`),e.digest("hex")}(t)}`:t}const F=k(w.finished);async function A(t){const{fileUrl:e,savePath:r,token:o,key:n,fileName:a}=t;return l.get(e,{...x({token:o,key:n}),responseType:"stream"}).then((async t=>{if(t.request?.path?.startsWith("/login"))throw new Error(`"${a}" need token`);if(200===t.status){const o=m(r);return t.data?.pipe(o),F(o).then((()=>({fileUrl:e,savePath:r})))}throw new Error(`response status ${t.status}`)}))}const j=/\[(.*?)\]\((.*?)\)/g,B=/\[(.*?)\]\((.*?\.yuque\.com\/attachments.*?)\)/;async function M(t){const{mdData:e,savePath:r,attachmentsDir:n,articleTitle:a,token:s,key:l}=t,h=(e.match(j)||[]).filter((t=>B.test(t)));if(0===h.length)return{mdData:e};const d=u({text:`下载 "${a}" 的附件中...`,stream:c});"test"!==i.NODE_ENV&&d.start();const p=o.resolve(r,n),f=h.map((t=>function(t,e){const[,r,n]=B.exec(t)||[];if(!n)return!1;const a=r||n.split("/").at(-1);if(!a)return!1;const s=o.join(e,a);return{fileName:a,url:n,rawMd:t,currentFilePath:s}}(t,p))).filter((t=>!1!==t));g(p,{recursive:!0});const m=f.map((t=>A({fileUrl:t.url,savePath:t.currentFilePath,token:s,key:l,fileName:t.fileName}))),w=await Promise.all(m).finally((function(){d&&d.stop()}));let k=e;return w.forEach((t=>{const e=f.find((e=>e.url===t.fileUrl));if(e){const t=`[附件: ${e.fileName}](${n}/${e.fileName})`;k=k.replace(e.rawMd,t)}})),{mdData:k}}function J(t){const e=[];return q(t,(function(t,r){"link"===t.type&&e.push({node:t,keyChain:r})})),e}function q(t,e,r=[]){e(t,r),Array.isArray(t.children)&&(r.push("children"),t.children.forEach(((t,o)=>{q(t,e,[...r,String(o)])})))}const K=/name="audio" value="data:(.*?audioId.*?)".*?><\/card>/gm;async function H(t){const{mdData:e,htmlData:r,savePath:n,attachmentsDir:a,articleTitle:s,token:l,key:h}=t,d=function(t){return y(t)}(e),p=J(d).filter((t=>/_lake_card.*?videoId/.test(t.node.url))),f=function(t){const e=t.match(K)||[];try{return e.map((t=>t.replace(K,"$1"))).map((t=>JSON.parse(decodeURIComponent(t))))}catch(t){return[]}}(r);if(0===p.length&&0===f.length)return{mdData:e};const m=u({text:`下载 "${s}" 的音视频中...`,stream:c});"test"!==i.NODE_ENV&&m.start();const w=o.resolve(n,a);g(w,{recursive:!0});let k=e;try{if(p.length>0){const e=await async function(t,e,r){const{key:n,token:a}=e,s=t.map((async t=>{const e=function(t){try{const e=new URL(t).searchParams.get("_lake_card")??"",r=decodeURIComponent(e),o=JSON.parse(r);return{name:o?.name||"",videoId:o?.videoId||""}}catch(t){return!1}}(t.node.url);if(!e)return!1;const s=await V({videoId:e.videoId,key:n,token:a});if(!s)return!1;const i=e.name??e.videoId.split("/").at(-1)??e.videoId;return{videoInfo:{...e,...s},astNode:t,fileName:i,currentFilePath:o.join(r,i)}})),i=await Promise.all(s);return i.filter(W)}(p,t,w),r=e.map((t=>A({fileUrl:t.videoInfo.video,savePath:t.currentFilePath,token:l,key:h,fileName:t.videoInfo.name})));(await Promise.all(r)).forEach((t=>{const r=e.find((e=>e.videoInfo.video===t.fileUrl));r&&(r.astNode.node.url=`${a}${o.sep}${r.fileName}`,r.astNode.node.children=[{type:"text",value:`音视频附件: ${r.videoInfo.name}`}])})),k=function(t){return $(t)}(d)}if(f.length>0){const e=await async function(t,e,r){const{key:n,token:a}=e,s=t.map((async t=>{const e=await V({videoId:t.audioId,key:n,token:a});if(!e)return!1;const s=t?.fileName??t.id;return{audioInfo:{...t,...e},fileName:s,currentFilePath:o.join(r,s)}})),i=await Promise.all(s);return i.filter(W)}(f,t,w),r=e.map((t=>A({fileUrl:t.audioInfo.audio,savePath:t.currentFilePath,token:l,key:h,fileName:t.audioInfo.fileName}))),n=await Promise.all(r);let s="\n\n> [yuque-dl warn]: 由于语雀markdown接口限制, 无法准确定位音频文件在文档中所在位置, 所以统一所有音频放到一起\n";n.forEach((t=>{const r=e.find((e=>e.audioInfo.audio===t.fileUrl));r&&(s+=`> - [音视频附件: ${r.audioInfo.fileName}](${a}${o.sep}${r.fileName})\n`)})),k+=s}}finally{m&&m.stop()}return{mdData:k}}function V(t){let e="https://www.yuque.com/api/video";const{videoId:r,token:o,key:n}=t,a=new URLSearchParams;return a.set("video_id",r),e=`${e}?${a.toString()}`,l.get(e,x({token:o,key:n})).then((({data:t,status:e})=>{const r=t.data;return 200===e&&"success"===r.status&&r.info})).catch((()=>!1))}function W(t){return!!t}async function G(t){const{articleInfo:r,progressBar:o,options:n}=t,{token:a,key:s}=n,{bookId:l,itemUrl:d,savePath:p,saveFilePath:f,uuid:m,articleUrl:g,articleTitle:w,ignoreImg:k,host:y,imageServiceDomains:$}=r,v={articleUrl:d,bookId:l,token:a,host:y,key:s},{httpStatus:E,apiUrl:L,response:N}=await C(v),P=N?.data?.type?.toLocaleLowerCase();let D="";if(P===I.SHEET){const{response:t}=await C(v,!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;D=o?R(o):""}catch(t){const e=b.get(P);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([I.BOARD,I.TABLE].includes(P)){const t=b.get(P);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof N?.data?.sourcecode)throw new Error(`download article Error: ${L}, http status ${E}`);D=N.data.sourcecode,D=function(t){const e=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=t.match(e);let o=t;const n=t;try{r?.forEach((t=>{e.lastIndex=0;const r=e.exec(t)?.[2]??"",{pathname:n,search:a}=new URL(r);if(!n.endsWith(".svg")&&a){const e=decodeURIComponent(a);o=o.replace(t,e.slice(1))}}))}catch(t){return n}return o}(D)}const S=function(t){if(!t)return[];const e=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(t.match(e)||[]);return r=r.map((t=>(t=t.replace(e,"$2"),/^http.*/g.test(t)?t:""))).filter((t=>Boolean(t))),r}(D),T=await C(v,!1),U=T.response?.data?.content??"";S.length&&!k&&(D=function(t,e,r){if(!r)return e;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let a,s=!0;for(;s||Boolean(a);)if(s=!1,a=o.exec(r),a?.[1])try{const t=decodeURIComponent(a[1]),e=JSON.parse(t);n.push(e?.src||"")}catch(t){n.push("")}const i=new Map;return t.forEach((t=>{const{origin:r,pathname:o}=new URL(t),a=`${r}${o}`,s=n.find(((t,e)=>{const r=new RegExp(`${a}.*?`).test(t);return r&&n.splice(e,1),r}));if(s){const r=new RegExp(t,"g"),o=i.get(t)||0;let n=0;e=e.replace(r,(t=>{let e=t;return n===o&&(e=s),n+=1,e})),i.set(t,o+1)}})),e}(S,D,U));const x={toc:n.toc,articleTitle:w,articleUrl:g},O=[];try{o.pause(),console.log("");D=(await M({mdData:D,savePath:p,attachmentsDir:`./attachments/${m}`,articleTitle:w,token:a,key:s})).mdData}catch(t){O.push(`附件下载失败: ${t.message||"unknown error"}`)}finally{o.continue()}try{o.pause(),console.log("");D=(await H({mdData:D,htmlData:U,savePath:p,attachmentsDir:`./attachments/${m}`,articleTitle:w,token:a,key:s})).mdData}catch(t){O.push(`音视频下载失败: ${t.message||"unknown error"}`)}finally{o.continue()}if(S.length&&!k){let t;o.pause(),console.log(""),"test"!==i.NODE_ENV&&(t=u({text:`下载 "${w}" 的图片中...`,stream:c}),t.start());let r=[],n=D;try{const t=await h.run(D,{dist:p,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:g||"",transform:t=>_(t=t.replace("x-oss-process=image%2Fwatermark%2C",""),$)});r=t.errorInfo,n=t.data}catch(t){r=[t]}D=n;const a=()=>{t&&t.stop(),o.continue()};if(r.length>0){const t=r[0];let o="图片下载失败(失败的以远程链接保存): ";throw o=t.url?`${o}${t.error?.message} ${t.url.slice(0,20)}...`:`${o}${t.message}`,await e(f,Q(D,x)),a(),new Error(`${o}`)}a()}try{if(await e(f,Q(D,x)),O.length>0)throw new Error(O[0]);return!0}catch(t){throw new Error(`${t.message}`)}}function Q(t,e){const{articleTitle:r,articleUrl:o,toc:n}=e;let a=t;a=a.replace(/\|\s?<br \/>- \[(x|\s)\]([\s\S\n]*?)\|/gm,((t,e,r)=>`| <br />- [${e}]${r}|`.replace(/\n/gm,"").replace(/<br(\s?)\/>/gm,""))),a=a.replace(/(\|?)(.*?)\|/gm,((t,e,r)=>`${e}${r.replace(/<br(\s?)\/>/gm,"")}|`)),a=a.replace(/<br(\s?)\/>/gm,"\n"),a=a.replace(/<a.*?>(\s*?)<\/a>/gm,"");const s=r?`# ${r}\n\n`:"";let i=n?d(a).content:"";i&&(i=`${i}\n\n---\n\n`);return a=`${s}${i}${a}${o?`\n\n> 原文: <${o}>`:""}`,a}async function z(t,e){if(!function(t){if("function"==typeof URL.canParse)return URL.canParse(t);try{return new URL(t),!0}catch(t){return!1}}(t))throw new Error("Please enter a valid URL");const{bookId:n,tocList:a,bookName:s,bookDesc:i,bookSlug:c,host:u,imageServiceDomains:h}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return l.get(t,x(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||L,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((t=>{const e=t?.message??"";if(!e)throw new Error("unknown error");if(["getaddrinfo ENOTFOUND","read ECONNRESET","Client network socket disconnected before secure TLS connection was established"].some((t=>e.startsWith(t))))throw new Error("请检查网络(是否正常联网/是否开启了代理软件)");throw new Error(e)}))})(t,{token:e.token,key:e.key});if(!n)throw new Error("No found book id");if(!a||0===a.length)throw new Error("No found toc list");const d=o.resolve(e.distDir,s?O(s):String(n));await r(d,{recursive:!0});const p=a.length,f=new D(d,p);if(await f.init(),f.curr===p)return f.bar&&f.bar.stop(),void P.info(`√ 已完成: ${d}`);const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((t=>{m.set(t.toc.uuid,t)}));const g=t.replace(new RegExp(`(.*?/${c}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:n,uuidMap:a,tocList:s,bookPath:i,bookId:c,progressBar:l,host:u,options:h,imageServiceDomains:d=[]}=t;let p=0,f=0,m=0;const g=[],w=[];for(let t=0;t<n;t++){const e=s[t];if("string"!=typeof e.type)continue;if(a.get(e.uuid))continue;const o=e.type.toLocaleLowerCase();if(o===v.TITLE||""!==e.child_uuid||o===v.LINK){let t=e;const n=[],s=[];for(;t;)n.unshift(O(t.title)),s.unshift(t.uuid),t=a.get(t.parent_uuid)?a.get(t.parent_uuid).toc:void 0;const c={path:n.map(O).join("/"),pathTitleList:n,pathIdList:s,toc:e};o===v.LINK?(m+=1,w.push(c)):await r(`${i}/${n.map(O).join("/")}`,{recursive:!0}),a.set(e.uuid,c),o===I.DOC?await k(e):await l.updateProgress(c,o!==v.LINK)}else e.url&&await k(e)}async function k(t){f+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=t.type.toLocaleLowerCase();a.get(t.parent_uuid)&&(r=a.get(t.parent_uuid));const s=O(t.title),m=[...r.pathTitleList,s],w=[...r.pathIdList,t.uuid];let k=[...r.pathTitleList,`${s}.md`].map(O).join("/"),y=r.pathTitleList.map(O).join("/");n===I.DOC&&t.child_uuid&&(k=[...r.pathTitleList,s,"index.md"].map(O).join("/"),y=m.map(O).join("/"));const $={path:k,savePath:y,pathTitleList:m,pathIdList:w,toc:t};let v=!0;const b=`${e}/${t.url}`;try{const e={bookId:c,itemUrl:t.url,savePath:o.resolve(i,$.savePath),saveFilePath:o.resolve(i,$.path),uuid:t.uuid,articleUrl:b,articleTitle:t.title,ignoreImg:h.ignoreImg,host:u,imageServiceDomains:d};await G({articleInfo:e,progressBar:l,options:h})}catch(t){v=!1,p+=1,g.push({articleUrl:b,errItem:$,errMsg:t.message,err:t})}a.set(t.uuid,$),await l.updateProgress($,v)}if(m>0){P.warn("该知识库存在以下外链文章");for(const t of w)P.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(p>0){P.error(`本次执行总数${f}篇，✕ 失败${p}篇`);for(const t of g)P.error(`《${t.errItem.path}》: ${t.articleUrl}`),t.errMsg.split("\n").forEach((t=>{P.error(`———— ✕ ${t}`)}));P.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:g,total:p,uuidMap:m,tocList:a,bookPath:d,bookId:n,progressBar:f,host:u,options:e,imageServiceDomains:h});const w=new T({bookPath:d,bookName:s,bookDesc:i,uuidMap:m});await w.genFile(),P.info(`√ 生成目录 ${o.resolve(d,"index.md")}`),f.curr===p&&P.info(`√ 已完成: ${d}`)}export{P as l,z as m};