yuque-dl 1.0.53 → 1.0.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/dist/cjs/cli.js +1 -1
- package/dist/cjs/index-Bgxi37ql.js +1 -0
- package/dist/cjs/index.js +1 -1
- package/dist/es/cli.js +1 -1
- package/dist/es/index-CUtW7Cgn.js +1 -0
- package/dist/es/index.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/index-J1Zg6WKx.js +0 -1
- package/dist/es/index-dTuOCG5L.js +0 -1
package/README.md
CHANGED
|
@@ -49,7 +49,7 @@ yuque-dl "https://www.yuque.com/yuque/thyzgp"
|
|
|
49
49
|
|
|
50
50
|
## Example
|
|
51
51
|
|
|
52
|
-
|
|
53
53
|
|
|
54
54
|
## 其他场景
|
|
55
55
|
|
|
@@ -73,7 +73,7 @@ yuque-dl "https://www.yuque.com/yuque/thyzgp" -t "abcd..."
|
|
|
73
73
|
|
|
74
74
|
### 公开密码访问的知识库
|
|
75
75
|
|
|
76
|
-
|
|
77
77
|
|
|
78
78
|
公开密码访问的知识库两种情况:
|
|
79
79
|
|
|
@@ -89,13 +89,13 @@ yuque-dl "url" -k "verified_books" -t "verified_books的值"
|
|
|
89
89
|
使用[`vitepress`](https://vitepress.dev/)快速启动一个web服务提供可预览下载的内容
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
yuque-dl server ./download
|
|
92
|
+
yuque-dl server ./download/知识库/
|
|
93
93
|
|
|
94
94
|
➜ Local: http://localhost:5173/
|
|
95
95
|
➜ Network: use --host to expose
|
|
96
96
|
```
|
|
97
97
|
|
|
98
|
-
|
|
99
99
|
|
|
100
100
|
## Feature
|
|
101
101
|
|
|
@@ -105,6 +105,7 @@ yuque-dl server ./download/
|
|
|
105
105
|
- [x] 支持转换表格类型的文档 (ps: 表格内插入图表暂不支持)
|
|
106
106
|
- [x] 添加toc目录功能
|
|
107
107
|
- [x] 添加测试
|
|
108
|
+
- [x] 添加附件下载
|
|
108
109
|
- [ ] 支持其他文档类型？🤔
|
|
109
110
|
- [ ] 直接打包成可执行文件 🤔
|
|
110
111
|
|
package/dist/cjs/cli.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var e=require("node:fs"),
|
|
1
|
+
"use strict";var e=require("node:fs"),n=require("cac"),r=require("semver"),i=require("./index-Bgxi37ql.js");require("rand-user-agent");var o=require("node:fs/promises");require("cli-progress");var t=require("vitepress"),s=require("node:path");require("log4js"),require("axios"),require("node:process"),require("ora"),require("pull-md-img"),require("markdown-toc"),require("pako"),require("node:crypto"),require("node:stream"),require("node:util");var a="undefined"!=typeof document?document.currentScript:null;let c;async function d(e){const n=s.resolve(e);if(!await u(n))throw new Error("server root not found");await async function(e){const n=e.split("/").filter(Boolean).at(-1)||"yuque-dl",r=s.join(e,"/.vitepress");await o.mkdir(r,{recursive:!0});const i=s.join(r,"config.ts"),t=await async function(e){const n=[],r=[".vitepress","img","index.md","progress.json"];let i=await o.readdir(e);i=i.filter((e=>!r.includes(e)));for(const r of i){const i=s.join(e,r);if((await o.stat(i)).isDirectory()){const o=await u(s.join(i,"index.md")),t={text:r,collapsed:!0,items:await l(e,r)};o&&(t.link=`/${r}/`),n.push(t)}else n.push({text:r.slice(0,r.lastIndexOf(".")),link:`/${r}`})}return n}(e),a=`\n export default {\n title: "${n}",\n themeConfig: {\n search: {\n provider: 'local'\n },\n sidebar: ${JSON.stringify(t)}\n },\n vite: {\n optimizeDeps: {\n include: []\n }\n }\n }\n `;await o.writeFile(i,a)}(n);const r=async()=>{const n=await t.createServer(e,{},(async()=>(c||(c=(async()=>{await n.close(),await r()})().finally((()=>{c=void 0}))),c)));await n.listen(),n.printUrls()};await r().catch((e=>{console.error(e),process.exit(1)}))}async function u(e){try{return await o.access(e),!0}catch(e){if("ENOENT"===e.code)return!1;throw e}}async function l(e,...n){const r=await o.readdir(s.join(e,...n)),i=[];for(const t of r){const r=s.join(e,...n,t);if((await o.stat(r)).isDirectory()){const o=await l(s.join(e),...n,t);if(o.length>0){const e={text:t,items:o,collapsed:false};await u(s.join(r,"index.md"))&&(e.link="/"+[...n,t].map(decodeURIComponent).join("/")+"/"),i.push(e)}}else{if("index.md"===t||/^-.*\.(md|MD)$/.test(t)||!t.endsWith(".md"))continue;const e=t.replace(/\.md$/,""),r={text:e,link:"/"+[...n.map(decodeURIComponent),`${encodeURI(e)}.html`].join("/")};i.push(r)}}return i}const p=n.cac("yuque-dl"),{version:f,engines:m}=JSON.parse(e.readFileSync(new URL("../../package.json","undefined"==typeof document?require("url").pathToFileURL(__filename).href:a&&a.src||new URL("cli.js",document.baseURI).href)).toString());!function(){const e=m.node;r.satisfies(process.version,e)||(i.logger.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}(),p.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--toc","是否输出文档toc目录",{default:!1}).action((async(e,n)=>{try{await i.main(e,n),process.exit(0)}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}})),p.command("server <serverPath>","启动web服务").action((async e=>{try{await d(e)}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}})),p.help(),p.version(f);try{p.parse()}catch(e){i.logger.error(e.message||"unknown exception"),process.exit(1)}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var e=require("node:fs/promises"),t=require("node:path"),r=require("rand-user-agent"),o=require("log4js"),n=require("cli-progress"),s=require("axios"),a=require("node:process"),i=require("ora"),c=require("pull-md-img"),l=require("markdown-toc"),u=require("pako"),h=require("node:crypto"),p=require("node:stream"),d=require("node:util"),f=require("node:fs");function m(e){var t=Object.create(null);return e&&Object.keys(e).forEach((function(r){if("default"!==r){var o=Object.getOwnPropertyDescriptor(e,r);Object.defineProperty(t,r,o.get?o:{enumerable:!0,get:function(){return e[r]}})}})),t.default=e,Object.freeze(t)}var g,w,y=m(p);!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(g||(g={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(w||(w={}));const k=new Map([[w.BOARD,"画板类型"],[w.TABLE,"数据表类型"],[w.SHEET,"表格类型"],[w.DOC,"文档类型"]]),$="_yuque_session",b="https://www.yuque.com",v="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",I=(o.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),o.getLogger("yuque-dl"));class L{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,I.info("根据上次数据继续断点下载")),this.bar=new n.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},n.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let t=[];try{const r=await e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"})}return t}async updateProgress(t,r){this.curr!==this.total&&(this.curr=this.curr+1,r&&(this.progressInfo.push(t),await e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function E({browser:e="chrome",os:t="mac os",device:o="desktop"}){o=o.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let n=r(o,e,t);if("chrome"===e)for(;n.includes("Chrome-Lighthouse")||n.includes("Gener8")||n.includes("HeadlessChrome")||n.includes("SMTBot");)n=r(o,e,t);if("safari"===e)for(;n.includes("Applebot");)n=r(o,e,t);return n}class P{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:r,bookPath:o,uuidMap:n}=this.summaryInfo;let s=`# ${t}\n\n`;r&&(s+=`> ${r}\n\n`);let a=s;const i=[];n.forEach((e=>{const t=e.toc,r=t.parent_uuid,o=this.findTree(i,r),n={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:"link"},s=t.type.toLocaleLowerCase();s===g.TITLE||""!==t.child_uuid?(n.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)),s===g.DOC&&(n.link=e.path)):(n.type="link",n.link=s===g.LINK?e.toc.url:e.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)))}));a+=this.genSummaryContent(i,"");try{await e.writeFile(`${o}/index.md`,a)}catch(e){I.error("Generate Summary Error")}return a}genSummaryContent(e,t){for(const r of e){if(r.type===g.TITLE)if(r.link){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${e})\n\n`}else t+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===g.LINK){const e=r.link?r.link.replace(/\s/g,"%20"):r.link;t+=`- [${r.text}](${e})\n`}Array.isArray(r.children)&&(t+=this.genSummaryContent(r.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}function S(e){const{key:t=$,token:r}=e,o={"user-agent":E({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function T(e){return{headers:S(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...S(e)}}}}const D=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:n,key:a,host:i=b}=e;let c=`${i}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,s.get(c,T({token:n,key:a})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))};function U(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const x=e=>{if(!e)return"";const t=u.inflate(e,{to:"string"}),r=JSON.parse(t);let o="";return r.forEach((e=>{const t=`## ${e.name}\n`,r=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),n=Math.max(...r.map((e=>Number(e))));if(o<0||n<0)return"";let s="";const a="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let i=Array(n+1).fill(" ").map(((e,t)=>{const r=t%a.length;return a[r]})).join(" | ");i=`| |${i}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,s=`${i}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<n+1;o++){const n=e?.[t]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(``):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}s=`${s}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return s}(e.data);o=`${o}\n${t}\n${r}`})),o};function N(e,t=[]){return function(e,t){try{const{host:r,pathname:o}=new URL(e);return!t.includes(r)&&Boolean(o)}catch(e){return!1}}(e,t)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(e)}&sign=${function(e){const t=h.createHash("sha256");return t.update(`${v}${e}`),t.digest("hex")}(e)}`:e}const O=/\[(.*?)\]\((.*?\.yuque\.com\/attachments.*?)\)/g;async function C(e){const{mdData:r,savePath:o,attachmentsDir:n,articleTitle:c,token:l,key:u}=e,h=r.match(O)||[];if(0===h.length)return{mdData:r};const p=i({text:`下载 "${c}" 的附件中...`,stream:a.stdout});"test"!==a.env.NODE_ENV&&p.start();const d=t.resolve(o,n),m=h.map((e=>function(e,r){O.lastIndex=0;const[,o,n]=O.exec(e)||[];if(!n)return!1;const s=o||n.split("/").at(-1);if(!s)return!1;const a=t.join(r,s);return{fileName:s,url:n,rawMd:e,currentFilePath:a}}(e,d))).filter((e=>!1!==e));f.mkdirSync(d,{recursive:!0});const g=m.map((e=>async function(e){const{fileUrl:t,savePath:r,token:o,key:n}=e;return s.get(t,{...T({token:o,key:n}),responseType:"stream"}).then((async e=>{if(200===e.status){const o=f.createWriteStream(r);return e.data.pipe(o),j(o).then((()=>({fileUrl:t,savePath:r})))}throw new Error(`response status ${e.status}`)}))}({fileUrl:e.url,savePath:e.currentFilePath,token:l,key:u}))),w=await Promise.all(g);let y=r;return w.forEach((e=>{const t=m.find((t=>t.url===e.fileUrl));if(t){const e=`[附件: ${t.fileName}](${n}/${t.fileName})`;y=y.replace(t.rawMd,e)}})),p&&p.stop(),{mdData:y}}const j=d.promisify(y.finished);async function q(t){const{articleInfo:r,progressBar:o,options:n}=t,{token:s,key:l}=n,{bookId:u,itemUrl:h,savePath:p,saveFilePath:d,uuid:f,articleUrl:m,articleTitle:g,ignoreImg:y,host:$,imageServiceDomains:b}=r,v={articleUrl:h,bookId:u,token:s,host:$,key:l},{httpStatus:I,apiUrl:L,response:E}=await D(v),P=E?.data?.type?.toLocaleLowerCase();let S="";if(P===w.SHEET){const{response:e}=await D(v,!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;S=o?x(o):""}catch(e){const t=k.get(P);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([w.BOARD,w.TABLE].includes(P)){const e=k.get(P);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof E?.data?.sourcecode)throw new Error(`download article Error: ${L}, http status ${I}`);S=E.data.sourcecode,S=function(e){const t=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=e.match(t);let o=e;const n=e;try{r?.forEach((e=>{t.lastIndex=0;const r=t.exec(e)?.[2]??"",{pathname:n,search:s}=new URL(r);if(!n.endsWith(".svg")&&s){const t=decodeURIComponent(s);o=o.replace(e,t.slice(1))}}))}catch(e){return n}return o}(S)}const T=function(e){if(!e)return[];const t=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(e.match(t)||[]);return r=r.map((e=>(e=e.replace(t,"$2"),/^http.*/g.test(e)?e:""))).filter((e=>Boolean(e))),r}(S);if(T.length&&!y){const e=await D(v,!1);S=function(e,t,r){if(!r)return t;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let s,a=!0;for(;a||Boolean(s);)if(a=!1,s=o.exec(r),s?.[1])try{const e=decodeURIComponent(s[1]),t=JSON.parse(e);n.push(t?.src||"")}catch(e){n.push("")}const i=new Map;return e.forEach((e=>{const{origin:r,pathname:o}=new URL(e),s=`${r}${o}`,a=n.find(((e,t)=>{const r=new RegExp(`${s}.*?`).test(e);return r&&n.splice(t,1),r}));if(a){const r=new RegExp(e,"g"),o=i.get(e)||0;let n=0;t=t.replace(r,(e=>{let t=e;return n===o&&(t=a),n+=1,t})),i.set(e,o+1)}})),t}(T,S,e.response?.data?.content??"")}const U={toc:n.toc,articleTitle:g,articleUrl:m},O=[];try{o.pause(),console.log("");S=(await C({mdData:S,savePath:p,attachmentsDir:`./attachments/${f}`,articleTitle:g,token:s,key:l})).mdData}catch(e){O.push(`附件下载失败: ${e.message||"unknown error"}`)}finally{o.continue()}if(T.length&&!y){let t;o.pause(),console.log(""),"test"!==a.env.NODE_ENV&&(t=i({text:`下载 "${g}" 的图片中...`,stream:a.stdout}),t.start());let r=[],n=S;try{const e=await c.run(S,{dist:p,imgDir:`./img/${f}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:m||"",transform:e=>N(e,b)});r=e.errorInfo,n=e.data}catch(e){r=[e]}S=n;const s=()=>{t&&t.stop(),o.continue()};if(r.length>0){const t=r[0];let o="图片下载失败(失败的以远程链接保存): ";throw o=t.url?`${o}${t.error?.message} ${t.url.slice(0,20)}...`:`${o}${t.message}`,await e.writeFile(d,A(S,U)),s(),new Error(`${o}`)}s()}try{if(await e.writeFile(d,A(S,U)),O.length>0)throw new Error(O[0]);return!0}catch(e){throw new Error(`download article Error ${m}: ${e.message}`)}}function A(e,t){const{articleTitle:r,articleUrl:o,toc:n}=t;let s=e;s=s.replace(/<br(\s?)\/>/gm,"\n"),s=s.replace(/<a.*?>(\s*?)<\/a>/gm,"");const a=r?`# ${r}\n\n`:"";let i=n?l(s).content:"";i&&(i=`${i}\n\n---\n\n`);return s=`${a}${i}${s}${o?`\n\n> 原文: <${o}>`:""}`,s}exports.logger=I,exports.main=async function(r,o){const{bookId:n,tocList:a,bookName:i,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:h}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return s.get(e,T(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||b,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((()=>({})))})(r,{token:o.token,key:o.key});if(!n)throw new Error("No found book id");if(!a||0===a.length)throw new Error("No found toc list");const p=t.resolve(o.distDir,i?U(i):String(n));await e.mkdir(p,{recursive:!0});const d=a.length,f=new L(p,d);if(await f.init(),f.curr===d)return f.bar&&f.bar.stop(),void I.info(`√ 已完成: ${p}`);const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((e=>{m.set(e.toc.uuid,e)}));const y=r.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(r){const{articleUrlPrefix:o,total:n,uuidMap:s,tocList:a,bookPath:i,bookId:c,progressBar:l,host:u,options:h,imageServiceDomains:p=[]}=r;let d=0,f=0,m=0;const y=[],k=[];for(let t=0;t<n;t++){const r=a[t];if("string"!=typeof r.type)continue;if(s.get(r.uuid))continue;const o=r.type.toLocaleLowerCase();if(o===g.TITLE||""!==r.child_uuid||o===g.LINK){let t=r;const n=[],a=[];for(;t;)n.unshift(U(t.title)),a.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const c={path:n.map(U).join("/"),pathTitleList:n,pathIdList:a,toc:r};o===g.LINK?(m+=1,k.push(c)):await e.mkdir(`${i}/${n.map(U).join("/")}`,{recursive:!0}),s.set(r.uuid,c),o===w.DOC?await $(r):await l.updateProgress(c,o!==g.LINK)}else r.url&&await $(r)}async function $(e){f+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=e.type.toLocaleLowerCase();s.get(e.parent_uuid)&&(r=s.get(e.parent_uuid));const a=U(e.title),m=[...r.pathTitleList,a],g=[...r.pathIdList,e.uuid];let k=[...r.pathTitleList,`${a}.md`].map(U).join("/"),$=r.pathTitleList.map(U).join("/");n===w.DOC&&e.child_uuid&&(k=[...r.pathTitleList,a,"index.md"].map(U).join("/"),$=m.map(U).join("/"));const b={path:k,savePath:$,pathTitleList:m,pathIdList:g,toc:e};let v=!0;const I=`${o}/${e.url}`;try{const r={bookId:c,itemUrl:e.url,savePath:t.resolve(i,b.savePath),saveFilePath:t.resolve(i,b.path),uuid:e.uuid,articleUrl:I,articleTitle:e.title,ignoreImg:h.ignoreImg,host:u,imageServiceDomains:p};await q({articleInfo:r,progressBar:l,options:h})}catch(e){v=!1,d+=1,y.push({articleUrl:I,errItem:b,errMsg:e.message,err:e})}s.set(e.uuid,b),await l.updateProgress(b,v)}if(m>0){I.warn("该知识库存在以下外链文章");for(const e of k)I.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(d>0){I.error(`本次执行总数${f}篇，✕ 失败${d}篇`);for(const e of y)I.error(`《${e.errItem.path}》: ${e.articleUrl}`),I.error(`———— ✕ ${e.errMsg}`);I.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:y,total:d,uuidMap:m,tocList:a,bookPath:p,bookId:n,progressBar:f,host:u,options:o,imageServiceDomains:h});const k=new P({bookPath:p,bookName:i,bookDesc:c,uuidMap:m});await k.genFile(),I.info(`√ 生成目录 ${t.resolve(p,"index.md")}`),f.curr===d&&I.info(`√ 已完成: ${p}`)};
|
package/dist/cjs/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";require("node:fs/promises"),require("node:path");var e=require("./index-
|
|
1
|
+
"use strict";require("node:fs/promises"),require("node:path");var e=require("./index-Bgxi37ql.js");require("rand-user-agent"),require("log4js"),require("cli-progress"),require("axios"),require("node:process"),require("ora"),require("pull-md-img"),require("markdown-toc"),require("pako"),require("node:crypto"),require("node:stream"),require("node:util"),require("node:fs"),exports.main=e.main;
|
package/dist/es/cli.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{readFileSync as o}from"node:fs";import{cac as
|
|
1
|
+
import{readFileSync as o}from"node:fs";import{cac as e}from"cac";import t from"semver";import{m as n,l as i}from"./index-CUtW7Cgn.js";import"rand-user-agent";import{mkdir as r,writeFile as s,access as a,readdir as c,stat as p}from"node:fs/promises";import"cli-progress";import{createServer as m}from"vitepress";import{resolve as d,join as l}from"node:path";import"log4js";import"axios";import"node:process";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:crypto";import"node:stream";import"node:util";let f;async function u(o){const e=d(o);if(!await w(e))throw new Error("server root not found");await async function(o){const e=o.split("/").filter(Boolean).at(-1)||"yuque-dl",t=l(o,"/.vitepress");await r(t,{recursive:!0});const n=l(t,"config.ts"),i=await async function(o){const e=[],t=[".vitepress","img","index.md","progress.json"];let n=await c(o);n=n.filter((o=>!t.includes(o)));for(const t of n){const n=l(o,t);if((await p(n)).isDirectory()){const i=await w(l(n,"index.md")),r={text:t,collapsed:!0,items:await y(o,t)};i&&(r.link=`/${t}/`),e.push(r)}else e.push({text:t.slice(0,t.lastIndexOf(".")),link:`/${t}`})}return e}(o),a=`\n export default {\n title: "${e}",\n themeConfig: {\n search: {\n provider: 'local'\n },\n sidebar: ${JSON.stringify(i)}\n },\n vite: {\n optimizeDeps: {\n include: []\n }\n }\n }\n `;await s(n,a)}(e);const t=async()=>{const e=await m(o,{},(async()=>(f||(f=(async()=>{await e.close(),await t()})().finally((()=>{f=void 0}))),f)));await e.listen(),e.printUrls()};await t().catch((o=>{console.error(o),process.exit(1)}))}async function w(o){try{return await a(o),!0}catch(o){if("ENOENT"===o.code)return!1;throw o}}async function y(o,...e){const t=await c(l(o,...e)),n=[];for(const i of t){const t=l(o,...e,i);if((await p(t)).isDirectory()){const r=await y(l(o),...e,i);if(r.length>0){const o={text:i,items:r,collapsed:false};await w(l(t,"index.md"))&&(o.link="/"+[...e,i].map(decodeURIComponent).join("/")+"/"),n.push(o)}}else{if("index.md"===i||/^-.*\.(md|MD)$/.test(i)||!i.endsWith(".md"))continue;const o=i.replace(/\.md$/,""),t={text:o,link:"/"+[...e.map(decodeURIComponent),`${encodeURI(o)}.html`].join("/")};n.push(t)}}return n}const x=e("yuque-dl"),{version:k,engines:g}=JSON.parse(o(new URL("../../package.json",import.meta.url)).toString());!function(){const o=g.node;t.satisfies(process.version,o)||(i.error(`✕ nodejs 版本需 ${o}, 当前版本为 ${process.version}`),process.exit(1))}(),x.command("<url>","语雀知识库url").option("-d, --dist-dir <dir>","下载的目录 eg: -d download",{default:"download"}).option("-i, --ignore-img","忽略图片不下载",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("--toc","是否输出文档toc目录",{default:!1}).action((async(o,e)=>{try{await n(o,e),process.exit(0)}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}})),x.command("server <serverPath>","启动web服务").action((async o=>{try{await u(o)}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}})),x.help(),x.version(k);try{x.parse()}catch(o){i.error(o.message||"unknown exception"),process.exit(1)}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"node:path";import n from"rand-user-agent";import s from"log4js";import a from"cli-progress";import i from"axios";import{stdout as c,env as l}from"node:process";import u from"ora";import h from"pull-md-img";import p from"markdown-toc";import d from"pako";import f from"node:crypto";import*as m from"node:stream";import{promisify as g}from"node:util";import{mkdirSync as w,createWriteStream as $}from"node:fs";var y,k;!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(y||(y={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(k||(k={}));const b=new Map([[k.BOARD,"画板类型"],[k.TABLE,"数据表类型"],[k.SHEET,"表格类型"],[k.DOC,"文档类型"]]),I="_yuque_session",v="https://www.yuque.com",L="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",E=(s.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),s.getLogger("yuque-dl"));class T{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,E.info("根据上次数据继续断点下载")),this.bar=new a.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},a.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr!==this.total&&(this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function P({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=n(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=n(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=n(r,t,e);return o}class S{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:n}=this.summaryInfo;let s=`# ${e}\n\n`;r&&(s+=`> ${r}\n\n`);let a=s;const i=[];n.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(i,r),n={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:"link"},s=e.type.toLocaleLowerCase();s===y.TITLE||""!==e.child_uuid?(n.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)),s===y.DOC&&(n.link=t.path)):(n.type="link",n.link=s===y.LINK?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),n.level=o.level+1,o.children.push(n)):(n.level=1,i.push(n)))}));a+=this.genSummaryContent(i,"");try{await t.writeFile(`${o}/index.md`,a)}catch(t){E.error("Generate Summary Error")}return a}genSummaryContent(t,e){for(const r of t){if(r.type===y.TITLE)if(r.link){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${t})\n\n`}else e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===y.LINK){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}function D(t){const{key:e=I,token:r}=t,o={"user-agent":P({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function U(t){return{headers:D(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...D(t)}}}}const N=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:n,key:s,host:a=v}=t;let c=`${a}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,i.get(c,U({token:n,key:s})).then((({data:t,status:e})=>({apiUrl:c,httpStatus:e,response:t})))};function x(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const C=t=>{if(!t)return"";const e=d.inflate(t,{to:"string"}),r=JSON.parse(e);let o="";return r.forEach((t=>{const e=`## ${t.name}\n`,r=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),n=Math.max(...r.map((t=>Number(t))));if(o<0||n<0)return"";let s="";const a="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let i=Array(n+1).fill(" ").map(((t,e)=>{const r=e%a.length;return a[r]})).join(" | ");i=`| |${i}|`;let c=Array(n+2).fill("---").join(" |");c=`|${c}|`,s=`${i}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<n+1;o++){const n=t?.[e]?.[o]?.v||null;n&&"string"==typeof n?r.push(n):n&&"object"==typeof n?"image"===n?.class&&n?.src?r.push(``):"checkbox"===n?.class?r.push(n?.value?"[x] ":"[ ] "):"link"===n?.class?r.push(`[${n?.text}](${n?.url})`):"select"===n?.class&&r.push(n?.value?.join(",")):r.push(null)}s=`${s}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return s}(t.data);o=`${o}\n${e}\n${r}`})),o};function O(t,e=[]){return function(t,e){try{const{host:r,pathname:o}=new URL(t);return!e.includes(r)&&Boolean(o)}catch(t){return!1}}(t,e)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(t)}&sign=${function(t){const e=f.createHash("sha256");return e.update(`${L}${t}`),e.digest("hex")}(t)}`:t}const A=/\[(.*?)\]\((.*?\.yuque\.com\/attachments.*?)\)/g;async function B(t){const{mdData:e,savePath:r,attachmentsDir:n,articleTitle:s,token:a,key:h}=t,p=e.match(A)||[];if(0===p.length)return{mdData:e};const d=u({text:`下载 "${s}" 的附件中...`,stream:c});"test"!==l.NODE_ENV&&d.start();const f=o.resolve(r,n),m=p.map((t=>function(t,e){A.lastIndex=0;const[,r,n]=A.exec(t)||[];if(!n)return!1;const s=r||n.split("/").at(-1);if(!s)return!1;const a=o.join(e,s);return{fileName:s,url:n,rawMd:t,currentFilePath:a}}(t,f))).filter((t=>!1!==t));w(f,{recursive:!0});const g=m.map((t=>async function(t){const{fileUrl:e,savePath:r,token:o,key:n}=t;return i.get(e,{...U({token:o,key:n}),responseType:"stream"}).then((async t=>{if(200===t.status){const o=$(r);return t.data.pipe(o),R(o).then((()=>({fileUrl:e,savePath:r})))}throw new Error(`response status ${t.status}`)}))}({fileUrl:t.url,savePath:t.currentFilePath,token:a,key:h}))),y=await Promise.all(g);let k=e;return y.forEach((t=>{const e=m.find((e=>e.url===t.fileUrl));if(e){const t=`[附件: ${e.fileName}](${n}/${e.fileName})`;k=k.replace(e.rawMd,t)}})),d&&d.stop(),{mdData:k}}const R=g(m.finished);async function j(t){const{articleInfo:r,progressBar:o,options:n}=t,{token:s,key:a}=n,{bookId:i,itemUrl:p,savePath:d,saveFilePath:f,uuid:m,articleUrl:g,articleTitle:w,ignoreImg:$,host:y,imageServiceDomains:I}=r,v={articleUrl:p,bookId:i,token:s,host:y,key:a},{httpStatus:L,apiUrl:E,response:T}=await N(v),P=T?.data?.type?.toLocaleLowerCase();let S="";if(P===k.SHEET){const{response:t}=await N(v,!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;S=o?C(o):""}catch(t){const e=b.get(P);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([k.BOARD,k.TABLE].includes(P)){const t=b.get(P);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof T?.data?.sourcecode)throw new Error(`download article Error: ${E}, http status ${L}`);S=T.data.sourcecode,S=function(t){const e=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=t.match(e);let o=t;const n=t;try{r?.forEach((t=>{e.lastIndex=0;const r=e.exec(t)?.[2]??"",{pathname:n,search:s}=new URL(r);if(!n.endsWith(".svg")&&s){const e=decodeURIComponent(s);o=o.replace(t,e.slice(1))}}))}catch(t){return n}return o}(S)}const D=function(t){if(!t)return[];const e=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(t.match(e)||[]);return r=r.map((t=>(t=t.replace(e,"$2"),/^http.*/g.test(t)?t:""))).filter((t=>Boolean(t))),r}(S);if(D.length&&!$){const t=await N(v,!1);S=function(t,e,r){if(!r)return e;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,n=[];let s,a=!0;for(;a||Boolean(s);)if(a=!1,s=o.exec(r),s?.[1])try{const t=decodeURIComponent(s[1]),e=JSON.parse(t);n.push(e?.src||"")}catch(t){n.push("")}const i=new Map;return t.forEach((t=>{const{origin:r,pathname:o}=new URL(t),s=`${r}${o}`,a=n.find(((t,e)=>{const r=new RegExp(`${s}.*?`).test(t);return r&&n.splice(e,1),r}));if(a){const r=new RegExp(t,"g"),o=i.get(t)||0;let n=0;e=e.replace(r,(t=>{let e=t;return n===o&&(e=a),n+=1,e})),i.set(t,o+1)}})),e}(D,S,t.response?.data?.content??"")}const U={toc:n.toc,articleTitle:w,articleUrl:g},x=[];try{o.pause(),console.log("");S=(await B({mdData:S,savePath:d,attachmentsDir:`./attachments/${m}`,articleTitle:w,token:s,key:a})).mdData}catch(t){x.push(`附件下载失败: ${t.message||"unknown error"}`)}finally{o.continue()}if(D.length&&!$){let t;o.pause(),console.log(""),"test"!==l.NODE_ENV&&(t=u({text:`下载 "${w}" 的图片中...`,stream:c}),t.start());let r=[],n=S;try{const t=await h.run(S,{dist:d,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:g||"",transform:t=>O(t,I)});r=t.errorInfo,n=t.data}catch(t){r=[t]}S=n;const s=()=>{t&&t.stop(),o.continue()};if(r.length>0){const t=r[0];let o="图片下载失败(失败的以远程链接保存): ";throw o=t.url?`${o}${t.error?.message} ${t.url.slice(0,20)}...`:`${o}${t.message}`,await e(f,F(S,U)),s(),new Error(`${o}`)}s()}try{if(await e(f,F(S,U)),x.length>0)throw new Error(x[0]);return!0}catch(t){throw new Error(`download article Error ${g}: ${t.message}`)}}function F(t,e){const{articleTitle:r,articleUrl:o,toc:n}=e;let s=t;s=s.replace(/<br(\s?)\/>/gm,"\n"),s=s.replace(/<a.*?>(\s*?)<\/a>/gm,"");const a=r?`# ${r}\n\n`:"";let i=n?p(s).content:"";i&&(i=`${i}\n\n---\n\n`);return s=`${a}${i}${s}${o?`\n\n> 原文: <${o}>`:""}`,s}async function M(t,e){const{bookId:n,tocList:s,bookName:a,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:h}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return i.get(t,U(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||v,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((()=>({})))})(t,{token:e.token,key:e.key});if(!n)throw new Error("No found book id");if(!s||0===s.length)throw new Error("No found toc list");const p=o.resolve(e.distDir,a?x(a):String(n));await r(p,{recursive:!0});const d=s.length,f=new T(p,d);if(await f.init(),f.curr===d)return f.bar&&f.bar.stop(),void E.info(`√ 已完成: ${p}`);const m=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((t=>{m.set(t.toc.uuid,t)}));const g=t.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:n,uuidMap:s,tocList:a,bookPath:i,bookId:c,progressBar:l,host:u,options:h,imageServiceDomains:p=[]}=t;let d=0,f=0,m=0;const g=[],w=[];for(let t=0;t<n;t++){const e=a[t];if("string"!=typeof e.type)continue;if(s.get(e.uuid))continue;const o=e.type.toLocaleLowerCase();if(o===y.TITLE||""!==e.child_uuid||o===y.LINK){let t=e;const n=[],a=[];for(;t;)n.unshift(x(t.title)),a.unshift(t.uuid),t=s.get(t.parent_uuid)?s.get(t.parent_uuid).toc:void 0;const c={path:n.map(x).join("/"),pathTitleList:n,pathIdList:a,toc:e};o===y.LINK?(m+=1,w.push(c)):await r(`${i}/${n.map(x).join("/")}`,{recursive:!0}),s.set(e.uuid,c),o===k.DOC?await $(e):await l.updateProgress(c,o!==y.LINK)}else e.url&&await $(e)}async function $(t){f+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const n=t.type.toLocaleLowerCase();s.get(t.parent_uuid)&&(r=s.get(t.parent_uuid));const a=x(t.title),m=[...r.pathTitleList,a],w=[...r.pathIdList,t.uuid];let $=[...r.pathTitleList,`${a}.md`].map(x).join("/"),y=r.pathTitleList.map(x).join("/");n===k.DOC&&t.child_uuid&&($=[...r.pathTitleList,a,"index.md"].map(x).join("/"),y=m.map(x).join("/"));const b={path:$,savePath:y,pathTitleList:m,pathIdList:w,toc:t};let I=!0;const v=`${e}/${t.url}`;try{const e={bookId:c,itemUrl:t.url,savePath:o.resolve(i,b.savePath),saveFilePath:o.resolve(i,b.path),uuid:t.uuid,articleUrl:v,articleTitle:t.title,ignoreImg:h.ignoreImg,host:u,imageServiceDomains:p};await j({articleInfo:e,progressBar:l,options:h})}catch(t){I=!1,d+=1,g.push({articleUrl:v,errItem:b,errMsg:t.message,err:t})}s.set(t.uuid,b),await l.updateProgress(b,I)}if(m>0){E.warn("该知识库存在以下外链文章");for(const t of w)E.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(d>0){E.error(`本次执行总数${f}篇，✕ 失败${d}篇`);for(const t of g)E.error(`《${t.errItem.path}》: ${t.articleUrl}`),E.error(`———— ✕ ${t.errMsg}`);E.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:g,total:d,uuidMap:m,tocList:s,bookPath:p,bookId:n,progressBar:f,host:u,options:e,imageServiceDomains:h});const w=new S({bookPath:p,bookName:a,bookDesc:c,uuidMap:m});await w.genFile(),E.info(`√ 生成目录 ${o.resolve(p,"index.md")}`),f.curr===d&&E.info(`√ 已完成: ${p}`)}export{E as l,M as m};
|
package/dist/es/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import"node:fs/promises";import"node:path";export{m as main}from"./index-
|
|
1
|
+
import"node:fs/promises";import"node:path";export{m as main}from"./index-CUtW7Cgn.js";import"rand-user-agent";import"log4js";import"cli-progress";import"axios";import"node:process";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:crypto";import"node:stream";import"node:util";import"node:fs";
|
package/package.json
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var e,t,r=require("node:fs/promises"),o=require("node:path"),s=require("rand-user-agent"),n=require("log4js"),i=require("cli-progress"),a=require("axios"),c=require("node:process"),l=require("ora"),u=require("pull-md-img"),h=require("markdown-toc"),p=require("pako"),d=require("node:crypto");!function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(e||(e={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(t||(t={}));const f=new Map([[t.BOARD,"画板类型"],[t.TABLE,"数据表类型"],[t.SHEET,"表格类型"],[t.DOC,"文档类型"]]),g="_yuque_session",m="https://www.yuque.com",w="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",$=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class k{constructor(e,t){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,$.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const t=await r.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(t)}catch(t){t&&"ENOENT"===t.code&&await r.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,t){this.curr!==this.total&&(this.curr=this.curr+1,t&&(this.progressInfo.push(e),await r.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(e){if(!(e<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function y({browser:e="chrome",os:t="mac os",device:r="desktop"}){r=r.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let o=s(r,e,t);if("chrome"===e)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=s(r,e,t);if("safari"===e)for(;o.includes("Applebot");)o=s(r,e,t);return o}class b{constructor(e){this.summaryInfo=e}async genFile(){const{bookName:t,bookDesc:o,bookPath:s,uuidMap:n}=this.summaryInfo;let i=`# ${t}\n\n`;o&&(i+=`> ${o}\n\n`);let a=i;const c=[];n.forEach((t=>{const r=t.toc,o=r.parent_uuid,s=this.findTree(c,o),n={text:r.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:r.uuid,level:1,type:"link"},i=r.type.toLocaleLowerCase();i===e.TITLE||""!==r.child_uuid?(n.type="title","boolean"!=typeof s?(Array.isArray(s.children)||(s.children=[]),n.level=s.level+1,s.children.push(n)):(n.level=1,c.push(n)),i===e.DOC&&(n.link=t.path)):(n.type="link",n.link=i===e.LINK?t.toc.url:t.path,"boolean"!=typeof s?(Array.isArray(s.children)||(s.children=[]),n.level=s.level+1,s.children.push(n)):(n.level=1,c.push(n)))}));a+=this.genSummaryContent(c,"");try{await r.writeFile(`${s}/index.md`,a)}catch(e){$.error("Generate Summary Error")}return a}genSummaryContent(t,r){for(const o of t){if(o.type===e.TITLE)if(o.link){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;r+=`\n${"".padStart(o.level+1,"#")} [${o.text}](${e})\n\n`}else r+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if(o.type===e.LINK){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;r+=`- [${o.text}](${e})\n`}Array.isArray(o.children)&&(r+=this.genSummaryContent(o.children,""))}return r}findIdItem(e,t){if(e.id===t)return e;if(e.children){const r=this.findTree(e.children,t);if(r)return r}return!1}findTree(e,t){if(!t)return!1;for(const r of e){const e=this.findIdItem(r,t);if(e)return e}return!1}}function I(e){const{key:t=g,token:r}=e,o={"user-agent":y({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${t}=${r};`),o}function L(e){return{headers:I(e),beforeRedirect:t=>{t.headers={...t?.headers||{},...I(e)}}}}const v=(e,t=!0)=>{const{articleUrl:r,bookId:o,token:s,key:n,host:i=m}=e;let c=`${i}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,L({token:s,key:n})).then((({data:e,status:t})=>({apiUrl:c,httpStatus:t,response:e})))};function E(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const S=e=>{if(!e)return"";const t=p.inflate(e,{to:"string"}),r=JSON.parse(t);let o="";return r.forEach((e=>{const t=`## ${e.name}\n`,r=function(e){let t=Object.keys(e);t=t.filter((t=>Object.keys(e[t]).some((r=>e?.[t]?.[r]?.v))));let r=[];t.forEach((t=>{const o=e[t];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...t.map((e=>Number(e)))),s=Math.max(...r.map((e=>Number(e))));if(o<0||s<0)return"";let n="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((e,t)=>{const r=t%i.length;return i[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,n=`${a}\n${c}\n`;for(let t=0;t<o+1;t++){const r=[];for(let o=0;o<s+1;o++){const s=e?.[t]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(``):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}n=`${n}${`| ${t+1} | ${r.join(" | ")}|`}\n`}return n}(e.data);o=`${o}\n${t}\n${r}`})),o};function T(e,t=[]){return function(e,t){try{const{host:r,pathname:o}=new URL(e);return!t.includes(r)&&Boolean(o)}catch(e){return!1}}(e,t)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(e)}&sign=${function(e){const t=d.createHash("sha256");return t.update(`${w}${e}`),t.digest("hex")}(e)}`:e}async function P(e){const{articleInfo:o,progressBar:s,options:n}=e,{token:i,key:a}=n,{bookId:h,itemUrl:p,savePath:d,saveFilePath:g,uuid:m,articleUrl:w,articleTitle:$,ignoreImg:k,host:y,imageServiceDomains:b}=o,I={articleUrl:p,bookId:h,token:i,host:y,key:a},{httpStatus:L,apiUrl:E,response:P}=await v(I),x=P?.data?.type?.toLocaleLowerCase();let U="";if(x===t.SHEET){const{response:e}=await v(I,!1);try{const t=e?.data?.content,r=t?JSON.parse(t):{},o=r?.sheet;U=o?S(o):""}catch(e){const t=f.get(x);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([t.BOARD,t.TABLE].includes(x)){const e=f.get(x);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof P?.data?.sourcecode)throw new Error(`download article Error: ${E}, http status ${L}`);U=P.data.sourcecode,U=function(e){const t=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=e.match(t);let o=e;const s=e;try{r?.forEach((e=>{t.lastIndex=0;const r=t.exec(e)?.[2]??"",{pathname:s,search:n}=new URL(r);if(!s.endsWith(".svg")&&n){const t=decodeURIComponent(n);o=o.replace(e,t.slice(1))}}))}catch(e){return s}return o}(U)}const C=function(e){if(!e)return[];const t=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(e.match(t)||[]);return r=r.map((e=>(e=e.replace(t,"$2"),/^http.*/g.test(e)?e:""))).filter((e=>Boolean(e))),r}(U);if(C.length&&!k){const e=await v(I,!1);U=function(e,t,r){if(!r)return t;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,s=[];let n,i=!0;for(;i||Boolean(n);)if(i=!1,n=o.exec(r),n?.[1])try{const e=decodeURIComponent(n[1]),t=JSON.parse(e);s.push(t?.src||"")}catch(e){s.push("")}const a=new Map;return e.forEach((e=>{const{origin:r,pathname:o}=new URL(e),n=`${r}${o}`,i=s.find(((e,t)=>{const r=new RegExp(`${n}.*?`).test(e);return r&&s.splice(t,1),r}));if(i){const r=new RegExp(e,"g"),o=a.get(e)||0;let s=0;t=t.replace(r,(e=>{let t=e;return s===o&&(t=i),s+=1,t})),a.set(e,o+1)}})),t}(C,U,e.response?.data?.content??"")}const N={toc:n.toc,articleTitle:$,articleUrl:w};if(C.length&&!k){let e;s.pause(),console.log(""),"test"!==c.env.NODE_ENV&&(e=l({text:`下载 "${$}" 的图片中...`,stream:c.stdout}),e.start());let t=[],o=U;try{const e=await u.run(U,{dist:d,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:w||"",transform:e=>T(e,b)});t=e.errorInfo,o=e.data}catch(e){t=[e]}U=o;const n=()=>{e&&e.stop(),s.continue()};if(t.length>0){const e=t[0];let o="图片下载失败(失败的以远程链接保存): ";throw o=e.url?`${o}${e.error?.message} ${e.url.slice(0,20)}...`:`${o}${e.message}`,await r.writeFile(g,D(U,N)),n(),new Error(`${o}`)}n()}try{return await r.writeFile(g,D(U,N)),!0}catch(e){throw new Error(`download article Error ${w}: ${e.message}`)}}function D(e,t){const{articleTitle:r,articleUrl:o,toc:s}=t;let n=e;n=n.replace(/<br(\s?)\/>/gm,"\n"),n=n.replace(/<a.*?>(\s*?)<\/a>/gm,"");const i=r?`# ${r}\n\n`:"";let a=s?h(n).content:"";a&&(a=`${a}\n\n---\n\n`);return n=`${i}${a}${n}${o?`\n\n> 原文: <${o}>`:""}`,n}exports.logger=$,exports.main=async function(s,n){const{bookId:i,tocList:c,bookName:l,bookDesc:u,bookSlug:h,host:p,imageServiceDomains:d}=await((e,t)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(e,L(t)).then((({data:e="",status:t})=>200===t?e:"")).then((e=>{const t=r.exec(e)??"";if(!t[1])return{};const o=JSON.parse(decodeURIComponent(t[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||m,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((()=>({})))})(s,{token:n.token,key:n.key});if(!i)throw new Error("No found book id");if(!c||0===c.length)throw new Error("No found toc list");const f=o.resolve(n.distDir,l?E(l):String(i));await r.mkdir(f,{recursive:!0});const g=c.length,w=new k(f,g);if(await w.init(),w.curr===g)return w.bar&&w.bar.stop(),void $.info(`√ 已完成: ${f}`);const y=new Map;w.isDownloadInterrupted&&w.progressInfo.forEach((e=>{y.set(e.toc.uuid,e)}));const I=s.replace(new RegExp(`(.*?/${h}).*`),"$1");await async function(s){const{articleUrlPrefix:n,total:i,uuidMap:a,tocList:c,bookPath:l,bookId:u,progressBar:h,host:p,options:d,imageServiceDomains:f=[]}=s;let g=0,m=0,w=0;const k=[],y=[];for(let o=0;o<i;o++){const s=c[o];if("string"!=typeof s.type)continue;if(a.get(s.uuid))continue;const n=s.type.toLocaleLowerCase();if(n===e.TITLE||""!==s.child_uuid||n===e.LINK){let o=s;const i=[],c=[];for(;o;)i.unshift(E(o.title)),c.unshift(o.uuid),o=a.get(o.parent_uuid)?a.get(o.parent_uuid).toc:void 0;const u={path:i.map(E).join("/"),pathTitleList:i,pathIdList:c,toc:s};n===e.LINK?(w+=1,y.push(u)):await r.mkdir(`${l}/${i.map(E).join("/")}`,{recursive:!0}),a.set(s.uuid,u),n===t.DOC?await b(s):await h.updateProgress(u,n!==e.LINK)}else s.url&&await b(s)}async function b(e){m+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const s=e.type.toLocaleLowerCase();a.get(e.parent_uuid)&&(r=a.get(e.parent_uuid));const i=E(e.title),c=[...r.pathTitleList,i],w=[...r.pathIdList,e.uuid];let $=[...r.pathTitleList,`${i}.md`].map(E).join("/"),y=r.pathTitleList.map(E).join("/");s===t.DOC&&e.child_uuid&&($=[...r.pathTitleList,i,"index.md"].map(E).join("/"),y=c.map(E).join("/"));const b={path:$,savePath:y,pathTitleList:c,pathIdList:w,toc:e};let I=!0;const L=`${n}/${e.url}`;try{const t={bookId:u,itemUrl:e.url,savePath:o.resolve(l,b.savePath),saveFilePath:o.resolve(l,b.path),uuid:e.uuid,articleUrl:L,articleTitle:e.title,ignoreImg:d.ignoreImg,host:p,imageServiceDomains:f};await P({articleInfo:t,progressBar:h,options:d})}catch(e){I=!1,g+=1,k.push({articleUrl:L,errItem:b,errMsg:e.message,err:e})}a.set(e.uuid,b),await h.updateProgress(b,I)}if(w>0){$.warn("该知识库存在以下外链文章");for(const e of y)$.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(g>0){$.error(`本次执行总数${m}篇，✕ 失败${g}篇`);for(const e of k)$.error(`《${e.errItem.path}》: ${e.articleUrl}`),$.error(`———— ✕ ${e.errMsg}`);$.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:I,total:g,uuidMap:y,tocList:c,bookPath:f,bookId:i,progressBar:w,host:p,options:n,imageServiceDomains:d});const v=new b({bookPath:f,bookName:l,bookDesc:u,uuidMap:y});await v.genFile(),$.info(`√ 生成目录 ${o.resolve(f,"index.md")}`),w.curr===g&&$.info(`√ 已完成: ${f}`)};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import t,{writeFile as e,mkdir as r}from"node:fs/promises";import o from"node:path";import s from"rand-user-agent";import n from"log4js";import i from"cli-progress";import a from"axios";import{env as c,stdout as l}from"node:process";import u from"ora";import p from"pull-md-img";import h from"markdown-toc";import d from"pako";import f from"node:crypto";var m,g;!function(t){t.TITLE="title",t.LINK="link",t.DOC="doc"}(m||(m={})),function(t){t.BOARD="board",t.TABLE="table",t.SHEET="sheet",t.DOC="doc"}(g||(g={}));const w=new Map([[g.BOARD,"画板类型"],[g.TABLE,"数据表类型"],[g.SHEET,"表格类型"],[g.DOC,"文档类型"]]),$="_yuque_session",k="https://www.yuque.com",y="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",b=(n.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),n.getLogger("yuque-dl"));class I{constructor(t,e){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=t,this.progressFilePath=`${t}/progress.json`,this.total=e}async init(){this.progressInfo=await this.getProgress(),this.curr=this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&(this.isDownloadInterrupted=!0,b.info("根据上次数据继续断点下载")),this.bar=new i.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},i.Presets.legacy),this.bar.start(this.total,this.curr))}async getProgress(){let e=[];try{const r=await t.readFile(this.progressFilePath,{encoding:"utf8"});e=JSON.parse(r)}catch(r){r&&"ENOENT"===r.code&&await t.writeFile(this.progressFilePath,JSON.stringify(e),{encoding:"utf8"})}return e}async updateProgress(e,r){this.curr!==this.total&&(this.curr=this.curr+1,r&&(this.progressInfo.push(e),await t.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"})),this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log(""))))}pause(){this.bar&&this.bar.stop()}continue(){this.clearLine(2),this.bar?.start(this.total,this.curr)}clearLine(t){if(!(t<=0)&&"function"==typeof process?.stderr?.cursorTo){process.stderr.cursorTo(0);for(let e=0;e<t;e++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}function L({browser:t="chrome",os:e="mac os",device:r="desktop"}){r=r.toLowerCase(),t=t.toLowerCase(),e=e.toLowerCase();let o=s(r,t,e);if("chrome"===t)for(;o.includes("Chrome-Lighthouse")||o.includes("Gener8")||o.includes("HeadlessChrome")||o.includes("SMTBot");)o=s(r,t,e);if("safari"===t)for(;o.includes("Applebot");)o=s(r,t,e);return o}class v{constructor(t){this.summaryInfo=t}async genFile(){const{bookName:e,bookDesc:r,bookPath:o,uuidMap:s}=this.summaryInfo;let n=`# ${e}\n\n`;r&&(n+=`> ${r}\n\n`);let i=n;const a=[];s.forEach((t=>{const e=t.toc,r=e.parent_uuid,o=this.findTree(a,r),s={text:e.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:e.uuid,level:1,type:"link"},n=e.type.toLocaleLowerCase();n===m.TITLE||""!==e.child_uuid?(s.type="title","boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)),n===m.DOC&&(s.link=t.path)):(s.type="link",s.link=n===m.LINK?t.toc.url:t.path,"boolean"!=typeof o?(Array.isArray(o.children)||(o.children=[]),s.level=o.level+1,o.children.push(s)):(s.level=1,a.push(s)))}));i+=this.genSummaryContent(a,"");try{await t.writeFile(`${o}/index.md`,i)}catch(t){b.error("Generate Summary Error")}return i}genSummaryContent(t,e){for(const r of t){if(r.type===m.TITLE)if(r.link){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`\n${"".padStart(r.level+1,"#")} [${r.text}](${t})\n\n`}else e+=`\n${"".padStart(r.level+1,"#")} ${r.text}\n\n`;else if(r.type===m.LINK){const t=r.link?r.link.replace(/\s/g,"%20"):r.link;e+=`- [${r.text}](${t})\n`}Array.isArray(r.children)&&(e+=this.genSummaryContent(r.children,""))}return e}findIdItem(t,e){if(t.id===e)return t;if(t.children){const r=this.findTree(t.children,e);if(r)return r}return!1}findTree(t,e){if(!e)return!1;for(const r of t){const t=this.findIdItem(r,e);if(t)return t}return!1}}function E(t){const{key:e=$,token:r}=t,o={"user-agent":L({browser:"chrome",device:"desktop"})};return r&&(o.cookie=`${e}=${r};`),o}function S(t){return{headers:E(t),beforeRedirect:e=>{e.headers={...e?.headers||{},...E(t)}}}}const T=(t,e=!0)=>{const{articleUrl:r,bookId:o,token:s,key:n,host:i=k}=t;let c=`${i}/api/docs/${r}`;const l={book_id:String(o),merge_dynamic_data:String(!1)};e&&(l.mode="markdown");const u=new URLSearchParams(l).toString();return c=`${c}?${u}`,a.get(c,S({token:s,key:n})).then((({data:t,status:e})=>({apiUrl:c,httpStatus:e,response:t})))};function P(t){if(!t)return"";return t.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}const D=t=>{if(!t)return"";const e=d.inflate(t,{to:"string"}),r=JSON.parse(e);let o="";return r.forEach((t=>{const e=`## ${t.name}\n`,r=function(t){let e=Object.keys(t);e=e.filter((e=>Object.keys(t[e]).some((r=>t?.[e]?.[r]?.v))));let r=[];e.forEach((e=>{const o=t[e];o&&(r=r.concat(Object.keys(o)))}));const o=Math.max(...e.map((t=>Number(t)))),s=Math.max(...r.map((t=>Number(t))));if(o<0||s<0)return"";let n="";const i="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let a=Array(s+1).fill(" ").map(((t,e)=>{const r=e%i.length;return i[r]})).join(" | ");a=`| |${a}|`;let c=Array(s+2).fill("---").join(" |");c=`|${c}|`,n=`${a}\n${c}\n`;for(let e=0;e<o+1;e++){const r=[];for(let o=0;o<s+1;o++){const s=t?.[e]?.[o]?.v||null;s&&"string"==typeof s?r.push(s):s&&"object"==typeof s?"image"===s?.class&&s?.src?r.push(``):"checkbox"===s?.class?r.push(s?.value?"[x] ":"[ ] "):"link"===s?.class?r.push(`[${s?.text}](${s?.url})`):"select"===s?.class&&r.push(s?.value?.join(",")):r.push(null)}n=`${n}${`| ${e+1} | ${r.join(" | ")}|`}\n`}return n}(t.data);o=`${o}\n${e}\n${r}`})),o};function U(t,e=[]){return function(t,e){try{const{host:r,pathname:o}=new URL(t);return!e.includes(r)&&Boolean(o)}catch(t){return!1}}(t,e)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(t)}&sign=${function(t){const e=f.createHash("sha256");return e.update(`${y}${t}`),e.digest("hex")}(t)}`:t}async function x(t){const{articleInfo:r,progressBar:o,options:s}=t,{token:n,key:i}=s,{bookId:a,itemUrl:h,savePath:d,saveFilePath:f,uuid:m,articleUrl:$,articleTitle:k,ignoreImg:y,host:b,imageServiceDomains:I}=r,L={articleUrl:h,bookId:a,token:n,host:b,key:i},{httpStatus:v,apiUrl:E,response:S}=await T(L),P=S?.data?.type?.toLocaleLowerCase();let x="";if(P===g.SHEET){const{response:t}=await T(L,!1);try{const e=t?.data?.content,r=e?JSON.parse(e):{},o=r?.sheet;x=o?D(o):""}catch(t){const e=w.get(P);throw new Error(`download article Error: “${e}”解析错误 ${t}`)}}else{if([g.BOARD,g.TABLE].includes(P)){const t=w.get(P);throw new Error(`download article Error: 暂不支持“${t}”的文档`)}if("string"!=typeof S?.data?.sourcecode)throw new Error(`download article Error: ${E}, http status ${v}`);x=S.data.sourcecode,x=function(t){const e=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,r=t.match(e);let o=t;const s=t;try{r?.forEach((t=>{e.lastIndex=0;const r=e.exec(t)?.[2]??"",{pathname:s,search:n}=new URL(r);if(!s.endsWith(".svg")&&n){const e=decodeURIComponent(n);o=o.replace(t,e.slice(1))}}))}catch(t){return s}return o}(x)}const N=function(t){if(!t)return[];const e=/!\[(.*?)\]\((.*?)\)/gm;let r=Array.from(t.match(e)||[]);return r=r.map((t=>(t=t.replace(e,"$2"),/^http.*/g.test(t)?t:""))).filter((t=>Boolean(t))),r}(x);if(N.length&&!y){const t=await T(L,!1);x=function(t,e,r){if(!r)return e;const o=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,s=[];let n,i=!0;for(;i||Boolean(n);)if(i=!1,n=o.exec(r),n?.[1])try{const t=decodeURIComponent(n[1]),e=JSON.parse(t);s.push(e?.src||"")}catch(t){s.push("")}const a=new Map;return t.forEach((t=>{const{origin:r,pathname:o}=new URL(t),n=`${r}${o}`,i=s.find(((t,e)=>{const r=new RegExp(`${n}.*?`).test(t);return r&&s.splice(e,1),r}));if(i){const r=new RegExp(t,"g"),o=a.get(t)||0;let s=0;e=e.replace(r,(t=>{let e=t;return s===o&&(e=i),s+=1,e})),a.set(t,o+1)}})),e}(N,x,t.response?.data?.content??"")}const O={toc:s.toc,articleTitle:k,articleUrl:$};if(N.length&&!y){let t;o.pause(),console.log(""),"test"!==c.NODE_ENV&&(t=u({text:`下载 "${k}" 的图片中...`,stream:l}),t.start());let r=[],s=x;try{const t=await p.run(x,{dist:d,imgDir:`./img/${m}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:$||"",transform:t=>U(t,I)});r=t.errorInfo,s=t.data}catch(t){r=[t]}x=s;const n=()=>{t&&t.stop(),o.continue()};if(r.length>0){const t=r[0];let o="图片下载失败(失败的以远程链接保存): ";throw o=t.url?`${o}${t.error?.message} ${t.url.slice(0,20)}...`:`${o}${t.message}`,await e(f,C(x,O)),n(),new Error(`${o}`)}n()}try{return await e(f,C(x,O)),!0}catch(t){throw new Error(`download article Error ${$}: ${t.message}`)}}function C(t,e){const{articleTitle:r,articleUrl:o,toc:s}=e;let n=t;n=n.replace(/<br(\s?)\/>/gm,"\n"),n=n.replace(/<a.*?>(\s*?)<\/a>/gm,"");const i=r?`# ${r}\n\n`:"";let a=s?h(n).content:"";a&&(a=`${a}\n\n---\n\n`);return n=`${i}${a}${n}${o?`\n\n> 原文: <${o}>`:""}`,n}async function N(t,e){const{bookId:s,tocList:n,bookName:i,bookDesc:c,bookSlug:l,host:u,imageServiceDomains:p}=await((t,e)=>{const r=/decodeURIComponent\("(.+)"\)\);/m;return a.get(t,S(e)).then((({data:t="",status:e})=>200===e?t:"")).then((t=>{const e=r.exec(t)??"";if(!e[1])return{};const o=JSON.parse(decodeURIComponent(e[1]));return o.book?{bookId:o.book.id,bookSlug:o.book.slug,tocList:o.book.toc||[],bookName:o.book.name||"",bookDesc:o.book.description||"",host:o.space?.host||k,imageServiceDomains:o.imageServiceDomains||[]}:{}})).catch((()=>({})))})(t,{token:e.token,key:e.key});if(!s)throw new Error("No found book id");if(!n||0===n.length)throw new Error("No found toc list");const h=o.resolve(e.distDir,i?P(i):String(s));await r(h,{recursive:!0});const d=n.length,f=new I(h,d);if(await f.init(),f.curr===d)return f.bar&&f.bar.stop(),void b.info(`√ 已完成: ${h}`);const w=new Map;f.isDownloadInterrupted&&f.progressInfo.forEach((t=>{w.set(t.toc.uuid,t)}));const $=t.replace(new RegExp(`(.*?/${l}).*`),"$1");await async function(t){const{articleUrlPrefix:e,total:s,uuidMap:n,tocList:i,bookPath:a,bookId:c,progressBar:l,host:u,options:p,imageServiceDomains:h=[]}=t;let d=0,f=0,w=0;const $=[],k=[];for(let t=0;t<s;t++){const e=i[t];if("string"!=typeof e.type)continue;if(n.get(e.uuid))continue;const o=e.type.toLocaleLowerCase();if(o===m.TITLE||""!==e.child_uuid||o===m.LINK){let t=e;const s=[],i=[];for(;t;)s.unshift(P(t.title)),i.unshift(t.uuid),t=n.get(t.parent_uuid)?n.get(t.parent_uuid).toc:void 0;const c={path:s.map(P).join("/"),pathTitleList:s,pathIdList:i,toc:e};o===m.LINK?(w+=1,k.push(c)):await r(`${a}/${s.map(P).join("/")}`,{recursive:!0}),n.set(e.uuid,c),o===g.DOC?await y(e):await l.updateProgress(c,o!==m.LINK)}else e.url&&await y(e)}async function y(t){f+=1;let r={path:"",pathTitleList:[],pathIdList:[]};const s=t.type.toLocaleLowerCase();n.get(t.parent_uuid)&&(r=n.get(t.parent_uuid));const i=P(t.title),m=[...r.pathTitleList,i],w=[...r.pathIdList,t.uuid];let k=[...r.pathTitleList,`${i}.md`].map(P).join("/"),y=r.pathTitleList.map(P).join("/");s===g.DOC&&t.child_uuid&&(k=[...r.pathTitleList,i,"index.md"].map(P).join("/"),y=m.map(P).join("/"));const b={path:k,savePath:y,pathTitleList:m,pathIdList:w,toc:t};let I=!0;const L=`${e}/${t.url}`;try{const e={bookId:c,itemUrl:t.url,savePath:o.resolve(a,b.savePath),saveFilePath:o.resolve(a,b.path),uuid:t.uuid,articleUrl:L,articleTitle:t.title,ignoreImg:p.ignoreImg,host:u,imageServiceDomains:h};await x({articleInfo:e,progressBar:l,options:p})}catch(t){I=!1,d+=1,$.push({articleUrl:L,errItem:b,errMsg:t.message,err:t})}n.set(t.uuid,b),await l.updateProgress(b,I)}if(w>0){b.warn("该知识库存在以下外链文章");for(const t of k)b.warn(`———— ✕ ${t.path} ${t.toc.url}`)}if(d>0){b.error(`本次执行总数${f}篇，✕ 失败${d}篇`);for(const t of $)b.error(`《${t.errItem.path}》: ${t.articleUrl}`),b.error(`———— ✕ ${t.errMsg}`);b.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}}({articleUrlPrefix:$,total:d,uuidMap:w,tocList:n,bookPath:h,bookId:s,progressBar:f,host:u,options:e,imageServiceDomains:p});const y=new v({bookPath:h,bookName:i,bookDesc:c,uuidMap:w});await y.genFile(),b.info(`√ 生成目录 ${o.resolve(h,"index.md")}`),f.curr===d&&b.info(`√ 已完成: ${h}`)}export{b as l,N as m};
|