yuque-dl 1.0.85 → 1.0.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/es/cli.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{_ as e,l as o,m as n,d as t,a as i,b as r}from"./index-
|
|
1
|
+
import{_ as e,l as o,m as n,d as t,a as i,b as r}from"./index-grIDsf7R.js";import{existsSync as s,readFileSync as d}from"node:fs";import{cac as c}from"cac";import l from"semver";import"rand-user-agent";import{mkdir as p,copyFile as a,readFile as m,writeFile as u,access as f,readdir as v,stat as h}from"node:fs/promises";import"cli-progress";import{resolve as y,join as x,sep as g,dirname as k}from"node:path";import{fileURLToPath as w}from"node:url";import{createServer as $}from"vitepress";import"log4js";import"node:crypto";import"node:process";import"axios";import"mdast-util-from-markdown";import"mdast-util-to-markdown";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:stream";import"node:util";let b;function M(o,n){return e(this,void 0,void 0,(function*(){const t=y(o);if(!(yield E(t)))throw new Error("server root not found");const i=x(t,"/.vitepress");s(i)&&!n.force||(yield function(o){return e(this,void 0,void 0,(function*(){const n=o.split(g).filter(Boolean).at(-1)||"yuque-dl",t=x(o,"/.vitepress");yield p(t,{recursive:!0});const i=k(w(import.meta.url)),r=x(i,"../../server-lib/bundle.js"),s=x(t,"bundle.mjs");yield a(r,s);const d=x(t,"config.mjs"),c=yield function(o){return e(this,void 0,void 0,(function*(){const e=[],n=[".vitepress","img","index.md","progress.json","attachments"];let t=yield v(o);t=t.filter((e=>!n.includes(e)));for(const n of t){const t=x(o,n);if((yield h(t)).isDirectory()){const i=yield E(x(t,"index.md")),r={text:I(n),collapsed:!0,items:yield j(o,n)};i&&(r.link=`/${n}/`),e.push(r)}else e.push({text:I(n.slice(0,n.lastIndexOf("."))),link:`/${n}`})}return e}))}(o);N((yield m(y(o,"index.md"),"utf8")).split("\n"),c),_(c);const l=`\n import {fixHtmlTags} from './bundle.mjs'\n export default {\n title: "${n}",\n themeConfig: {\n search: {\n provider: 'local'\n },\n sidebar: ${JSON.stringify(c)}\n },\n vite: {\n optimizeDeps: {\n include: []\n }\n },\n markdown: {\n html: true,\n breaks: true,\n config(md) {\n // 包装原始 render 方法，捕获解析异常\n const originalRender = md.render.bind(md);\n md.render = (src, env) => {\n try {\n let newMd = fixHtmlTags(originalRender(src, env))\n newMd = newMd.replace('<html><head></head><body>', '')\n newMd = newMd.replace('</body></html>', '')\n return newMd\n } catch (error) {\n console.error("Markdown/HTML parsing error:", error);\n return src\n }\n }\n }\n }\n }\n `;yield u(d,l)}))}(t));const r=()=>e(this,void 0,void 0,(function*(){const t=yield $(o,{host:n.host,port:n.port},(()=>e(this,void 0,void 0,(function*(){return b||(b=(()=>e(this,void 0,void 0,(function*(){yield t.close(),yield r()})))().finally((()=>{b=void 0}))),b}))));yield t.listen(),t.printUrls()}));yield r().catch((e=>{console.error(e),process.exit(1)}))}))}function E(o){return e(this,void 0,void 0,(function*(){try{return yield f(o),!0}catch(e){if("ENOENT"===e.code)return!1;throw e}}))}function N(e,o){o.map((o=>{"items"in o&&N(e,o.items);const n=e.findIndex((e=>e.includes(`[${o.text}]`)||e.includes(`# ${o.text}`)));o.index=n}))}function _(e){e.length<=1||(e.forEach((e=>{"items"in e&&_(e.items)})),e.sort(((e,o)=>e.index&&o.index?e.index>o.index?1:-1:0)))}function j(o,...n){return e(this,void 0,void 0,(function*(){const e=yield v(x(o,...n)),t=[];for(const i of e){const e=x(o,...n,i);if((yield h(e)).isDirectory()){const r=yield j(x(o),...n,i);if(r.length>0){const o={text:I(i),items:r,collapsed:false};(yield E(x(e,"index.md")))&&(o.link="/"+[...n,i].map(decodeURIComponent).join("/")+"/"),t.push(o)}}else{if("index.md"===i||/^-.*\.(md|MD)$/.test(i)||!i.endsWith(".md"))continue;const e=i.replace(/\.md$/,""),o={text:I(e),link:"/"+[...n.map(decodeURIComponent),`${encodeURI(e)}.html`].join("/")};t.push(o)}}return t}))}function I(e){return e.replace(/_.*?$/g,"")}const O=c("yuque-dl"),{version:U,engines:S}=JSON.parse(d(new URL("../../package.json",import.meta.url)).toString());function T(e){return e.option("-d, --distDir <dir>","下载的目录{CUSTOM_NEW_LINE(eg: -d download)}",{default:"download"}).option("-i, --ignoreImg","忽略图片不下载",{default:!1}).option("--ignoreAttachments [fileExtension]","忽略附件, 可选带上忽略的附件文件后缀(多种后缀逗号分割){CUSTOM_NEW_LINE(eg: --ignoreAttachments mp4,pdf // 忽略后缀名mp4,pdf的附件)}{CUSTOM_NEW_LINE(eg: --ignoreAttachments // 忽略所有附件)}",{default:!1}).option("-k, --key <key>",'语雀的cookie key， 默认是 "_yuque_session"， 在某些企业版本中 key 不一样').option("-t, --token <token>","语雀的cookie key 对应的值").option("-p, --password <password>","公开密码访问的知识库/文档密码").option("--toc","是否输出文档toc目录",{default:!1}).option("--incremental","开启增量下载[初次下载加不加该参数没区别]",{default:!1}).option("--convertMarkdownVideoLinks","转化markdown视频链接为video标签",{default:!1}).option("--hideFooter","是否禁用页脚显示[更新时间、原文地址...]",{default:!1})}!function(){const e=S.node;l.satisfies(process.version,e)||(o.error(`✕ nodejs 版本需 ${e}, 当前版本为 ${process.version}`),process.exit(1))}();T(O.command("<url>","语雀知识库url")).action(((t,i)=>e(void 0,void 0,void 0,(function*(){try{yield n(t,i),process.exit(0)}catch(e){o.error(e.message||"unknown exception"),process.exit(1)}}))));T(O.command("doc <...urls>","下载单个或多个文档(必须指定-t参数)")).action(((n,i)=>e(void 0,void 0,void 0,(function*(){try{yield t(n,i),process.exit(0)}catch(e){o.error(e.message||"unknown exception"),process.exit(1)}}))));T(O.command("batch <...urls>","批量下载多个知识库")).action(((n,t)=>e(void 0,void 0,void 0,(function*(){try{yield i(n,t),process.exit(0)}catch(e){o.error(e.message||"unknown exception"),process.exit(1)}}))));T(O.command("user","下载当前账号的所有知识库")).action((n=>e(void 0,void 0,void 0,(function*(){try{yield r(n),process.exit(0)}catch(e){o.error(e.message||"unknown exception"),process.exit(1)}})))),O.command("server <serverPath>","启动web服务").option("-p, --port <port>"," --port 1234",{default:5173}).option("--host [host]"," --host 0.0.0.0 或 --host",{default:"localhost"}).option("--force","强制重新生成.vitepress",{default:!1}).action(((n,t)=>e(void 0,void 0,void 0,(function*(){try{yield M(n,t)}catch(e){o.error(e.message||"unknown exception"),process.exit(1)}})))),O.help((e=>{var o;const n=e.find((e=>"Options"===e.title));if(n){let e=(null==n?void 0:n.body)||"";e=e.replace(/^(\s*-h, --help\s*)(.*)$/gm,((e,o)=>`${o}显示帮助信息`)),e=e.replace(/^(\s*-v, --version\s*)(.*)$/gm,((e,o)=>`${o}显示当前版本`)),e=e.replace(/\(default:/gm,(()=>"(默认值:"));const t=(null===(o=e.match(/^(\s*-h, --help\s*)/gm))||void 0===o?void 0:o[0].length)||0;e=e.replace(/\{CUSTOM_NEW_LINE\((.*?)\)\}/g,((e,o)=>`\n${"".padStart(t," ")} └─ ${o}`)),n.body=e}})),O.version(U);try{O.parse()}catch(e){o.error(e.message||"unknown exception"),process.exit(1)}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import e,{writeFile as t,mkdir as o}from"node:fs/promises";import n from"node:path";import r from"rand-user-agent";import i from"log4js";import s from"cli-progress";import a,{publicEncrypt as l,constants as c}from"node:crypto";import{env as d,stdout as u}from"node:process";import h from"axios";import{fromMarkdown as f}from"mdast-util-from-markdown";import{toMarkdown as p}from"mdast-util-to-markdown";import v from"ora";import m from"pull-md-img";import g from"markdown-toc";import y from"pako";import{createWriteStream as k,mkdirSync as w}from"node:fs";import*as $ from"node:stream";import{promisify as I}from"node:util";function b(e,t,o,n){return new(o||(o=Promise))((function(r,i){function s(e){try{l(n.next(e))}catch(e){i(e)}}function a(e){try{l(n.throw(e))}catch(e){i(e)}}function l(e){var t;e.done?r(e.value):(t=e.value,t instanceof o?t:new o((function(e){e(t)}))).then(s,a)}l((n=n.apply(e,t||[])).next())}))}var D,E;"function"==typeof SuppressedError&&SuppressedError,function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(D||(D={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(E||(E={}));const P=new Map([[E.BOARD,"画板类型"],[E.TABLE,"数据表类型"],[E.SHEET,"表格类型"],[E.DOC,"文档类型"]]),U="_yuque_session",S="https://www.yuque.com",N="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",L="\n-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCfwyOyncSrUTmkaUPsXT6UUdXx\nTQ6a0wgPShvebfwq8XeNj575bUlXxVa/ExIn4nOUwx6iR7vJ2fvz5Ls750D051S7\nq70sevcmc8SsBNoaMQtyF/gETPBSsyWv3ccBJFrzZ5hxFdlVUfg6tXARtEI8rbIH\nsu6TBkVjk+n1Pw/ihQIDAQAB\n-----END PUBLIC KEY-----",T={Book:"verified_books",Doc:"verified_docs"},O=(i.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),i.getLogger("yuque-dl"));class A{constructor(e,t,o=!1,n=!1){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t,this.incremental=o,this.disableProgressJSON=n}init(){return b(this,void 0,void 0,(function*(){this.progressInfo=this.disableProgressJSON?[]:yield this.getProgress(),this.curr=this.incremental?0:this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&!this.incremental&&(this.isDownloadInterrupted=!0,O.info("根据上次数据继续断点下载")),this.bar=new s.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},s.Presets.legacy),this.bar.start(this.total,this.curr))}))}getProgress(){return b(this,void 0,void 0,(function*(){let t=[];try{const o=yield e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(o)}catch(o){o&&"ENOENT"===o.code&&(yield e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"}))}return t}))}updateProgress(t,o){return b(this,void 0,void 0,(function*(){if(this.curr!==this.total){if(this.curr=this.curr+1,o){const o=t.toc.uuid,n=this.progressInfo.find((e=>e.toc.uuid===o));n?Object.assign(n,t):this.progressInfo.push(t),this.disableProgressJSON||(yield e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"}))}this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}}))}pause(){this.bar&&this.bar.stop()}continue(){var e;this.clearLine(2),null===(e=this.bar)||void 0===e||e.start(this.total,this.curr)}clearLine(e){var t;if(!(e<=0)&&"function"==typeof(null===(t=null===process||void 0===process?void 0:process.stderr)||void 0===t?void 0:t.cursorTo)){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}const x={bold:[1,22],italic:[3,23],underline:[4,24],inverse:[7,27],white:[37,39],grey:[90,39],black:[90,39],blue:[34,39],cyan:[36,39],green:[32,39],magenta:[35,39],red:[91,39],yellow:[33,39]};function _(e,t){return function(e){return e?`[${x[e][0]}m`:""}(t)+e+function(e){return e?`[${x[e][1]}m`:""}(t)}function C({browser:e="chrome",os:t="mac os",device:o="desktop"}){o=o.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let n=r(o,e,t);if("chrome"===e)for(;n.includes("Chrome-Lighthouse")||n.includes("Gener8")||n.includes("HeadlessChrome")||n.includes("SMTBot");)n=r(o,e,t);if("safari"===e)for(;n.includes("Applebot");)n=r(o,e,t);return n}function R(e){if("function"==typeof URL.canParse)return URL.canParse(e);try{return new URL(e),!0}catch(e){return!1}}function F(e){return e.toString().padStart(2,"0")}function j(e){const t=new Date(e);if(isNaN(t.getTime()))return"";const o=t.getFullYear(),n=t.getMonth()+1,r=t.getDate(),i=t.getHours(),s=t.getMinutes(),a=t.getSeconds();return`${o}-${F(n)}-${F(r)} ${F(i)}:${F(s)}:${F(a)}`}function B(e){return e instanceof Date&&!isNaN(e.getTime())}class M{constructor(e){this.summaryInfo=e}genFile(){return b(this,void 0,void 0,(function*(){const{bookName:t,bookDesc:o,bookPath:n,uuidMap:r}=this.summaryInfo;let i=`# ${t}\n\n`;o&&(i+=`> ${o}\n\n`);let s=i;const a=[];r.forEach((e=>{const t=e.toc,o=t.parent_uuid,n=this.findTree(a,o),r={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:"link"},i=t.type.toLocaleLowerCase();i===D.TITLE||""!==t.child_uuid?(r.type="title","boolean"!=typeof n?(Array.isArray(n.children)||(n.children=[]),r.level=n.level+1,n.children.push(r)):(r.level=1,a.push(r)),i===D.DOC&&(r.link=e.path)):(r.type="link",r.link=i===D.LINK?e.toc.url:e.path,"boolean"!=typeof n?(Array.isArray(n.children)||(n.children=[]),r.level=n.level+1,n.children.push(r)):(r.level=1,a.push(r)))}));s+=this.genSummaryContent(a,"");try{yield e.writeFile(`${n}/index.md`,s)}catch(e){O.error("Generate Summary Error")}return s}))}genSummaryContent(e,t){for(const o of e){if(o.type===D.TITLE)if(o.link){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`\n${"".padStart(o.level+1,"#")} [${o.text}](${e})\n\n`}else t+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if(o.type===D.LINK){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`${1===o.level?"\n##":"-"} [${o.text}](${e})\n`}Array.isArray(o.children)&&(t+=this.genSummaryContent(o.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const o=this.findTree(e.children,t);if(o)return o}return!1}findTree(e,t){if(!t)return!1;for(const o of e){const e=this.findIdItem(o,t);if(e)return e}return!1}}function J(e){const{key:t=U,token:o}=e,n={"user-agent":C({browser:"chrome",device:"desktop"})};return o&&(n.cookie=`${t}=${o};`),n}function q(e){const t={headers:J(e),beforeRedirect:t=>{t.headers=Object.assign(Object.assign({},(null==t?void 0:t.headers)||{}),J(e))}};return"test"===d.NODE_ENV&&(t.proxy=!1),t}function V(e){var t;const o=null!==(t=/decodeURIComponent\("(.+)"\)\);/m.exec(e))&&void 0!==t?t:"";if(o[1])return JSON.parse(decodeURIComponent(o[1]))}function K(e,t=Date.now()){const o=Date.now()-t;return(`${Date.now()-o}:${e}`.match(/.{1,100}/g)||[]).map((e=>l({key:L,padding:c.RSA_PKCS1_PADDING},Buffer.from(e)).toString("base64"))).join(":")}function Q(e,t,o){return b(this,void 0,void 0,(function*(){const{html:n}=yield H(e,o),r=V(n),{targetType:i,needVerifyTargetId:s}=(null==r?void 0:r.matchCondition)||{};if(!i||!s)return!1;const a=T[i];if(!a)return!1;const l=`${new URL(e).origin}/api/${"Book"===i?"books":"docs"}/${s}/verify`,c=q(o);c.headers||(c.headers={}),c.headers.referer=e;const{headers:d}=yield h.put(l,{password:K(t,null==r?void 0:r.timestamp)},c),u=Reflect.get(d,"set-cookie");if(!u)return!1;if(Array.isArray(u)&&0===u.length)return!1;return{key:a,token:Reflect.get(function(e){const t={};return(Array.isArray(e)?e:e?[e]:[]).forEach((e=>{const o=e.split(";")[0],[n,...r]=o.split("=");n&&r.length>0&&(t[n]=r.join("="))})),t}(u),a)}}))}function H(e,t){return b(this,void 0,void 0,(function*(){const{data:o="",status:n,headers:r}=yield h.get(e,q(t));return{html:200===n?o:"",resHeaders:r,status:n}}))}function G(e){var t;const o=null!==(t=null==e?void 0:e.message)&&void 0!==t?t:"";if(!o)return new Error("unknown error");return["getaddrinfo ENOTFOUND","read ECONNRESET","Client network socket disconnected before secure TLS connection was established"].some((e=>o.startsWith(e)))?new Error("Please check the network connection or proxy settings"):new Error(o)}const W=(e,t)=>b(void 0,void 0,void 0,(function*(){var o;try{const{html:n}=yield H(e,t),r=V(n);return(null==r?void 0:r.book)?{bookId:r.book.id,bookSlug:r.book.slug,tocList:r.book.toc||[],bookName:r.book.name||"",bookDesc:r.book.description||"",host:(null===(o=r.space)||void 0===o?void 0:o.host)||S,imageServiceDomains:r.imageServiceDomains||[]}:{}}catch(e){throw G(e)}})),X=(e,t=!0)=>{const{articleUrl:o,bookId:n,token:r,key:i,host:s=S}=e;let a=`${s}/api/docs/${o}`;const l={book_id:String(n),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const c=new URLSearchParams(l).toString();return a=`${a}?${c}`,h.get(a,q({token:r,key:i})).then((({data:e,status:t})=>({apiUrl:a,httpStatus:t,response:e})))},z=(e,t)=>b(void 0,void 0,void 0,(function*(){var o,n,r;try{const{html:i}=yield H(e,t),s=V(i);return(null==s?void 0:s.doc)?{docId:s.doc.id,docSlug:s.doc.slug,docTitle:s.doc.title||"",bookId:s.doc.book_id,bookSlug:(null===(o=s.book)||void 0===o?void 0:o.slug)||"",bookName:(null===(n=s.book)||void 0===n?void 0:n.name)||"",host:(null===(r=s.space)||void 0===r?void 0:r.host)||S,imageServiceDomains:s.imageServiceDomains||[]}:{}}catch(e){throw G(e)}}));function Y(e){return f(e)}function Z(e){return p(e)}function ee(e,t,o=[]){t(e,o),Array.isArray(e.children)&&(o.push("children"),e.children.forEach(((e,n)=>{ee(e,t,[...o,String(n)])})))}function te(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}function oe(e,t){let o=e;const n=Y(e),r=function(e){const t=[];return ee(e,(function(e,o){"inlineCode"===e.type&&t.push({node:e,keyChain:o})})),t}(n);return 0===r.length?e:(r.forEach((e=>{const o=e.node,n=(r=o.value,/<([a-z][\s\S]*?)>/i.test(r));var r;const i=function(e){return/(~~|\*\*|_)/g.test(e)}(o.value);if(!n&&!i)return;const s=o.value.replace(/</g,"<").replace(/>/g,">").replace(/"/g,""");t.includes(s)||(o.type="html",o.value=`<code>${o.value}</code>`)})),o=Z(n),o)}const ne=e=>{if(!e)return"";const t=y.inflate(e,{to:"string"}),o=JSON.parse(t);let n="";return o.forEach((e=>{const t=`## ${e.name}\n`,o=function(e){var t,o,n;let r=Object.keys(e);r=r.filter((t=>Object.keys(e[t]).some((o=>{var n,r;return null===(r=null===(n=null==e?void 0:e[t])||void 0===n?void 0:n[o])||void 0===r?void 0:r.v}))));let i=[];r.forEach((t=>{const o=e[t];o&&(i=i.concat(Object.keys(o)))}));const s=Math.max(...r.map((e=>Number(e)))),a=Math.max(...i.map((e=>Number(e))));if(s<0||a<0)return"";let l="";const c="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let d=Array(a+1).fill(" ").map(((e,t)=>{const o=t%c.length;return c[o]})).join(" | ");d=`| |${d}|`;let u=Array(a+2).fill("---").join(" |");u=`|${u}|`,l=`${d}\n${u}\n`;for(let r=0;r<s+1;r++){const i=[];for(let s=0;s<a+1;s++){const a=(null===(o=null===(t=null==e?void 0:e[r])||void 0===t?void 0:t[s])||void 0===o?void 0:o.v)||null;a&&"string"==typeof a?i.push(a):a&&"object"==typeof a?"image"===(null==a?void 0:a.class)&&(null==a?void 0:a.src)?i.push(``):"checkbox"===(null==a?void 0:a.class)?i.push((null==a?void 0:a.value)?"[x] ":"[ ] "):"link"===(null==a?void 0:a.class)?i.push(`[${null==a?void 0:a.text}](${null==a?void 0:a.url})`):"select"===(null==a?void 0:a.class)&&i.push(null===(n=null==a?void 0:a.value)||void 0===n?void 0:n.join(",")):i.push(null)}l=`${l}${`| ${r+1} | ${i.join(" | ")}|`}\n`}return l}(e.data);n=`${n}\n${t}\n${o}`})),n};function re(e,t=[]){return function(e,t){try{const{host:o,pathname:n}=new URL(e);return!t.includes(o)&&Boolean(n)}catch(e){return!1}}(e,t)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(e)}&sign=${function(e){const t=a.createHash("sha256");return t.update(`${N}${e}`),t.digest("hex")}(e)}`:e}const ie=I($.finished);function se(e){return b(this,void 0,void 0,(function*(){const{fileUrl:t,savePath:o,token:n,key:r,fileName:i}=e;return h.get(t,Object.assign(Object.assign({},q({token:n,key:r})),{responseType:"stream"})).then((e=>b(this,void 0,void 0,(function*(){var n,r,s;if(null===(r=null===(n=e.request)||void 0===n?void 0:n.path)||void 0===r?void 0:r.startsWith("/login"))throw new Error(`"${i}" need token`);if(200===e.status){const n=k(o);return null===(s=e.data)||void 0===s||s.pipe(n),ie(n).then((()=>({fileUrl:t,savePath:o})))}throw new Error(`response status ${e.status}`)}))))}))}const ae=/\[(.*?)\]\((.*?)\)/g,le=/\[(.*?)\]\((.*?\.yuque\.com\/attachments.*?)\)/;function ce(e){return b(this,void 0,void 0,(function*(){const{mdData:t,savePath:o,attachmentsDir:r,articleTitle:i,token:s,key:a,ignoreAttachments:l}=e,c=(t.match(ae)||[]).filter((e=>le.test(e)));if(0===c.length)return{mdData:t};const h=n.resolve(o,r);let f=c.map((e=>function(e,t){const[,o,r]=le.exec(e)||[];if(!r)return!1;const i=o||r.split("/").at(-1);if(!i)return!1;const s=n.join(t,i);return{fileName:i,url:r,rawMd:e,currentFilePath:s}}(e,h))).filter((e=>!1!==e));if("string"==typeof l){const e=l.split(",");f=f.filter((t=>{const o=t.url.lastIndexOf(".");if(-1===o)return!0;const n=t.url.slice(o+1);return!e.find((e=>e===n))}))}if(0===f.length)return{mdData:t};const p=v({text:`下载 "${i}" 的附件中...`,stream:u});"test"!==d.NODE_ENV&&p.start(),w(h,{recursive:!0});const m=f.map((e=>se({fileUrl:e.url,savePath:e.currentFilePath,token:s,key:a,fileName:e.fileName}))),g=yield Promise.all(m).finally((function(){p&&p.stop()}));let y=t;return g.forEach((e=>{const t=f.find((t=>t.url===e.fileUrl));if(t){const e=`[附件: ${t.fileName}](${r}/${t.fileName})`;y=y.replace(t.rawMd,e)}})),{mdData:y}}))}function de(e){return b(this,void 0,void 0,(function*(){const{mdData:t,htmlData:o,savePath:r,attachmentsDir:i,articleTitle:s,token:a,key:l,ignoreAttachments:c}=e,h=Y(t),f=function(e){const t=[];return ee(e,(function(e,o){"link"===e.type&&t.push({node:e,keyChain:o})})),t}(h);let p=f.filter((e=>/_lake_card.*?videoId/.test(e.node.url))),m=ve(o,"audio"),g=ve(o,"video");if(0===p.length&&0===m.length&&0===g.length)return{mdData:t};if("string"==typeof c){const e=c.split(",");if(p=p.filter((t=>{const o=t.node.url,n=o.lastIndexOf(".");if(-1===n)return!0;const r=o.slice(n+1);return!e.find((e=>e===r))})),m=m.filter((t=>{const o=t.audioId.lastIndexOf(".");if(-1===o)return!0;const n=t.audioId.slice(o+1);return!e.find((e=>e===n))})),g=g.filter((t=>{const o=t.videoId.lastIndexOf(".");if(-1===o)return!0;const n=t.videoId.slice(o+1);return!e.find((e=>e===n))})),0===p.length&&0===m.length&&0===g.length)return{mdData:t}}const y=v({text:`下载 "${s}" 的音视频中...`,stream:u});"test"!==d.NODE_ENV&&y.start();const k=n.resolve(r,i);w(k,{recursive:!0});let $=t;try{if(p.length>0){const t=yield function(e,t,o){return b(this,void 0,void 0,(function*(){const{key:r,token:i}=t,s=e.map((e=>b(this,void 0,void 0,(function*(){var t,s;const a=function(e){var t;try{const o=null!==(t=new URL(e).searchParams.get("_lake_card"))&&void 0!==t?t:"",n=decodeURIComponent(o),r=JSON.parse(n);return{name:(null==r?void 0:r.name)||"",videoId:(null==r?void 0:r.videoId)||""}}catch(e){return!1}}(e.node.url);if(!a)return!1;const l=yield ue({videoId:a.videoId,key:r,token:i});if(!l)return!1;const c=null!==(s=null!==(t=a.name)&&void 0!==t?t:a.videoId.split("/").at(-1))&&void 0!==s?s:a.videoId;return{videoInfo:Object.assign(Object.assign({},a),l),astNode:e,fileName:c,currentFilePath:n.join(o,c)}}))));return(yield Promise.all(s)).filter(he)}))}(p,e,k),o=t.map((e=>se({fileUrl:e.videoInfo.video,savePath:e.currentFilePath,token:a,key:l,fileName:e.videoInfo.name})));(yield Promise.all(o)).forEach((e=>{const o=t.find((t=>t.videoInfo.video===e.fileUrl));o&&(o.astNode.node.url=`${i}${n.sep}${o.fileName}`,o.astNode.node.children=[{type:"text",value:`音视频附件: ${o.videoInfo.name}`}])})),$=Z(h)}if(m.length>0||g.length>0){const t=yield me(m,e,k,"audio"),o=yield me(g,e,k,"video"),r=[...t,...o].map((e=>b(this,void 0,void 0,(function*(){const t="audio"===e.type?e.videoInfo.fileName:e.videoInfo.name,o={fileUrl:"audio"===e.type?e.videoInfo.audio:e.videoInfo.video,savePath:e.currentFilePath,token:a,key:l,fileName:t};return Object.assign(Object.assign({},yield se(o)),{id:e.videoInfo.id,fileName:t})}))));(yield Promise.all(r)).forEach((e=>{const t=f.find((t=>new RegExp(`#${e.id}`,"gm").test(t.node.url)));t&&(t.node.url=`${i}${n.sep}${e.fileName}`,t.node.children=[{type:"text",value:`音视频附件: ${e.fileName}`}])})),$=Z(h)}}finally{y&&y.stop()}return{mdData:$}}))}function ue(e){let t="https://www.yuque.com/api/video";const{videoId:o,token:n,key:r}=e,i=new URLSearchParams;return i.set("video_id",o),t=`${t}?${i.toString()}`,h.get(t,q({token:n,key:r})).then((({data:e,status:t})=>{const o=e.data;return 200===t&&"success"===o.status&&o.info})).catch((()=>!1))}function he(e){return!!e}const fe=/name="audio" value="data:(.*?audioId.*?)".*?><\/card>/gm,pe=/name="video" value="data:(.*?videoId.*?)".*?><\/card>/gm;function ve(e,t){const o="video"===t?pe:fe,n=e.match(o)||[];try{return n.map((e=>e.replace(o,"$1"))).map((e=>JSON.parse(decodeURIComponent(e))))}catch(e){return[]}}function me(e,t,o,r){return b(this,void 0,void 0,(function*(){const{key:i,token:s}=t,a=e.map((e=>b(this,void 0,void 0,(function*(){var t;const a=yield ue({videoId:"audio"===r?e.audioId:e.videoId,key:i,token:s});if(!a)return!1;const l=null!==(t="audio"===r?null==e?void 0:e.fileName:e.name)&&void 0!==t?t:e.id;return{videoInfo:Object.assign(Object.assign({},e),a),type:r,fileName:l,currentFilePath:n.join(o,l)}}))));return(yield Promise.all(a)).filter(he)}))}function ge(e){return b(this,void 0,void 0,(function*(){var o,n,r,i,s,a,l,c,h,f;const{articleInfo:p,progressBar:g,options:y,progressItem:k,oldProgressItem:w}=e,{token:$,key:I,convertMarkdownVideoLinks:b,hideFooter:D,ignoreImg:U,ignoreAttachments:S}=y,{bookId:N,itemUrl:L,savePath:T,saveFilePath:O,uuid:A,articleUrl:x,articleTitle:_,host:C,imageServiceDomains:R}=p,F={articleUrl:L,bookId:N,token:$,host:C,key:I},{httpStatus:M,apiUrl:J,response:q}=yield X(F);!function(){var e,t,o,n;const r=(null===(e=null==q?void 0:q.data)||void 0===e?void 0:e.created_at)||"",i=(null===(t=null==q?void 0:q.data)||void 0===t?void 0:t.content_updated_at)||"",s=(null===(o=null==q?void 0:q.data)||void 0===o?void 0:o.published_at)||"",a=(null===(n=null==q?void 0:q.data)||void 0===n?void 0:n.first_published_at)||"";k.createAt=r,k.contentUpdatedAt=i,k.publishedAt=s,k.firstPublishedAt=a}();const{needDownload:V,isUpdateDownload:K}=function(e,t){const o={isFirstDownload:!0,isUpdateDownload:!1,needDownload:!0};if(!e.contentUpdatedAt||!t||!(null==t?void 0:t.contentUpdatedAt))return o;const n=new Date(e.contentUpdatedAt),r=new Date(t.contentUpdatedAt);if(!B(n)||!B(r))return o;if(n.getTime()>r.getTime())return{needDownload:!0,isUpdateDownload:!0,isFirstDownload:!1};if(n.getTime()===r.getTime())return{needDownload:!1,isUpdateDownload:!1,isFirstDownload:!1};return o}(k,w);if(!V)return{needDownload:V,isUpdateDownload:K,isDownloadFinish:!0};const Q=null===(n=null===(o=null==q?void 0:q.data)||void 0===o?void 0:o.type)||void 0===n?void 0:n.toLocaleLowerCase();let H="";if(Q===E.SHEET){const{response:e}=yield X(F,!1);try{const t=null===(r=null==e?void 0:e.data)||void 0===r?void 0:r.content,o=t?JSON.parse(t):{},n=null==o?void 0:o.sheet;H=n?ne(n):""}catch(e){const t=P.get(Q);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([E.BOARD,E.TABLE].includes(Q)){const e=P.get(Q);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof(null===(i=null==q?void 0:q.data)||void 0===i?void 0:i.sourcecode))throw new Error(`download article Error: ${J}, http status ${M}`);H=q.data.sourcecode,H=function(e){const t=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,o=e.match(t);let n=e;const r=e;try{null==o||o.forEach((e=>{var o,r;t.lastIndex=0;const i=null!==(r=null===(o=t.exec(e))||void 0===o?void 0:o[2])&&void 0!==r?r:"",{pathname:s,search:a}=new URL(i);if(!s.endsWith(".svg")&&a){const t=decodeURIComponent(a);n=n.replace(e,t.slice(1))}}))}catch(e){return r}return n}(H)}const G=function(e){if(!e)return[];const t=/!\[(.*?)\]\((.*?)\)/gm;let o=Array.from(e.match(t)||[]);return o=o.map((e=>(e=e.replace(t,"$2"),/^http.*/g.test(e)?e:""))).filter((e=>Boolean(e))),o}(H),W=null!==(l=null===(a=null===(s=(yield X(F,!1)).response)||void 0===s?void 0:s.data)||void 0===a?void 0:a.content)&&void 0!==l?l:"";G.length&&!U&&(H=function(e,t,o){if(!o)return t;const n=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,r=[];let i,s=!0;for(;s||Boolean(i);)if(s=!1,i=n.exec(o),null==i?void 0:i[1])try{const e=decodeURIComponent(i[1]),t=JSON.parse(e);r.push((null==t?void 0:t.src)||"")}catch(e){r.push("")}const a=new Map;return e.forEach((e=>{const{origin:o,pathname:n}=new URL(e),i=`${o}${n}`,s=r.find(((e,t)=>{const o=new RegExp(`${i}.*?`).test(e);return o&&r.splice(t,1),o}));if(s){const o=new RegExp(e,"g"),n=a.get(e)||0;let r=0;t=t.replace(o,(e=>{let t=e;return r===n&&(t=s),r+=1,t})),a.set(e,n+1)}})),t}(G,H,W));const z={toc:y.toc,articleTitle:_,articleUrl:x,articleUpdateTime:j(null!==(h=null===(c=null==q?void 0:q.data)||void 0===c?void 0:c.content_updated_at)&&void 0!==h?h:""),convertMarkdownVideoLinks:b,hideFooter:D},Y=[];if(!S||"string"==typeof S)try{g.pause(),console.log("");H=(yield ce({mdData:H,savePath:T,attachmentsDir:`./attachments/${te(A)}`,articleTitle:_,token:$,key:I,ignoreAttachments:S})).mdData}catch(e){Y.push(`附件下载失败: ${e.message||"unknown error"}`)}finally{g.continue()}if(!S||"string"==typeof S)try{g.pause(),console.log("");H=(yield de({mdData:H,htmlData:W,savePath:T,attachmentsDir:`./attachments/${te(A)}`,articleTitle:_,token:$,key:I,ignoreAttachments:S})).mdData}catch(e){Y.push(`音视频下载失败: ${e.message||"unknown error"}`)}finally{g.continue()}if(G.length&&!U){let e;g.pause(),console.log(""),"test"!==d.NODE_ENV&&(e=v({text:`下载 "${_}" 的图片中...`,stream:u}),e.start());let o=[],n=H;try{const e=yield m.run(H,{dist:T,imgDir:`./img/${A}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:x||"",transform:e=>re(e=e.replace("x-oss-process=image%2Fwatermark%2C",""),R),timeout:18e4});o=e.errorInfo,n=e.data}catch(e){o=[e]}H=n;const r=()=>{e&&e.stop(),g.continue()};if(o.length>0){const e=o[0];let n="图片下载失败(失败的以远程链接保存): ";throw n=e.url?`${n}${null===(f=e.error)||void 0===f?void 0:f.message} ${e.url.slice(0,20)}...`:`${n}${e.message}`,yield t(O,ye(H,z)),r(),new Error(`${n}`)}r()}try{if(H=oe(H,W),yield t(O,ye(H,z)),Y.length>0)throw new Error(Y[0]);return{needDownload:V,isUpdateDownload:K,isDownloadFinish:!0}}catch(e){throw new Error(`${e.message}`)}}))}function ye(e,t){const{articleTitle:o,articleUrl:n,toc:r,convertMarkdownVideoLinks:i,hideFooter:s}=t;let a=e;a=a.replace(/<a\b[^>]*?>(\s*?)<\/a>/gm,"");const l=o?`# ${o}\n\n`:"";let c=r?g(a).content:"";c&&(c=`${c}\n\n---\n\n`);let d="";return s||(d="\n\n",t.articleUpdateTime&&(d+=`> 更新: ${t.articleUpdateTime} \n`),n&&(d+=`> 原文: <${n}>`)),a=`${l}${c}${a}${d}`,i&&(a=function(e){return e.replace(/\[(.*?)\]\((.*?)\.(mp4|mp3)\)/gm,((e,t,o,n)=>{const r="mp3"===n?"audio":"video";return`<${r} controls width="800" alt="${t}" src="${o+"."+n}"></${r}>`}))}(a)),a}function ke(e,t){return b(this,void 0,void 0,(function*(){if(!R(e))throw new Error("Please enter a valid URL");if(t.password){const o=yield Q(e,t.password,{token:t.token,key:t.key});if(!o)throw new Error("Password validation failed");t.key=o.key,t.token=o.token}const{bookId:r,tocList:i,bookName:s,bookDesc:a,bookSlug:l,host:c,imageServiceDomains:d}=yield W(e,{token:t.token,key:t.key});if(!r)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const u=n.resolve(t.distDir,s?te(s):String(r));yield o(u,{recursive:!0});const h=i.length,f=new A(u,h,t.incremental);if(yield f.init(),!t.incremental&&f.curr==h)return f.bar&&f.bar.stop(),void O.info(`√ 已完成: ${u}`);const p=new Map;(f.isDownloadInterrupted||t.incremental)&&f.progressInfo.forEach((e=>{p.set(e.toc.uuid,e)}));const v=e.replace(new RegExp(`(.*?/${l}).*`),"$1");yield function(e){return b(this,void 0,void 0,(function*(){const{articleUrlPrefix:t,total:r,uuidMap:i,tocList:s,bookPath:a,bookId:l,progressBar:c,host:d,options:u,imageServiceDomains:h=[]}=e;let f=0,p=0,v=0;const m=[],g=[],y=[];for(let e=0;e<r;e++){const t=s[e];if("string"!=typeof t.type)continue;const n=t.type.toLocaleLowerCase();if(n===D.TITLE||""!==t.child_uuid||n===D.LINK){let e=t;const r=[],s=[];for(;e;)r.unshift(te(e.title)),s.unshift(e.uuid),e=i.get(e.parent_uuid)?i.get(e.parent_uuid).toc:void 0;const l={path:r.map(te).join("/"),pathTitleList:r,pathIdList:s,toc:t};n===D.LINK?(v+=1,g.push(l)):yield o(`${a}/${r.map(te).join("/")}`,{recursive:!0}),n===E.DOC?yield k(t):(i.set(t.uuid,l),yield c.updateProgress(l,n!==D.LINK))}else t.url&&(yield k(t))}function k(e){return b(this,void 0,void 0,(function*(){p+=1;let o={path:"",pathTitleList:[],pathIdList:[]};const r=e.type.toLocaleLowerCase();i.get(e.parent_uuid)&&(o=i.get(e.parent_uuid));const s=te(e.title),v=[...o.pathTitleList,s],g=[...o.pathIdList,e.uuid];let k=[...o.pathTitleList,`${s}.md`].map(te).join("/"),w=o.pathTitleList.map(te).join("/");r===E.DOC&&e.child_uuid&&(k=[...o.pathTitleList,s,"index.md"].map(te).join("/"),w=v.map(te).join("/"));const $={path:k,savePath:w,pathTitleList:v,pathIdList:g,toc:e};let I=!0;const b=`${t}/${e.url}`;try{const t={bookId:l,itemUrl:e.url,savePath:n.resolve(a,$.savePath),saveFilePath:n.resolve(a,$.path),uuid:e.uuid,articleUrl:b,articleTitle:e.title,host:d,imageServiceDomains:h},{isUpdateDownload:o}=yield ge({articleInfo:t,progressBar:c,options:u,progressItem:$,oldProgressItem:i.get(e.uuid)});o&&y.push({progressItem:$,articleInfo:t})}catch(e){I=!1,f+=1,m.push({articleUrl:b,errItem:$,errMsg:e.message,err:e})}i.set(e.uuid,$),yield c.updateProgress($,I)}))}if(v>0){O.warn("该知识库存在以下外链文章");for(const e of g)O.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(f>0){O.error(`本次执行总数${p}篇，✕ 失败${f}篇`);for(const e of m)O.error(`《${e.errItem.path}》: ${e.articleUrl}`),e.errMsg.split("\n").forEach((e=>{O.error(`———— ✕ ${e}`)}));O.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}y.length>0&&(O.info("以下文档有更新: "),y.forEach((e=>{O.info(`———— √ ${e.articleInfo.saveFilePath}`)})))}))}({articleUrlPrefix:v,total:h,uuidMap:p,tocList:i,bookPath:u,bookId:r,progressBar:f,host:c,options:t,imageServiceDomains:d});const m=new M({bookPath:u,bookName:s,bookDesc:a,uuidMap:p});yield m.genFile(),O.info(`√ 生成目录 ${n.resolve(u,"index.md")}`),f.curr===h&&O.info(`√ 已完成: ${u}`)}))}function we(e,t){return b(this,void 0,void 0,(function*(){const o=Array.isArray(e)?e:[e];if(!o||0===o.length)throw new Error("Please provide at least one book URL");for(const e of o)if(!R(e))throw new Error(`Invalid URL: ${e}`);const n=o.length;O.info(`批量下载 ${n} 个知识库\n`),yield Ie(o,t)}))}function $e(e){return b(this,void 0,void 0,(function*(){var t;if(!e.token)throw new Error("Token is required for downloading all books. Use -t <token>");const o=yield function(e){return b(this,void 0,void 0,(function*(){const t=[];let o=0;try{for(;;){const n=`${S}/api/mine/books?limit=50&offset=${o}`,{data:r}=yield h.get(n,q(e)),i=(null==r?void 0:r.data)||[];if(t.push(...i),i.length<50)break;o+=50}return t}catch(e){throw G(e)}}))}({token:e.token,key:e.key});if(0===o.length)return void O.warn("未找到任何知识库");const n=[],r=[],i=[];for(let e=0;e<o.length;e++){const s=o[e],a=null===(t=s.user)||void 0===t?void 0:t.login,l=`${S}/${a}/${s.slug}`;a?"Book"===s.type&&(r.push(l),i.push(s)):n.push({url:l,error:"无法获取用户登录名"})}const s=r.length,a=i.reduce(((e,t)=>e+(t.items_count||0)),0);O.info(`找到 ${s} 个知识库，共 ${a} 篇文档\n`),yield Ie(r,e,n)}))}function Ie(e,t){return b(this,arguments,void 0,(function*(e,t,o=[]){const n=[],r=e.length+o.length;for(let i=0;i<e.length;i++){const s=e[i];console.log(_(_(`[${i+1}/${r}] 下载: `,"bold")+`${s}`,"magenta")),console.log("");try{yield ke(s,t),n.push(s)}catch(e){const t=e.message||"unknown error";O.error(`✕ 下载失败: ${s} — ${t}`),o.push({url:s,error:t})}}console.log(`\n${"=".repeat(100)}\n`),O.info(`下载完成: ${n.length}/${r} 个知识库成功`),o.length>0&&(O.error(`失败 ${o.length} 个:`),o.forEach((({url:e,error:t})=>{O.error(`———— ✕ ${e}: ${t}`)})))}))}function be(e,t){return b(this,void 0,void 0,(function*(){const r=Array.isArray(e)?e:[e];if(!r||0===r.length)throw new Error("Please provide at least one document URL");for(const e of r)if(!R(e))throw new Error(`Invalid URL: ${e}`);const i=r.length,s=n.resolve(t.distDir);yield o(s,{recursive:!0});const a=new A(s,i,!1,!0);yield a.init();let l=0;const c=[],d=[];for(let e=0;e<r.length;e++){const o=r[e];let i;try{const e=yield z(o,{token:t.token,key:t.key}),{docId:r,docSlug:l,docTitle:c,bookId:u,bookSlug:h,host:f,imageServiceDomains:p=[]}=e;if(!r||!u||!l)throw new Error("Failed to get document info from URL");const v=te(c||l),m=s,g=n.resolve(s,`${v}.md`);i={path:`${v}.md`,pathTitleList:[v],pathIdList:[String(r)],toc:{type:"DOC",title:c||l,uuid:String(r),url:l,prev_uuid:"",sibling_uuid:"",child_uuid:"",parent_uuid:"",doc_id:r,level:0,id:r,open_window:0,visible:1}};const y=h?`${f}/${h}/${l}`:o,k={bookId:u,itemUrl:l,savePath:m,saveFilePath:g,uuid:String(r),articleUrl:y,articleTitle:c||l,host:f,imageServiceDomains:p};yield ge({articleInfo:k,progressBar:a,options:t,progressItem:i}),yield a.updateProgress(i,!0),d.push(g)}catch(e){i&&(yield a.updateProgress(i,!1)),l+=1;const t=e.message||"unknown error";c.push({url:o,error:t})}}a.bar&&a.bar.stop(),d.forEach((e=>{O.info(`√ 已完成: ${e}`)})),l>0&&c.forEach((({url:e,error:t})=>{O.error(`✕ 下载失败: ${e}`),O.error(`———— ${t}`)}))}))}export{b as _,we as a,$e as b,be as d,O as l,ke as m};
|
|
1
|
+
import e,{writeFile as t,mkdir as o}from"node:fs/promises";import n from"node:path";import r from"rand-user-agent";import i from"log4js";import s from"cli-progress";import a,{publicEncrypt as l,constants as d}from"node:crypto";import{env as c,stdout as u}from"node:process";import h from"axios";import{fromMarkdown as f}from"mdast-util-from-markdown";import{toMarkdown as p}from"mdast-util-to-markdown";import v from"ora";import m from"pull-md-img";import g from"markdown-toc";import y from"pako";import{createWriteStream as k,mkdirSync as $}from"node:fs";import*as w from"node:stream";import{promisify as I}from"node:util";function b(e,t,o,n){return new(o||(o=Promise))((function(r,i){function s(e){try{l(n.next(e))}catch(e){i(e)}}function a(e){try{l(n.throw(e))}catch(e){i(e)}}function l(e){var t;e.done?r(e.value):(t=e.value,t instanceof o?t:new o((function(e){e(t)}))).then(s,a)}l((n=n.apply(e,t||[])).next())}))}var D,P;"function"==typeof SuppressedError&&SuppressedError,function(e){e.TITLE="title",e.LINK="link",e.DOC="doc"}(D||(D={})),function(e){e.BOARD="board",e.TABLE="table",e.SHEET="sheet",e.DOC="doc"}(P||(P={}));const E=new Map([[P.BOARD,"画板类型"],[P.TABLE,"数据表类型"],[P.SHEET,"表格类型"],[P.DOC,"文档类型"]]),U="_yuque_session",L="https://www.yuque.com",S="UXO91eVnUveQn8suOJaYMvBcWs9KptS8N5HoP8ezSeU4vqApZpy1CkPaTpkpQEx2W2mlhxL8zwS8UePwBgksUM0CTtAODbTTTDFD",T="\n-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCfwyOyncSrUTmkaUPsXT6UUdXx\nTQ6a0wgPShvebfwq8XeNj575bUlXxVa/ExIn4nOUwx6iR7vJ2fvz5Ls750D051S7\nq70sevcmc8SsBNoaMQtyF/gETPBSsyWv3ccBJFrzZ5hxFdlVUfg6tXARtEI8rbIH\nsu6TBkVjk+n1Pw/ihQIDAQAB\n-----END PUBLIC KEY-----",N={Book:"verified_books",Doc:"verified_docs"},O=(i.configure({appenders:{cheese:{type:"console",layout:{type:"pattern",pattern:"%[%c [%p]:%] %m%n"}}},categories:{default:{appenders:["cheese"],level:"trace"}}}),i.getLogger("yuque-dl"));class A{constructor(e,t,o=!1,n=!1){this.bookPath="",this.progressFilePath="",this.progressInfo=[],this.curr=0,this.total=0,this.isDownloadInterrupted=!1,this.bar=null,this.completePromise=null,this.bookPath=e,this.progressFilePath=`${e}/progress.json`,this.total=t,this.incremental=o,this.disableProgressJSON=n}init(){return b(this,void 0,void 0,(function*(){this.progressInfo=this.disableProgressJSON?[]:yield this.getProgress(),this.curr=this.incremental?0:this.progressInfo.length,this.curr!==this.total&&(this.curr>0&&this.curr!==this.total&&!this.incremental&&(this.isDownloadInterrupted=!0,O.info("根据上次数据继续断点下载")),this.bar=new s.SingleBar({format:"Download [{bar}] {percentage}% | {value}/{total}"},s.Presets.legacy),this.bar.start(this.total,this.curr))}))}getProgress(){return b(this,void 0,void 0,(function*(){let t=[];try{const o=yield e.readFile(this.progressFilePath,{encoding:"utf8"});t=JSON.parse(o)}catch(o){o&&"ENOENT"===o.code&&(yield e.writeFile(this.progressFilePath,JSON.stringify(t),{encoding:"utf8"}))}return t}))}updateProgress(t,o){return b(this,void 0,void 0,(function*(){if(this.curr!==this.total){if(this.curr=this.curr+1,o){const o=t.toc.uuid,n=this.progressInfo.find((e=>e.toc.uuid===o));n?Object.assign(n,t):this.progressInfo.push(t),this.disableProgressJSON||(yield e.writeFile(this.progressFilePath,JSON.stringify(this.progressInfo),{encoding:"utf8"}))}this.bar&&(this.bar.update(this.curr),this.curr>=this.total&&(this.bar.stop(),console.log("")))}}))}pause(){this.bar&&this.bar.stop()}continue(){var e;this.clearLine(2),null===(e=this.bar)||void 0===e||e.start(this.total,this.curr)}clearLine(e){var t;if(!(e<=0)&&"function"==typeof(null===(t=null===process||void 0===process?void 0:process.stderr)||void 0===t?void 0:t.cursorTo)){process.stderr.cursorTo(0);for(let t=0;t<e;t++)process.stderr.moveCursor(0,-1),process.stderr.clearLine(1)}}}const _={bold:[1,22],italic:[3,23],underline:[4,24],inverse:[7,27],white:[37,39],grey:[90,39],black:[90,39],blue:[34,39],cyan:[36,39],green:[32,39],magenta:[35,39],red:[91,39],yellow:[33,39]};function x(e,t){return function(e){return e?`[${_[e][0]}m`:""}(t)+e+function(e){return e?`[${_[e][1]}m`:""}(t)}function C({browser:e="chrome",os:t="mac os",device:o="desktop"}){o=o.toLowerCase(),e=e.toLowerCase(),t=t.toLowerCase();let n=r(o,e,t);if("chrome"===e)for(;n.includes("Chrome-Lighthouse")||n.includes("Gener8")||n.includes("HeadlessChrome")||n.includes("SMTBot");)n=r(o,e,t);if("safari"===e)for(;n.includes("Applebot");)n=r(o,e,t);return n}function R(e){if("function"==typeof URL.canParse)return URL.canParse(e);try{return new URL(e),!0}catch(e){return!1}}function F(e){return e.toString().padStart(2,"0")}function j(e){const t=new Date(e);if(isNaN(t.getTime()))return"";const o=t.getFullYear(),n=t.getMonth()+1,r=t.getDate(),i=t.getHours(),s=t.getMinutes(),a=t.getSeconds();return`${o}-${F(n)}-${F(r)} ${F(i)}:${F(s)}:${F(a)}`}function B(e){return e instanceof Date&&!isNaN(e.getTime())}class M{constructor(e){this.summaryInfo=e}genFile(){return b(this,void 0,void 0,(function*(){const{bookName:t,bookDesc:o,bookPath:n,uuidMap:r}=this.summaryInfo;let i=`# ${t}\n\n`;o&&(i+=`> ${o}\n\n`);let s=i;const a=[];r.forEach((e=>{const t=e.toc,o=t.parent_uuid,n=this.findTree(a,o),r={text:t.title.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/,""),id:t.uuid,level:1,type:"link"},i=t.type.toLocaleLowerCase();i===D.TITLE||""!==t.child_uuid?(r.type="title","boolean"!=typeof n?(Array.isArray(n.children)||(n.children=[]),r.level=n.level+1,n.children.push(r)):(r.level=1,a.push(r)),i===D.DOC&&(r.link=e.path)):(r.type="link",r.link=i===D.LINK?e.toc.url:e.path,"boolean"!=typeof n?(Array.isArray(n.children)||(n.children=[]),r.level=n.level+1,n.children.push(r)):(r.level=1,a.push(r)))}));s+=this.genSummaryContent(a,"");try{yield e.writeFile(`${n}/index.md`,s)}catch(e){O.error("Generate Summary Error")}return s}))}genSummaryContent(e,t){for(const o of e){if(o.type===D.TITLE)if(o.link){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`\n${"".padStart(o.level+1,"#")} [${o.text}](${e})\n\n`}else t+=`\n${"".padStart(o.level+1,"#")} ${o.text}\n\n`;else if(o.type===D.LINK){const e=o.link?o.link.replace(/\s/g,"%20"):o.link;t+=`${1===o.level?"\n##":"-"} [${o.text}](${e})\n`}Array.isArray(o.children)&&(t+=this.genSummaryContent(o.children,""))}return t}findIdItem(e,t){if(e.id===t)return e;if(e.children){const o=this.findTree(e.children,t);if(o)return o}return!1}findTree(e,t){if(!t)return!1;for(const o of e){const e=this.findIdItem(o,t);if(e)return e}return!1}}function J(e){const{key:t=U,token:o}=e,n={"user-agent":C({browser:"chrome",device:"desktop"})};return o&&(n.cookie=`${t}=${o};`),n}function q(e){const t={headers:J(e),beforeRedirect:t=>{t.headers=Object.assign(Object.assign({},(null==t?void 0:t.headers)||{}),J(e))}};return"test"===c.NODE_ENV&&(t.proxy=!1),t}function V(e){var t;const o=null!==(t=/decodeURIComponent\("(.+)"\)\);/m.exec(e))&&void 0!==t?t:"";if(o[1])return JSON.parse(decodeURIComponent(o[1]))}function K(e,t=Date.now()){const o=Date.now()-t;return(`${Date.now()-o}:${e}`.match(/.{1,100}/g)||[]).map((e=>l({key:T,padding:d.RSA_PKCS1_PADDING},Buffer.from(e)).toString("base64"))).join(":")}function Q(e,t,o){return b(this,void 0,void 0,(function*(){const{html:n}=yield H(e,o),r=V(n),{targetType:i,needVerifyTargetId:s}=(null==r?void 0:r.matchCondition)||{};if(!i||!s)return!1;const a=N[i];if(!a)return!1;const l=`${new URL(e).origin}/api/${"Book"===i?"books":"docs"}/${s}/verify`,d=q(o);d.headers||(d.headers={}),d.headers.referer=e;const{headers:c}=yield h.put(l,{password:K(t,null==r?void 0:r.timestamp)},d),u=Reflect.get(c,"set-cookie");if(!u)return!1;if(Array.isArray(u)&&0===u.length)return!1;return{key:a,token:Reflect.get(function(e){const t={};return(Array.isArray(e)?e:e?[e]:[]).forEach((e=>{const o=e.split(";")[0],[n,...r]=o.split("=");n&&r.length>0&&(t[n]=r.join("="))})),t}(u),a)}}))}function H(e,t){return b(this,void 0,void 0,(function*(){const{data:o="",status:n,headers:r}=yield h.get(e,q(t));return{html:200===n?o:"",resHeaders:r,status:n}}))}function G(e){var t;const o=null!==(t=null==e?void 0:e.message)&&void 0!==t?t:"";if(!o)return new Error("unknown error");return["getaddrinfo ENOTFOUND","read ECONNRESET","Client network socket disconnected before secure TLS connection was established"].some((e=>o.startsWith(e)))?new Error("Please check the network connection or proxy settings"):new Error(o)}const W=(e,t)=>b(void 0,void 0,void 0,(function*(){var o;try{const{html:n}=yield H(e,t),r=V(n);return(null==r?void 0:r.book)?{bookId:r.book.id,bookSlug:r.book.slug,tocList:r.book.toc||[],bookName:r.book.name||"",bookDesc:r.book.description||"",host:(null===(o=r.space)||void 0===o?void 0:o.host)||L,imageServiceDomains:r.imageServiceDomains||[]}:{}}catch(e){throw G(e)}})),X=(e,t=!0)=>{const{articleUrl:o,bookId:n,token:r,key:i,host:s=L}=e;let a=`${s}/api/docs/${o}`;const l={book_id:String(n),merge_dynamic_data:String(!1)};t&&(l.mode="markdown");const d=new URLSearchParams(l).toString();return a=`${a}?${d}`,h.get(a,q({token:r,key:i})).then((({data:e,status:t})=>({apiUrl:a,httpStatus:t,response:e})))},z=(e,t)=>b(void 0,void 0,void 0,(function*(){var o,n,r;try{const{html:i}=yield H(e,t),s=V(i);return(null==s?void 0:s.doc)?{docId:s.doc.id,docSlug:s.doc.slug,docTitle:s.doc.title||"",bookId:s.doc.book_id,bookSlug:(null===(o=s.book)||void 0===o?void 0:o.slug)||"",bookName:(null===(n=s.book)||void 0===n?void 0:n.name)||"",host:(null===(r=s.space)||void 0===r?void 0:r.host)||L,imageServiceDomains:s.imageServiceDomains||[]}:{}}catch(e){throw G(e)}}));function Y(e){return f(e)}function Z(e){return p(e)}function ee(e,t,o=[]){t(e,o),Array.isArray(e.children)&&(o.push("children"),e.children.forEach(((e,n)=>{ee(e,t,[...o,String(n)])})))}function te(e){if(!e)return"";return e.replace(/[\\/:*?"<>|\n\r]/g,"_").replace(/\s/g,"").replace(/[\ud800-\udbff][\udc00-\udfff]/g,"")}function oe(e,t){let o=e;const n=Y(e),r=function(e){const t=[];return ee(e,(function(e,o){"inlineCode"===e.type&&t.push({node:e,keyChain:o})})),t}(n);return 0===r.length?e:(r.forEach((e=>{const o=e.node,n=(r=o.value,/<([a-z][\s\S]*?)>/i.test(r));var r;const i=function(e){return/(~~|\*\*|_)/g.test(e)}(o.value);if(!n&&!i)return;const s=o.value.replace(/</g,"<").replace(/>/g,">").replace(/"/g,""");t.includes(s)||(o.type="html",o.value=`<code>${o.value}</code>`)})),o=Z(n),o)}const ne=e=>{if(!e)return"";const t=y.inflate(e,{to:"string"}),o=JSON.parse(t);let n="";return o.forEach((e=>{const t=`## ${e.name}\n`,o=function(e){var t,o,n;let r=Object.keys(e);r=r.filter((t=>Object.keys(e[t]).some((o=>{var n,r;return null===(r=null===(n=null==e?void 0:e[t])||void 0===n?void 0:n[o])||void 0===r?void 0:r.v}))));let i=[];r.forEach((t=>{const o=e[t];o&&(i=i.concat(Object.keys(o)))}));const s=Math.max(...r.map((e=>Number(e)))),a=Math.max(...i.map((e=>Number(e))));if(s<0||a<0)return"";let l="";const d="ABCDEFGHIJKLMNOPQRSTUVWXYZ";let c=Array(a+1).fill(" ").map(((e,t)=>{const o=t%d.length;return d[o]})).join(" | ");c=`| |${c}|`;let u=Array(a+2).fill("---").join(" |");u=`|${u}|`,l=`${c}\n${u}\n`;for(let r=0;r<s+1;r++){const i=[];for(let s=0;s<a+1;s++){const a=(null===(o=null===(t=null==e?void 0:e[r])||void 0===t?void 0:t[s])||void 0===o?void 0:o.v)||null;a&&"string"==typeof a?i.push(a):a&&"object"==typeof a?"image"===(null==a?void 0:a.class)&&(null==a?void 0:a.src)?i.push(``):"checkbox"===(null==a?void 0:a.class)?i.push((null==a?void 0:a.value)?"[x] ":"[ ] "):"link"===(null==a?void 0:a.class)?i.push(`[${null==a?void 0:a.text}](${null==a?void 0:a.url})`):"select"===(null==a?void 0:a.class)&&i.push(null===(n=null==a?void 0:a.value)||void 0===n?void 0:n.join(",")):i.push(null)}l=`${l}${`| ${r+1} | ${i.join(" | ")}|`}\n`}return l}(e.data);n=`${n}\n${t}\n${o}`})),n};function re(e,t=[]){return function(e,t){try{const{host:o,pathname:n}=new URL(e);return!t.includes(o)&&Boolean(n)}catch(e){return!1}}(e,t)?`https://www.yuque.com/api/filetransfer/images?url=${encodeURIComponent(e)}&sign=${function(e){const t=a.createHash("sha256");return t.update(`${S}${e}`),t.digest("hex")}(e)}`:e}const ie=I(w.finished);function se(e){return b(this,void 0,void 0,(function*(){const{fileUrl:t,savePath:o,token:n,key:r,fileName:i}=e;return h.get(t,Object.assign(Object.assign({},q({token:n,key:r})),{responseType:"stream"})).then((e=>b(this,void 0,void 0,(function*(){var n,r,s;if(null===(r=null===(n=e.request)||void 0===n?void 0:n.path)||void 0===r?void 0:r.startsWith("/login"))throw new Error(`"${i}" need token`);if(200===e.status){const n=k(o);return null===(s=e.data)||void 0===s||s.pipe(n),ie(n).then((()=>({fileUrl:t,savePath:o})))}throw new Error(`response status ${e.status}`)}))))}))}const ae=/\[(.*?)\]\((.*?)\)/g,le=/\[(.*?)\]\((.*?\.yuque\.com\/attachments.*?)\)/;function de(e){return b(this,void 0,void 0,(function*(){const{mdData:t,savePath:o,attachmentsDir:r,articleTitle:i,token:s,key:a,ignoreAttachments:l}=e,d=(t.match(ae)||[]).filter((e=>le.test(e)));if(0===d.length)return{mdData:t};const h=n.resolve(o,r);let f=d.map((e=>function(e,t){const[,o,r]=le.exec(e)||[];if(!r)return!1;const i=o||r.split("/").at(-1);if(!i)return!1;const s=n.join(t,i);return{fileName:i,url:r,rawMd:e,currentFilePath:s}}(e,h))).filter((e=>!1!==e));if("string"==typeof l){const e=l.split(",");f=f.filter((t=>{const o=t.url.lastIndexOf(".");if(-1===o)return!0;const n=t.url.slice(o+1);return!e.find((e=>e===n))}))}if(0===f.length)return{mdData:t};const p=v({text:`下载 "${i}" 的附件中...`,stream:u});"test"!==c.NODE_ENV&&p.start(),$(h,{recursive:!0});const m=f.map((e=>se({fileUrl:e.url,savePath:e.currentFilePath,token:s,key:a,fileName:e.fileName}))),g=yield Promise.all(m).finally((function(){p&&p.stop()}));let y=t;return g.forEach((e=>{const t=f.find((t=>t.url===e.fileUrl));if(t){const e=`[附件: ${t.fileName}](${r}/${t.fileName})`;y=y.replace(t.rawMd,e)}})),{mdData:y}}))}function ce(e){return b(this,void 0,void 0,(function*(){const{mdData:t,htmlData:o,savePath:r,attachmentsDir:i,articleTitle:s,token:a,key:l,ignoreAttachments:d}=e,h=Y(t),f=function(e){const t=[];return ee(e,(function(e,o){"link"===e.type&&t.push({node:e,keyChain:o})})),t}(h);let p=f.filter((e=>/_lake_card.*?videoId/.test(e.node.url))),m=ve(o,"audio"),g=ve(o,"video");if(0===p.length&&0===m.length&&0===g.length)return{mdData:t};if("string"==typeof d){const e=d.split(",");if(p=p.filter((t=>{const o=t.node.url,n=o.lastIndexOf(".");if(-1===n)return!0;const r=o.slice(n+1);return!e.find((e=>e===r))})),m=m.filter((t=>{const o=t.audioId.lastIndexOf(".");if(-1===o)return!0;const n=t.audioId.slice(o+1);return!e.find((e=>e===n))})),g=g.filter((t=>{const o=t.videoId.lastIndexOf(".");if(-1===o)return!0;const n=t.videoId.slice(o+1);return!e.find((e=>e===n))})),0===p.length&&0===m.length&&0===g.length)return{mdData:t}}const y=v({text:`下载 "${s}" 的音视频中...`,stream:u});"test"!==c.NODE_ENV&&y.start();const k=n.resolve(r,i);$(k,{recursive:!0});let w=t;try{if(p.length>0){const t=yield function(e,t,o){return b(this,void 0,void 0,(function*(){const{key:r,token:i}=t,s=e.map((e=>b(this,void 0,void 0,(function*(){var t,s;const a=function(e){var t;try{const o=null!==(t=new URL(e).searchParams.get("_lake_card"))&&void 0!==t?t:"",n=decodeURIComponent(o),r=JSON.parse(n);return{name:(null==r?void 0:r.name)||"",videoId:(null==r?void 0:r.videoId)||""}}catch(e){return!1}}(e.node.url);if(!a)return!1;const l=yield ue({videoId:a.videoId,key:r,token:i});if(!l)return!1;const d=null!==(s=null!==(t=a.name)&&void 0!==t?t:a.videoId.split("/").at(-1))&&void 0!==s?s:a.videoId;return{videoInfo:Object.assign(Object.assign({},a),l),astNode:e,fileName:d,currentFilePath:n.join(o,d)}}))));return(yield Promise.all(s)).filter(he)}))}(p,e,k),o=t.map((e=>se({fileUrl:e.videoInfo.video,savePath:e.currentFilePath,token:a,key:l,fileName:e.videoInfo.name})));(yield Promise.all(o)).forEach((e=>{const o=t.find((t=>t.videoInfo.video===e.fileUrl));o&&(o.astNode.node.url=`${i}${n.sep}${o.fileName}`,o.astNode.node.children=[{type:"text",value:`音视频附件: ${o.videoInfo.name}`}])})),w=Z(h)}if(m.length>0||g.length>0){const t=yield me(m,e,k,"audio"),o=yield me(g,e,k,"video"),r=[...t,...o].map((e=>b(this,void 0,void 0,(function*(){const t="audio"===e.type?e.videoInfo.fileName:e.videoInfo.name,o={fileUrl:"audio"===e.type?e.videoInfo.audio:e.videoInfo.video,savePath:e.currentFilePath,token:a,key:l,fileName:t};return Object.assign(Object.assign({},yield se(o)),{id:e.videoInfo.id,fileName:t})}))));(yield Promise.all(r)).forEach((e=>{const t=f.find((t=>new RegExp(`#${e.id}`,"gm").test(t.node.url)));t&&(t.node.url=`${i}${n.sep}${e.fileName}`,t.node.children=[{type:"text",value:`音视频附件: ${e.fileName}`}])})),w=Z(h)}}finally{y&&y.stop()}return{mdData:w}}))}function ue(e){let t="https://www.yuque.com/api/video";const{videoId:o,token:n,key:r}=e,i=new URLSearchParams;return i.set("video_id",o),t=`${t}?${i.toString()}`,h.get(t,q({token:n,key:r})).then((({data:e,status:t})=>{const o=e.data;return 200===t&&"success"===o.status&&o.info})).catch((()=>!1))}function he(e){return!!e}const fe=/name="audio" value="data:(.*?audioId.*?)".*?><\/card>/gm,pe=/name="video" value="data:(.*?videoId.*?)".*?><\/card>/gm;function ve(e,t){const o="video"===t?pe:fe,n=e.match(o)||[];try{return n.map((e=>e.replace(o,"$1"))).map((e=>JSON.parse(decodeURIComponent(e))))}catch(e){return[]}}function me(e,t,o,r){return b(this,void 0,void 0,(function*(){const{key:i,token:s}=t,a=e.map((e=>b(this,void 0,void 0,(function*(){var t;const a=yield ue({videoId:"audio"===r?e.audioId:e.videoId,key:i,token:s});if(!a)return!1;const l=null!==(t="audio"===r?null==e?void 0:e.fileName:e.name)&&void 0!==t?t:e.id;return{videoInfo:Object.assign(Object.assign({},e),a),type:r,fileName:l,currentFilePath:n.join(o,l)}}))));return(yield Promise.all(a)).filter(he)}))}function ge(e){return b(this,void 0,void 0,(function*(){var o,n,r,i,s,a,l,d,h,f;const{articleInfo:p,progressBar:g,options:y,progressItem:k,oldProgressItem:$}=e,{token:w,key:I,convertMarkdownVideoLinks:b,hideFooter:D,ignoreImg:U,ignoreAttachments:L}=y,{bookId:S,itemUrl:T,savePath:N,saveFilePath:O,uuid:A,articleUrl:_,articleTitle:x,host:C,imageServiceDomains:R}=p,F={articleUrl:T,bookId:S,token:w,host:C,key:I},{httpStatus:M,apiUrl:J,response:q}=yield X(F);!function(){var e,t,o,n;const r=(null===(e=null==q?void 0:q.data)||void 0===e?void 0:e.created_at)||"",i=(null===(t=null==q?void 0:q.data)||void 0===t?void 0:t.content_updated_at)||"",s=(null===(o=null==q?void 0:q.data)||void 0===o?void 0:o.published_at)||"",a=(null===(n=null==q?void 0:q.data)||void 0===n?void 0:n.first_published_at)||"";k.createAt=r,k.contentUpdatedAt=i,k.publishedAt=s,k.firstPublishedAt=a}();const{needDownload:V,isUpdateDownload:K}=function(e,t){const o={isFirstDownload:!0,isUpdateDownload:!1,needDownload:!0};if(!e.contentUpdatedAt||!t||!(null==t?void 0:t.contentUpdatedAt))return o;const n=new Date(e.contentUpdatedAt),r=new Date(t.contentUpdatedAt);if(!B(n)||!B(r))return o;if(n.getTime()>r.getTime())return{needDownload:!0,isUpdateDownload:!0,isFirstDownload:!1};if(n.getTime()===r.getTime())return{needDownload:!1,isUpdateDownload:!1,isFirstDownload:!1};return o}(k,$);if(!V)return{needDownload:V,isUpdateDownload:K,isDownloadFinish:!0};const Q=null===(n=null===(o=null==q?void 0:q.data)||void 0===o?void 0:o.type)||void 0===n?void 0:n.toLocaleLowerCase();let H="";if(Q===P.SHEET){const{response:e}=yield X(F,!1);try{const t=null===(r=null==e?void 0:e.data)||void 0===r?void 0:r.content,o=t?JSON.parse(t):{},n=null==o?void 0:o.sheet;H=n?ne(n):""}catch(e){const t=E.get(Q);throw new Error(`download article Error: “${t}”解析错误 ${e}`)}}else{if([P.BOARD,P.TABLE].includes(Q)){const e=E.get(Q);throw new Error(`download article Error: 暂不支持“${e}”的文档`)}if("string"!=typeof(null===(i=null==q?void 0:q.data)||void 0===i?void 0:i.sourcecode))throw new Error(`download article Error: ${J}, http status ${M}`);H=q.data.sourcecode,H=function(e){const t=/!\[(.*?)\]\((http.*?latex.*?)\)/gm,o=e.match(t);let n=e;const r=e;try{null==o||o.forEach((e=>{var o,r;t.lastIndex=0;const i=null!==(r=null===(o=t.exec(e))||void 0===o?void 0:o[2])&&void 0!==r?r:"",{pathname:s,search:a}=new URL(i);if(!s.endsWith(".svg")&&a){const t=decodeURIComponent(a);n=n.replace(e,t.slice(1))}}))}catch(e){return r}return n}(H)}const G=function(e){if(!e)return[];const t=/!\[(.*?)\]\((.*?)\)/gm;let o=Array.from(e.match(t)||[]);return o=o.map((e=>(e=e.replace(t,"$2"),/^http.*/g.test(e)?e:""))).filter((e=>Boolean(e))),o}(H),W=null!==(l=null===(a=null===(s=(yield X(F,!1)).response)||void 0===s?void 0:s.data)||void 0===a?void 0:a.content)&&void 0!==l?l:"";G.length&&!U&&(H=function(e,t,o){if(!o)return t;const n=/<card.*?name="image".*?value="data:(.*?)">(.*?)<\/card>/gm,r=[];let i,s=!0;for(;s||Boolean(i);)if(s=!1,i=n.exec(o),null==i?void 0:i[1])try{const e=decodeURIComponent(i[1]),t=JSON.parse(e);r.push((null==t?void 0:t.src)||"")}catch(e){r.push("")}const a=new Map;return e.forEach((e=>{const{origin:o,pathname:n}=new URL(e),i=`${o}${n}`,s=r.find(((e,t)=>{const o=new RegExp(`${i}.*?`).test(e);return o&&r.splice(t,1),o}));if(s){const o=new RegExp(e,"g"),n=a.get(e)||0;let r=0;t=t.replace(o,(e=>{let t=e;return r===n&&(t=s),r+=1,t})),a.set(e,n+1)}})),t}(G,H,W));const z={toc:y.toc,articleTitle:x,articleUrl:_,articleUpdateTime:j(null!==(h=null===(d=null==q?void 0:q.data)||void 0===d?void 0:d.content_updated_at)&&void 0!==h?h:""),convertMarkdownVideoLinks:b,hideFooter:D},Y=[];if(!L||"string"==typeof L)try{g.pause(),console.log("");H=(yield de({mdData:H,savePath:N,attachmentsDir:`./attachments/${te(A)}`,articleTitle:x,token:w,key:I,ignoreAttachments:L})).mdData}catch(e){Y.push(`附件下载失败: ${e.message||"unknown error"}`)}finally{g.continue()}if(!L||"string"==typeof L)try{g.pause(),console.log("");H=(yield ce({mdData:H,htmlData:W,savePath:N,attachmentsDir:`./attachments/${te(A)}`,articleTitle:x,token:w,key:I,ignoreAttachments:L})).mdData}catch(e){Y.push(`音视频下载失败: ${e.message||"unknown error"}`)}finally{g.continue()}if(G.length&&!U){let e;g.pause(),console.log(""),"test"!==c.NODE_ENV&&(e=v({text:`下载 "${x}" 的图片中...`,stream:u}),e.start());let o=[],n=H;try{const e=yield m.run(H,{dist:N,imgDir:`./img/${A}`,isIgnoreConsole:!0,errorStillReturn:!0,referer:_||"",transform:e=>re(e=e.replace("x-oss-process=image%2Fwatermark%2C",""),R),timeout:18e4});o=e.errorInfo,n=e.data}catch(e){o=[e]}H=n;const r=()=>{e&&e.stop(),g.continue()};if(o.length>0){const e=o[0];let n="图片下载失败(失败的以远程链接保存): ";throw n=e.url?`${n}${null===(f=e.error)||void 0===f?void 0:f.message} ${e.url.slice(0,20)}...`:`${n}${e.message}`,yield t(O,ye(H,z)),r(),new Error(`${n}`)}r()}try{if(H=oe(H,W),yield t(O,ye(H,z)),Y.length>0)throw new Error(Y[0]);return{needDownload:V,isUpdateDownload:K,isDownloadFinish:!0}}catch(e){throw new Error(`${e.message}`)}}))}function ye(e,t){const{articleTitle:o,articleUrl:n,toc:r,convertMarkdownVideoLinks:i,hideFooter:s}=t;let a=e;a=a.replace(/<a\b[^>]*?>(\s*?)<\/a>/gm,"");const l=o?`# ${o}\n\n`:"";let d=r?g(a).content:"";d&&(d=`${d}\n\n---\n\n`);let c="";return s||(c="\n\n",t.articleUpdateTime&&(c+=`> 更新: ${t.articleUpdateTime} \n`),n&&(c+=`> 原文: <${n}>`)),a=`${l}${d}${a}${c}`,i&&(a=function(e){return e.replace(/\[(.*?)\]\((.*?)\.(mp4|mp3)\)/gm,((e,t,o,n)=>{const r="mp3"===n?"audio":"video";return`<${r} controls width="800" alt="${t}" src="${o+"."+n}"></${r}>`}))}(a)),a}function ke(e,t){return b(this,void 0,void 0,(function*(){if(!R(e))throw new Error("Please enter a valid URL");if(t.password){const o=yield Q(e,t.password,{token:t.token,key:t.key});if(!o)throw new Error("Password validation failed");t.key=o.key,t.token=o.token}const{bookId:r,tocList:i,bookName:s,bookDesc:a,bookSlug:l,host:d,imageServiceDomains:c}=yield W(e,{token:t.token,key:t.key});if(!r)throw new Error("No found book id");if(!i||0===i.length)throw new Error("No found toc list");const u=n.resolve(t.distDir,s?te(s):String(r));yield o(u,{recursive:!0});const h=i.length,f=new A(u,h,t.incremental);if(yield f.init(),!t.incremental&&f.curr==h)return f.bar&&f.bar.stop(),void O.info(`√ 已完成: ${u}`);const p=new Map;(f.isDownloadInterrupted||t.incremental)&&f.progressInfo.forEach((e=>{p.set(e.toc.uuid,e)}));const v=e.replace(new RegExp(`(.*?/${l}).*`),"$1");yield function(e){return b(this,void 0,void 0,(function*(){const{articleUrlPrefix:t,total:r,uuidMap:i,tocList:s,bookPath:a,bookId:l,progressBar:d,host:c,options:u,imageServiceDomains:h=[]}=e;let f=0,p=0,v=0;const m=[],g=[],y=[];for(let e=0;e<r;e++){const t=s[e];if("string"!=typeof t.type)continue;const n=t.type.toLocaleLowerCase();if(n===D.TITLE||""!==t.child_uuid||n===D.LINK){let e=t;const r=[],s=[],l=[];for(;e;)r.unshift(`${te(e.title)}_${e.uuid}`),s.unshift(`${te(e.title)}`),l.unshift(e.uuid),e=i.get(e.parent_uuid)?i.get(e.parent_uuid).toc:void 0;const c={path:r.map(te).join("/"),rawPath:s.map(te).join("/"),pathTitleList:r,rawPathTitleList:s,pathIdList:l,toc:t};n===D.LINK?(v+=1,g.push(c)):yield o(`${a}/${r.map(te).join("/")}`,{recursive:!0}),n===P.DOC?yield k(t):(i.set(t.uuid,c),yield d.updateProgress(c,n!==D.LINK))}else t.url&&(yield k(t))}function k(e){return b(this,void 0,void 0,(function*(){p+=1;let o={path:"",pathTitleList:[],rawPathTitleList:[],pathIdList:[]};const r=e.type.toLocaleLowerCase();i.get(e.parent_uuid)&&(o=i.get(e.parent_uuid));const s=te(e.title),v=[...o.pathTitleList,`${s}_${e.uuid}`],g=[...o.rawPathTitleList||[],s],k=[...o.pathIdList,e.uuid];let $=[...o.pathTitleList,`${s}_${e.uuid}.md`].map(te).join("/"),w=[...o.pathTitleList,`${s}.md`].map(te).join("/"),I=o.pathTitleList.map(te).join("/");r===P.DOC&&e.child_uuid&&($=[...o.pathTitleList,`${s}_${e.uuid}`,"index.md"].map(te).join("/"),w=[...o.pathTitleList,`${s}`,"index.md"].map(te).join("/"),I=v.map(te).join("/"));const b={path:$,savePath:I,pathTitleList:v,rawPathTitleList:g,rawMdPath:w,pathIdList:k,toc:e};let D=!0;const E=`${t}/${e.url}`;try{const t={bookId:l,itemUrl:e.url,savePath:n.resolve(a,b.savePath),saveFilePath:n.resolve(a,b.path),uuid:e.uuid,articleUrl:E,articleTitle:e.title,host:c,imageServiceDomains:h},{isUpdateDownload:o}=yield ge({articleInfo:t,progressBar:d,options:u,progressItem:b,oldProgressItem:i.get(e.uuid)});o&&y.push({progressItem:b,articleInfo:t})}catch(e){D=!1,f+=1,m.push({articleUrl:E,errItem:b,errMsg:e.message,err:e})}i.set(e.uuid,b),yield d.updateProgress(b,D)}))}if(v>0){O.warn("该知识库存在以下外链文章");for(const e of g)O.warn(`———— ✕ ${e.path} ${e.toc.url}`)}if(f>0){O.error(`本次执行总数${p}篇，✕ 失败${f}篇`);for(const e of m)O.error(`《${e.errItem.path}》: ${e.articleUrl}`),e.errMsg.split("\n").forEach((e=>{O.error(`———— ✕ ${e}`)}));O.error("o(╥﹏╥)o 由于网络波动或链接失效以上下载失败，可重新执行命令重试(PS:不会影响已下载成功的数据)")}y.length>0&&(O.info("以下文档有更新: "),y.forEach((e=>{O.info(`———— √ ${e.articleInfo.saveFilePath}`)})))}))}({articleUrlPrefix:v,total:h,uuidMap:p,tocList:i,bookPath:u,bookId:r,progressBar:f,host:d,options:t,imageServiceDomains:c});const m=new M({bookPath:u,bookName:s,bookDesc:a,uuidMap:p});yield m.genFile(),O.info(`√ 生成目录 ${n.resolve(u,"index.md")}`),f.curr===h&&O.info(`√ 已完成: ${u}`)}))}function $e(e,t){return b(this,void 0,void 0,(function*(){const o=Array.isArray(e)?e:[e];if(!o||0===o.length)throw new Error("Please provide at least one book URL");for(const e of o)if(!R(e))throw new Error(`Invalid URL: ${e}`);const n=o.length;O.info(`批量下载 ${n} 个知识库\n`),yield Ie(o,t)}))}function we(e){return b(this,void 0,void 0,(function*(){var t;if(!e.token)throw new Error("Token is required for downloading all books. Use -t <token>");const o=yield function(e){return b(this,void 0,void 0,(function*(){const t=[];let o=0;try{for(;;){const n=`${L}/api/mine/books?limit=50&offset=${o}`,{data:r}=yield h.get(n,q(e)),i=(null==r?void 0:r.data)||[];if(t.push(...i),i.length<50)break;o+=50}return t}catch(e){throw G(e)}}))}({token:e.token,key:e.key});if(0===o.length)return void O.warn("未找到任何知识库");const n=[],r=[],i=[];for(let e=0;e<o.length;e++){const s=o[e],a=null===(t=s.user)||void 0===t?void 0:t.login,l=`${L}/${a}/${s.slug}`;a?"Book"===s.type&&(r.push(l),i.push(s)):n.push({url:l,error:"无法获取用户登录名"})}const s=r.length,a=i.reduce(((e,t)=>e+(t.items_count||0)),0);O.info(`找到 ${s} 个知识库，共 ${a} 篇文档\n`),yield Ie(r,e,n)}))}function Ie(e,t){return b(this,arguments,void 0,(function*(e,t,o=[]){const n=[],r=e.length+o.length;for(let i=0;i<e.length;i++){const s=e[i];console.log(x(x(`[${i+1}/${r}] 下载: `,"bold")+`${s}`,"magenta")),console.log("");try{yield ke(s,t),n.push(s)}catch(e){const t=e.message||"unknown error";O.error(`✕ 下载失败: ${s} — ${t}`),o.push({url:s,error:t})}}console.log(`\n${"=".repeat(100)}\n`),O.info(`下载完成: ${n.length}/${r} 个知识库成功`),o.length>0&&(O.error(`失败 ${o.length} 个:`),o.forEach((({url:e,error:t})=>{O.error(`———— ✕ ${e}: ${t}`)})))}))}function be(e,t){return b(this,void 0,void 0,(function*(){const r=Array.isArray(e)?e:[e];if(!r||0===r.length)throw new Error("Please provide at least one document URL");for(const e of r)if(!R(e))throw new Error(`Invalid URL: ${e}`);const i=r.length,s=n.resolve(t.distDir);yield o(s,{recursive:!0});const a=new A(s,i,!1,!0);yield a.init();let l=0;const d=[],c=[];for(let e=0;e<r.length;e++){const o=r[e];let i;try{const e=yield z(o,{token:t.token,key:t.key}),{docId:r,docSlug:l,docTitle:d,bookId:u,bookSlug:h,host:f,imageServiceDomains:p=[]}=e;if(!r||!u||!l)throw new Error("Failed to get document info from URL");const v=te(d||l),m=s,g=n.resolve(s,`${v}.md`);i={path:`${v}.md`,pathTitleList:[v],pathIdList:[String(r)],toc:{type:"DOC",title:d||l,uuid:String(r),url:l,prev_uuid:"",sibling_uuid:"",child_uuid:"",parent_uuid:"",doc_id:r,level:0,id:r,open_window:0,visible:1}};const y=h?`${f}/${h}/${l}`:o,k={bookId:u,itemUrl:l,savePath:m,saveFilePath:g,uuid:String(r),articleUrl:y,articleTitle:d||l,host:f,imageServiceDomains:p};yield ge({articleInfo:k,progressBar:a,options:t,progressItem:i}),yield a.updateProgress(i,!0),c.push(g)}catch(e){i&&(yield a.updateProgress(i,!1)),l+=1;const t=e.message||"unknown error";d.push({url:o,error:t})}}a.bar&&a.bar.stop(),c.forEach((e=>{O.info(`√ 已完成: ${e}`)})),l>0&&d.forEach((({url:e,error:t})=>{O.error(`✕ 下载失败: ${e}`),O.error(`———— ${t}`)}))}))}export{b as _,$e as a,we as b,be as d,O as l,ke as m};
|
package/dist/es/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export{a as downloadBooksFromUrls,d as downloadDocsFromUrls,b as downloadUserBooks,m as main}from"./index-
|
|
1
|
+
export{a as downloadBooksFromUrls,d as downloadDocsFromUrls,b as downloadUserBooks,m as main}from"./index-grIDsf7R.js";import"node:fs/promises";import"node:path";import"rand-user-agent";import"log4js";import"cli-progress";import"node:crypto";import"node:process";import"axios";import"mdast-util-from-markdown";import"mdast-util-to-markdown";import"ora";import"pull-md-img";import"markdown-toc";import"pako";import"node:fs";import"node:stream";import"node:util";
|
package/package.json
CHANGED
package/types/types/index.d.ts
CHANGED