publishport-opencli 1.8.5-pp.2 → 1.8.5-pp.21

package/dist/src/download/article-download.js

@@ -1,305 +1,21 @@
-/**
- * Article download helper — shared logic for downloading articles as Markdown.
- *
- * Used by: zhihu/download, weixin/download, and future article adapters.
- *
- * Flow: ArticleData → TurndownService → image download → frontmatter → .md file
- */
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import TurndownService from 'turndown';
-import { gfm } from 'turndown-plugin-gfm';
-import { httpDownload, sanitizeFilename } from './index.js';
-import { formatBytes } from './progress.js';
-const IMAGE_CONCURRENCY = 5;
-const DEFAULT_LABELS = {
-    author: '作者',
-    publishTime: '发布时间',
-    sourceUrl: '原文链接',
-};
-// ============================================================
-// Markdown Conversion
-// ============================================================
-// Nodes that never carry article content. Turndown keeps them by default — if an
-// adapter's contentHtml extraction misses one, CSS / scripts / widget markup
-// ends up inline in the .md. Strip them unconditionally at the converter level.
-// `svg` is not in HTMLElementTagNameMap, so we type-narrow manually.
-// `header/footer/nav/aside` cover page chrome that adapters occasionally
-// forget to trim — the article's own title/author/publishTime are supplied
-// as separate fields on ArticleData, so duplicated nodes are redundant.
-// `iframe` is NOT in this set — it's handled by a dedicated rule below that
-// degrades to a link so embedded content (YouTube, Twitter, CodePen …) keeps
-// a reachable URL in the exported markdown.
-const STRIPPED_TAGS = [
-    'script', 'style', 'noscript',
-    'canvas',
-    'form', 'button', 'dialog',
-    'header', 'footer', 'nav', 'aside',
-];
-function createTurndown(configure, cleanSelectors) {
-    const td = new TurndownService({
-        headingStyle: 'atx',
-        codeBlockStyle: 'fenced',
-        bulletListMarker: '-',
-    });
-    td.use(gfm);
-    td.remove(STRIPPED_TAGS);
-    // turndown-plugin-gfm@1.0.2 emits single-tilde strikethrough (`~x~`), which
-    // is not the canonical GFM form. Override it so exported markdown is
-    // portable across common renderers.
-    td.addRule('canonicalStrikethrough', {
-        filter: (node) => ['DEL', 'S', 'STRIKE'].includes(node.nodeName),
-        replacement: (content) => `~~${content}~~`,
-    });
-    // SVG isn't in the static HTML tag map; match by name with a custom filter.
-    td.addRule('stripSvg', {
-        filter: (node) => node.nodeName === 'svg' || node.nodeName === 'SVG',
-        replacement: () => '',
-    });
-    td.addRule('linebreak', {
-        filter: 'br',
-        replacement: () => '\n',
-    });
-    // Inline base64 images would land as huge `![](data:image/...;base64,...)`
-    // strings that the image downloader can't localize. Drop them.
-    td.addRule('ignoreBase64Images', {
-        filter: (node) => {
-            if (node.nodeName !== 'IMG')
-                return false;
-            const src = node.getAttribute?.('src') ?? '';
-            return src.startsWith('data:');
-        },
-        replacement: () => '',
-    });
-    // Markdown has no native video/audio primitive. Emit inline HTML so
-    // renderers that support it (GitHub, VS Code preview …) still play the
-    // media; viewers that don't simply show the tag as text, which is still
-    // more information than dropping the node outright.
-    td.addRule('videoElement', {
-        filter: (node) => node.nodeName === 'VIDEO',
-        replacement: (_content, node) => {
-            const el = node;
-            const src = el.getAttribute('src')
-                || el.querySelector('source')?.getAttribute('src')
-                || '';
-            if (!src)
-                return '';
-            const poster = el.getAttribute('poster') || '';
-            return `\n<video src="${src}" controls${poster ? ` poster="${poster}"` : ''}></video>\n`;
-        },
-    });
-    td.addRule('audioElement', {
-        filter: (node) => node.nodeName === 'AUDIO',
-        replacement: (_content, node) => {
-            const el = node;
-            const src = el.getAttribute('src')
-                || el.querySelector('source')?.getAttribute('src')
-                || '';
-            return src ? `\n<audio src="${src}" controls></audio>\n` : '';
-        },
-    });
-    // Iframes (YouTube, Twitter, CodePen …) degrade to a markdown link so the
-    // embedded resource is still reachable from the exported file.
-    td.addRule('iframeToLink', {
-        filter: (node) => node.nodeName === 'IFRAME',
-        replacement: (_content, node) => {
-            const el = node;
-            const src = el.getAttribute('src') || '';
-            if (!src)
-                return '';
-            const title = el.getAttribute('title') || 'Embedded content';
-            return `\n[${title}](${src})\n`;
-        },
-    });
-    // Per-adapter dirty-node removal. Adapters know their site's specific noise
-    // (zhihu 折叠卡, weixin 赞赏栏, wiki 折叠 infobox …); we keep the default set
-    // empty so the generic converter stays untouched.
-    const selectorRules = (cleanSelectors ?? [])
-        .map(sel => sel.trim())
-        .filter(Boolean);
-    if (selectorRules.length > 0) {
-        td.addRule('cleanSelectors', {
-            filter: (node) => {
-                const match = node.matches;
-                if (typeof match !== 'function')
-                    return false;
-                return selectorRules.some((sel) => {
-                    try {
-                        return match.call(node, sel);
-                    }
-                    catch {
-                        return false;
-                    }
-                });
-            },
-            replacement: () => '',
-        });
-    }
-    if (configure)
-        configure(td);
-    return td;
-}
-function convertToMarkdown(contentHtml, codeBlocks, configure, cleanSelectors) {
-    const td = createTurndown(configure, cleanSelectors);
-    let md = td.turndown(contentHtml);
-    // Restore code block placeholders
-    codeBlocks.forEach((block, i) => {
-        const placeholder = `CODEBLOCK-PLACEHOLDER-${i}`;
-        const fenced = `\n\`\`\`${block.lang}\n${block.code}\n\`\`\`\n`;
-        md = md.replace(placeholder, fenced);
-    });
-    // Clean up
-    md = md.replace(/\u00a0/g, ' ');
-    // Turndown leaves behind lone dashes / middle dots when list bullets or
-    // decorative separators lose their surrounding inline context.
-    md = md.replace(/^[ \t]*[-·][ \t]*$/gm, '');
-    md = md.replace(/^[ \t]+$/gm, '');
-    md = md.replace(/[ \t]+$/gm, '');
-    md = md.replace(/\n{3,}/g, '\n\n');
-    return md;
-}
-function replaceImageUrls(md, urlMap) {
-    return md.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, imgUrl) => {
-        const local = urlMap[imgUrl];
-        return local ? `![${alt}](${local})` : match;
-    });
-}
-// ============================================================
-// Image Downloading
-// ============================================================
-function defaultDetectImageExt(url) {
-    const extMatch = url.match(/\.(\w{3,4})(?:\?|$)/);
-    return extMatch ? extMatch[1] : 'jpg';
-}
-async function downloadImages(imgUrls, imgDir, headers, detectExt) {
-    const urlMap = {};
-    if (imgUrls.length === 0)
-        return urlMap;
-    const detect = detectExt || defaultDetectImageExt;
-    // Deduplicate image URLs
-    const seen = new Set();
-    const uniqueUrls = imgUrls.filter(url => {
-        if (seen.has(url))
-            return false;
-        seen.add(url);
-        return true;
-    });
-    for (let i = 0; i < uniqueUrls.length; i += IMAGE_CONCURRENCY) {
-        const batch = uniqueUrls.slice(i, i + IMAGE_CONCURRENCY);
-        const results = await Promise.all(batch.map(async (rawUrl, j) => {
-            const index = i + j + 1;
-            let imgUrl = rawUrl;
-            if (imgUrl.startsWith('//'))
-                imgUrl = `https:${imgUrl}`;
-            const ext = detect(imgUrl);
-            const filename = `img_${String(index).padStart(3, '0')}.${ext}`;
-            const filepath = path.join(imgDir, filename);
-            try {
-                const result = await httpDownload(imgUrl, filepath, {
-                    headers,
-                    timeout: 15000,
-                });
-                if (result.success) {
-                    return { remoteUrl: rawUrl, localPath: `images/${filename}` };
-                }
-            }
-            catch {
-                // Skip failed downloads
-            }
-            return null;
-        }));
-        for (const r of results) {
-            if (r)
-                urlMap[r.remoteUrl] = r.localPath;
-        }
-    }
-    return urlMap;
-}
-// ============================================================
-// Main API
-// ============================================================
-/**
- * Download an article to Markdown with optional image localization.
- *
- * Handles the full pipeline:
- * 1. HTML → Markdown (via TurndownService)
- * 2. Code block placeholder restoration
- * 3. Batch image downloading with concurrency + deduplication
- * 4. Image URL replacement in Markdown
- * 5. Frontmatter generation (customizable labels)
- * 6. File write
- */
-export async function downloadArticle(data, options) {
-    const { output, downloadImages: shouldDownloadImages = true, imageHeaders, maxTitleLength = 80, configureTurndown, detectImageExt, frontmatterLabels, cleanSelectors, stdout = false, } = options;
-    const labels = { ...DEFAULT_LABELS, ...frontmatterLabels };
-    if (!data.title) {
-        return [{
-                title: 'Error',
-                author: '-',
-                publish_time: '-',
-                status: 'failed — no title',
-                size: '-',
-                saved: '-',
-            }];
-    }
-    if (!data.contentHtml) {
-        return [{
-                title: data.title,
-                author: data.author || '-',
-                publish_time: data.publishTime || '-',
-                status: 'failed — no content',
-                size: '-',
-                saved: '-',
-            }];
-    }
-    // Convert HTML to Markdown
-    let markdown = convertToMarkdown(data.contentHtml, data.codeBlocks || [], configureTurndown, cleanSelectors);
-    const safeTitle = sanitizeFilename(data.title, maxTitleLength);
-    // Download images only when writing to disk. In stdout mode remote URLs
-    // stay intact so the piped output is self-contained.
-    if (!stdout && shouldDownloadImages && data.imageUrls && data.imageUrls.length > 0) {
-        const articleDir = path.join(output, safeTitle);
-        fs.mkdirSync(articleDir, { recursive: true });
-        const imagesDir = path.join(articleDir, 'images');
-        fs.mkdirSync(imagesDir, { recursive: true });
-        const urlMap = await downloadImages(data.imageUrls, imagesDir, imageHeaders, detectImageExt);
-        markdown = replaceImageUrls(markdown, urlMap);
-    }
-    // Build frontmatter with customizable labels.
-    // Shape: `# Title\n[> meta\n...]\n---\n\n<markdown>` — exactly one blank
-    // line separates every section, so we never produce ≥3 consecutive newlines.
-    const headerLines = [`# ${data.title}`];
-    if (data.author)
-        headerLines.push(`> ${labels.author}: ${data.author}`);
-    if (data.publishTime)
-        headerLines.push(`> ${labels.publishTime}: ${data.publishTime}`);
-    if (data.sourceUrl)
-        headerLines.push(`> ${labels.sourceUrl}: ${data.sourceUrl}`);
-    const frontmatter = headerLines.join('\n') + '\n\n---\n\n';
-    const fullContent = frontmatter + markdown;
-    const size = Buffer.byteLength(fullContent, 'utf-8');
-    if (stdout) {
-        process.stdout.write(fullContent.endsWith('\n') ? fullContent : fullContent + '\n');
-        return [{
-                title: data.title,
-                author: data.author || '-',
-                publish_time: data.publishTime || '-',
-                status: 'success',
-                size: formatBytes(size),
-                saved: '-',
-            }];
-    }
-    const articleDir = path.join(output, safeTitle);
-    fs.mkdirSync(articleDir, { recursive: true });
-    const filename = `${safeTitle}.md`;
-    const filePath = path.join(articleDir, filename);
-    fs.writeFileSync(filePath, fullContent, 'utf-8');
-    return [{
-            title: data.title,
-            author: data.author || '-',
-            publish_time: data.publishTime || '-',
-            status: 'success',
-            size: formatBytes(size),
-            saved: filePath,
-        }];
-}
+import*as P from"node:fs";import*as H from"node:path";import k from"turndown";import{gfm as x}from"turndown-plugin-gfm";import{httpDownload as C,sanitizeFilename as E}from"./index.js";import{formatBytes as I}from"./progress.js";const b=5,M={author:"作者",publishTime:"发布时间",sourceUrl:"原文链接"},T=["script","style","noscript","canvas","form","button","dialog","header","footer","nav","aside"];function D(J,$){const V=new k({headingStyle:"atx",codeBlockStyle:"fenced",bulletListMarker:"-"});V.use(x);V.remove(T);V.addRule("canonicalStrikethrough",{filter:(K)=>["DEL","S","STRIKE"].includes(K.nodeName),replacement:(K)=>`~~${K}~~`});V.addRule("stripSvg",{filter:(K)=>K.nodeName==="svg"||K.nodeName==="SVG",replacement:()=>""});V.addRule("linebreak",{filter:"br",replacement:()=>`
+`});V.addRule("ignoreBase64Images",{filter:(K)=>{if(K.nodeName!=="IMG")return!1;return(K.getAttribute?.("src")??"").startsWith("data:")},replacement:()=>""});V.addRule("videoElement",{filter:(K)=>K.nodeName==="VIDEO",replacement:(K,Q)=>{const W=Q,Z=W.getAttribute("src")||W.querySelector("source")?.getAttribute("src")||"";if(!Z)return"";const X=W.getAttribute("poster")||"";return`
+<video src="${Z}" controls${X?` poster="${X}"`:""}></video>
+`}});V.addRule("audioElement",{filter:(K)=>K.nodeName==="AUDIO",replacement:(K,Q)=>{const W=Q,Z=W.getAttribute("src")||W.querySelector("source")?.getAttribute("src")||"";return Z?`
+<audio src="${Z}" controls></audio>
+`:""}});V.addRule("iframeToLink",{filter:(K)=>K.nodeName==="IFRAME",replacement:(K,Q)=>{const W=Q,Z=W.getAttribute("src")||"";if(!Z)return"";return`
+[${W.getAttribute("title")||"Embedded content"}](${Z})
+`}});const q=($??[]).map((K)=>K.trim()).filter(Boolean);if(q.length>0)V.addRule("cleanSelectors",{filter:(K)=>{const Q=K.matches;if(typeof Q!=="function")return!1;return q.some((W)=>{try{return Q.call(K,W)}catch{return!1}})},replacement:()=>""});if(J)J(V);return V}function w(J,$,V,q){let Q=D(V,q).turndown(J);$.forEach((W,Z)=>{const X=`CODEBLOCK-PLACEHOLDER-${Z}`,y=`
+\`\`\`${W.lang}
+${W.code}
+\`\`\`
+`;Q=Q.replace(X,y)});Q=Q.replace(/\u00a0/g," ");Q=Q.replace(/^[ \t]*[-·][ \t]*$/gm,"");Q=Q.replace(/^[ \t]+$/gm,"");Q=Q.replace(/[ \t]+$/gm,"");Q=Q.replace(/\n{3,}/g,`
+
+`);return Q}function U(J,$){return J.replace(/!\[([^\]]*)\]\(([^)]+)\)/g,(V,q,K)=>{const Q=$[K];return Q?`![${q}](${Q})`:V})}function h(J){const $=J.match(/\.(\w{3,4})(?:\?|$)/);return $?$[1]:"jpg"}async function p(J,$,V,q){const K={};if(J.length===0)return K;const Q=q||h,W=new Set,Z=J.filter((X)=>{if(W.has(X))return!1;W.add(X);return!0});for(let X=0;X<Z.length;X+=b){const y=Z.slice(X,X+b),j=await Promise.all(y.map(async(F,z)=>{const G=X+z+1;let O=F;if(O.startsWith("//"))O=`https:${O}`;const v=Q(O),Y=`img_${String(G).padStart(3,"0")}.${v}`,N=H.join($,Y);try{if((await C(O,N,{headers:V,timeout:15000})).success)return{remoteUrl:F,localPath:`images/${Y}`}}catch{}return null}));for(const F of j)if(F)K[F.remoteUrl]=F.localPath}return K}export async function downloadArticle(J,$){const{output:V,downloadImages:q=!0,imageHeaders:K,maxTitleLength:Q=80,configureTurndown:W,detectImageExt:Z,frontmatterLabels:X,cleanSelectors:y,stdout:j=!1}=$,F={...M,...X};if(!J.title)return[{title:"Error",author:"-",publish_time:"-",status:"failed — no title",size:"-",saved:"-"}];if(!J.contentHtml)return[{title:J.title,author:J.author||"-",publish_time:J.publishTime||"-",status:"failed — no content",size:"-",saved:"-"}];let z=w(J.contentHtml,J.codeBlocks||[],W,y);const G=E(J.title,Q);if(!j&&q&&J.imageUrls&&J.imageUrls.length>0){const B=H.join(V,G);P.mkdirSync(B,{recursive:!0});const R=H.join(B,"images");P.mkdirSync(R,{recursive:!0});const S=await p(J.imageUrls,R,K,Z);z=U(z,S)}const O=[`# ${J.title}`];if(J.author)O.push(`> ${F.author}: ${J.author}`);if(J.publishTime)O.push(`> ${F.publishTime}: ${J.publishTime}`);if(J.sourceUrl)O.push(`> ${F.sourceUrl}: ${J.sourceUrl}`);const Y=O.join(`
+`)+`
+
+---
+
+`+z,N=Buffer.byteLength(Y,"utf-8");if(j){process.stdout.write(Y.endsWith(`
+`)?Y:Y+`
+`);return[{title:J.title,author:J.author||"-",publish_time:J.publishTime||"-",status:"success",size:I(N),saved:"-"}]}const _=H.join(V,G);P.mkdirSync(_,{recursive:!0});const L=`${G}.md`,A=H.join(_,L);P.writeFileSync(A,Y,"utf-8");return[{title:J.title,author:J.author||"-",publish_time:J.publishTime||"-",status:"success",size:I(N),saved:A}]}