w-dwdata-hub 1.0.0

package/src/fetchAisixiang.mjs

@@ -0,0 +1,966 @@
+import get from 'lodash-es/get.js'
+import isestr from 'wsemi/src/isestr.mjs'
+import ispint from 'wsemi/src/ispint.mjs'
+import isp0int from 'wsemi/src/isp0int.mjs'
+import cint from 'wsemi/src/cint.mjs'
+import delay from 'wsemi/src/delay.mjs'
+import fetchWithRetry from './fetchWithRetry.mjs'
+import getOptFetch from './getOptFetch.mjs'
+import decodeEntities from './decodeEntities.mjs'
+import htmlToMarkdown from './htmlToMarkdown.mjs'
+import extractDivContent from './extractDivContent.mjs'
+import safeFilename from './safeFilename.mjs'
+
+
+//愛思想網站根網址
+let BASE_URL = 'https://www.aisixiang.com'
+
+
+//愛思想對User-Agent較敏感, 故固定送出桌面瀏覽器標頭
+let USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
+    '(KHTML, like Gecko) Chrome/120.0 Safari/537.36'
+
+
+//頁間延遲毫秒, 降低被封鎖機率
+let PAGE_DELAY_MS = 1000
+
+
+//翻頁安全上限, 50頁×30筆=1500筆
+let MAX_PAGES = 50
+
+
+//本模組之預設抓取設定
+let DFLT = {
+    responseType: 'text',
+    headers: { 'User-Agent': USER_AGENT, 'Accept-Language': 'zh-CN,zh;q=0.9' },
+    timeout: 30000,
+    maxRetries: 5,
+    baseDelayMs: 5000,
+    maxDelayMs: 30000,
+    label: 'fetch-aisixiang',
+}
+
+
+/**
+ * 正規化本模組共用設定
+ *
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @returns {Object} 回傳已正規化之設定物件，內含baseUrl、showLog與optFetch
+ */
+function getCfg(opt = {}) {
+
+    //baseUrl
+    let baseUrl = get(opt, 'baseUrl')
+    if (!isestr(baseUrl)) {
+        baseUrl = BASE_URL
+    }
+    baseUrl = baseUrl.replace(/\/+$/, '')
+
+    //optFetch
+    let optFetch = getOptFetch(opt, DFLT)
+
+    //pageDelayMs
+    let pageDelayMs = get(opt, 'pageDelayMs')
+    if (!isp0int(pageDelayMs)) {
+        pageDelayMs = PAGE_DELAY_MS
+    }
+    else {
+        pageDelayMs = cint(pageDelayMs)
+    }
+
+    //maxPages
+    let maxPages = get(opt, 'maxPages')
+    if (!ispint(maxPages)) {
+        maxPages = MAX_PAGES
+    }
+    else {
+        maxPages = Math.min(cint(maxPages), MAX_PAGES)
+    }
+
+    return { baseUrl, showLog: optFetch.showLog, pageDelayMs, maxPages, optFetch }
+}
+
+
+/**
+ * 由HTML抽出title標籤內容
+ *
+ * @param {String} html 輸入HTML字串
+ * @returns {String|null} 回傳標題字串，無title標籤時回傳null
+ */
+function extractTitle(html) {
+    let m = html.match(/<title>([\s\S]*?)<\/title>/i)
+    return m ? decodeEntities(m[1].trim()) : null
+}
+
+
+/**
+ * 解析作者專欄頁之文章清單
+ *
+ * 專欄頁以h3標記分類(论文、时评、随笔等), 分類區塊內為/data/<aid>.html之文章連結
+ *
+ * @param {String} html 輸入作者專欄頁HTML字串
+ * @param {String} baseUrl 輸入網站根網址字串
+ * @returns {Array} 回傳文章物件陣列，各物件為{aid,url,title,category}
+ */
+function parseAuthorColumnPage(html, baseUrl) {
+
+    let cats = ['论文', '时评', '随笔', '著作', '演讲', '读书', '访谈', '未分类']
+    let positions = []
+    for (let cat of cats) {
+        let re = new RegExp(`<h3>${cat}<\\/h3>`, 'g')
+        let m
+        while ((m = re.exec(html)) !== null) {
+            positions.push({ cat, pos: m.index })
+        }
+    }
+    positions.sort((a, b) => a.pos - b.pos)
+    positions.push({ cat: null, pos: html.length })
+
+    let items = []
+    for (let i = 0; i < positions.length - 1; i++) {
+        let { cat, pos } = positions[i]
+        let block = html.slice(pos, positions[i + 1].pos)
+        let linkRe = /<a\s+href="\/data\/(\d+)\.html"[^>]*>([\s\S]*?)<\/a>/g
+        let m
+        while ((m = linkRe.exec(block)) !== null) {
+            let title = m[2].replace(/<[^>]+>/g, '').trim()
+            if (!title) {
+                continue
+            }
+            items.push({
+                aid: m[1],
+                url: `${baseUrl}/data/${m[1]}.html`,
+                title,
+                category: cat,
+            })
+        }
+    }
+
+    return items
+}
+
+
+/**
+ * 解析搜尋結果頁之文章清單
+ *
+ * 精確切出search_list區塊再解析, 避免0結果時誤抓側邊欄連結
+ *
+ * @param {String} html 輸入搜尋結果頁HTML字串
+ * @param {String} baseUrl 輸入網站根網址字串
+ * @returns {Array} 回傳文章物件陣列，各物件為{aid,url,title,author}
+ */
+function parseSearchResults(html, baseUrl) {
+
+    let block = extractDivContent(html, 'search_list')
+    if (block === null) {
+        return []
+    }
+
+    let items = []
+    let seen = new Set()
+    let re = /href="\/data\/(\d+)\.html"[^>]*title="([^"]+)"/g
+    let m
+    while ((m = re.exec(block)) !== null) {
+        if (seen.has(m[1])) {
+            continue
+        }
+        seen.add(m[1])
+        let raw = m[2]
+        let idx = raw.indexOf('：')
+        items.push({
+            aid: m[1],
+            url: `${baseUrl}/data/${m[1]}.html`,
+            title: idx > 0 ? raw.slice(idx + 1).trim() : raw.trim(),
+            author: idx > 0 ? raw.slice(0, idx).trim() : '',
+        })
+    }
+
+    return items
+}
+
+
+/**
+ * 解析策展主題頁之文章清單
+ *
+ * 主題頁之連結無title屬性, 標題在anchor內文, 格式為<a>作者：標題</a>
+ *
+ * @param {String} html 輸入主題頁HTML字串
+ * @param {String} baseUrl 輸入網站根網址字串
+ * @returns {Array} 回傳文章物件陣列，各物件為{aid,url,title,author}
+ */
+function parseZhuantiArticles(html, baseUrl) {
+
+    let m = html.match(/([\s\S]+?)<div class="list_page">/)
+    let block = m ? m[1] : html
+
+    let items = []
+    let seen = new Set()
+    let re = /<a\s+href="\/data\/(\d+)\.html"[^>]*>([^<]+)<\/a>/g
+    let mm
+    while ((mm = re.exec(block)) !== null) {
+        if (seen.has(mm[1])) {
+            continue
+        }
+        seen.add(mm[1])
+        let raw = mm[2].trim()
+        let idx = raw.indexOf('：')
+        items.push({
+            aid: mm[1],
+            url: `${baseUrl}/data/${mm[1]}.html`,
+            title: idx > 0 ? raw.slice(idx + 1).trim() : raw,
+            author: idx > 0 ? raw.slice(0, idx).trim() : '',
+        })
+    }
+
+    return items
+}
+
+
+/**
+ * 由list_page區塊解析最大頁碼
+ *
+ * @param {String} html 輸入列表頁HTML字串
+ * @returns {Integer} 回傳總頁數整數，無分頁區塊時回傳1
+ */
+function parseSearchTotalPages(html) {
+
+    let m = html.match(/<div class="list_page">([\s\S]+?)<\/div>/)
+    if (!m) {
+        return 1
+    }
+
+    let nums = [...m[1].matchAll(/page=(\d+)/g)].map((x) => Number(x[1]))
+
+    return nums.length ? Math.max(...nums) : 1
+}
+
+
+/**
+ * 自動翻頁抓取全部列表頁
+ *
+ * 由第1頁解析總頁數後逐頁抓取, 頁間延遲1秒; 該頁無新項目即停止翻頁, 並以MAX_PAGES為安全上限
+ *
+ * @param {Function} buildUrl 輸入回傳指定頁碼網址之函數
+ * @param {Function} parser 輸入解析單頁HTML為項目陣列之函數
+ * @param {Object} cfg 輸入已正規化之設定物件
+ * @returns {Promise} 回傳Promise，resolve回傳{items,total_pages,pages_fetched}
+ */
+async function fetchAllSearchPages(buildUrl, parser, cfg) {
+
+    let all = []
+    let seen = new Set()
+    let totalPages = 1
+    let pagesFetched = 0
+
+    for (let page = 1; page <= cfg.maxPages; page++) {
+
+        let url = buildUrl(page)
+        if (cfg.showLog) {
+            process.stderr.write(`[info] fetching page ${page}${totalPages > 1 ? `/${totalPages}` : ''} ...\n`)
+        }
+        let html = await fetchWithRetry(url, cfg.optFetch)
+        pagesFetched++
+
+        if (page === 1) {
+            totalPages = parseSearchTotalPages(html)
+        }
+
+        let items = parser(html, cfg.baseUrl)
+        let fresh = items.filter((it) => !seen.has(it.aid))
+        if (fresh.length === 0) {
+            if (cfg.showLog) {
+                process.stderr.write(`[info] page ${page} 無新項目，停止翻頁\n`)
+            }
+            break
+        }
+
+        for (let it of fresh) {
+            seen.add(it.aid)
+        }
+        all.push(...fresh)
+
+        if (page >= totalPages) {
+            break
+        }
+        await delay(cfg.pageDelayMs)
+
+    }
+
+    return { items: all, total_pages: totalPages, pages_fetched: pagesFetched }
+}
+
+
+/**
+ * 抓取愛思想專欄作者清單
+ *
+ * 即時解析/thinktank/頁面, 回傳約960位專欄作者之slug與中文名
+ *
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
+ * @returns {Promise} 回傳Promise，resolve回傳作者物件陣列，各物件為{slug,name}
+ * @example
+ *
+ * import { fetchAuthorsList } from './src/fetchAisixiang.mjs'
+ *
+ * let test = async () => {
+ *
+ *     let rs = await fetchAuthorsList()
+ *     console.log(rs.length, rs[0])
+ *     // => 963 { slug: 'gezhaoguang', name: '葛兆光' }
+ *
+ * }
+ * await test()
+ *     .catch((err) => {
+ *         console.log(err)
+ *     })
+ *
+ */
+async function fetchAuthorsList(opt = {}) {
+
+    let cfg = getCfg(opt)
+
+    let html = await fetchWithRetry(`${cfg.baseUrl}/thinktank/`, cfg.optFetch)
+
+    let matches = [...html.matchAll(/href="\/thinktank\/([^"]+?)\.html"[^>]*>([^<]+)</g)]
+    let uniq = new Map()
+    for (let [, slug, rawName] of matches) {
+        let key = slug.toLowerCase()
+        if (uniq.has(key)) {
+            continue
+        }
+        let name = rawName.split(/[，、]/)[0].trim()
+        if (!name || name.length > 30) {
+            continue
+        }
+        uniq.set(key, { slug, name })
+    }
+
+    return [...uniq.values()]
+}
+
+
+/**
+ * 由作者清單查找指定中文名之作者
+ *
+ * 本函數不轉繁簡, 呼叫端須自行使用站方登錄字形(通常為簡體)
+ *
+ * @param {Array} authors 輸入作者物件陣列
+ * @param {String} name 輸入作者中文名字串
+ * @returns {Object|null} 回傳作者物件，查無時回傳null
+ * @example
+ *
+ * import { lookupAuthor } from './src/fetchAisixiang.mjs'
+ *
+ * console.log(lookupAuthor([{ slug: 'gezhaoguang', name: '葛兆光' }], '葛兆光'))
+ * // => { slug: 'gezhaoguang', name: '葛兆光' }
+ *
+ */
+function lookupAuthor(authors, name) {
+
+    if (!Array.isArray(authors)) {
+        return null
+    }
+
+    return authors.find((a) => a.name === name) || null
+}
+
+
+/**
+ * 抓取愛思想策展主題清單
+ *
+ * 即時解析/zhuanti/頁面, 回傳約800個策展主題之id、名稱與分類(学科、事件、人物)
+ *
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
+ * @returns {Promise} 回傳Promise，resolve回傳主題物件陣列，各物件為{id,name,category}
+ * @example
+ *
+ * import { fetchTopicsList } from './src/fetchAisixiang.mjs'
+ *
+ * let test = async () => {
+ *
+ *     let rs = await fetchTopicsList()
+ *     console.log(rs.length, rs[0])
+ *     // => 803 { id: '301', name: '大数据', category: '学科' }
+ *
+ * }
+ * await test()
+ *     .catch((err) => {
+ *         console.log(err)
+ *     })
+ *
+ */
+async function fetchTopicsList(opt = {}) {
+
+    let cfg = getCfg(opt)
+
+    let html = await fetchWithRetry(`${cfg.baseUrl}/zhuanti/`, cfg.optFetch)
+
+    let cats = [
+        { name: '学科', start: html.indexOf('<h3>学科关键词</h3>') },
+        { name: '事件', start: html.indexOf('<h3>事件关键词</h3>') },
+        { name: '人物', start: html.indexOf('<h3>人物关键词</h3>') },
+    ].filter((c) => c.start >= 0).sort((a, b) => a.start - b.start)
+    cats.push({ name: null, start: html.length })
+
+    let seen = new Set()
+    let topics = []
+    for (let i = 0; i < cats.length - 1; i++) {
+        let block = html.slice(cats[i].start, cats[i + 1].start)
+        let re = /href="\/zhuanti\/(\d+)\.html"[^>]*>([^<]+)</g
+        let m
+        while ((m = re.exec(block)) !== null) {
+            if (seen.has(m[1])) {
+                continue
+            }
+            seen.add(m[1])
+            let name = m[2].trim().replace(/^\[|\]$/g, '').trim()
+            if (!name || name.length > 50) {
+                continue
+            }
+            topics.push({ id: m[1], name, category: cats[i].name })
+        }
+    }
+
+    return topics
+}
+
+
+/**
+ * 由主題清單查找指定名稱之主題
+ *
+ * @param {Array} topics 輸入主題物件陣列
+ * @param {String} name 輸入主題名稱字串
+ * @returns {Object|null} 回傳主題物件，查無時回傳null
+ * @example
+ *
+ * import { lookupTopic } from './src/fetchAisixiang.mjs'
+ *
+ * console.log(lookupTopic([{ id: '301', name: '大数据' }], '大数据'))
+ * // => { id: '301', name: '大数据' }
+ *
+ */
+function lookupTopic(topics, name) {
+
+    if (!Array.isArray(topics)) {
+        return null
+    }
+
+    return topics.find((t) => t.name === name) || null
+}
+
+
+/**
+ * 抓取愛思想指定作者之文章清單
+ *
+ * 給name時先抓站方作者清單解析出slug再抓專欄頁(兩跳), 給slug則直接抓專欄頁;
+ * 查無此作者或該欄頁0篇時仍回status為'success'且count為0, 並以message說明, 以與抓取失敗區分
+ *
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.name] 輸入作者中文名字串，與slug擇一必填
+ * @param {String} [opt.slug] 輸入作者slug字串，與name擇一必填
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，name與slug皆未給時reject回傳錯誤物件
+ * @example
+ *
+ * import { fetchAuthorArticles } from './src/fetchAisixiang.mjs'
+ *
+ * let test = async () => {
+ *
+ *     let r = await fetchAuthorArticles({ name: '葛兆光' })
+ *     console.log(r.status, r.count, r.resolved.slug)
+ *     // => 'success' 123 'gezhaoguang'
+ *
+ * }
+ * await test()
+ *     .catch((err) => {
+ *         console.log(err)
+ *     })
+ *
+ */
+async function fetchAuthorArticles(opt = {}) {
+
+    let cfg = getCfg(opt)
+
+    //「有提供才檢」, 傳了但非有效字串視為未提供, 讓下方「至少一個」檢查接手
+    let name = get(opt, 'name')
+    if (!isestr(name)) {
+        name = ''
+    }
+    let slug = get(opt, 'slug')
+    if (!isestr(slug)) {
+        slug = ''
+    }
+    if (!name && !slug) {
+        throw new Error('需要 name 或 slug')
+    }
+
+    let resolved = null
+    let chineseName = name
+    let usedSlug = slug
+
+    if (!usedSlug) {
+
+        if (cfg.showLog) {
+            process.stderr.write('[info] fetching authors list from /thinktank/ ...\n')
+        }
+        let authors = await fetchAuthorsList(opt)
+        let author = lookupAuthor(authors, name)
+        if (!author) {
+            //抓到真實作者清單且確定無此作者, 屬成功查詢、結果0筆(非抓取失敗)
+            return {
+                status: 'success',
+                site: 'aisixiang',
+                mode: 'author',
+                query: name,
+                fetched_at: new Date().toISOString(),
+                authors_count: authors.length,
+                count: 0,
+                items: [],
+                message:
+                    `"${name}" 不在愛思想專欄作者清單中（共 ${authors.length} 位）。` +
+                    `尚無此作者文章（此為查詢成功的真實結果、0 筆，非抓取失敗）。` +
+                    `提醒：本函數不轉簡繁，呼叫端負責用站方登錄字形（通常是簡體）。`,
+            }
+        }
+
+        usedSlug = author.slug
+        chineseName = chineseName || author.name
+        resolved = { slug: usedSlug, url: `${cfg.baseUrl}/thinktank/${usedSlug}.html`, name: author.name }
+    }
+    else {
+        resolved = { slug: usedSlug, url: `${cfg.baseUrl}/thinktank/${usedSlug}.html` }
+    }
+
+    let html = await fetchWithRetry(resolved.url, cfg.optFetch)
+    let items = parseAuthorColumnPage(html, cfg.baseUrl)
+
+    let payload = {
+        status: 'success',
+        site: 'aisixiang',
+        mode: 'author',
+        query: chineseName || usedSlug,
+        resolved,
+        fetched_at: new Date().toISOString(),
+        count: items.length,
+        items,
+    }
+
+    //命中作者欄頁(含slug捷徑)但該欄頁實際0篇文章, 加message讓呼叫端能區分「真的查清楚、就是0篇」與「抓取失敗」
+    if (items.length === 0) {
+        payload.message =
+            `已命中愛思想作者欄頁（${resolved.url}）但該欄頁目前 0 篇文章。` +
+            `此為查詢成功的真實結果（非抓取失敗）；若 slug 為手動指定，請確認 slug 是否正確。`
+    }
+
+    return payload
+}
+
+
+/**
+ * 抓取愛思想指定關鍵字tag之文章清單
+ *
+ * 走站方keywords搜尋並自動翻頁全抓; 站方搜尋只認簡體, 呼叫端負責簡體化
+ *
+ * @param {String} keyword 輸入關鍵字字串
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，keyword非有效字串時reject回傳錯誤物件
+ * @example
+ *
+ * import { fetchKeywordArticles } from './src/fetchAisixiang.mjs'
+ *
+ * let test = async () => {
+ *
+ *     let r = await fetchKeywordArticles('老庄')
+ *     console.log(r.status, r.count)
+ *     // => 'success' 42
+ *
+ * }
+ * await test()
+ *     .catch((err) => {
+ *         console.log(err)
+ *     })
+ *
+ */
+async function fetchKeywordArticles(keyword, opt = {}) {
+
+    //check
+    if (!isestr(keyword)) {
+        throw new Error('需要 keyword')
+    }
+
+    let cfg = getCfg(opt)
+
+    let buildUrl = (page) => `${cfg.baseUrl}/data/search?searchfield=keywords&keywords=${encodeURIComponent(keyword)}&page=${page}`
+    let { items, total_pages, pages_fetched } = await fetchAllSearchPages(buildUrl, parseSearchResults, cfg)
+
+    if (items.length === 0) {
+        return {
+            status: 'success',
+            site: 'aisixiang',
+            mode: 'keyword',
+            query: keyword,
+            resolved: { search_url: buildUrl(1) },
+            fetched_at: new Date().toISOString(),
+            count: 0,
+            items: [],
+            message:
+                `關鍵字 "${keyword}" 在愛思想無相關文章。` +
+                `提醒：站方搜尋只認簡體，呼叫端負責簡體化；若已是簡體仍 0 筆，該主題可能無 tag 索引。`,
+        }
+    }
+
+    return {
+        status: 'success',
+        site: 'aisixiang',
+        mode: 'keyword',
+        query: keyword,
+        resolved: { search_url: buildUrl(1), total_pages, pages_fetched },
+        fetched_at: new Date().toISOString(),
+        count: items.length,
+        items,
+    }
+}
+
+
+/**
+ * 抓取愛思想指定標題關鍵字之文章清單
+ *
+ * 走站方title搜尋並自動翻頁全抓; 站方搜尋只認簡體, 呼叫端負責簡體化
+ *
+ * @param {String} keyword 輸入標題關鍵字字串
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，keyword非有效字串時reject回傳錯誤物件
+ * @example
+ *
+ * import { fetchTitleArticles } from './src/fetchAisixiang.mjs'
+ *
+ * let test = async () => {
+ *
+ *     let r = await fetchTitleArticles('禅宗与中国文化')
+ *     console.log(r.status, r.count)
+ *     // => 'success' 3
+ *
+ * }
+ * await test()
+ *     .catch((err) => {
+ *         console.log(err)
+ *     })
+ *
+ */
+async function fetchTitleArticles(keyword, opt = {}) {
+
+    //check
+    if (!isestr(keyword)) {
+        throw new Error('需要 keyword')
+    }
+
+    let cfg = getCfg(opt)
+
+    let buildUrl = (page) => `${cfg.baseUrl}/data/search?searchfield=title&keywords=${encodeURIComponent(keyword)}&page=${page}`
+    let { items, total_pages, pages_fetched } = await fetchAllSearchPages(buildUrl, parseSearchResults, cfg)
+
+    if (items.length === 0) {
+        return {
+            status: 'success',
+            site: 'aisixiang',
+            mode: 'title',
+            query: keyword,
+            resolved: { search_url: buildUrl(1) },
+            fetched_at: new Date().toISOString(),
+            count: 0,
+            items: [],
+            message:
+                `標題關鍵字 "${keyword}" 在愛思想無相關文章。` +
+                `提醒：站方搜尋只認簡體，呼叫端負責簡體化。`,
+        }
+    }
+
+    return {
+        status: 'success',
+        site: 'aisixiang',
+        mode: 'title',
+        query: keyword,
+        resolved: { search_url: buildUrl(1), total_pages, pages_fetched },
+        fetched_at: new Date().toISOString(),
+        count: items.length,
+        items,
+    }
+}
+
+
+/**
+ * 抓取愛思想指定策展主題之文章清單
+ *
+ * 給keyword時先抓站方主題清單解析出id再抓主題頁(兩跳), 給id則直接抓主題頁並自動翻頁全抓;
+ * 查無此主題時仍回status為'success'且count為0, 並以message建議改用fetchKeywordArticles(本函數不自動轉向)
+ *
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.keyword] 輸入主題名稱字串，與id擇一必填
+ * @param {String} [opt.id] 輸入主題id字串，與keyword擇一必填
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，keyword與id皆未給時reject回傳錯誤物件
+ * @example
+ *
+ * import { fetchTopicArticles } from './src/fetchAisixiang.mjs'
+ *
+ * let test = async () => {
+ *
+ *     let r = await fetchTopicArticles({ keyword: '大数据' })
+ *     console.log(r.status, r.count, r.resolved.id)
+ *     // => 'success' 60 '301'
+ *
+ * }
+ * await test()
+ *     .catch((err) => {
+ *         console.log(err)
+ *     })
+ *
+ */
+async function fetchTopicArticles(opt = {}) {
+
+    let cfg = getCfg(opt)
+
+    //「有提供才檢」, 傳了但非有效字串視為未提供, 讓下方「至少一個」檢查接手
+    let keyword = get(opt, 'keyword')
+    if (!isestr(keyword)) {
+        keyword = ''
+    }
+    let id = get(opt, 'id')
+    if (!isestr(id)) {
+        id = ''
+    }
+    if (!keyword && !id) {
+        throw new Error('需要 keyword 或 id')
+    }
+
+    let topicId = id
+    let topicName = keyword
+    let resolved = null
+
+    if (!topicId) {
+
+        if (cfg.showLog) {
+            process.stderr.write('[info] fetching topics list from /zhuanti/ ...\n')
+        }
+        let topics = await fetchTopicsList(opt)
+        let hit = lookupTopic(topics, keyword)
+        if (!hit) {
+            //抓到真實主題清單且確定不是策展主題, 屬成功查詢、結果0筆(非抓取失敗)
+            return {
+                status: 'success',
+                site: 'aisixiang',
+                mode: 'topic',
+                query: keyword,
+                fetched_at: new Date().toISOString(),
+                topics_count: topics.length,
+                count: 0,
+                items: [],
+                message:
+                    `"${keyword}" 不在愛思想策展主題清單中（共 ${topics.length} 個主題）。` +
+                    `建議改用 fetchKeywordArticles('${keyword}') 查 keyword tag 結果。` +
+                    `本函數不自動轉向，請呼叫端決定是否重試。`,
+            }
+        }
+
+        topicId = hit.id
+        topicName = hit.name
+        resolved = {
+            id: topicId,
+            name: hit.name,
+            category: hit.category,
+            url: `${cfg.baseUrl}/zhuanti/${topicId}.html`,
+        }
+    }
+    else {
+        resolved = { id: topicId, url: `${cfg.baseUrl}/zhuanti/${topicId}.html` }
+    }
+
+    let buildUrl = (page) => `${cfg.baseUrl}/zhuanti/${topicId}.html?page=${page}`
+    let { items, total_pages, pages_fetched } = await fetchAllSearchPages(buildUrl, parseZhuantiArticles, cfg)
+
+    let payload = {
+        status: 'success',
+        site: 'aisixiang',
+        mode: 'topic',
+        query: topicName || topicId,
+        resolved: { ...resolved, total_pages, pages_fetched },
+        fetched_at: new Date().toISOString(),
+        count: items.length,
+        items,
+    }
+
+    //命中主題(含id捷徑)但該主題頁實際0篇文章, 加message讓呼叫端能區分「真的查清楚、就是0篇」與「抓取失敗」
+    if (items.length === 0) {
+        payload.message =
+            `已命中愛思想策展主題（${resolved.url}）但該主題頁目前 0 篇文章。` +
+            `此為查詢成功的真實結果（非抓取失敗）；若需更多結果可改用 fetchKeywordArticles 查 keyword tag。`
+    }
+
+    return payload
+}
+
+
+/**
+ * 抓取愛思想單篇文章並轉為Markdown
+ *
+ * 由article-content區塊切出正文轉Markdown, 並產生含title、source與created之YAML frontmatter;
+ * 標題會去除站名後綴, 例如「葛兆光：禅宗与中国文化_爱思想」轉為「葛兆光：禅宗与中国文化」
+ *
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.aid] 輸入文章aid字串，與url擇一必填
+ * @param {String} [opt.url] 輸入文章網址字串，與aid擇一必填
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,url,title,chars,markdown}，aid與url皆未給或解析失敗時reject回傳錯誤物件
+ * @example
+ *
+ * import { fetchArticle } from './src/fetchAisixiang.mjs'
+ *
+ * let test = async () => {
+ *
+ *     let r = await fetchArticle({ aid: '146669' })
+ *     console.log(r.status, r.title, r.chars)
+ *     // => 'success' '葛兆光：禅宗与中国文化' 12345
+ *
+ * }
+ * await test()
+ *     .catch((err) => {
+ *         console.log(err)
+ *     })
+ *
+ */
+async function fetchArticle(opt = {}) {
+
+    let cfg = getCfg(opt)
+
+    //「有提供才檢」, 傳了但非有效字串視為未提供, 讓下方「至少一個」檢查接手
+    let aid = get(opt, 'aid')
+    if (!isestr(aid)) {
+        aid = ''
+    }
+    let url = get(opt, 'url')
+    if (!isestr(url)) {
+        url = ''
+    }
+    let targetUrl = aid ? `${cfg.baseUrl}/data/${aid}.html` : url
+    if (!targetUrl) {
+        throw new Error('需要 aid 或 url')
+    }
+
+    let html = await fetchWithRetry(targetUrl, cfg.optFetch)
+    let titleRaw = extractTitle(html)
+    let contentHtml = extractDivContent(html, 'article-content')
+    if (!titleRaw || !contentHtml) {
+        throw new Error(`解析失敗：title=${!!titleRaw}, content=${!!contentHtml}`)
+    }
+
+    let body = htmlToMarkdown(contentHtml)
+
+    //去掉站名後綴
+    let title = titleRaw.replace(/[_\-\s]*(?:_)?(?:爱思想|愛思想)\s*$/u, '').trim()
+    let today = new Date().toISOString().slice(0, 10)
+
+    let markdown =
+`---
+title: "${title}"
+source: "${targetUrl}"
+author:
+published:
+created: ${today}
+description:
+---
+${body}
+`
+
+    return {
+        status: 'success',
+        site: 'aisixiang',
+        mode: 'fetch',
+        url: targetUrl,
+        title,
+        chars: body.length,
+        markdown,
+    }
+}
+
+
+/**
+ * 抓取愛思想(aisixiang.com)
+ *
+ * @returns {Object} 回傳物件，其內可呼叫fetchAuthorsList、lookupAuthor、fetchTopicsList、lookupTopic、fetchAuthorArticles、fetchKeywordArticles、fetchTitleArticles、fetchTopicArticles、fetchArticle
+ * @example
+ *
+ * 詳見fetchAuthorArticles、fetchKeywordArticles、fetchTitleArticles、fetchTopicArticles、fetchArticle範例
+ *
+ */
+let fetchAisixiang = {
+    BASE_URL,
+    USER_AGENT,
+    MAX_PAGES,
+    safeFilename,
+    fetchAuthorsList,
+    lookupAuthor,
+    fetchTopicsList,
+    lookupTopic,
+    fetchAuthorArticles,
+    fetchKeywordArticles,
+    fetchTitleArticles,
+    fetchTopicArticles,
+    fetchArticle,
+}
+
+
+export {
+    BASE_URL,
+    USER_AGENT,
+    PAGE_DELAY_MS,
+    MAX_PAGES,
+    safeFilename,
+    fetchAuthorsList,
+    lookupAuthor,
+    fetchTopicsList,
+    lookupTopic,
+    fetchAuthorArticles,
+    fetchKeywordArticles,
+    fetchTitleArticles,
+    fetchTopicArticles,
+    fetchArticle
+}
+export default fetchAisixiang