w-dwdata-hub 1.0.0

package/src/toRocDate.mjs

@@ -0,0 +1,35 @@
+import isestr from 'wsemi/src/isestr.mjs'
+
+
+/**
+ * 西元YYYYMMDD日期字串轉民國日期字串
+ *
+ * 民國年為西元年減1911, 月與日保留原始2碼不補零亦不去零, 以符合證交所與櫃買中心API之查詢格式
+ *
+ * @param {String} dateStr 輸入西元日期YYYYMMDD字串
+ * @returns {String} 回傳民國日期字串，格式為'YYY/MM/DD'，輸入非有效字串時回傳''
+ * @example
+ *
+ * import toRocDate from './src/toRocDate.mjs'
+ *
+ * console.log(toRocDate('20260807'), toRocDate('19110101'))
+ * // => '115/08/07' '0/01/01'
+ *
+ */
+function toRocDate(dateStr) {
+
+    //check
+    if (!isestr(dateStr)) {
+        return ''
+    }
+
+    //y, m, d
+    let y = parseInt(dateStr.substring(0, 4), 10) - 1911
+    let m = dateStr.substring(4, 6)
+    let d = dateStr.substring(6, 8)
+
+    return `${y}/${m}/${d}`
+}
+
+
+export default toRocDate

package/test/WDwdataHub.test.mjs

@@ -0,0 +1,63 @@
+import assert from 'assert'
+import WDwdataHub from '../src/WDwdataHub.mjs'
+
+
+describe('WDwdataHub', function() {
+
+    it('匯出全部抓取函數', function() {
+        let r = Object.keys(WDwdataHub).sort()
+        let rr = [
+            'fetchAiNewsAggregator',
+            'fetchAisixiang',
+            'fetchCnyes',
+            'fetchGuancha',
+            'fetchHackerNews',
+            'fetchMoneydj',
+            'fetchMops',
+            'fetchRSS',
+            'fetchStatementdog',
+            'fetchTaifex',
+            'fetchTpex3insti',
+            'fetchTpexMargin',
+            'fetchTpexStock',
+            'fetchTwDataHoliday',
+            'fetchTwseMargin',
+            'fetchTwseStock',
+            'fetchTwseT86',
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('單一來源者為函數', function() {
+        let ks = [
+            'fetchAiNewsAggregator', 'fetchCnyes', 'fetchHackerNews', 'fetchMoneydj', 'fetchMops',
+            'fetchRSS', 'fetchStatementdog', 'fetchTaifex', 'fetchTpex3insti', 'fetchTpexMargin',
+            'fetchTpexStock', 'fetchTwDataHoliday', 'fetchTwseMargin', 'fetchTwseStock', 'fetchTwseT86',
+        ]
+        let r = ks.map((k) => typeof WDwdataHub[k])
+        let rr = ks.map(() => 'function')
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('多函數來源者為物件, 其內各項為函數或常數', function() {
+        let r = [
+            typeof WDwdataHub.fetchAisixiang,
+            typeof WDwdataHub.fetchAisixiang.fetchAuthorArticles,
+            typeof WDwdataHub.fetchAisixiang.fetchArticle,
+            typeof WDwdataHub.fetchGuancha,
+            typeof WDwdataHub.fetchGuancha.fetchAuthorArticles,
+            typeof WDwdataHub.fetchGuancha.fetchArticle,
+            Array.isArray(WDwdataHub.fetchGuancha.KNOWN_TOPICS),
+        ]
+        let rr = ['object', 'function', 'function', 'object', 'function', 'function', true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('不匯出內部共用函數', function() {
+        let ks = ['fetchWithRetry', 'getOptFetch', 'isYmd', 'toRocDate', 'toDatetimeUTC8', 'decodeEntities', 'htmlToMarkdown', 'extractDivContent', 'parseIntComma']
+        let r = ks.map((k) => Object.prototype.hasOwnProperty.call(WDwdataHub, k))
+        let rr = ks.map(() => false)
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/decodeEntities.test.mjs

@@ -0,0 +1,42 @@
+import assert from 'assert'
+import decodeEntities from '../src/decodeEntities.mjs'
+
+
+describe('decodeEntities', function() {
+
+    it('解碼具名實體', function() {
+        let r = decodeEntities('a&b<c>d"e'f g')
+        let rr = 'a&b<c>d"e\'f g'
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('解碼中文網站正文常見之標點實體', function() {
+        let r = decodeEntities('&hellip;&mdash;&ndash;&middot;&laquo;&raquo;&copy;&reg;')
+        let rr = '…—–·«»©®'
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('解碼十進位與十六進位數值實體', function() {
+        let r = decodeEntities('&#65;&#66;&#x43;&#x44;&#20013;&#x6587;')
+        let rr = 'ABCD中文'
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('無對照之具名實體保留原字串', function() {
+        let r = decodeEntities('a&unknown;b&amp;c')
+        let rr = 'a&unknown;b&c'
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('非字串回傳空字串', function() {
+        let r = [
+            decodeEntities(null),
+            decodeEntities(undefined),
+            decodeEntities(123),
+            decodeEntities({}),
+        ]
+        let rr = ['', '', '', '']
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/extractDivContent.test.mjs

@@ -0,0 +1,48 @@
+import assert from 'assert'
+import extractDivContent from '../src/extractDivContent.mjs'
+
+
+describe('extractDivContent', function() {
+
+    it('以深度追蹤切出含巢狀div之完整內容', function() {
+        let r = extractDivContent('<body><div class="ct">a<div>b</div>c</div><p>外部</p></body>', 'ct')
+        let rr = 'a<div>b</div>c'
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('切出空容器回傳空字串而非null', function() {
+        let r = extractDivContent('<div class="ct"></div><div class="side"><a>不應被抓</a></div>', 'ct')
+        let rr = ''
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('起始標籤採前綴比對, div帶其他屬性亦可命中', function() {
+        let r = extractDivContent('<div class="content all-txt" id="cont"><p>內文</p></div>', 'content all-txt')
+        let rr = '<p>內文</p>'
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('找不到指定class回傳null', function() {
+        let r = extractDivContent('<div class="other">a</div>', 'ct')
+        let rr = null
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('標籤未閉合回傳null', function() {
+        let r = extractDivContent('<div class="ct">a<div>b', 'ct')
+        let rr = null
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('非有效輸入回傳null', function() {
+        let r = [
+            extractDivContent(null, 'ct'),
+            extractDivContent('<div class="ct">a</div>', ''),
+            extractDivContent('<div class="ct">a</div>', null),
+            extractDivContent(123, 'ct'),
+        ]
+        let rr = [null, null, null, null]
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/fetchAiNewsAggregator.test.mjs

@@ -0,0 +1,79 @@
+import assert from 'assert'
+import fetchAiNewsAggregator from '../src/fetchAiNewsAggregator.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+
+
+describe('fetchAiNewsAggregator', function() {
+
+    let svr = null
+
+    before(async function() {
+        svr = await serverForTest()
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('轉為統一格式並去除欄位前後空白', async function() {
+        let r = await fetchAiNewsAggregator({ url: svr.url('/ai-news/latest-24h.json'), showLog: false })
+        let rr = [
+            {
+                url: 'https://ai.example/a1',
+                time: '2026-08-07 09:02:03',
+                title: 'AI標題一',
+                description: '',
+                from: 'Example News',
+            },
+            {
+                url: 'https://ai.example/a2',
+                time: '',
+                title: 'AI標題二',
+                description: '',
+                from: 'Other News',
+            },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('published_at無法解析時time退回空字串', async function() {
+        let t = await fetchAiNewsAggregator({ url: svr.url('/ai-news/latest-24h.json'), showLog: false })
+        let r = t[1].time
+        let rr = ''
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('回傳非JSON時reject', async function() {
+        let r = null
+        await fetchAiNewsAggregator({ url: svr.url('/json/bad'), maxRetries: 0, showLog: false })
+            .then(() => {
+                r = 'resolved'
+            })
+            .catch((err) => {
+                r = err.message.startsWith('回傳非JSON格式')
+            })
+        let rr = true
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('HTTP 404不重試並reject', async function() {
+        let c0 = svr.getCount('/status/404')
+        let e = null
+        await fetchAiNewsAggregator({ url: svr.url('/status/404'), maxRetries: 3, showLog: false })
+            .catch((err) => {
+                e = err
+            })
+        let r = [e.statusCode, svr.getCount('/status/404') - c0]
+        let rr = [404, 1]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('回應無items欄位時回傳空陣列', async function() {
+        let r = await fetchAiNewsAggregator({ url: svr.url('/json/ok'), showLog: false })
+        let rr = []
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/fetchAisixiang.test.mjs

@@ -0,0 +1,225 @@
+import assert from 'assert'
+import fetchAisixiang, {
+    BASE_URL,
+    MAX_PAGES,
+    fetchAuthorsList,
+    lookupAuthor,
+    fetchTopicsList,
+    lookupTopic,
+    fetchAuthorArticles,
+    fetchKeywordArticles,
+    fetchTitleArticles,
+    fetchTopicArticles,
+    fetchArticle
+} from '../src/fetchAisixiang.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+
+
+describe('fetchAisixiang', function() {
+
+    let svr = null
+    let optFast = null
+
+    before(async function() {
+        svr = await serverForTest()
+        optFast = { baseUrl: svr.url(''), baseDelayMs: 10, maxDelayMs: 10, pageDelayMs: 0, showLog: false }
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('模組常數與預設匯出之函數齊備', function() {
+        let r = [
+            BASE_URL,
+            MAX_PAGES,
+            Object.keys(fetchAisixiang).sort(),
+        ]
+        let rr = [
+            'https://www.aisixiang.com',
+            50,
+            [
+                'BASE_URL', 'MAX_PAGES', 'USER_AGENT', 'fetchArticle', 'fetchAuthorArticles',
+                'fetchAuthorsList', 'fetchKeywordArticles', 'fetchTitleArticles', 'fetchTopicArticles',
+                'fetchTopicsList', 'lookupAuthor', 'lookupTopic', 'safeFilename',
+            ],
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorsList解析作者清單並去除重複slug', async function() {
+        let r = await fetchAuthorsList(optFast)
+        let rr = [
+            { slug: 'gezhaoguang', name: '葛兆光' },
+            { slug: 'qinhui', name: '秦晖' },
+            { slug: 'zhangwei', name: '张维' },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('lookupAuthor查無時回傳null', function() {
+        let authors = [{ slug: 'gezhaoguang', name: '葛兆光' }]
+        let r = [
+            lookupAuthor(authors, '葛兆光'),
+            lookupAuthor(authors, '葛兆光'.replace('葛', '楊')),
+            lookupAuthor(authors, ''),
+            lookupAuthor(null, '葛兆光'),
+        ]
+        let rr = [{ slug: 'gezhaoguang', name: '葛兆光' }, null, null, null]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchTopicsList解析主題清單並標記分類', async function() {
+        let r = await fetchTopicsList(optFast)
+        let rr = [
+            { id: '301', name: '大数据', category: '学科' },
+            { id: '302', name: '政治学', category: '学科' },
+            { id: '401', name: '中美贸易战', category: '事件' },
+            { id: '501', name: '康德', category: '人物' },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('lookupTopic查無時回傳null', function() {
+        let topics = [{ id: '301', name: '大数据', category: '学科' }]
+        let r = [
+            lookupTopic(topics, '大数据').id,
+            lookupTopic(topics, '大數據'),
+            lookupTopic(null, '大数据'),
+        ]
+        let rr = ['301', null, null]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorArticles以name兩跳查得專欄文章並標記分類', async function() {
+        let t = await fetchAuthorArticles({ ...optFast, name: '葛兆光' })
+        let r = [t.status, t.site, t.mode, t.resolved.slug, t.resolved.name, t.count, t.items[0], t.items[2].category]
+        let rr = [
+            'success', 'aisixiang', 'author', 'gezhaoguang', '葛兆光', 3,
+            { aid: '146669', url: svr.url('/data/146669.html'), title: '禅宗与中国文化', category: '论文' },
+            '时评',
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorArticles以slug捷徑跳過清單查找', async function() {
+        let c0 = svr.getCount('/thinktank/')
+        let t = await fetchAuthorArticles({ ...optFast, slug: 'gezhaoguang' })
+        let r = [t.status, t.count, svr.getCount('/thinktank/') - c0]
+        let rr = ['success', 3, 0]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorArticles查無作者時回success且count為0並附說明', async function() {
+        let t = await fetchAuthorArticles({ ...optFast, name: '不存在的作者XYZ' })
+        let r = [t.status, t.count, t.items, t.authors_count, t.message.includes('尚無此作者文章'), t.message.includes('簡繁')]
+        let rr = ['success', 0, [], 3, true, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorArticles命中欄頁但0篇時回success且附說明', async function() {
+        let t = await fetchAuthorArticles({ ...optFast, slug: 'nobody' })
+        let r = [t.status, t.count, t.message.includes('0 篇文章')]
+        let rr = ['success', 0, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchKeywordArticles自動翻頁並解析搜尋結果', async function() {
+        let t = await fetchKeywordArticles('老庄', optFast)
+        let r = [t.status, t.mode, t.count, t.items[0], t.resolved.search_url.includes('searchfield=keywords')]
+        let rr = [
+            'success', 'keyword', 2,
+            { aid: '300001', url: svr.url('/data/300001.html'), title: '老庄之道', author: '作者甲' },
+            true,
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchKeywordArticles無結果時回success且count為0, 不誤抓側邊欄連結', async function() {
+        let t = await fetchKeywordArticles('老莊', optFast)
+        let r = [t.status, t.count, t.items, t.message.includes('無相關文章'), t.message.includes('簡體')]
+        let rr = ['success', 0, [], true, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchTitleArticles走title搜尋端點', async function() {
+        let t = await fetchTitleArticles('老庄', optFast)
+        let r = [t.status, t.mode, t.count, t.resolved.search_url.includes('searchfield=title')]
+        let rr = ['success', 'title', 2, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchTopicArticles以keyword兩跳並自動翻頁全抓', async function() {
+        let t = await fetchTopicArticles({ ...optFast, keyword: '大数据' })
+        let r = [t.status, t.mode, t.resolved.id, t.resolved.category, t.resolved.total_pages, t.resolved.pages_fetched, t.count]
+        let rr = ['success', 'topic', '301', '学科', 2, 2, 3]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchTopicArticles以id捷徑跳過清單查找', async function() {
+        let c0 = svr.getCount('/zhuanti/')
+        let t = await fetchTopicArticles({ ...optFast, id: '301' })
+        let r = [t.status, t.count, svr.getCount('/zhuanti/') - c0]
+        let rr = ['success', 3, 0]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchTopicArticles查無主題時回success且count為0並建議改查keyword', async function() {
+        let t = await fetchTopicArticles({ ...optFast, keyword: '不存在的主題ABC' })
+        let r = [t.status, t.count, t.topics_count, t.message.includes('fetchKeywordArticles')]
+        let rr = ['success', 0, 4, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchArticle轉為含frontmatter之Markdown並去除站名後綴', async function() {
+        let t = await fetchArticle({ ...optFast, aid: '146669' })
+        let r = [
+            t.status,
+            t.mode,
+            t.url,
+            t.title,
+            t.markdown.startsWith('---\n'),
+            t.markdown.includes(`source: "${svr.url('/data/146669.html')}"`),
+            t.markdown.includes('**禅宗**'),
+            t.markdown.includes('[士大夫](/data/1.html)'),
+            t.markdown.includes('&思想史脉络'),
+            t.markdown.includes('嵌套段落亦應被完整切出。'),
+        ]
+        let rr = ['success', 'fetch', svr.url('/data/146669.html'), '葛兆光：禅宗与中国文化', true, true, true, true, true, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchArticle以url為入口', async function() {
+        let t = await fetchArticle({ ...optFast, url: svr.url('/data/146669.html') })
+        let r = [t.status, t.title, t.chars > 30]
+        let rr = ['success', '葛兆光：禅宗与中国文化', true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchArticle解析失敗時reject', async function() {
+        let r = null
+        await fetchArticle({ ...optFast, aid: '999999' })
+            .then(() => {
+                r = 'resolved'
+            })
+            .catch((err) => {
+                r = err.message.startsWith('解析失敗')
+            })
+        let rr = true
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('必要參數缺失時拋錯', async function() {
+        let r = []
+        await fetchAuthorArticles(optFast).catch((err) => r.push(err.message))
+        await fetchKeywordArticles(undefined, optFast).catch((err) => r.push(err.message))
+        await fetchTitleArticles(undefined, optFast).catch((err) => r.push(err.message))
+        await fetchTopicArticles(optFast).catch((err) => r.push(err.message))
+        await fetchArticle(optFast).catch((err) => r.push(err.message))
+        let rr = ['需要 name 或 slug', '需要 keyword', '需要 keyword', '需要 keyword 或 id', '需要 aid 或 url']
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/fetchCnyes.test.mjs

@@ -0,0 +1,61 @@
+import assert from 'assert'
+import fetchCnyes from '../src/fetchCnyes.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+
+
+describe('fetchCnyes', function() {
+
+    let svr = null
+
+    before(async function() {
+        svr = await serverForTest()
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('轉為統一格式並組出新聞內容頁網址', async function() {
+        let r = await fetchCnyes({ url: svr.url('/cnyes/newslist'), pageDelayMs: 0, showLog: false })
+        let rr = [
+            { time: '2026-08-06 15:06:40', title: '鉅亨標題一', link: 'https://news.cnyes.com/news/id/5001' },
+            { time: '', title: '鉅亨標題二', link: 'https://news.cnyes.com/news/id/5002' },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('publishAt缺漏時該筆time退回空字串, 不影響其餘新聞', async function() {
+        let t = await fetchCnyes({ url: svr.url('/cnyes/newslist'), pageDelayMs: 0, showLog: false })
+        let r = [t.length, t[1].time, t[1].title]
+        let rr = [2, '', '鉅亨標題二']
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('無更多資料時停止翻頁', async function() {
+        let c0 = svr.getCount('/cnyes/newslist')
+        await fetchCnyes({ url: svr.url('/cnyes/newslist'), pageDelayMs: 0, showLog: false })
+        let r = svr.getCount('/cnyes/newslist') - c0
+        let rr = 2
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('targetTotal達標即停止翻頁', async function() {
+        let c0 = svr.getCount('/cnyes/newslist')
+        let t = await fetchCnyes({ url: svr.url('/cnyes/newslist'), targetTotal: 1, pageDelayMs: 0, showLog: false })
+        let r = [t.length, svr.getCount('/cnyes/newslist') - c0]
+        let rr = [1, 1]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('HTTP 403納入重試範圍, 含初始共執行maxRetries+1次', async function() {
+        let c0 = svr.getCount('/status/403')
+        await fetchCnyes({ url: svr.url('/status/403'), maxRetries: 2, baseDelayMs: 10, maxDelayMs: 10, pageDelayMs: 0, showLog: false })
+            .catch(() => {})
+        let r = svr.getCount('/status/403') - c0
+        let rr = 3
+        assert.strict.deepEqual(r, rr)
+    })
+
+})