w-dwdata-hub 1.0.0

package/test/fetchGuancha.test.mjs

@@ -0,0 +1,190 @@
+import assert from 'assert'
+import fetchGuancha, {
+    BASE_URL,
+    KNOWN_TOPICS,
+    isHomepageHtml,
+    parseListPage,
+    fetchAuthorsList,
+    lookupAuthor,
+    lookupTopic,
+    fetchAuthorArticles,
+    fetchKeywordArticles,
+    fetchTitleArticles,
+    fetchTopicArticles,
+    fetchArticle
+} from '../src/fetchGuancha.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+import { guanchaHomeHtml, guanchaListPage1Html } from './tools/fixtures.mjs'
+
+
+describe('fetchGuancha', function() {
+
+    let svr = null
+    let optFast = null
+
+    before(async function() {
+        svr = await serverForTest()
+        optFast = { baseUrl: svr.url(''), maxRetries: 0, pageDelayMs: 0, showLog: false }
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('模組常數與預設匯出之函數齊備', function() {
+        let r = [
+            BASE_URL,
+            Array.isArray(KNOWN_TOPICS),
+            KNOWN_TOPICS.every((t) => typeof t.slug === 'string' && typeof t.name === 'string' && t.slug.length > 0 && t.name.length > 0),
+            Object.keys(fetchGuancha).sort(),
+        ]
+        let rr = [
+            'https://www.guancha.cn',
+            true,
+            true,
+            [
+                'BASE_URL', 'KNOWN_TOPICS', 'MAX_PAGES', 'USER_AGENT', 'fetchArticle', 'fetchAuthorArticles',
+                'fetchAuthorsList', 'fetchKeywordArticles', 'fetchTitleArticles', 'fetchTopicArticles',
+                'lookupAuthor', 'lookupTopic', 'safeFilename',
+            ],
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('KNOWN_TOPICS之分頁版slug排在大頻道版之前, 令lookupTopic之first-match取到可翻頁者', function() {
+        let r = [
+            lookupTopic('财经').slug,
+            lookupTopic('军事').slug,
+            lookupTopic('俄罗斯之声').slug,
+            lookupTopic('不存在的主題XYZ'),
+            lookupTopic(''),
+            lookupTopic(null),
+        ]
+        let rr = ['CaiJing', 'JunShi', 'ELuoSiZhiSheng', null, null, null]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('isHomepageHtml以A-Z作者索引區塊辨識首頁', function() {
+        let r = [
+            isHomepageHtml(guanchaHomeHtml),
+            isHomepageHtml(guanchaListPage1Html),
+        ]
+        let rr = [true, false]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('parseListPage不限定文章URL之slug', function() {
+        let r = parseListPage(guanchaListPage1Html, 'https://www.guancha.cn')
+        let rr = [
+            { url: 'https://www.guancha.cn/AnSheng/2026_08_07_800001.shtml', title: '觀察者網標題一', slug: 'AnSheng' },
+            { url: 'https://www.guancha.cn/AnSheng/2026_08_06_800002.shtml', title: '觀察者網標題二', slug: 'AnSheng' },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorsList由首頁A-Z索引解析作者對照', async function() {
+        let t = await fetchAuthorsList(optFast)
+        let r = [t.length, t[0], lookupAuthor(t, '张维为').slug, lookupAuthor(t, '不存在XYZ')]
+        let rr = [6, { slug: 'AnSheng', name: '安生', letter: 'A' }, 'ZhangWeiWei', null]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorArticles以name兩跳並自動翻頁至無新項目為止', async function() {
+        let t = await fetchAuthorArticles({ ...optFast, name: '安生' })
+        let r = [t.status, t.site, t.mode, t.resolved.slug, t.resolved.name, t.resolved.pages_fetched, t.count]
+        let rr = ['success', 'guancha', 'author', 'AnSheng', '安生', 3, 3]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorArticles以slug捷徑跳過索引查找', async function() {
+        let c0 = svr.getCount('/')
+        let t = await fetchAuthorArticles({ ...optFast, slug: 'AnSheng' })
+        let r = [t.status, t.count, svr.getCount('/') - c0]
+        let rr = ['success', 3, 0]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchAuthorArticles查無作者時回success且count為0並附說明', async function() {
+        let t = await fetchAuthorArticles({ ...optFast, name: '完全不存在的作者XYZ123' })
+        let r = [t.status, t.count, t.items, t.authors_count, t.message.includes('尚無此作者文章')]
+        let rr = ['success', 0, [], 6, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('slug不存在時第1頁被導向首頁, 回status為error且reason為redirected-to-homepage', async function() {
+        let t = await fetchAuthorArticles({ ...optFast, slug: 'NotExistSlug' })
+        let r = [t.status, t.mode, t.reason, t.error.includes('不存在')]
+        let rr = ['error', 'author', 'redirected-to-homepage', true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchKeywordArticles與fetchTitleArticles因站方簽名混淆一律回error', async function() {
+        let t1 = await fetchKeywordArticles('人工智能', optFast)
+        let t2 = await fetchTitleArticles('人工智能', optFast)
+        let r = [
+            t1.status, t1.mode, t1.error, t1.message.includes('sojson'),
+            t2.status, t2.mode, t2.error,
+        ]
+        let rr = ['error', 'keyword', 'unsupported-by-curl', true, 'error', 'title', 'unsupported-by-curl']
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchTopicArticles以name查對照表後翻頁全抓', async function() {
+        let t = await fetchTopicArticles({ ...optFast, name: '财经' })
+        let r = [t.status, t.mode, t.resolved.slug, t.count]
+        let rr = ['success', 'topic', 'CaiJing', 2]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchTopicArticles不在對照表時fail-fast', async function() {
+        let t = await fetchTopicArticles({ ...optFast, name: '完全不存在的主題ABC123' })
+        let r = [t.status, t.mode, t.error, t.topics_count, t.message.includes('slug')]
+        let rr = ['error', 'topic', 'topic-not-in-table', KNOWN_TOPICS.length, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('fetchArticle轉為含frontmatter之Markdown, 保留圖片並抽出作者與發布時間', async function() {
+        let t = await fetchArticle({ ...optFast, url: svr.url('/AnSheng/2026_08_07_800001.shtml') })
+        let r = [
+            t.status,
+            t.mode,
+            t.title,
+            t.author,
+            t.published,
+            t.markdown.startsWith('---\n'),
+            t.markdown.includes(`source: "${svr.url('/AnSheng/2026_08_07_800001.shtml')}"`),
+            t.markdown.includes('![](https://i.guancha.cn/p1.jpg)'),
+            t.markdown.includes('**粗體**'),
+        ]
+        let rr = ['success', 'fetch', 'OPEC+宣布增产', '文/观察者网 王五', '2026-08-07 09:02:03', true, true, true, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('文章已下架被導向首頁時回status為error且error為article-not-found', async function() {
+        let t = await fetchArticle({ ...optFast, url: svr.url('/AnSheng/2026_01_01_000000.shtml') })
+        let r = [t.status, t.mode, t.error, t.message.includes('無此文章')]
+        let rr = ['error', 'fetch', 'article-not-found', true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('抓取失敗時回status為error', async function() {
+        let t = await fetchArticle({ ...optFast, url: svr.url('/nothing.shtml') })
+        let r = [t.status, t.mode, typeof t.error]
+        let rr = ['error', 'fetch', 'string']
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('必要參數缺失時拋錯', async function() {
+        let r = []
+        await fetchAuthorArticles(optFast).catch((err) => r.push(err.message))
+        await fetchKeywordArticles(undefined, optFast).catch((err) => r.push(err.message))
+        await fetchTitleArticles(undefined, optFast).catch((err) => r.push(err.message))
+        await fetchTopicArticles(optFast).catch((err) => r.push(err.message))
+        await fetchArticle(optFast).catch((err) => r.push(err.message))
+        let rr = ['需要 name 或 slug', '需要 keyword', '需要 keyword', '需要 name 或 slug', '需要 url']
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/fetchHackerNews.test.mjs

@@ -0,0 +1,78 @@
+import assert from 'assert'
+import fetchHackerNews from '../src/fetchHackerNews.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+
+
+describe('fetchHackerNews', function() {
+
+    let svr = null
+
+    before(async function() {
+        svr = await serverForTest()
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('僅保留story, 無url者退回站內討論頁網址', async function() {
+        let r = await fetchHackerNews(30, { apiBase: svr.url('/hn'), showLog: false })
+        let rr = [
+            {
+                url: 'https://hn.example/1',
+                time: '2026-08-06 15:06:40',
+                title: 'HN標題一',
+                description: '',
+                from: 'Hacker News',
+            },
+            {
+                url: 'https://news.ycombinator.com/item?id=102',
+                time: '2026-08-06 15:07:40',
+                title: 'HN標題二',
+                description: '',
+                from: 'Hacker News',
+            },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('limit限制取回篇數', async function() {
+        let t = await fetchHackerNews(1, { apiBase: svr.url('/hn'), showLog: false })
+        let r = [t.length, t[0].title]
+        let rr = [1, 'HN標題一']
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('limit非正整數時改用預設值30而非回傳空陣列', async function() {
+        let r = []
+        for (let limit of [0, -1, NaN, null, undefined, 'abc']) {
+            let t = await fetchHackerNews(limit, { apiBase: svr.url('/hn'), showLog: false })
+            r.push(t.length)
+        }
+        let rr = [2, 2, 2, 2, 2, 2]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('apiBase尾端斜線會被正規化', async function() {
+        let t = await fetchHackerNews(30, { apiBase: svr.url('/hn') + '/', showLog: false })
+        let r = t.length
+        let rr = 2
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('取ID清單失敗時reject', async function() {
+        let r = null
+        await fetchHackerNews(30, { apiBase: svr.url('/nothing'), maxRetries: 0, showLog: false })
+            .then(() => {
+                r = 'resolved'
+            })
+            .catch((err) => {
+                r = err.statusCode
+            })
+        let rr = 404
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/fetchMoneydj.test.mjs

@@ -0,0 +1,82 @@
+import assert from 'assert'
+import fetchMoneydj from '../src/fetchMoneydj.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+
+
+describe('fetchMoneydj', function() {
+
+    let svr = null
+
+    //optFast, 令重試與頁間等待縮至最短, 避免測試耗時
+    let optFast = { baseDelayMs: 10, maxDelayMs: 10, pageDelayMs: 0, showLog: false }
+
+    before(async function() {
+        svr = await serverForTest()
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('逐頁抓取並合併, 相對路徑連結補上列表頁origin', async function() {
+        let r = await fetchMoneydj({ ...optFast, baseUrl: svr.url('/moneydj/newsreallist.aspx?a=mb06&index1='), totalPages: 2 })
+        let rr = [
+            { time: '08/07 09:02', title: 'MoneyDJ標題一', link: svr.url('/kmdj/news/newsviewer.aspx?a=m1') },
+            { time: '09:05', title: 'MoneyDJ標題二', link: 'https://www.moneydj.com/m2' },
+            { time: '昨 23:59', title: 'MoneyDJ標題三', link: svr.url('/kmdj/news/newsviewer.aspx?a=m4') },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('時間格式不符之列被略過', async function() {
+        let t = await fetchMoneydj({ ...optFast, baseUrl: svr.url('/moneydj/newsreallist.aspx?a=mb06&index1='), totalPages: 1 })
+        let r = [t.length, t.some((v) => v.title === '不應出現')]
+        let rr = [2, false]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('totalPages控制抓取頁數', async function() {
+        let c0 = svr.getCount('/moneydj/newsreallist.aspx')
+        await fetchMoneydj({ ...optFast, baseUrl: svr.url('/moneydj/newsreallist.aspx?a=mb06&index1='), totalPages: 3 })
+        let r = svr.getCount('/moneydj/newsreallist.aspx') - c0
+        let rr = 3
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('onPageDone於每頁完成時被呼叫', async function() {
+        let rs = []
+        await fetchMoneydj({
+            ...optFast,
+            baseUrl: svr.url('/moneydj/newsreallist.aspx?a=mb06&index1='),
+            totalPages: 2,
+            onPageDone: (pageIndex, itemCount, totalPages) => {
+                rs.push([pageIndex, itemCount, totalPages])
+            },
+        })
+        let rr = [[1, 2, 2], [2, 1, 2]]
+        assert.strict.deepEqual(rs, rr)
+    })
+
+    it('單頁抓取失敗時跳過該頁不中斷整體抓取', async function() {
+        let t = await fetchMoneydj({ ...optFast, baseUrl: svr.url('/moneydj-flaky/newsreallist.aspx?index1='), totalPages: 2, maxRetries: 0 })
+        let r = [t.length, t[0].title]
+        let rr = [1, 'MoneyDJ標題三']
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('全部頁面皆0筆時reject', async function() {
+        let r = null
+        await fetchMoneydj({ ...optFast, baseUrl: svr.url('/moneydj-empty/newsreallist.aspx?index1='), totalPages: 2 })
+            .then(() => {
+                r = 'resolved'
+            })
+            .catch((err) => {
+                r = err.message.startsWith('抓取到 0 筆新聞')
+            })
+        let rr = true
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/fetchMops.test.mjs

@@ -0,0 +1,106 @@
+import assert from 'assert'
+import fetchMops, { MARKET_KINDS, getPayload } from '../src/fetchMops.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+import hasChrome from './tools/hasChrome.mjs'
+
+
+describe('fetchMops', function() {
+
+    let svr = null
+    let chromeOk = false
+
+    before(async function() {
+        this.timeout(120000)
+        svr = await serverForTest()
+        chromeOk = await hasChrome()
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('目標市場別為上市、上櫃、興櫃與公開發行共4種', function() {
+        let r = MARKET_KINDS
+        let rr = [
+            { name: '上市', marketKind: 'sii' },
+            { name: '上櫃', marketKind: 'otc' },
+            { name: '興櫃', marketKind: 'rotc' },
+            { name: '公開發行', marketKind: 'pub' },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('查詢payload帶入指定市場別且其餘欄位固定', function() {
+        let r = getPayload('sii')
+        let rr = {
+            scopeType: '2',
+            companyId: '',
+            dateType: '2',
+            firstDate: '',
+            lastDate: '',
+            marketKind: 'sii',
+            announcementBasis: '0',
+            dateRangeType: '1',
+            announcementType: '1',
+            sort: '1',
+            encodeURIComponent: 1,
+            step: 1,
+            firstin: 1,
+            off: 1,
+        }
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('逐一查詢4種市場別並回傳結果與hasError', async function() {
+        if (!chromeOk) {
+            this.skip()
+        }
+        this.timeout(180000)
+        let t = await fetchMops({
+            pageUrl: svr.url('/mops/page'),
+            apiUrl: svr.url('/mops/api/t146sb10'),
+            maxRetries: 0,
+            showLog: false,
+        })
+        let r = [
+            t.hasError,
+            t.results.length,
+            t.results.map((v) => v.market),
+            t.results.map((v) => v.marketKind),
+            t.results[0].data.result[0].companyId,
+            t.results[2].data.code,
+        ]
+        let rr = [
+            false,
+            4,
+            ['上市', '上櫃', '興櫃', '公開發行'],
+            ['sii', 'otc', 'rotc', 'pub'],
+            '2330',
+            406,
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('application-level錯誤(code非200亦非406)時該筆帶error且hasError為true', async function() {
+        if (!chromeOk) {
+            this.skip()
+        }
+        this.timeout(180000)
+        let t = await fetchMops({
+            pageUrl: svr.url('/mops/page'),
+            apiUrl: svr.url('/mops-error/api/t146sb10'),
+            maxRetries: 0,
+            showLog: false,
+        })
+        let r = [
+            t.hasError,
+            t.results.length,
+            t.results.every((v) => String(v.error).startsWith('MOPS code 500')),
+        ]
+        let rr = [true, 4, true]
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/fetchRSS.test.mjs

@@ -0,0 +1,85 @@
+import assert from 'assert'
+import fetchRSS from '../src/fetchRSS.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+
+
+describe('fetchRSS', function() {
+
+    let svr = null
+
+    before(async function() {
+        svr = await serverForTest()
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('非http與https網址拋錯', async function() {
+        let r = []
+        for (let url of [null, undefined, '', 123, 'abc', 'ftp://a.com/feed.xml']) {
+            await fetchRSS(url)
+                .then(() => {
+                    r.push('resolved')
+                })
+                .catch((err) => {
+                    r.push(err.message)
+                })
+        }
+        let rr = [
+            'rssUrl 須為有效的 http/https RSS 網址',
+            'rssUrl 須為有效的 http/https RSS 網址',
+            'rssUrl 須為有效的 http/https RSS 網址',
+            'rssUrl 須為有效的 http/https RSS 網址',
+            'rssUrl 須為有效的 http/https RSS 網址',
+            'rssUrl 須為有效的 http/https RSS 網址',
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('解析RSS並轉為統一格式, from優先取creator否則退回訂閱源標題', async function() {
+        let r = await fetchRSS(svr.url('/rss/feed.xml'), { showLog: false })
+        let rr = [
+            {
+                url: 'https://rss.example/1',
+                time: '2026-08-07 09:02:03',
+                title: 'RSS標題一',
+                description: '描述一',
+                from: '作者甲',
+            },
+            {
+                url: 'https://rss.example/2',
+                time: '2026-08-07 10:03:04',
+                title: 'RSS標題二',
+                description: '描述二',
+                from: '測試訂閱源',
+            },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('HTTP 404納入重試範圍, 含初始共執行maxRetries+1次', async function() {
+        let c0 = svr.getCount('/status/404')
+        await fetchRSS(svr.url('/status/404'), { maxRetries: 2, baseDelayMs: 10, maxDelayMs: 10, showLog: false })
+            .catch(() => {})
+        let r = svr.getCount('/status/404') - c0
+        let rr = 3
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('內容非XML時reject', async function() {
+        let r = null
+        await fetchRSS(svr.url('/json/ok'), { maxRetries: 0, showLog: false })
+            .then(() => {
+                r = 'resolved'
+            })
+            .catch(() => {
+                r = 'rejected'
+            })
+        let rr = 'rejected'
+        assert.strict.deepEqual(r, rr)
+    })
+
+})

package/test/fetchStatementdog.test.mjs

@@ -0,0 +1,69 @@
+import assert from 'assert'
+import fetchStatementdog from '../src/fetchStatementdog.mjs'
+import serverForTest from './tools/serverForTest.mjs'
+
+
+describe('fetchStatementdog', function() {
+
+    let svr = null
+
+    //optFast, 令重試等待縮至10毫秒, 避免測試耗時
+    let optFast = { baseDelayMs: 10, maxDelayMs: 10, showLog: false }
+
+    before(async function() {
+        svr = await serverForTest()
+    })
+
+    after(async function() {
+        if (svr) {
+            await svr.close()
+        }
+    })
+
+    it('以主要selector解析, 相對路徑連結補上來源網址origin', async function() {
+        let r = await fetchStatementdog({ ...optFast, url: svr.url('/statementdog/news/latest') })
+        let rr = [
+            { time: '2026-08-07', title: '財報狗標題一', link: svr.url('/news/n1') },
+            { time: '2026-08-06', title: '財報狗標題二', link: 'https://other.example/n2' },
+        ]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('無標題之項目被略過', async function() {
+        let t = await fetchStatementdog({ ...optFast, url: svr.url('/statementdog/news/latest') })
+        let r = t.length
+        let rr = 2
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('主要selector未匹配時改用備援selector', async function() {
+        let t = await fetchStatementdog({ ...optFast, url: svr.url('/statementdog-fallback/news/latest') })
+        let r = [t.length, t[0].title, t[0].link]
+        let rr = [2, '備援標題一', svr.url('/news/f1')]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('解析到0筆時reject且不重試', async function() {
+        let c0 = svr.getCount('/statementdog-empty/news/latest')
+        let r = null
+        await fetchStatementdog({ ...optFast, url: svr.url('/statementdog-empty/news/latest'), maxRetries: 3 })
+            .then(() => {
+                r = 'resolved'
+            })
+            .catch((err) => {
+                r = [err.message.startsWith('抓取到 0 筆新聞'), svr.getCount('/statementdog-empty/news/latest') - c0]
+            })
+        let rr = [true, 1]
+        assert.strict.deepEqual(r, rr)
+    })
+
+    it('HTTP 403納入重試範圍, 含初始共執行maxRetries+1次', async function() {
+        let c0 = svr.getCount('/status/403')
+        await fetchStatementdog({ ...optFast, url: svr.url('/status/403'), maxRetries: 2 })
+            .catch(() => {})
+        let r = svr.getCount('/status/403') - c0
+        let rr = 3
+        assert.strict.deepEqual(r, rr)
+    })
+
+})