w-dwdata-hub 1.0.13 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/w-dwdata-hub.umd.js +2 -2
- package/dist/w-dwdata-hub.umd.js.map +1 -1
- package/docs/WDwdataHub.mjs.html +1 -1
- package/docs/decodeEntities.mjs.html +1 -1
- package/docs/extractDivContent.mjs.html +1 -1
- package/docs/fetchAiNewsAggregator.mjs.html +2 -2
- package/docs/fetchAisixiang.mjs.html +13 -13
- package/docs/fetchCnyes.mjs.html +8 -8
- package/docs/fetchGuancha.mjs.html +15 -15
- package/docs/fetchHackerNews.mjs.html +2 -2
- package/docs/fetchMoneydj.mjs.html +4 -4
- package/docs/fetchMops.mjs.html +14 -14
- package/docs/fetchRSS.mjs.html +2 -2
- package/docs/fetchStatementdog.mjs.html +6 -6
- package/docs/fetchTaifex.mjs.html +14 -14
- package/docs/fetchTpex3insti.mjs.html +5 -5
- package/docs/fetchTpexMargin.mjs.html +5 -5
- package/docs/fetchTpexStock.mjs.html +4 -4
- package/docs/fetchTwDataHoliday.mjs.html +2 -2
- package/docs/fetchTwseMargin.mjs.html +5 -5
- package/docs/fetchTwseStock.mjs.html +4 -4
- package/docs/fetchTwseT86.mjs.html +5 -5
- package/docs/fetchWithRetry.mjs.html +7 -7
- package/docs/getOptFetch.mjs.html +9 -9
- package/docs/global.html +29 -29
- package/docs/htmlToMarkdown.mjs.html +1 -1
- package/docs/index.html +1 -1
- package/docs/isYmd.mjs.html +1 -1
- package/docs/parseIntComma.mjs.html +1 -1
- package/docs/safeFilename.mjs.html +1 -1
- package/docs/stripHtml.mjs.html +1 -1
- package/docs/toDatetimeUTC8.mjs.html +1 -1
- package/docs/toRocDate.mjs.html +1 -1
- package/g.mjs +13 -13
- package/package.json +5 -5
- package/src/fetchAiNewsAggregator.mjs +1 -1
- package/src/fetchAisixiang.mjs +12 -12
- package/src/fetchCnyes.mjs +7 -7
- package/src/fetchGuancha.mjs +14 -14
- package/src/fetchHackerNews.mjs +1 -1
- package/src/fetchMoneydj.mjs +3 -3
- package/src/fetchMops.mjs +13 -13
- package/src/fetchRSS.mjs +1 -1
- package/src/fetchStatementdog.mjs +5 -5
- package/src/fetchTaifex.mjs +13 -13
- package/src/fetchTpex3insti.mjs +4 -4
- package/src/fetchTpexMargin.mjs +4 -4
- package/src/fetchTpexStock.mjs +3 -3
- package/src/fetchTwDataHoliday.mjs +1 -1
- package/src/fetchTwseMargin.mjs +4 -4
- package/src/fetchTwseStock.mjs +3 -3
- package/src/fetchTwseT86.mjs +4 -4
- package/src/fetchWithRetry.mjs +6 -6
- package/src/getOptFetch.mjs +8 -8
- package/test/fetchAiNewsAggregator.test.mjs +5 -5
- package/test/fetchAisixiang.test.mjs +1 -1
- package/test/fetchCnyes.test.mjs +5 -5
- package/test/fetchGuancha.test.mjs +1 -1
- package/test/fetchHackerNews.test.mjs +5 -5
- package/test/fetchMoneydj.test.mjs +1 -1
- package/test/fetchMops.test.mjs +2 -2
- package/test/fetchRSS.test.mjs +11 -11
- package/test/fetchStatementdog.test.mjs +1 -1
- package/test/fetchTaifex.test.mjs +1 -1
- package/test/fetchTpex3insti.test.mjs +1 -1
- package/test/fetchTpexMargin.test.mjs +1 -1
- package/test/fetchTpexStock.test.mjs +1 -1
- package/test/fetchTwDataHoliday.test.mjs +1 -1
- package/test/fetchTwseMargin.test.mjs +1 -1
- package/test/fetchTwseStock.test.mjs +1 -1
- package/test/fetchTwseT86.test.mjs +1 -1
- package/test/fetchWithRetry.test.mjs +1 -1
|
@@ -87,7 +87,7 @@ export default parseIntComma
|
|
|
87
87
|
<br class="clear">
|
|
88
88
|
|
|
89
89
|
<footer>
|
|
90
|
-
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on
|
|
90
|
+
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Sun Sep 13 2026 14:29:36 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
|
|
91
91
|
</footer>
|
|
92
92
|
|
|
93
93
|
<script>prettyPrint();</script>
|
|
@@ -89,7 +89,7 @@ export default safeFilename
|
|
|
89
89
|
<br class="clear">
|
|
90
90
|
|
|
91
91
|
<footer>
|
|
92
|
-
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on
|
|
92
|
+
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Sun Sep 13 2026 14:29:36 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
|
|
93
93
|
</footer>
|
|
94
94
|
|
|
95
95
|
<script>prettyPrint();</script>
|
package/docs/stripHtml.mjs.html
CHANGED
|
@@ -117,7 +117,7 @@ export default stripHtml
|
|
|
117
117
|
<br class="clear">
|
|
118
118
|
|
|
119
119
|
<footer>
|
|
120
|
-
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on
|
|
120
|
+
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Sun Sep 13 2026 14:29:36 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
|
|
121
121
|
</footer>
|
|
122
122
|
|
|
123
123
|
<script>prettyPrint();</script>
|
|
@@ -147,7 +147,7 @@ export default toDatetimeUTC8
|
|
|
147
147
|
<br class="clear">
|
|
148
148
|
|
|
149
149
|
<footer>
|
|
150
|
-
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on
|
|
150
|
+
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Sun Sep 13 2026 14:29:36 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
|
|
151
151
|
</footer>
|
|
152
152
|
|
|
153
153
|
<script>prettyPrint();</script>
|
package/docs/toRocDate.mjs.html
CHANGED
|
@@ -94,7 +94,7 @@ export default toRocDate
|
|
|
94
94
|
<br class="clear">
|
|
95
95
|
|
|
96
96
|
<footer>
|
|
97
|
-
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on
|
|
97
|
+
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Sun Sep 13 2026 14:29:36 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
|
|
98
98
|
</footer>
|
|
99
99
|
|
|
100
100
|
<script>prettyPrint();</script>
|
package/g.mjs
CHANGED
|
@@ -4,67 +4,67 @@ import wi from './src/WDwdataHub.mjs'
|
|
|
4
4
|
let test = async () => {
|
|
5
5
|
|
|
6
6
|
//綜合新聞: RSS訂閱源, 回傳統一格式{url,time,title,description,from}
|
|
7
|
-
let r1 = await wi.fetchRSS('https://www.ptt.cc/atom/Gossiping.xml', {
|
|
7
|
+
let r1 = await wi.fetchRSS('https://www.ptt.cc/atom/Gossiping.xml', { useShowLog: false })
|
|
8
8
|
console.log('rss:', r1.length, r1[0].time, r1[0].title)
|
|
9
9
|
// => rss: 19 2026-08-07 09:30:19 [問卦] 公視是怎麼走到今天這一步的?
|
|
10
10
|
|
|
11
11
|
//綜合新聞: 擋node fetch之站台(403)改走curl抓取, 並以withContent附feed內嵌全文(去標籤純文字)
|
|
12
|
-
let r1b = await wi.fetchRSS('https://alphaarchitect.com/feed', { method: 'curl', withContent: true,
|
|
12
|
+
let r1b = await wi.fetchRSS('https://alphaarchitect.com/feed', { method: 'curl', withContent: true, useShowLog: false })
|
|
13
13
|
console.log('rss(curl):', r1b.length, r1b[0].content.length, r1b[0].description.length)
|
|
14
14
|
// => rss(curl): 5 6339 545
|
|
15
15
|
|
|
16
16
|
//綜合新聞: Hacker News最新文章, 可指定取回篇數
|
|
17
|
-
let r2 = await wi.fetchHackerNews(5, {
|
|
17
|
+
let r2 = await wi.fetchHackerNews(5, { useShowLog: false })
|
|
18
18
|
console.log('hackerNews:', r2.length, r2[0].from, r2[0].title)
|
|
19
19
|
// => hackerNews: 5 Hacker News Romania blasts rock to divert water from drought-hit Danube to nuclear reactor
|
|
20
20
|
|
|
21
21
|
//綜合新聞: AI News Aggregator近24小時彙整
|
|
22
|
-
let r3 = await wi.fetchAiNewsAggregator({
|
|
22
|
+
let r3 = await wi.fetchAiNewsAggregator({ useShowLog: false })
|
|
23
23
|
console.log('aiNews:', r3.length, r3[0].from)
|
|
24
24
|
// => aiNews: 914 夕小瑶科技说
|
|
25
25
|
|
|
26
26
|
//台股新聞: 鉅亨網, 逐頁抓至targetTotal則
|
|
27
|
-
let r4 = await wi.fetchCnyes({ targetTotal: 30,
|
|
27
|
+
let r4 = await wi.fetchCnyes({ targetTotal: 30, useShowLog: false })
|
|
28
28
|
console.log('cnyes:', r4.length, r4[0].time, r4[0].link)
|
|
29
29
|
// => cnyes: 30 2026-08-07 09:20:46 https://news.cnyes.com/news/id/6563766
|
|
30
30
|
|
|
31
31
|
//台股資料: 證交所上市個股日成交資訊, 指定日期會由整月資料過濾為當日單筆
|
|
32
|
-
let r5 = await wi.fetchTwseStock('20260806', '2330', {
|
|
32
|
+
let r5 = await wi.fetchTwseStock('20260806', '2330', { useShowLog: false })
|
|
33
33
|
console.log('twseStock:', r5.stat, r5.data.length, r5.data[0][6])
|
|
34
34
|
// => twseStock: OK 1 2,365.00
|
|
35
35
|
|
|
36
36
|
//台股資料: 櫃買中心上櫃融資融券, 未指定個股則回全市場
|
|
37
|
-
let r6 = await wi.fetchTpexMargin('20260806', ['6488'], {
|
|
37
|
+
let r6 = await wi.fetchTpexMargin('20260806', ['6488'], { useShowLog: false })
|
|
38
38
|
console.log('tpexMargin:', r6.source, r6.count, r6.data[0].marginChange)
|
|
39
39
|
// => tpexMargin: tpex_margin 1 -694
|
|
40
40
|
|
|
41
41
|
//台股資料: 期交所台指期、三大法人未平倉與Put/Call Ratio, 單支失敗僅記於errors
|
|
42
|
-
let r7 = await wi.fetchTaifex('20260806', {
|
|
42
|
+
let r7 = await wi.fetchTaifex('20260806', { useShowLog: false })
|
|
43
43
|
console.log('taifex:', r7.futures.tx.close, r7.pcRatio.ratio, r7.errors)
|
|
44
44
|
// => taifex: 44280 109.14 []
|
|
45
45
|
|
|
46
46
|
//台股資料: 證交所休市日期, 給定日期則另回傳該日是否休市
|
|
47
|
-
let r8 = await wi.fetchTwDataHoliday('20260101', {
|
|
47
|
+
let r8 = await wi.fetchTwDataHoliday('20260101', { useShowLog: false })
|
|
48
48
|
console.log('holiday:', r8.dataYear, r8.totalHolidays, r8.isHoliday, r8.holidayName)
|
|
49
49
|
// => holiday: 2026 22 true 中華民國開國紀念日
|
|
50
50
|
|
|
51
51
|
//中文評論網站: 愛思想, 依作者中文名查專欄文章清單
|
|
52
|
-
let r9 = await wi.fetchAisixiang.fetchAuthorArticles({ name: '葛兆光',
|
|
52
|
+
let r9 = await wi.fetchAisixiang.fetchAuthorArticles({ name: '葛兆光', useShowLog: false })
|
|
53
53
|
console.log('aisixiang:', r9.status, r9.resolved.slug, r9.count)
|
|
54
54
|
// => aisixiang: success gezhaoguang 170
|
|
55
55
|
|
|
56
56
|
//中文評論網站: 愛思想, 抓單篇文章轉為含frontmatter之Markdown
|
|
57
|
-
let r10 = await wi.fetchAisixiang.fetchArticle({ aid: '146669',
|
|
57
|
+
let r10 = await wi.fetchAisixiang.fetchArticle({ aid: '146669', useShowLog: false })
|
|
58
58
|
console.log('aisixiang:', r10.status, r10.title, r10.chars)
|
|
59
59
|
// => aisixiang: success 葛兆光:禅宗与中国文化 16986
|
|
60
60
|
|
|
61
61
|
//中文評論網站: 觀察者網, 依主題中文名查文章清單, 自動選用可翻頁之欄目slug
|
|
62
|
-
let r11 = await wi.fetchGuancha.fetchTopicArticles({ name: '财经', maxPages: 1,
|
|
62
|
+
let r11 = await wi.fetchGuancha.fetchTopicArticles({ name: '财经', maxPages: 1, useShowLog: false })
|
|
63
63
|
console.log('guancha:', r11.status, r11.resolved.slug, r11.count)
|
|
64
64
|
// => guancha: success CaiJing 30
|
|
65
65
|
|
|
66
66
|
//查無資料時回status為success且count為0, 抓取失敗才回status為error, 兩者可明確區分
|
|
67
|
-
let r12 = await wi.fetchGuancha.fetchAuthorArticles({ name: '不存在的作者XYZ',
|
|
67
|
+
let r12 = await wi.fetchGuancha.fetchAuthorArticles({ name: '不存在的作者XYZ', useShowLog: false })
|
|
68
68
|
console.log('guancha:', r12.status, r12.count, r12.message)
|
|
69
69
|
// => guancha: success 0 "不存在的作者XYZ" 不在觀察者網作者索引中(共 641 位)。尚無此作者文章。提醒:本函數不轉繁簡,呼叫端負責用站方登錄字形(簡體)。
|
|
70
70
|
|
package/package.json
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "w-dwdata-hub",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.15",
|
|
4
4
|
"main": "dist/w-dwdata-hub.umd.js",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"cheerio": "^1.2.0",
|
|
7
7
|
"rss-parser": "^3.13.0",
|
|
8
|
-
"w-fetch-web": "^1.0.
|
|
9
|
-
"wsemi": "^1.8.
|
|
8
|
+
"w-fetch-web": "^1.0.19",
|
|
9
|
+
"wsemi": "^1.8.92"
|
|
10
10
|
},
|
|
11
11
|
"devDependencies": {
|
|
12
|
-
"w-package-tools": "^1.1.
|
|
12
|
+
"w-package-tools": "^1.1.16"
|
|
13
13
|
},
|
|
14
14
|
"scripts": {
|
|
15
|
-
"test": "mocha --parallel --timeout 180000",
|
|
15
|
+
"test": "mocha --parallel \"test/*.test.mjs\" --timeout 180000",
|
|
16
16
|
"deploy": "gh-pages -d docs"
|
|
17
17
|
},
|
|
18
18
|
"repository": {
|
|
@@ -32,7 +32,7 @@ let DFLT = {
|
|
|
32
32
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設5
|
|
33
33
|
* @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數,預設3000
|
|
34
34
|
* @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數,預設15000
|
|
35
|
-
* @param {Boolean} [opt.
|
|
35
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示重試訊息布林值,預設true
|
|
36
36
|
* @returns {Promise} 回傳Promise,resolve回傳新聞物件陣列,各物件為{url,time,title,description,from},抓取失敗時reject回傳錯誤物件
|
|
37
37
|
* @example
|
|
38
38
|
*
|
package/src/fetchAisixiang.mjs
CHANGED
|
@@ -77,7 +77,7 @@ function getCfg(opt = {}) {
|
|
|
77
77
|
maxPages = Math.min(cint(maxPages), MAX_PAGES)
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
-
return { baseUrl,
|
|
80
|
+
return { baseUrl, useShowLog: optFetch.useShowLog, pageDelayMs, maxPages, optFetch }
|
|
81
81
|
}
|
|
82
82
|
|
|
83
83
|
|
|
@@ -255,7 +255,7 @@ async function fetchAllSearchPages(buildUrl, parser, cfg) {
|
|
|
255
255
|
for (let page = 1; page <= cfg.maxPages; page++) {
|
|
256
256
|
|
|
257
257
|
let url = buildUrl(page)
|
|
258
|
-
if (cfg.
|
|
258
|
+
if (cfg.useShowLog) {
|
|
259
259
|
process.stderr.write(`[info] fetching page ${page}${totalPages > 1 ? `/${totalPages}` : ''} ...\n`)
|
|
260
260
|
}
|
|
261
261
|
let html = await fetchWithRetry(url, cfg.optFetch)
|
|
@@ -268,7 +268,7 @@ async function fetchAllSearchPages(buildUrl, parser, cfg) {
|
|
|
268
268
|
let items = parser(html, cfg.baseUrl)
|
|
269
269
|
let fresh = items.filter((it) => !seen.has(it.aid))
|
|
270
270
|
if (fresh.length === 0) {
|
|
271
|
-
if (cfg.
|
|
271
|
+
if (cfg.useShowLog) {
|
|
272
272
|
process.stderr.write(`[info] page ${page} 無新項目,停止翻頁\n`)
|
|
273
273
|
}
|
|
274
274
|
break
|
|
@@ -301,7 +301,7 @@ async function fetchAllSearchPages(buildUrl, parser, cfg) {
|
|
|
301
301
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
302
302
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
303
303
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
304
|
-
* @param {Boolean} [opt.
|
|
304
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
305
305
|
* @returns {Promise} 回傳Promise,resolve回傳作者物件陣列,各物件為{slug,name}
|
|
306
306
|
* @example
|
|
307
307
|
*
|
|
@@ -381,7 +381,7 @@ function lookupAuthor(authors, name) {
|
|
|
381
381
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
382
382
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
383
383
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
384
|
-
* @param {Boolean} [opt.
|
|
384
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
385
385
|
* @returns {Promise} 回傳Promise,resolve回傳主題物件陣列,各物件為{id,name,category}
|
|
386
386
|
* @example
|
|
387
387
|
*
|
|
@@ -474,7 +474,7 @@ function lookupTopic(topics, name) {
|
|
|
474
474
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
475
475
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
476
476
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
477
|
-
* @param {Boolean} [opt.
|
|
477
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
478
478
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items},name與slug皆未給時reject回傳錯誤物件
|
|
479
479
|
* @example
|
|
480
480
|
*
|
|
@@ -516,7 +516,7 @@ async function fetchAuthorArticles(opt = {}) {
|
|
|
516
516
|
|
|
517
517
|
if (!usedSlug) {
|
|
518
518
|
|
|
519
|
-
if (cfg.
|
|
519
|
+
if (cfg.useShowLog) {
|
|
520
520
|
process.stderr.write('[info] fetching authors list from /thinktank/ ...\n')
|
|
521
521
|
}
|
|
522
522
|
let authors = await fetchAuthorsList(opt)
|
|
@@ -584,7 +584,7 @@ async function fetchAuthorArticles(opt = {}) {
|
|
|
584
584
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
585
585
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
586
586
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
587
|
-
* @param {Boolean} [opt.
|
|
587
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
588
588
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items},keyword非有效字串時reject回傳錯誤物件
|
|
589
589
|
* @example
|
|
590
590
|
*
|
|
@@ -656,7 +656,7 @@ async function fetchKeywordArticles(keyword, opt = {}) {
|
|
|
656
656
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
657
657
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
658
658
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
659
|
-
* @param {Boolean} [opt.
|
|
659
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
660
660
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items},keyword非有效字串時reject回傳錯誤物件
|
|
661
661
|
* @example
|
|
662
662
|
*
|
|
@@ -730,7 +730,7 @@ async function fetchTitleArticles(keyword, opt = {}) {
|
|
|
730
730
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
731
731
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
732
732
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
733
|
-
* @param {Boolean} [opt.
|
|
733
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
734
734
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items},keyword與id皆未給時reject回傳錯誤物件
|
|
735
735
|
* @example
|
|
736
736
|
*
|
|
@@ -772,7 +772,7 @@ async function fetchTopicArticles(opt = {}) {
|
|
|
772
772
|
|
|
773
773
|
if (!topicId) {
|
|
774
774
|
|
|
775
|
-
if (cfg.
|
|
775
|
+
if (cfg.useShowLog) {
|
|
776
776
|
process.stderr.write('[info] fetching topics list from /zhuanti/ ...\n')
|
|
777
777
|
}
|
|
778
778
|
let topics = await fetchTopicsList(opt)
|
|
@@ -847,7 +847,7 @@ async function fetchTopicArticles(opt = {}) {
|
|
|
847
847
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
848
848
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
849
849
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
850
|
-
* @param {Boolean} [opt.
|
|
850
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
851
851
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{status,site,mode,url,title,chars,markdown},aid與url皆未給或解析失敗時reject回傳錯誤物件
|
|
852
852
|
* @example
|
|
853
853
|
*
|
package/src/fetchCnyes.mjs
CHANGED
|
@@ -52,7 +52,7 @@ let DEFAULT_DAYS_BACK = 10
|
|
|
52
52
|
* @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設10
|
|
53
53
|
* @param {Integer} [opt.baseDelayMs=5000] 輸入重試之線性退避基礎毫秒整數,預設5000
|
|
54
54
|
* @param {Integer} [opt.maxDelayMs=30000] 輸入重試之線性退避上限毫秒整數,預設30000
|
|
55
|
-
* @param {Boolean} [opt.
|
|
55
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
56
56
|
* @returns {Promise} 回傳Promise,resolve回傳新聞物件陣列,各物件為{time,title,link},抓取失敗時reject回傳錯誤物件
|
|
57
57
|
* @example
|
|
58
58
|
*
|
|
@@ -117,13 +117,13 @@ async function fetchCnyes(opt = {}) {
|
|
|
117
117
|
|
|
118
118
|
//optFetch
|
|
119
119
|
let optFetch = getOptFetch(opt, DFLT)
|
|
120
|
-
let
|
|
120
|
+
let useShowLog = optFetch.useShowLog
|
|
121
121
|
|
|
122
122
|
//now, startAt
|
|
123
123
|
let now = Math.floor(Date.now() / 1000)
|
|
124
124
|
let startAt = now - 86400 * daysBack
|
|
125
125
|
|
|
126
|
-
if (
|
|
126
|
+
if (useShowLog) {
|
|
127
127
|
console.log('Starting to fetch Anue (tw_stock) news...')
|
|
128
128
|
}
|
|
129
129
|
|
|
@@ -146,14 +146,14 @@ async function fetchCnyes(opt = {}) {
|
|
|
146
146
|
//items
|
|
147
147
|
let items = get(data, 'items.data')
|
|
148
148
|
if (!isearr(items)) {
|
|
149
|
-
if (
|
|
149
|
+
if (useShowLog) {
|
|
150
150
|
console.log('No more items found.')
|
|
151
151
|
}
|
|
152
152
|
break
|
|
153
153
|
}
|
|
154
154
|
|
|
155
155
|
allItems = allItems.concat(items)
|
|
156
|
-
if (
|
|
156
|
+
if (useShowLog) {
|
|
157
157
|
console.log(`Page ${page}: Fetched ${items.length} items. Total so far: ${allItems.length}`)
|
|
158
158
|
}
|
|
159
159
|
page++
|
|
@@ -164,7 +164,7 @@ async function fetchCnyes(opt = {}) {
|
|
|
164
164
|
|
|
165
165
|
//finalItems
|
|
166
166
|
let finalItems = allItems.slice(0, targetTotal)
|
|
167
|
-
if (
|
|
167
|
+
if (useShowLog) {
|
|
168
168
|
console.log(`Total items collected: ${finalItems.length}`)
|
|
169
169
|
}
|
|
170
170
|
|
|
@@ -172,7 +172,7 @@ async function fetchCnyes(opt = {}) {
|
|
|
172
172
|
|
|
173
173
|
//time, 單筆publishAt缺漏或異常時退回空字串並記錄, 保留title與link避免拖垮其餘正常新聞
|
|
174
174
|
let time = toDatetimeUTC8(get(item, 'publishAt', 0))
|
|
175
|
-
if (time === '' &&
|
|
175
|
+
if (time === '' && useShowLog) {
|
|
176
176
|
console.warn(`[fetch-cnyes] 時間格式化失敗(newsId=${get(item, 'newsId', '')}, publishAt=${get(item, 'publishAt', '')}) — 該筆time退回空字串`)
|
|
177
177
|
}
|
|
178
178
|
|
package/src/fetchGuancha.mjs
CHANGED
|
@@ -113,10 +113,10 @@ function getCfg(opt = {}) {
|
|
|
113
113
|
timeoutMs = cint(timeoutMs)
|
|
114
114
|
}
|
|
115
115
|
|
|
116
|
-
//
|
|
117
|
-
let
|
|
118
|
-
if (!isbol(
|
|
119
|
-
|
|
116
|
+
//useShowLog
|
|
117
|
+
let useShowLog = get(opt, 'useShowLog')
|
|
118
|
+
if (!isbol(useShowLog)) {
|
|
119
|
+
useShowLog = true
|
|
120
120
|
}
|
|
121
121
|
|
|
122
122
|
//optFetch
|
|
@@ -143,7 +143,7 @@ function getCfg(opt = {}) {
|
|
|
143
143
|
maxPages = Math.min(cint(maxPages), MAX_PAGES)
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
return { baseUrl,
|
|
146
|
+
return { baseUrl, useShowLog, pageDelayMs, maxPages, optFetch }
|
|
147
147
|
}
|
|
148
148
|
|
|
149
149
|
|
|
@@ -316,7 +316,7 @@ async function fetchAllListPages(slug, cfg) {
|
|
|
316
316
|
for (let page = 1; page <= cfg.maxPages; page++) {
|
|
317
317
|
|
|
318
318
|
let url = `${cfg.baseUrl}/${slug}/list_${page}.shtml`
|
|
319
|
-
if (cfg.
|
|
319
|
+
if (cfg.useShowLog) {
|
|
320
320
|
process.stderr.write(`[info] fetching ${slug} list_${page} ...\n`)
|
|
321
321
|
}
|
|
322
322
|
|
|
@@ -329,7 +329,7 @@ async function fetchAllListPages(slug, cfg) {
|
|
|
329
329
|
if (page === 1) {
|
|
330
330
|
throw err
|
|
331
331
|
}
|
|
332
|
-
if (cfg.
|
|
332
|
+
if (cfg.useShowLog) {
|
|
333
333
|
process.stderr.write(`[info] page ${page} fetch error (assumed end of list): ${err.message}\n`)
|
|
334
334
|
}
|
|
335
335
|
break
|
|
@@ -342,7 +342,7 @@ async function fetchAllListPages(slug, cfg) {
|
|
|
342
342
|
err.reason = 'redirected-to-homepage'
|
|
343
343
|
throw err
|
|
344
344
|
}
|
|
345
|
-
if (cfg.
|
|
345
|
+
if (cfg.useShowLog) {
|
|
346
346
|
process.stderr.write(`[info] page ${page} 被導向首頁,視為到底,停止翻頁\n`)
|
|
347
347
|
}
|
|
348
348
|
break
|
|
@@ -353,7 +353,7 @@ async function fetchAllListPages(slug, cfg) {
|
|
|
353
353
|
let items = parseListPage(html, cfg.baseUrl)
|
|
354
354
|
let fresh = items.filter((it) => !seen.has(it.url))
|
|
355
355
|
if (fresh.length === 0) {
|
|
356
|
-
if (cfg.
|
|
356
|
+
if (cfg.useShowLog) {
|
|
357
357
|
process.stderr.write(`[info] page ${page} 無新項目,停止翻頁\n`)
|
|
358
358
|
}
|
|
359
359
|
break
|
|
@@ -384,7 +384,7 @@ async function fetchAllListPages(slug, cfg) {
|
|
|
384
384
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
385
385
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
386
386
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
387
|
-
* @param {Boolean} [opt.
|
|
387
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
388
388
|
* @returns {Promise} 回傳Promise,resolve回傳作者物件陣列,各物件為{slug,name,letter}
|
|
389
389
|
* @example
|
|
390
390
|
*
|
|
@@ -477,7 +477,7 @@ function lookupTopic(name) {
|
|
|
477
477
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
478
478
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
479
479
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
480
|
-
* @param {Boolean} [opt.
|
|
480
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
481
481
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items},name與slug皆未給時reject回傳錯誤物件
|
|
482
482
|
* @example
|
|
483
483
|
*
|
|
@@ -518,7 +518,7 @@ async function fetchAuthorArticles(opt = {}) {
|
|
|
518
518
|
|
|
519
519
|
if (!resolvedSlug) {
|
|
520
520
|
|
|
521
|
-
if (cfg.
|
|
521
|
+
if (cfg.useShowLog) {
|
|
522
522
|
process.stderr.write('[info] fetching authors index from / ...\n')
|
|
523
523
|
}
|
|
524
524
|
let authors = await fetchAuthorsList(opt)
|
|
@@ -685,7 +685,7 @@ async function fetchTitleArticles(keyword, opt = {}) {
|
|
|
685
685
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
686
686
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
687
687
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
688
|
-
* @param {Boolean} [opt.
|
|
688
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
689
689
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items},name與slug皆未給時reject回傳錯誤物件
|
|
690
690
|
* @example
|
|
691
691
|
*
|
|
@@ -790,7 +790,7 @@ async function fetchTopicArticles(opt = {}) {
|
|
|
790
790
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,預設5
|
|
791
791
|
* @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數,預設1000
|
|
792
792
|
* @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數,上限為50,預設50
|
|
793
|
-
* @param {Boolean} [opt.
|
|
793
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
794
794
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{status,site,mode,url,title,author,published,chars,markdown},url非有效字串時reject回傳錯誤物件
|
|
795
795
|
* @example
|
|
796
796
|
*
|
package/src/fetchHackerNews.mjs
CHANGED
|
@@ -41,7 +41,7 @@ let CONCURRENCY = 10
|
|
|
41
41
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設5
|
|
42
42
|
* @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數,預設3000
|
|
43
43
|
* @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數,預設15000
|
|
44
|
-
* @param {Boolean} [opt.
|
|
44
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示重試訊息布林值,預設true
|
|
45
45
|
* @returns {Promise} 回傳Promise,resolve回傳文章物件陣列,各物件為{url,time,title,description,from},抓取ID清單失敗時reject回傳錯誤物件
|
|
46
46
|
* @example
|
|
47
47
|
*
|
package/src/fetchMoneydj.mjs
CHANGED
|
@@ -53,7 +53,7 @@ let REG_TIME = /^(\d{2}\/\d{2}\s+\d{2}:\d{2}|\d{2}:\d{2}|昨\s*\d{2}:\d{2})$/
|
|
|
53
53
|
* @param {Integer} [opt.maxDelayMs=30000] 輸入重試最大延遲毫秒整數,預設30000
|
|
54
54
|
* @param {Integer} [opt.pageDelayMs] 輸入頁間延遲毫秒整數,未給時採隨機1000至3000毫秒
|
|
55
55
|
* @param {Function} [opt.onPageDone] 輸入每頁完成時之回呼函數,傳入(pageIndex,itemCount,totalPages)
|
|
56
|
-
* @param {Boolean} [opt.
|
|
56
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
57
57
|
* @returns {Promise} 回傳Promise,resolve回傳新聞物件陣列,各物件為{time,title,link},抓取到0筆時reject回傳錯誤物件
|
|
58
58
|
* @example
|
|
59
59
|
*
|
|
@@ -97,7 +97,7 @@ async function fetchMoneydj(opt = {}) {
|
|
|
97
97
|
|
|
98
98
|
//optFetch
|
|
99
99
|
let optFetch = getOptFetch(opt, DFLT)
|
|
100
|
-
let
|
|
100
|
+
let useShowLog = optFetch.useShowLog
|
|
101
101
|
|
|
102
102
|
//pageDelayMs, 頁間隨機延遲毫秒
|
|
103
103
|
let pageDelayMs = get(opt, 'pageDelayMs')
|
|
@@ -151,7 +151,7 @@ async function fetchMoneydj(opt = {}) {
|
|
|
151
151
|
items = await fetchPage(i)
|
|
152
152
|
}
|
|
153
153
|
catch (err) {
|
|
154
|
-
if (
|
|
154
|
+
if (useShowLog) {
|
|
155
155
|
console.warn(`[Page ${i}] 抓取失敗,已跳過:${err.message}`)
|
|
156
156
|
}
|
|
157
157
|
continue
|
package/src/fetchMops.mjs
CHANGED
|
@@ -119,7 +119,7 @@ async function fetchTargetWithRetry(page, target, cfg) {
|
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
let ms = Math.min(cfg.baseDelayMs * attempt, cfg.maxDelayMs)
|
|
122
|
-
if (cfg.
|
|
122
|
+
if (cfg.useShowLog) {
|
|
123
123
|
console.warn(`[${target.name}][Retry ${attempt}/${cfg.maxRetries}] ${data.error} — 等待 ${ms / 1000}s 後重試...`)
|
|
124
124
|
}
|
|
125
125
|
await delay(ms)
|
|
@@ -141,7 +141,7 @@ async function fetchTargetWithRetry(page, target, cfg) {
|
|
|
141
141
|
* @param {String} [opt.pageUrl='https://mops.twse.com.tw/mops/#/web/t146sb10'] 輸入MOPS重大訊息頁網址字串,供測試或改指向鏡像時覆寫
|
|
142
142
|
* @param {String} [opt.apiUrl='https://mops.twse.com.tw/mops/api/t146sb10'] 輸入MOPS查詢API網址字串,供測試或改指向鏡像時覆寫
|
|
143
143
|
* @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設10
|
|
144
|
-
* @param {Boolean} [opt.
|
|
144
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
145
145
|
* @returns {Promise} 回傳Promise,resolve回傳結果物件{results,hasError},results各筆為{market,marketKind,data,error,timestamp},瀏覽器啟動失敗時reject回傳錯誤物件
|
|
146
146
|
* @example
|
|
147
147
|
*
|
|
@@ -183,10 +183,10 @@ async function fetchMops(opt = {}) {
|
|
|
183
183
|
maxRetries = cint(maxRetries)
|
|
184
184
|
}
|
|
185
185
|
|
|
186
|
-
//
|
|
187
|
-
let
|
|
188
|
-
if (!isbol(
|
|
189
|
-
|
|
186
|
+
//useShowLog
|
|
187
|
+
let useShowLog = get(opt, 'useShowLog')
|
|
188
|
+
if (!isbol(useShowLog)) {
|
|
189
|
+
useShowLog = true
|
|
190
190
|
}
|
|
191
191
|
|
|
192
192
|
//cfg
|
|
@@ -194,7 +194,7 @@ async function fetchMops(opt = {}) {
|
|
|
194
194
|
maxRetries,
|
|
195
195
|
baseDelayMs: DEFAULT_BASE_DELAY_MS,
|
|
196
196
|
maxDelayMs: DEFAULT_MAX_DELAY_MS,
|
|
197
|
-
|
|
197
|
+
useShowLog,
|
|
198
198
|
}
|
|
199
199
|
|
|
200
200
|
//targets
|
|
@@ -207,7 +207,7 @@ async function fetchMops(opt = {}) {
|
|
|
207
207
|
}
|
|
208
208
|
})
|
|
209
209
|
|
|
210
|
-
if (
|
|
210
|
+
if (useShowLog) {
|
|
211
211
|
console.log('啟動瀏覽器...')
|
|
212
212
|
}
|
|
213
213
|
|
|
@@ -232,7 +232,7 @@ async function fetchMops(opt = {}) {
|
|
|
232
232
|
throw new Error(`瀏覽器啟動失敗(已重試 ${maxRetries} 次): ${err.message}`)
|
|
233
233
|
}
|
|
234
234
|
let ms = Math.min(DEFAULT_BASE_DELAY_MS * attempt, DEFAULT_MAX_DELAY_MS)
|
|
235
|
-
if (
|
|
235
|
+
if (useShowLog) {
|
|
236
236
|
console.warn(`[browser.launch][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
|
|
237
237
|
}
|
|
238
238
|
await delay(ms)
|
|
@@ -244,7 +244,7 @@ async function fetchMops(opt = {}) {
|
|
|
244
244
|
let page = await browser.newPage()
|
|
245
245
|
|
|
246
246
|
//帶重試之頁面導航
|
|
247
|
-
if (
|
|
247
|
+
if (useShowLog) {
|
|
248
248
|
console.log('前往 MOPS 重大訊息頁面 (t146sb10)...')
|
|
249
249
|
}
|
|
250
250
|
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
|
|
@@ -258,7 +258,7 @@ async function fetchMops(opt = {}) {
|
|
|
258
258
|
throw err
|
|
259
259
|
}
|
|
260
260
|
let ms = Math.min(DEFAULT_BASE_DELAY_MS * attempt, DEFAULT_MAX_DELAY_MS)
|
|
261
|
-
if (
|
|
261
|
+
if (useShowLog) {
|
|
262
262
|
console.warn(`[page.goto][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
|
|
263
263
|
}
|
|
264
264
|
await delay(ms)
|
|
@@ -271,7 +271,7 @@ async function fetchMops(opt = {}) {
|
|
|
271
271
|
let results = []
|
|
272
272
|
for (let target of targets) {
|
|
273
273
|
|
|
274
|
-
if (
|
|
274
|
+
if (useShowLog) {
|
|
275
275
|
console.log(`正在抓取 [${target.name}] 資料...`)
|
|
276
276
|
}
|
|
277
277
|
|
|
@@ -288,7 +288,7 @@ async function fetchMops(opt = {}) {
|
|
|
288
288
|
await page.waitForTimeout(1000)
|
|
289
289
|
}
|
|
290
290
|
|
|
291
|
-
if (
|
|
291
|
+
if (useShowLog) {
|
|
292
292
|
let summary = results.map((r) => {
|
|
293
293
|
let resultData = get(r, 'data.result', get(r, 'data'))
|
|
294
294
|
let count = Array.isArray(resultData) ? resultData.length : 0
|
package/src/fetchRSS.mjs
CHANGED
|
@@ -98,7 +98,7 @@ async function getXml(rssUrl, method, cfg) {
|
|
|
98
98
|
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設5
|
|
99
99
|
* @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數(僅fetch模式生效),預設3000
|
|
100
100
|
* @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數(僅fetch模式生效),預設15000
|
|
101
|
-
* @param {Boolean} [opt.
|
|
101
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示重試訊息布林值,預設true
|
|
102
102
|
* @returns {Promise} 回傳Promise,resolve回傳項目物件陣列,各物件為{url,time,title,description,from},withContent為true時另含content欄,網址無效或抓取失敗時reject回傳錯誤物件
|
|
103
103
|
* @example
|
|
104
104
|
*
|
|
@@ -47,7 +47,7 @@ let FALLBACK_DATE = 'time, [class*="date"], [class*="time"]'
|
|
|
47
47
|
* @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設10
|
|
48
48
|
* @param {Integer} [opt.baseDelayMs=5000] 輸入重試之線性退避基礎毫秒整數,預設5000
|
|
49
49
|
* @param {Integer} [opt.maxDelayMs=30000] 輸入重試之線性退避上限毫秒整數,預設30000
|
|
50
|
-
* @param {Boolean} [opt.
|
|
50
|
+
* @param {Boolean} [opt.useShowLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
51
51
|
* @returns {Promise} 回傳Promise,resolve回傳新聞物件陣列,各物件為{time,title,link},抓取失敗或解析到0筆時reject回傳錯誤物件
|
|
52
52
|
* @example
|
|
53
53
|
*
|
|
@@ -76,9 +76,9 @@ async function fetchStatementdog(opt = {}) {
|
|
|
76
76
|
|
|
77
77
|
//optFetch
|
|
78
78
|
let optFetch = getOptFetch(opt, DFLT)
|
|
79
|
-
let
|
|
79
|
+
let useShowLog = optFetch.useShowLog
|
|
80
80
|
|
|
81
|
-
if (
|
|
81
|
+
if (useShowLog) {
|
|
82
82
|
console.log(`Fetching ${url}...`)
|
|
83
83
|
}
|
|
84
84
|
|
|
@@ -97,7 +97,7 @@ async function fetchStatementdog(opt = {}) {
|
|
|
97
97
|
let titleSel = usePrimary ? PRIMARY_TITLE : FALLBACK_TITLE
|
|
98
98
|
let linkSel = usePrimary ? PRIMARY_LINK : FALLBACK_LINK
|
|
99
99
|
let dateSel = usePrimary ? PRIMARY_DATE : FALLBACK_DATE
|
|
100
|
-
if (!usePrimary &&
|
|
100
|
+
if (!usePrimary && useShowLog) {
|
|
101
101
|
console.warn('主要 CSS selector 未匹配,嘗試 fallback selector...')
|
|
102
102
|
}
|
|
103
103
|
|
|
@@ -125,7 +125,7 @@ async function fetchStatementdog(opt = {}) {
|
|
|
125
125
|
newsItems.push({ time, title, link })
|
|
126
126
|
})
|
|
127
127
|
|
|
128
|
-
if (
|
|
128
|
+
if (useShowLog) {
|
|
129
129
|
console.log(`Extracted News Items: ${newsItems.length}`)
|
|
130
130
|
}
|
|
131
131
|
|