w-dwdata-hub 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +9 -0
- package/.eslintignore +3 -0
- package/.eslintrc.js +55 -0
- package/.jsdoc +25 -0
- package/LICENSE +21 -0
- package/README.md +168 -0
- package/SECURITY.md +5 -0
- package/babel.config.js +16 -0
- package/dist/w-dwdata-hub.umd.js +7 -0
- package/dist/w-dwdata-hub.umd.js.map +1 -0
- package/docs/WDwdataHub.mjs.html +136 -0
- package/docs/decodeEntities.mjs.html +129 -0
- package/docs/extractDivContent.mjs.html +147 -0
- package/docs/fetchAiNewsAggregator.mjs.html +155 -0
- package/docs/fetchAisixiang.mjs.html +1038 -0
- package/docs/fetchCnyes.mjs.html +260 -0
- package/docs/fetchGuancha.mjs.html +1007 -0
- package/docs/fetchHackerNews.mjs.html +189 -0
- package/docs/fetchMoneydj.mjs.html +251 -0
- package/docs/fetchMops.mjs.html +385 -0
- package/docs/fetchRSS.mjs.html +160 -0
- package/docs/fetchStatementdog.mjs.html +213 -0
- package/docs/fetchTaifex.mjs.html +513 -0
- package/docs/fetchTpex3insti.mjs.html +278 -0
- package/docs/fetchTpexMargin.mjs.html +248 -0
- package/docs/fetchTpexStock.mjs.html +199 -0
- package/docs/fetchTwDataHoliday.mjs.html +236 -0
- package/docs/fetchTwseMargin.mjs.html +253 -0
- package/docs/fetchTwseStock.mjs.html +188 -0
- package/docs/fetchTwseT86.mjs.html +208 -0
- package/docs/fetchWithRetry.mjs.html +376 -0
- package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
- package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
- package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
- package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
- package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
- package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
- package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
- package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
- package/docs/getOptFetch.mjs.html +185 -0
- package/docs/global.html +22649 -0
- package/docs/htmlToMarkdown.mjs.html +168 -0
- package/docs/index.html +84 -0
- package/docs/isYmd.mjs.html +114 -0
- package/docs/parseIntComma.mjs.html +100 -0
- package/docs/safeFilename.mjs.html +102 -0
- package/docs/scripts/collapse.js +39 -0
- package/docs/scripts/commonNav.js +28 -0
- package/docs/scripts/linenumber.js +25 -0
- package/docs/scripts/nav.js +12 -0
- package/docs/scripts/polyfill.js +4 -0
- package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
- package/docs/scripts/prettify/lang-css.js +2 -0
- package/docs/scripts/prettify/prettify.js +28 -0
- package/docs/scripts/search.js +99 -0
- package/docs/styles/jsdoc.css +776 -0
- package/docs/styles/prettify.css +80 -0
- package/docs/toDatetimeUTC8.mjs.html +143 -0
- package/docs/toRocDate.mjs.html +107 -0
- package/g.mjs +73 -0
- package/package.json +31 -0
- package/script.txt +17 -0
- package/src/WDwdataHub.mjs +64 -0
- package/src/decodeEntities.mjs +57 -0
- package/src/extractDivContent.mjs +75 -0
- package/src/fetchAiNewsAggregator.mjs +83 -0
- package/src/fetchAisixiang.mjs +966 -0
- package/src/fetchCnyes.mjs +188 -0
- package/src/fetchGuancha.mjs +935 -0
- package/src/fetchHackerNews.mjs +117 -0
- package/src/fetchMoneydj.mjs +179 -0
- package/src/fetchMops.mjs +313 -0
- package/src/fetchRSS.mjs +88 -0
- package/src/fetchStatementdog.mjs +141 -0
- package/src/fetchTaifex.mjs +441 -0
- package/src/fetchTpex3insti.mjs +206 -0
- package/src/fetchTpexMargin.mjs +176 -0
- package/src/fetchTpexStock.mjs +127 -0
- package/src/fetchTwDataHoliday.mjs +164 -0
- package/src/fetchTwseMargin.mjs +181 -0
- package/src/fetchTwseStock.mjs +116 -0
- package/src/fetchTwseT86.mjs +136 -0
- package/src/fetchWithRetry.mjs +304 -0
- package/src/getOptFetch.mjs +113 -0
- package/src/htmlToMarkdown.mjs +96 -0
- package/src/isYmd.mjs +42 -0
- package/src/parseIntComma.mjs +28 -0
- package/src/safeFilename.mjs +30 -0
- package/src/toDatetimeUTC8.mjs +71 -0
- package/src/toRocDate.mjs +35 -0
- package/test/WDwdataHub.test.mjs +63 -0
- package/test/decodeEntities.test.mjs +42 -0
- package/test/extractDivContent.test.mjs +48 -0
- package/test/fetchAiNewsAggregator.test.mjs +79 -0
- package/test/fetchAisixiang.test.mjs +225 -0
- package/test/fetchCnyes.test.mjs +61 -0
- package/test/fetchGuancha.test.mjs +190 -0
- package/test/fetchHackerNews.test.mjs +78 -0
- package/test/fetchMoneydj.test.mjs +82 -0
- package/test/fetchMops.test.mjs +106 -0
- package/test/fetchRSS.test.mjs +85 -0
- package/test/fetchStatementdog.test.mjs +69 -0
- package/test/fetchTaifex.test.mjs +99 -0
- package/test/fetchTpex3insti.test.mjs +90 -0
- package/test/fetchTpexMargin.test.mjs +94 -0
- package/test/fetchTpexStock.test.mjs +68 -0
- package/test/fetchTwDataHoliday.test.mjs +100 -0
- package/test/fetchTwseMargin.test.mjs +81 -0
- package/test/fetchTwseStock.test.mjs +80 -0
- package/test/fetchTwseT86.test.mjs +76 -0
- package/test/fetchWithRetry.test.mjs +174 -0
- package/test/htmlToMarkdown.test.mjs +71 -0
- package/test/isYmd.test.mjs +56 -0
- package/test/parseIntComma.test.mjs +38 -0
- package/test/safeFilename.test.mjs +35 -0
- package/test/toDatetimeUTC8.test.mjs +65 -0
- package/test/toRocDate.test.mjs +38 -0
- package/test/tools/encodeBig5.mjs +80 -0
- package/test/tools/fixtures.mjs +539 -0
- package/test/tools/hasChrome.mjs +39 -0
- package/test/tools/serverForTest.mjs +397 -0
- package/toolg/addVersion.mjs +4 -0
- package/toolg/cleanFolder.mjs +5 -0
- package/toolg/gDistRollup.mjs +49 -0
- package/toolg/modifyReadme.mjs +4 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import get from 'lodash-es/get.js'
|
|
2
|
+
import isestr from 'wsemi/src/isestr.mjs'
|
|
3
|
+
import isearr from 'wsemi/src/isearr.mjs'
|
|
4
|
+
import ispint from 'wsemi/src/ispint.mjs'
|
|
5
|
+
import cint from 'wsemi/src/cint.mjs'
|
|
6
|
+
import fetchWithRetry from './fetchWithRetry.mjs'
|
|
7
|
+
import getOptFetch from './getOptFetch.mjs'
|
|
8
|
+
import toDatetimeUTC8 from './toDatetimeUTC8.mjs'
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
//Hacker News官方Firebase API
|
|
12
|
+
let API_BASE = 'https://hacker-news.firebaseio.com/v0'
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
//本函數之預設抓取設定
|
|
16
|
+
let DFLT = {
|
|
17
|
+
timeout: 30000,
|
|
18
|
+
maxRetries: 5,
|
|
19
|
+
baseDelayMs: 3000,
|
|
20
|
+
maxDelayMs: 15000,
|
|
21
|
+
label: 'fetch-hacker-news',
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
//預設取回篇數與取詳情之併發數
|
|
26
|
+
let DEFAULT_LIMIT = 30
|
|
27
|
+
let CONCURRENCY = 10
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* 抓取Hacker News最新文章
|
|
32
|
+
*
|
|
33
|
+
* 先取newstories.json之ID清單, 再分批(每批10筆)取各篇詳情並轉為統一格式;
|
|
34
|
+
* 分批採Promise.allSettled, 單篇重試耗盡不會拖垮同批其他已成功文章;
|
|
35
|
+
* 僅保留type為story者, 無外部連結時以Hacker News站內討論頁網址替代
|
|
36
|
+
*
|
|
37
|
+
* @param {Integer} [limit=30] 輸入取回文章數量正整數,非正整數時改用預設值,預設30
|
|
38
|
+
* @param {Object} [opt={}] 輸入設定物件,預設{}
|
|
39
|
+
* @param {String} [opt.apiBase='https://hacker-news.firebaseio.com/v0'] 輸入API根網址字串,供測試或改指向鏡像時覆寫
|
|
40
|
+
* @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數,預設30000
|
|
41
|
+
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設5
|
|
42
|
+
* @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數,預設3000
|
|
43
|
+
* @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數,預設15000
|
|
44
|
+
* @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值,預設true
|
|
45
|
+
* @returns {Promise} 回傳Promise,resolve回傳文章物件陣列,各物件為{url,time,title,description,from},抓取ID清單失敗時reject回傳錯誤物件
|
|
46
|
+
* @example
|
|
47
|
+
*
|
|
48
|
+
* import fetchHackerNews from './src/fetchHackerNews.mjs'
|
|
49
|
+
*
|
|
50
|
+
* let test = async () => {
|
|
51
|
+
*
|
|
52
|
+
* let rs = await fetchHackerNews(5)
|
|
53
|
+
* console.log(rs.length, rs[0])
|
|
54
|
+
* // => 5 { url: 'https://...', time: '2026-08-07 09:02:03', title: '...', description: '', from: 'Hacker News' }
|
|
55
|
+
*
|
|
56
|
+
* }
|
|
57
|
+
* await test()
|
|
58
|
+
* .catch((err) => {
|
|
59
|
+
* console.log(err)
|
|
60
|
+
* })
|
|
61
|
+
*
|
|
62
|
+
*/
|
|
63
|
+
async function fetchHackerNews(limit = DEFAULT_LIMIT, opt = {}) {
|
|
64
|
+
|
|
65
|
+
//limit, 非正整數(如NaN或<=0)時改用預設值, 避免slice(0,NaN)靜默回傳空陣列
|
|
66
|
+
limit = ispint(limit) ? cint(limit) : DEFAULT_LIMIT
|
|
67
|
+
|
|
68
|
+
//apiBase
|
|
69
|
+
let apiBase = get(opt, 'apiBase')
|
|
70
|
+
if (!isestr(apiBase)) {
|
|
71
|
+
apiBase = API_BASE
|
|
72
|
+
}
|
|
73
|
+
apiBase = apiBase.replace(/\/+$/, '')
|
|
74
|
+
|
|
75
|
+
//optFetch
|
|
76
|
+
let optFetch = getOptFetch(opt, DFLT)
|
|
77
|
+
|
|
78
|
+
//ids, 取得最新文章ID列表
|
|
79
|
+
let ids = await fetchWithRetry(`${apiBase}/newstories.json`, optFetch)
|
|
80
|
+
if (!isearr(ids)) {
|
|
81
|
+
ids = []
|
|
82
|
+
}
|
|
83
|
+
let selected = ids.slice(0, limit)
|
|
84
|
+
|
|
85
|
+
//items, 批次取得文章詳情並控制併發數
|
|
86
|
+
let items = []
|
|
87
|
+
for (let i = 0; i < selected.length; i += CONCURRENCY) {
|
|
88
|
+
let batch = selected.slice(i, i + CONCURRENCY)
|
|
89
|
+
|
|
90
|
+
//allSettled, 單一item重試耗盡而reject時不丟失整批其他已成功之文章
|
|
91
|
+
let rs = await Promise.allSettled(batch.map((id) => {
|
|
92
|
+
return fetchWithRetry(`${apiBase}/item/${id}.json`, optFetch)
|
|
93
|
+
}))
|
|
94
|
+
for (let r of rs) {
|
|
95
|
+
if (r.status === 'fulfilled') {
|
|
96
|
+
items.push(r.value)
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
//轉換為統一格式
|
|
103
|
+
return items
|
|
104
|
+
.filter((item) => item && item.type === 'story')
|
|
105
|
+
.map((item) => {
|
|
106
|
+
return {
|
|
107
|
+
url: String(get(item, 'url', '') || `https://news.ycombinator.com/item?id=${get(item, 'id', '')}`).trim(),
|
|
108
|
+
time: toDatetimeUTC8(get(item, 'time', 0)),
|
|
109
|
+
title: String(get(item, 'title', '') || '').trim(),
|
|
110
|
+
description: '',
|
|
111
|
+
from: 'Hacker News',
|
|
112
|
+
}
|
|
113
|
+
})
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
export default fetchHackerNews
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import * as cheerio from 'cheerio'
|
|
2
|
+
import get from 'lodash-es/get.js'
|
|
3
|
+
import isfun from 'wsemi/src/isfun.mjs'
|
|
4
|
+
import isestr from 'wsemi/src/isestr.mjs'
|
|
5
|
+
import ispint from 'wsemi/src/ispint.mjs'
|
|
6
|
+
import cint from 'wsemi/src/cint.mjs'
|
|
7
|
+
import delay from 'wsemi/src/delay.mjs'
|
|
8
|
+
import fetchWithRetry from './fetchWithRetry.mjs'
|
|
9
|
+
import getOptFetch from './getOptFetch.mjs'
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
//MoneyDJ台股新聞(MB06)列表頁, 頁碼接於index1之後
|
|
13
|
+
let BASE_URL = 'https://www.moneydj.com/kmdj/news/newsreallist.aspx?a=mb06&index1='
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
//本函數之預設抓取設定
|
|
17
|
+
let DFLT = {
|
|
18
|
+
responseType: 'text',
|
|
19
|
+
timeout: 30000,
|
|
20
|
+
maxRetries: 10,
|
|
21
|
+
baseDelayMs: 5000,
|
|
22
|
+
maxDelayMs: 30000,
|
|
23
|
+
retryStatus: [403],
|
|
24
|
+
label: 'fetch-moneydj',
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
//預設抓取頁數
|
|
29
|
+
let DEFAULT_TOTAL_PAGES = 50
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
//頁間隨機延遲之下限與變動範圍毫秒
|
|
33
|
+
let PAGE_DELAY_MIN_MS = 1000
|
|
34
|
+
let PAGE_DELAY_RANGE_MS = 2000
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
//列表頁時間欄位格式, 例如'08/07 09:02'、'09:02'、'昨 09:02'
|
|
38
|
+
let REG_TIME = /^(\d{2}\/\d{2}\s+\d{2}:\d{2}|\d{2}:\d{2}|昨\s*\d{2}:\d{2})$/
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* 抓取MoneyDJ台股新聞
|
|
43
|
+
*
|
|
44
|
+
* 逐頁抓取MoneyDJ台股新聞(MB06)列表, 頁間隨機延遲1至3秒以降低被封鎖風險;
|
|
45
|
+
* 單頁抓取失敗(重試耗盡)時記錄並跳過該頁, 不中斷整體抓取; 全部頁面皆0筆時拋出錯誤
|
|
46
|
+
*
|
|
47
|
+
* @param {Object} [opt={}] 輸入設定物件,預設{}
|
|
48
|
+
* @param {String} [opt.baseUrl='https://www.moneydj.com/kmdj/news/newsreallist.aspx?a=mb06&index1='] 輸入列表頁網址前綴字串,頁碼會直接串接於後,供測試或改指向鏡像時覆寫
|
|
49
|
+
* @param {Integer} [opt.totalPages=50] 輸入要抓取的頁數正整數,預設50
|
|
50
|
+
* @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數,預設30000
|
|
51
|
+
* @param {Integer} [opt.maxRetries=10] 輸入每頁最大重試次數整數,含初始共執行maxRetries+1次,預設10
|
|
52
|
+
* @param {Integer} [opt.baseDelayMs=5000] 輸入重試基礎延遲毫秒整數,預設5000
|
|
53
|
+
* @param {Integer} [opt.maxDelayMs=30000] 輸入重試最大延遲毫秒整數,預設30000
|
|
54
|
+
* @param {Integer} [opt.pageDelayMs] 輸入頁間延遲毫秒整數,未給時採隨機1000至3000毫秒
|
|
55
|
+
* @param {Function} [opt.onPageDone] 輸入每頁完成時之回呼函數,傳入(pageIndex,itemCount,totalPages)
|
|
56
|
+
* @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
57
|
+
* @returns {Promise} 回傳Promise,resolve回傳新聞物件陣列,各物件為{time,title,link},抓取到0筆時reject回傳錯誤物件
|
|
58
|
+
* @example
|
|
59
|
+
*
|
|
60
|
+
* import fetchMoneydj from './src/fetchMoneydj.mjs'
|
|
61
|
+
*
|
|
62
|
+
* let test = async () => {
|
|
63
|
+
*
|
|
64
|
+
* let rs = await fetchMoneydj({ totalPages: 2 })
|
|
65
|
+
* console.log(rs.length, rs[0])
|
|
66
|
+
* // => 60 { time: '08/07 09:02', title: '...', link: 'https://www.moneydj.com/kmdj/news/...' }
|
|
67
|
+
*
|
|
68
|
+
* }
|
|
69
|
+
* await test()
|
|
70
|
+
* .catch((err) => {
|
|
71
|
+
* console.log(err)
|
|
72
|
+
* })
|
|
73
|
+
*
|
|
74
|
+
*/
|
|
75
|
+
async function fetchMoneydj(opt = {}) {
|
|
76
|
+
|
|
77
|
+
//baseUrl
|
|
78
|
+
let baseUrl = get(opt, 'baseUrl')
|
|
79
|
+
if (!isestr(baseUrl)) {
|
|
80
|
+
baseUrl = BASE_URL
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
//totalPages
|
|
84
|
+
let totalPages = get(opt, 'totalPages')
|
|
85
|
+
if (!ispint(totalPages)) {
|
|
86
|
+
totalPages = DEFAULT_TOTAL_PAGES
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
totalPages = cint(totalPages)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
//onPageDone
|
|
93
|
+
let onPageDone = get(opt, 'onPageDone')
|
|
94
|
+
if (!isfun(onPageDone)) {
|
|
95
|
+
onPageDone = () => {}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
//optFetch
|
|
99
|
+
let optFetch = getOptFetch(opt, DFLT)
|
|
100
|
+
let showLog = optFetch.showLog
|
|
101
|
+
|
|
102
|
+
//pageDelayMs, 頁間隨機延遲毫秒
|
|
103
|
+
let pageDelayMs = get(opt, 'pageDelayMs')
|
|
104
|
+
if (!ispint(pageDelayMs) && pageDelayMs !== 0) {
|
|
105
|
+
pageDelayMs = null
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
pageDelayMs = cint(pageDelayMs)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
//domain, 由列表頁網址推得, 供補齊相對路徑連結
|
|
112
|
+
let domain = new URL(baseUrl).origin
|
|
113
|
+
|
|
114
|
+
//fetchPage
|
|
115
|
+
let fetchPage = async (pageIndex) => {
|
|
116
|
+
|
|
117
|
+
//html, MoneyDJ已確認使用UTF-8編碼(多次實測結果正確), 無需Big5解碼
|
|
118
|
+
let html = await fetchWithRetry(`${baseUrl}${pageIndex}`, { ...optFetch, label: `${DFLT.label} Page ${pageIndex}` })
|
|
119
|
+
|
|
120
|
+
//$
|
|
121
|
+
let $ = cheerio.load(html)
|
|
122
|
+
|
|
123
|
+
//newsItems
|
|
124
|
+
let newsItems = []
|
|
125
|
+
$('tr').each((i, el) => {
|
|
126
|
+
|
|
127
|
+
let $row = $(el)
|
|
128
|
+
let timeText = $row.find('td').eq(0).text().trim()
|
|
129
|
+
let $link = $row.find('td').eq(1).find('a')
|
|
130
|
+
|
|
131
|
+
if (timeText && REG_TIME.test(timeText) && $link.length > 0) {
|
|
132
|
+
let title = $link.attr('title') || $link.text().trim()
|
|
133
|
+
let linkRel = $link.attr('href')
|
|
134
|
+
if (linkRel) {
|
|
135
|
+
let link = linkRel.startsWith('http') ? linkRel : domain + linkRel
|
|
136
|
+
newsItems.push({ time: timeText, title, link })
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
})
|
|
141
|
+
|
|
142
|
+
return newsItems
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
let allNewsItems = []
|
|
146
|
+
for (let i = 1; i <= totalPages; i++) {
|
|
147
|
+
|
|
148
|
+
//items, 單頁失敗記錄並跳過, 不中斷整體抓取
|
|
149
|
+
let items = null
|
|
150
|
+
try {
|
|
151
|
+
items = await fetchPage(i)
|
|
152
|
+
}
|
|
153
|
+
catch (err) {
|
|
154
|
+
if (showLog) {
|
|
155
|
+
console.warn(`[Page ${i}] 抓取失敗,已跳過:${err.message}`)
|
|
156
|
+
}
|
|
157
|
+
continue
|
|
158
|
+
}
|
|
159
|
+
allNewsItems = allNewsItems.concat(items)
|
|
160
|
+
|
|
161
|
+
onPageDone(i, items.length, totalPages)
|
|
162
|
+
|
|
163
|
+
//頁間延遲, 未指定pageDelayMs時採隨機1至3秒以降低被封鎖風險
|
|
164
|
+
if (i < totalPages) {
|
|
165
|
+
await delay(pageDelayMs === null ? Math.floor(Math.random() * PAGE_DELAY_RANGE_MS) + PAGE_DELAY_MIN_MS : pageDelayMs)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
//check, 0筆代表版型改變或selector失效
|
|
171
|
+
if (allNewsItems.length === 0) {
|
|
172
|
+
throw new Error('抓取到 0 筆新聞,可能是頁面結構改變或 selector 失效,請確認 MoneyDJ 頁面是否正常。')
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return allNewsItems
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
export default fetchMoneydj
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
import { chromium } from 'playwright'
|
|
2
|
+
import get from 'lodash-es/get.js'
|
|
3
|
+
import isbol from 'wsemi/src/isbol.mjs'
|
|
4
|
+
import isestr from 'wsemi/src/isestr.mjs'
|
|
5
|
+
import isp0int from 'wsemi/src/isp0int.mjs'
|
|
6
|
+
import cint from 'wsemi/src/cint.mjs'
|
|
7
|
+
import delay from 'wsemi/src/delay.mjs'
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
//公開資訊觀測站重大訊息頁與其查詢API
|
|
11
|
+
let PAGE_URL = 'https://mops.twse.com.tw/mops/#/web/t146sb10'
|
|
12
|
+
let API_URL = 'https://mops.twse.com.tw/mops/api/t146sb10'
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
//預設值
|
|
16
|
+
let DEFAULT_MAX_RETRIES = 10
|
|
17
|
+
let DEFAULT_BASE_DELAY_MS = 5000
|
|
18
|
+
let DEFAULT_MAX_DELAY_MS = 30000
|
|
19
|
+
let LAUNCH_TIMEOUT_MS = 360000
|
|
20
|
+
let GOTO_TIMEOUT_MS = 60000
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
//目標市場別, marketKind為API查詢參數
|
|
24
|
+
let MARKET_KINDS = [
|
|
25
|
+
{ name: '上市', marketKind: 'sii' },
|
|
26
|
+
{ name: '上櫃', marketKind: 'otc' },
|
|
27
|
+
{ name: '興櫃', marketKind: 'rotc' },
|
|
28
|
+
{ name: '公開發行', marketKind: 'pub' },
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* 組出單一市場別之API查詢payload
|
|
34
|
+
*
|
|
35
|
+
* @param {String} marketKind 輸入市場別字串,可為'sii'、'otc'、'rotc'、'pub'
|
|
36
|
+
* @returns {Object} 回傳API查詢payload物件
|
|
37
|
+
*/
|
|
38
|
+
function getPayload(marketKind) {
|
|
39
|
+
return {
|
|
40
|
+
scopeType: '2',
|
|
41
|
+
companyId: '',
|
|
42
|
+
dateType: '2',
|
|
43
|
+
firstDate: '',
|
|
44
|
+
lastDate: '',
|
|
45
|
+
marketKind,
|
|
46
|
+
announcementBasis: '0',
|
|
47
|
+
dateRangeType: '1',
|
|
48
|
+
announcementType: '1',
|
|
49
|
+
sort: '1',
|
|
50
|
+
encodeURIComponent: 1,
|
|
51
|
+
step: 1,
|
|
52
|
+
firstin: 1,
|
|
53
|
+
off: 1,
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* 於瀏覽器頁面內以fetch呼叫MOPS查詢API並自動重試
|
|
60
|
+
*
|
|
61
|
+
* 需於MOPS頁面context內發送請求(同源且帶站方cookie), 故採page.evaluate而非node端直接請求;
|
|
62
|
+
* 重試條件為HTTP 5xx、403、429或網路錯誤, application-level錯誤(如MOPS code 500)屬非暫時性故不重試
|
|
63
|
+
*
|
|
64
|
+
* @param {Object} page 輸入playwright之page物件
|
|
65
|
+
* @param {Object} target 輸入目標物件,內含name、marketKind、url與payload
|
|
66
|
+
* @param {Object} cfg 輸入已正規化之設定物件
|
|
67
|
+
* @returns {Promise} 回傳Promise,resolve回傳{json,raw}或{error,retryable}
|
|
68
|
+
*/
|
|
69
|
+
async function fetchTargetWithRetry(page, target, cfg) {
|
|
70
|
+
|
|
71
|
+
let data = null
|
|
72
|
+
for (let attempt = 1; attempt <= cfg.maxRetries + 1; attempt++) {
|
|
73
|
+
|
|
74
|
+
data = await page.evaluate(async (t) => {
|
|
75
|
+
try {
|
|
76
|
+
|
|
77
|
+
let response = await fetch(t.url, {
|
|
78
|
+
method: 'POST',
|
|
79
|
+
headers: { 'Content-Type': 'application/json' },
|
|
80
|
+
body: JSON.stringify(t.payload),
|
|
81
|
+
})
|
|
82
|
+
if (response.status >= 500 || response.status === 403 || response.status === 429) {
|
|
83
|
+
return { error: `HTTP ${response.status}`, retryable: true }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
let text = await response.text()
|
|
87
|
+
try {
|
|
88
|
+
|
|
89
|
+
let json = JSON.parse(text)
|
|
90
|
+
|
|
91
|
+
//驗證MOPS application-level狀態碼(HTTP 200不代表查詢成功)
|
|
92
|
+
//實測契約: code=200「查詢成功」、code=406「查無相符資料」(正常空結果)、其餘(如500)為application-level錯誤
|
|
93
|
+
let code = json && json.code
|
|
94
|
+
if (code !== 200 && code !== 406) {
|
|
95
|
+
//非成功且非「查無資料」→ application-level錯誤, 帶原始回應供除錯, 屬非暫時性故不重試
|
|
96
|
+
return { error: `MOPS code ${code}: ${(json && json.message) || ''}`, json, raw: text.substring(0, 500), retryable: false }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return { json, raw: text.substring(0, 500) }
|
|
100
|
+
}
|
|
101
|
+
catch (e) {
|
|
102
|
+
return { error: 'Parse Error', raw: text, retryable: false }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
return { error: err.toString(), retryable: true }
|
|
108
|
+
}
|
|
109
|
+
}, target)
|
|
110
|
+
|
|
111
|
+
if (!data.error) {
|
|
112
|
+
return data
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
let retryable = data.retryable !== false
|
|
116
|
+
let attemptsLeft = cfg.maxRetries + 1 - attempt
|
|
117
|
+
if (!retryable || attemptsLeft <= 0) {
|
|
118
|
+
return data //回傳錯誤結果, 由呼叫方處理
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
let ms = Math.min(cfg.baseDelayMs * attempt, cfg.maxDelayMs)
|
|
122
|
+
if (cfg.showLog) {
|
|
123
|
+
console.warn(`[${target.name}][Retry ${attempt}/${cfg.maxRetries}] ${data.error} — 等待 ${ms / 1000}s 後重試...`)
|
|
124
|
+
}
|
|
125
|
+
await delay(ms)
|
|
126
|
+
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return data
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* 抓取公開資訊觀測站(MOPS)今日重大訊息
|
|
135
|
+
*
|
|
136
|
+
* 以playwright啟動本機Chrome(channel為'chrome', 可由環境變數CHROME_PATH指定執行檔)開啟MOPS重大訊息頁,
|
|
137
|
+
* 再於頁面context內逐一查詢上市、上櫃、興櫃與公開發行之今日重大訊息;
|
|
138
|
+
* 單一市場別失敗時記錄於該筆之error並續抓其餘市場別, 由hasError告知是否有失敗
|
|
139
|
+
*
|
|
140
|
+
* @param {Object} [opt={}] 輸入設定物件,預設{}
|
|
141
|
+
* @param {String} [opt.pageUrl='https://mops.twse.com.tw/mops/#/web/t146sb10'] 輸入MOPS重大訊息頁網址字串,供測試或改指向鏡像時覆寫
|
|
142
|
+
* @param {String} [opt.apiUrl='https://mops.twse.com.tw/mops/api/t146sb10'] 輸入MOPS查詢API網址字串,供測試或改指向鏡像時覆寫
|
|
143
|
+
* @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設10
|
|
144
|
+
* @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值,預設true
|
|
145
|
+
* @returns {Promise} 回傳Promise,resolve回傳結果物件{results,hasError},results各筆為{market,marketKind,data,error,timestamp},瀏覽器啟動失敗時reject回傳錯誤物件
|
|
146
|
+
* @example
|
|
147
|
+
*
|
|
148
|
+
* import fetchMops from './src/fetchMops.mjs'
|
|
149
|
+
*
|
|
150
|
+
* let test = async () => {
|
|
151
|
+
*
|
|
152
|
+
* let r = await fetchMops()
|
|
153
|
+
* console.log(r.hasError, r.results.length, r.results[0].market)
|
|
154
|
+
* // => false 4 '上市'
|
|
155
|
+
*
|
|
156
|
+
* }
|
|
157
|
+
* await test()
|
|
158
|
+
* .catch((err) => {
|
|
159
|
+
* console.log(err)
|
|
160
|
+
* })
|
|
161
|
+
*
|
|
162
|
+
*/
|
|
163
|
+
async function fetchMops(opt = {}) {
|
|
164
|
+
|
|
165
|
+
//pageUrl
|
|
166
|
+
let pageUrl = get(opt, 'pageUrl')
|
|
167
|
+
if (!isestr(pageUrl)) {
|
|
168
|
+
pageUrl = PAGE_URL
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
//apiUrl
|
|
172
|
+
let apiUrl = get(opt, 'apiUrl')
|
|
173
|
+
if (!isestr(apiUrl)) {
|
|
174
|
+
apiUrl = API_URL
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
//maxRetries
|
|
178
|
+
let maxRetries = get(opt, 'maxRetries')
|
|
179
|
+
if (!isp0int(maxRetries)) {
|
|
180
|
+
maxRetries = DEFAULT_MAX_RETRIES
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
maxRetries = cint(maxRetries)
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
//showLog
|
|
187
|
+
let showLog = get(opt, 'showLog')
|
|
188
|
+
if (!isbol(showLog)) {
|
|
189
|
+
showLog = true
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
//cfg
|
|
193
|
+
let cfg = {
|
|
194
|
+
maxRetries,
|
|
195
|
+
baseDelayMs: DEFAULT_BASE_DELAY_MS,
|
|
196
|
+
maxDelayMs: DEFAULT_MAX_DELAY_MS,
|
|
197
|
+
showLog,
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
//targets
|
|
201
|
+
let targets = MARKET_KINDS.map((v) => {
|
|
202
|
+
return {
|
|
203
|
+
name: v.name,
|
|
204
|
+
marketKind: v.marketKind,
|
|
205
|
+
url: apiUrl,
|
|
206
|
+
payload: getPayload(v.marketKind),
|
|
207
|
+
}
|
|
208
|
+
})
|
|
209
|
+
|
|
210
|
+
if (showLog) {
|
|
211
|
+
console.log('啟動瀏覽器...')
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
//browser, 帶重試之瀏覽器啟動
|
|
215
|
+
let browser = null
|
|
216
|
+
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
|
|
217
|
+
try {
|
|
218
|
+
browser = await chromium.launch({
|
|
219
|
+
headless: true,
|
|
220
|
+
//設定環境變數CHROME_PATH則用該路徑啟動瀏覽器
|
|
221
|
+
//未設定時為undefined, playwright會忽略executablePath改走channel為'chrome', 行為不變
|
|
222
|
+
executablePath: process.env.CHROME_PATH || undefined,
|
|
223
|
+
channel: 'chrome',
|
|
224
|
+
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
|
225
|
+
timeout: LAUNCH_TIMEOUT_MS,
|
|
226
|
+
})
|
|
227
|
+
break
|
|
228
|
+
}
|
|
229
|
+
catch (err) {
|
|
230
|
+
let attemptsLeft = maxRetries + 1 - attempt
|
|
231
|
+
if (attemptsLeft <= 0) {
|
|
232
|
+
throw new Error(`瀏覽器啟動失敗(已重試 ${maxRetries} 次): ${err.message}`)
|
|
233
|
+
}
|
|
234
|
+
let ms = Math.min(DEFAULT_BASE_DELAY_MS * attempt, DEFAULT_MAX_DELAY_MS)
|
|
235
|
+
if (showLog) {
|
|
236
|
+
console.warn(`[browser.launch][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
|
|
237
|
+
}
|
|
238
|
+
await delay(ms)
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
try {
|
|
243
|
+
|
|
244
|
+
let page = await browser.newPage()
|
|
245
|
+
|
|
246
|
+
//帶重試之頁面導航
|
|
247
|
+
if (showLog) {
|
|
248
|
+
console.log('前往 MOPS 重大訊息頁面 (t146sb10)...')
|
|
249
|
+
}
|
|
250
|
+
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
|
|
251
|
+
try {
|
|
252
|
+
await page.goto(pageUrl, { waitUntil: 'networkidle', timeout: GOTO_TIMEOUT_MS })
|
|
253
|
+
break
|
|
254
|
+
}
|
|
255
|
+
catch (err) {
|
|
256
|
+
let attemptsLeft = maxRetries + 1 - attempt
|
|
257
|
+
if (attemptsLeft <= 0) {
|
|
258
|
+
throw err
|
|
259
|
+
}
|
|
260
|
+
let ms = Math.min(DEFAULT_BASE_DELAY_MS * attempt, DEFAULT_MAX_DELAY_MS)
|
|
261
|
+
if (showLog) {
|
|
262
|
+
console.warn(`[page.goto][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
|
|
263
|
+
}
|
|
264
|
+
await delay(ms)
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
await page.waitForTimeout(2000)
|
|
269
|
+
|
|
270
|
+
//results
|
|
271
|
+
let results = []
|
|
272
|
+
for (let target of targets) {
|
|
273
|
+
|
|
274
|
+
if (showLog) {
|
|
275
|
+
console.log(`正在抓取 [${target.name}] 資料...`)
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
let data = await fetchTargetWithRetry(page, target, cfg)
|
|
279
|
+
|
|
280
|
+
results.push({
|
|
281
|
+
market: target.name,
|
|
282
|
+
marketKind: target.marketKind,
|
|
283
|
+
data: data.json || data,
|
|
284
|
+
...(data.error && { error: data.error }),
|
|
285
|
+
timestamp: new Date().toISOString(),
|
|
286
|
+
})
|
|
287
|
+
|
|
288
|
+
await page.waitForTimeout(1000)
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
if (showLog) {
|
|
292
|
+
let summary = results.map((r) => {
|
|
293
|
+
let resultData = get(r, 'data.result', get(r, 'data'))
|
|
294
|
+
let count = Array.isArray(resultData) ? resultData.length : 0
|
|
295
|
+
return `${r.market}: 取得 ${count} 筆資料`
|
|
296
|
+
})
|
|
297
|
+
console.log('抓取完成。摘要:', summary)
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
//hasError
|
|
301
|
+
let hasError = results.some((r) => r.error)
|
|
302
|
+
|
|
303
|
+
return { results, hasError }
|
|
304
|
+
|
|
305
|
+
}
|
|
306
|
+
finally {
|
|
307
|
+
await browser.close()
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
export { MARKET_KINDS, getPayload }
|
|
313
|
+
export default fetchMops
|
package/src/fetchRSS.mjs
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import Parser from 'rss-parser'
|
|
2
|
+
import get from 'lodash-es/get.js'
|
|
3
|
+
import isestr from 'wsemi/src/isestr.mjs'
|
|
4
|
+
import isearr from 'wsemi/src/isearr.mjs'
|
|
5
|
+
import fetchWithRetry from './fetchWithRetry.mjs'
|
|
6
|
+
import getOptFetch from './getOptFetch.mjs'
|
|
7
|
+
import toDatetimeUTC8 from './toDatetimeUTC8.mjs'
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
//本函數之預設抓取設定, 因YouTube等站台之RSS會回傳暫時性404故將404納入重試範圍
|
|
11
|
+
let DFLT = {
|
|
12
|
+
responseType: 'text',
|
|
13
|
+
timeout: 30000,
|
|
14
|
+
maxRetries: 5,
|
|
15
|
+
baseDelayMs: 3000,
|
|
16
|
+
maxDelayMs: 15000,
|
|
17
|
+
retryStatus: [404],
|
|
18
|
+
label: 'fetch-rss',
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* 抓取RSS訂閱源
|
|
24
|
+
*
|
|
25
|
+
* 支援RSS與Atom, 並轉為統一格式, 時間欄位一律轉為UTC+8之'YYYY-MM-DD HH:mm:ss'字串;
|
|
26
|
+
* 因YouTube等站台之RSS會回傳暫時性404, 故404亦納入重試範圍
|
|
27
|
+
*
|
|
28
|
+
* @param {String} rssUrl 輸入RSS網址字串,須為http或https
|
|
29
|
+
* @param {Object} [opt={}] 輸入設定物件,預設{}
|
|
30
|
+
* @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數,預設30000
|
|
31
|
+
* @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設5
|
|
32
|
+
* @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數,預設3000
|
|
33
|
+
* @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數,預設15000
|
|
34
|
+
* @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值,預設true
|
|
35
|
+
* @returns {Promise} 回傳Promise,resolve回傳項目物件陣列,各物件為{url,time,title,description,from},網址無效或抓取失敗時reject回傳錯誤物件
|
|
36
|
+
* @example
|
|
37
|
+
*
|
|
38
|
+
* import fetchRSS from './src/fetchRSS.mjs'
|
|
39
|
+
*
|
|
40
|
+
* let test = async () => {
|
|
41
|
+
*
|
|
42
|
+
* let rs = await fetchRSS('https://www.ptt.cc/atom/Gossiping.xml')
|
|
43
|
+
* console.log(rs.length, rs[0])
|
|
44
|
+
* // => 20 { url: 'https://...', time: '2026-08-07 09:02:03', title: '...', description: '...', from: '...' }
|
|
45
|
+
*
|
|
46
|
+
* }
|
|
47
|
+
* await test()
|
|
48
|
+
* .catch((err) => {
|
|
49
|
+
* console.log(err)
|
|
50
|
+
* })
|
|
51
|
+
*
|
|
52
|
+
*/
|
|
53
|
+
async function fetchRSS(rssUrl, opt = {}) {
|
|
54
|
+
|
|
55
|
+
//check
|
|
56
|
+
if (!isestr(rssUrl) || !/^https?:\/\//i.test(rssUrl)) {
|
|
57
|
+
throw new Error('rssUrl 須為有效的 http/https RSS 網址')
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
//xml
|
|
61
|
+
let xml = await fetchWithRetry(rssUrl, getOptFetch(opt, DFLT))
|
|
62
|
+
|
|
63
|
+
//feed
|
|
64
|
+
let parser = new Parser()
|
|
65
|
+
let feed = await parser.parseString(xml)
|
|
66
|
+
|
|
67
|
+
//feedFrom
|
|
68
|
+
let feedFrom = String(get(feed, 'title', '') || '').trim()
|
|
69
|
+
|
|
70
|
+
//items
|
|
71
|
+
let items = get(feed, 'items')
|
|
72
|
+
if (!isearr(items)) {
|
|
73
|
+
items = []
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
return items.map((item) => {
|
|
77
|
+
return {
|
|
78
|
+
url: String(get(item, 'link', '') || '').trim(),
|
|
79
|
+
time: toDatetimeUTC8(get(item, 'isoDate', '') || get(item, 'pubDate', '')),
|
|
80
|
+
title: String(get(item, 'title', '') || '').trim(),
|
|
81
|
+
description: String(get(item, 'contentSnippet', '') || get(item, 'summary', '') || '').trim(),
|
|
82
|
+
from: String(get(item, 'creator', '') || feedFrom).trim(),
|
|
83
|
+
}
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
export default fetchRSS
|