w-dwdata-hub 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/.editorconfig +9 -0
  2. package/.eslintignore +3 -0
  3. package/.eslintrc.js +55 -0
  4. package/.jsdoc +25 -0
  5. package/LICENSE +21 -0
  6. package/README.md +168 -0
  7. package/SECURITY.md +5 -0
  8. package/babel.config.js +16 -0
  9. package/dist/w-dwdata-hub.umd.js +7 -0
  10. package/dist/w-dwdata-hub.umd.js.map +1 -0
  11. package/docs/WDwdataHub.mjs.html +136 -0
  12. package/docs/decodeEntities.mjs.html +129 -0
  13. package/docs/extractDivContent.mjs.html +147 -0
  14. package/docs/fetchAiNewsAggregator.mjs.html +155 -0
  15. package/docs/fetchAisixiang.mjs.html +1038 -0
  16. package/docs/fetchCnyes.mjs.html +260 -0
  17. package/docs/fetchGuancha.mjs.html +1007 -0
  18. package/docs/fetchHackerNews.mjs.html +189 -0
  19. package/docs/fetchMoneydj.mjs.html +251 -0
  20. package/docs/fetchMops.mjs.html +385 -0
  21. package/docs/fetchRSS.mjs.html +160 -0
  22. package/docs/fetchStatementdog.mjs.html +213 -0
  23. package/docs/fetchTaifex.mjs.html +513 -0
  24. package/docs/fetchTpex3insti.mjs.html +278 -0
  25. package/docs/fetchTpexMargin.mjs.html +248 -0
  26. package/docs/fetchTpexStock.mjs.html +199 -0
  27. package/docs/fetchTwDataHoliday.mjs.html +236 -0
  28. package/docs/fetchTwseMargin.mjs.html +253 -0
  29. package/docs/fetchTwseStock.mjs.html +188 -0
  30. package/docs/fetchTwseT86.mjs.html +208 -0
  31. package/docs/fetchWithRetry.mjs.html +376 -0
  32. package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
  33. package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
  34. package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
  35. package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
  36. package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
  37. package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
  38. package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
  39. package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
  40. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
  41. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
  42. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
  43. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
  44. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
  45. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
  46. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
  47. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
  48. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
  49. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
  50. package/docs/getOptFetch.mjs.html +185 -0
  51. package/docs/global.html +22649 -0
  52. package/docs/htmlToMarkdown.mjs.html +168 -0
  53. package/docs/index.html +84 -0
  54. package/docs/isYmd.mjs.html +114 -0
  55. package/docs/parseIntComma.mjs.html +100 -0
  56. package/docs/safeFilename.mjs.html +102 -0
  57. package/docs/scripts/collapse.js +39 -0
  58. package/docs/scripts/commonNav.js +28 -0
  59. package/docs/scripts/linenumber.js +25 -0
  60. package/docs/scripts/nav.js +12 -0
  61. package/docs/scripts/polyfill.js +4 -0
  62. package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
  63. package/docs/scripts/prettify/lang-css.js +2 -0
  64. package/docs/scripts/prettify/prettify.js +28 -0
  65. package/docs/scripts/search.js +99 -0
  66. package/docs/styles/jsdoc.css +776 -0
  67. package/docs/styles/prettify.css +80 -0
  68. package/docs/toDatetimeUTC8.mjs.html +143 -0
  69. package/docs/toRocDate.mjs.html +107 -0
  70. package/g.mjs +73 -0
  71. package/package.json +31 -0
  72. package/script.txt +17 -0
  73. package/src/WDwdataHub.mjs +64 -0
  74. package/src/decodeEntities.mjs +57 -0
  75. package/src/extractDivContent.mjs +75 -0
  76. package/src/fetchAiNewsAggregator.mjs +83 -0
  77. package/src/fetchAisixiang.mjs +966 -0
  78. package/src/fetchCnyes.mjs +188 -0
  79. package/src/fetchGuancha.mjs +935 -0
  80. package/src/fetchHackerNews.mjs +117 -0
  81. package/src/fetchMoneydj.mjs +179 -0
  82. package/src/fetchMops.mjs +313 -0
  83. package/src/fetchRSS.mjs +88 -0
  84. package/src/fetchStatementdog.mjs +141 -0
  85. package/src/fetchTaifex.mjs +441 -0
  86. package/src/fetchTpex3insti.mjs +206 -0
  87. package/src/fetchTpexMargin.mjs +176 -0
  88. package/src/fetchTpexStock.mjs +127 -0
  89. package/src/fetchTwDataHoliday.mjs +164 -0
  90. package/src/fetchTwseMargin.mjs +181 -0
  91. package/src/fetchTwseStock.mjs +116 -0
  92. package/src/fetchTwseT86.mjs +136 -0
  93. package/src/fetchWithRetry.mjs +304 -0
  94. package/src/getOptFetch.mjs +113 -0
  95. package/src/htmlToMarkdown.mjs +96 -0
  96. package/src/isYmd.mjs +42 -0
  97. package/src/parseIntComma.mjs +28 -0
  98. package/src/safeFilename.mjs +30 -0
  99. package/src/toDatetimeUTC8.mjs +71 -0
  100. package/src/toRocDate.mjs +35 -0
  101. package/test/WDwdataHub.test.mjs +63 -0
  102. package/test/decodeEntities.test.mjs +42 -0
  103. package/test/extractDivContent.test.mjs +48 -0
  104. package/test/fetchAiNewsAggregator.test.mjs +79 -0
  105. package/test/fetchAisixiang.test.mjs +225 -0
  106. package/test/fetchCnyes.test.mjs +61 -0
  107. package/test/fetchGuancha.test.mjs +190 -0
  108. package/test/fetchHackerNews.test.mjs +78 -0
  109. package/test/fetchMoneydj.test.mjs +82 -0
  110. package/test/fetchMops.test.mjs +106 -0
  111. package/test/fetchRSS.test.mjs +85 -0
  112. package/test/fetchStatementdog.test.mjs +69 -0
  113. package/test/fetchTaifex.test.mjs +99 -0
  114. package/test/fetchTpex3insti.test.mjs +90 -0
  115. package/test/fetchTpexMargin.test.mjs +94 -0
  116. package/test/fetchTpexStock.test.mjs +68 -0
  117. package/test/fetchTwDataHoliday.test.mjs +100 -0
  118. package/test/fetchTwseMargin.test.mjs +81 -0
  119. package/test/fetchTwseStock.test.mjs +80 -0
  120. package/test/fetchTwseT86.test.mjs +76 -0
  121. package/test/fetchWithRetry.test.mjs +174 -0
  122. package/test/htmlToMarkdown.test.mjs +71 -0
  123. package/test/isYmd.test.mjs +56 -0
  124. package/test/parseIntComma.test.mjs +38 -0
  125. package/test/safeFilename.test.mjs +35 -0
  126. package/test/toDatetimeUTC8.test.mjs +65 -0
  127. package/test/toRocDate.test.mjs +38 -0
  128. package/test/tools/encodeBig5.mjs +80 -0
  129. package/test/tools/fixtures.mjs +539 -0
  130. package/test/tools/hasChrome.mjs +39 -0
  131. package/test/tools/serverForTest.mjs +397 -0
  132. package/toolg/addVersion.mjs +4 -0
  133. package/toolg/cleanFolder.mjs +5 -0
  134. package/toolg/gDistRollup.mjs +49 -0
  135. package/toolg/modifyReadme.mjs +4 -0
@@ -0,0 +1,117 @@
1
+ import get from 'lodash-es/get.js'
2
+ import isestr from 'wsemi/src/isestr.mjs'
3
+ import isearr from 'wsemi/src/isearr.mjs'
4
+ import ispint from 'wsemi/src/ispint.mjs'
5
+ import cint from 'wsemi/src/cint.mjs'
6
+ import fetchWithRetry from './fetchWithRetry.mjs'
7
+ import getOptFetch from './getOptFetch.mjs'
8
+ import toDatetimeUTC8 from './toDatetimeUTC8.mjs'
9
+
10
+
11
+ //Hacker News官方Firebase API
12
+ let API_BASE = 'https://hacker-news.firebaseio.com/v0'
13
+
14
+
15
+ //本函數之預設抓取設定
16
+ let DFLT = {
17
+ timeout: 30000,
18
+ maxRetries: 5,
19
+ baseDelayMs: 3000,
20
+ maxDelayMs: 15000,
21
+ label: 'fetch-hacker-news',
22
+ }
23
+
24
+
25
+ //預設取回篇數與取詳情之併發數
26
+ let DEFAULT_LIMIT = 30
27
+ let CONCURRENCY = 10
28
+
29
+
30
+ /**
31
+ * 抓取Hacker News最新文章
32
+ *
33
+ * 先取newstories.json之ID清單, 再分批(每批10筆)取各篇詳情並轉為統一格式;
34
+ * 分批採Promise.allSettled, 單篇重試耗盡不會拖垮同批其他已成功文章;
35
+ * 僅保留type為story者, 無外部連結時以Hacker News站內討論頁網址替代
36
+ *
37
+ * @param {Integer} [limit=30] 輸入取回文章數量正整數,非正整數時改用預設值,預設30
38
+ * @param {Object} [opt={}] 輸入設定物件,預設{}
39
+ * @param {String} [opt.apiBase='https://hacker-news.firebaseio.com/v0'] 輸入API根網址字串,供測試或改指向鏡像時覆寫
40
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數,預設30000
41
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設5
42
+ * @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數,預設3000
43
+ * @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數,預設15000
44
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值,預設true
45
+ * @returns {Promise} 回傳Promise,resolve回傳文章物件陣列,各物件為{url,time,title,description,from},抓取ID清單失敗時reject回傳錯誤物件
46
+ * @example
47
+ *
48
+ * import fetchHackerNews from './src/fetchHackerNews.mjs'
49
+ *
50
+ * let test = async () => {
51
+ *
52
+ * let rs = await fetchHackerNews(5)
53
+ * console.log(rs.length, rs[0])
54
+ * // => 5 { url: 'https://...', time: '2026-08-07 09:02:03', title: '...', description: '', from: 'Hacker News' }
55
+ *
56
+ * }
57
+ * await test()
58
+ * .catch((err) => {
59
+ * console.log(err)
60
+ * })
61
+ *
62
+ */
63
+ async function fetchHackerNews(limit = DEFAULT_LIMIT, opt = {}) {
64
+
65
+ //limit, 非正整數(如NaN或<=0)時改用預設值, 避免slice(0,NaN)靜默回傳空陣列
66
+ limit = ispint(limit) ? cint(limit) : DEFAULT_LIMIT
67
+
68
+ //apiBase
69
+ let apiBase = get(opt, 'apiBase')
70
+ if (!isestr(apiBase)) {
71
+ apiBase = API_BASE
72
+ }
73
+ apiBase = apiBase.replace(/\/+$/, '')
74
+
75
+ //optFetch
76
+ let optFetch = getOptFetch(opt, DFLT)
77
+
78
+ //ids, 取得最新文章ID列表
79
+ let ids = await fetchWithRetry(`${apiBase}/newstories.json`, optFetch)
80
+ if (!isearr(ids)) {
81
+ ids = []
82
+ }
83
+ let selected = ids.slice(0, limit)
84
+
85
+ //items, 批次取得文章詳情並控制併發數
86
+ let items = []
87
+ for (let i = 0; i < selected.length; i += CONCURRENCY) {
88
+ let batch = selected.slice(i, i + CONCURRENCY)
89
+
90
+ //allSettled, 單一item重試耗盡而reject時不丟失整批其他已成功之文章
91
+ let rs = await Promise.allSettled(batch.map((id) => {
92
+ return fetchWithRetry(`${apiBase}/item/${id}.json`, optFetch)
93
+ }))
94
+ for (let r of rs) {
95
+ if (r.status === 'fulfilled') {
96
+ items.push(r.value)
97
+ }
98
+ }
99
+
100
+ }
101
+
102
+ //轉換為統一格式
103
+ return items
104
+ .filter((item) => item && item.type === 'story')
105
+ .map((item) => {
106
+ return {
107
+ url: String(get(item, 'url', '') || `https://news.ycombinator.com/item?id=${get(item, 'id', '')}`).trim(),
108
+ time: toDatetimeUTC8(get(item, 'time', 0)),
109
+ title: String(get(item, 'title', '') || '').trim(),
110
+ description: '',
111
+ from: 'Hacker News',
112
+ }
113
+ })
114
+ }
115
+
116
+
117
+ export default fetchHackerNews
@@ -0,0 +1,179 @@
1
+ import * as cheerio from 'cheerio'
2
+ import get from 'lodash-es/get.js'
3
+ import isfun from 'wsemi/src/isfun.mjs'
4
+ import isestr from 'wsemi/src/isestr.mjs'
5
+ import ispint from 'wsemi/src/ispint.mjs'
6
+ import cint from 'wsemi/src/cint.mjs'
7
+ import delay from 'wsemi/src/delay.mjs'
8
+ import fetchWithRetry from './fetchWithRetry.mjs'
9
+ import getOptFetch from './getOptFetch.mjs'
10
+
11
+
12
+ //MoneyDJ台股新聞(MB06)列表頁, 頁碼接於index1之後
13
+ let BASE_URL = 'https://www.moneydj.com/kmdj/news/newsreallist.aspx?a=mb06&index1='
14
+
15
+
16
+ //本函數之預設抓取設定
17
+ let DFLT = {
18
+ responseType: 'text',
19
+ timeout: 30000,
20
+ maxRetries: 10,
21
+ baseDelayMs: 5000,
22
+ maxDelayMs: 30000,
23
+ retryStatus: [403],
24
+ label: 'fetch-moneydj',
25
+ }
26
+
27
+
28
+ //預設抓取頁數
29
+ let DEFAULT_TOTAL_PAGES = 50
30
+
31
+
32
+ //頁間隨機延遲之下限與變動範圍毫秒
33
+ let PAGE_DELAY_MIN_MS = 1000
34
+ let PAGE_DELAY_RANGE_MS = 2000
35
+
36
+
37
+ //列表頁時間欄位格式, 例如'08/07 09:02'、'09:02'、'昨 09:02'
38
+ let REG_TIME = /^(\d{2}\/\d{2}\s+\d{2}:\d{2}|\d{2}:\d{2}|昨\s*\d{2}:\d{2})$/
39
+
40
+
41
+ /**
42
+ * 抓取MoneyDJ台股新聞
43
+ *
44
+ * 逐頁抓取MoneyDJ台股新聞(MB06)列表, 頁間隨機延遲1至3秒以降低被封鎖風險;
45
+ * 單頁抓取失敗(重試耗盡)時記錄並跳過該頁, 不中斷整體抓取; 全部頁面皆0筆時拋出錯誤
46
+ *
47
+ * @param {Object} [opt={}] 輸入設定物件,預設{}
48
+ * @param {String} [opt.baseUrl='https://www.moneydj.com/kmdj/news/newsreallist.aspx?a=mb06&index1='] 輸入列表頁網址前綴字串,頁碼會直接串接於後,供測試或改指向鏡像時覆寫
49
+ * @param {Integer} [opt.totalPages=50] 輸入要抓取的頁數正整數,預設50
50
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數,預設30000
51
+ * @param {Integer} [opt.maxRetries=10] 輸入每頁最大重試次數整數,含初始共執行maxRetries+1次,預設10
52
+ * @param {Integer} [opt.baseDelayMs=5000] 輸入重試基礎延遲毫秒整數,預設5000
53
+ * @param {Integer} [opt.maxDelayMs=30000] 輸入重試最大延遲毫秒整數,預設30000
54
+ * @param {Integer} [opt.pageDelayMs] 輸入頁間延遲毫秒整數,未給時採隨機1000至3000毫秒
55
+ * @param {Function} [opt.onPageDone] 輸入每頁完成時之回呼函數,傳入(pageIndex,itemCount,totalPages)
56
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值,預設true
57
+ * @returns {Promise} 回傳Promise,resolve回傳新聞物件陣列,各物件為{time,title,link},抓取到0筆時reject回傳錯誤物件
58
+ * @example
59
+ *
60
+ * import fetchMoneydj from './src/fetchMoneydj.mjs'
61
+ *
62
+ * let test = async () => {
63
+ *
64
+ * let rs = await fetchMoneydj({ totalPages: 2 })
65
+ * console.log(rs.length, rs[0])
66
+ * // => 60 { time: '08/07 09:02', title: '...', link: 'https://www.moneydj.com/kmdj/news/...' }
67
+ *
68
+ * }
69
+ * await test()
70
+ * .catch((err) => {
71
+ * console.log(err)
72
+ * })
73
+ *
74
+ */
75
+ async function fetchMoneydj(opt = {}) {
76
+
77
+ //baseUrl
78
+ let baseUrl = get(opt, 'baseUrl')
79
+ if (!isestr(baseUrl)) {
80
+ baseUrl = BASE_URL
81
+ }
82
+
83
+ //totalPages
84
+ let totalPages = get(opt, 'totalPages')
85
+ if (!ispint(totalPages)) {
86
+ totalPages = DEFAULT_TOTAL_PAGES
87
+ }
88
+ else {
89
+ totalPages = cint(totalPages)
90
+ }
91
+
92
+ //onPageDone
93
+ let onPageDone = get(opt, 'onPageDone')
94
+ if (!isfun(onPageDone)) {
95
+ onPageDone = () => {}
96
+ }
97
+
98
+ //optFetch
99
+ let optFetch = getOptFetch(opt, DFLT)
100
+ let showLog = optFetch.showLog
101
+
102
+ //pageDelayMs, 頁間隨機延遲毫秒
103
+ let pageDelayMs = get(opt, 'pageDelayMs')
104
+ if (!ispint(pageDelayMs) && pageDelayMs !== 0) {
105
+ pageDelayMs = null
106
+ }
107
+ else {
108
+ pageDelayMs = cint(pageDelayMs)
109
+ }
110
+
111
+ //domain, 由列表頁網址推得, 供補齊相對路徑連結
112
+ let domain = new URL(baseUrl).origin
113
+
114
+ //fetchPage
115
+ let fetchPage = async (pageIndex) => {
116
+
117
+ //html, MoneyDJ已確認使用UTF-8編碼(多次實測結果正確), 無需Big5解碼
118
+ let html = await fetchWithRetry(`${baseUrl}${pageIndex}`, { ...optFetch, label: `${DFLT.label} Page ${pageIndex}` })
119
+
120
+ //$
121
+ let $ = cheerio.load(html)
122
+
123
+ //newsItems
124
+ let newsItems = []
125
+ $('tr').each((i, el) => {
126
+
127
+ let $row = $(el)
128
+ let timeText = $row.find('td').eq(0).text().trim()
129
+ let $link = $row.find('td').eq(1).find('a')
130
+
131
+ if (timeText && REG_TIME.test(timeText) && $link.length > 0) {
132
+ let title = $link.attr('title') || $link.text().trim()
133
+ let linkRel = $link.attr('href')
134
+ if (linkRel) {
135
+ let link = linkRel.startsWith('http') ? linkRel : domain + linkRel
136
+ newsItems.push({ time: timeText, title, link })
137
+ }
138
+ }
139
+
140
+ })
141
+
142
+ return newsItems
143
+ }
144
+
145
+ let allNewsItems = []
146
+ for (let i = 1; i <= totalPages; i++) {
147
+
148
+ //items, 單頁失敗記錄並跳過, 不中斷整體抓取
149
+ let items = null
150
+ try {
151
+ items = await fetchPage(i)
152
+ }
153
+ catch (err) {
154
+ if (showLog) {
155
+ console.warn(`[Page ${i}] 抓取失敗,已跳過:${err.message}`)
156
+ }
157
+ continue
158
+ }
159
+ allNewsItems = allNewsItems.concat(items)
160
+
161
+ onPageDone(i, items.length, totalPages)
162
+
163
+ //頁間延遲, 未指定pageDelayMs時採隨機1至3秒以降低被封鎖風險
164
+ if (i < totalPages) {
165
+ await delay(pageDelayMs === null ? Math.floor(Math.random() * PAGE_DELAY_RANGE_MS) + PAGE_DELAY_MIN_MS : pageDelayMs)
166
+ }
167
+
168
+ }
169
+
170
+ //check, 0筆代表版型改變或selector失效
171
+ if (allNewsItems.length === 0) {
172
+ throw new Error('抓取到 0 筆新聞,可能是頁面結構改變或 selector 失效,請確認 MoneyDJ 頁面是否正常。')
173
+ }
174
+
175
+ return allNewsItems
176
+ }
177
+
178
+
179
+ export default fetchMoneydj
@@ -0,0 +1,313 @@
1
+ import { chromium } from 'playwright'
2
+ import get from 'lodash-es/get.js'
3
+ import isbol from 'wsemi/src/isbol.mjs'
4
+ import isestr from 'wsemi/src/isestr.mjs'
5
+ import isp0int from 'wsemi/src/isp0int.mjs'
6
+ import cint from 'wsemi/src/cint.mjs'
7
+ import delay from 'wsemi/src/delay.mjs'
8
+
9
+
10
+ //公開資訊觀測站重大訊息頁與其查詢API
11
+ let PAGE_URL = 'https://mops.twse.com.tw/mops/#/web/t146sb10'
12
+ let API_URL = 'https://mops.twse.com.tw/mops/api/t146sb10'
13
+
14
+
15
+ //預設值
16
+ let DEFAULT_MAX_RETRIES = 10
17
+ let DEFAULT_BASE_DELAY_MS = 5000
18
+ let DEFAULT_MAX_DELAY_MS = 30000
19
+ let LAUNCH_TIMEOUT_MS = 360000
20
+ let GOTO_TIMEOUT_MS = 60000
21
+
22
+
23
+ //目標市場別, marketKind為API查詢參數
24
+ let MARKET_KINDS = [
25
+ { name: '上市', marketKind: 'sii' },
26
+ { name: '上櫃', marketKind: 'otc' },
27
+ { name: '興櫃', marketKind: 'rotc' },
28
+ { name: '公開發行', marketKind: 'pub' },
29
+ ]
30
+
31
+
32
+ /**
33
+ * 組出單一市場別之API查詢payload
34
+ *
35
+ * @param {String} marketKind 輸入市場別字串,可為'sii'、'otc'、'rotc'、'pub'
36
+ * @returns {Object} 回傳API查詢payload物件
37
+ */
38
+ function getPayload(marketKind) {
39
+ return {
40
+ scopeType: '2',
41
+ companyId: '',
42
+ dateType: '2',
43
+ firstDate: '',
44
+ lastDate: '',
45
+ marketKind,
46
+ announcementBasis: '0',
47
+ dateRangeType: '1',
48
+ announcementType: '1',
49
+ sort: '1',
50
+ encodeURIComponent: 1,
51
+ step: 1,
52
+ firstin: 1,
53
+ off: 1,
54
+ }
55
+ }
56
+
57
+
58
+ /**
59
+ * 於瀏覽器頁面內以fetch呼叫MOPS查詢API並自動重試
60
+ *
61
+ * 需於MOPS頁面context內發送請求(同源且帶站方cookie), 故採page.evaluate而非node端直接請求;
62
+ * 重試條件為HTTP 5xx、403、429或網路錯誤, application-level錯誤(如MOPS code 500)屬非暫時性故不重試
63
+ *
64
+ * @param {Object} page 輸入playwright之page物件
65
+ * @param {Object} target 輸入目標物件,內含name、marketKind、url與payload
66
+ * @param {Object} cfg 輸入已正規化之設定物件
67
+ * @returns {Promise} 回傳Promise,resolve回傳{json,raw}或{error,retryable}
68
+ */
69
+ async function fetchTargetWithRetry(page, target, cfg) {
70
+
71
+ let data = null
72
+ for (let attempt = 1; attempt <= cfg.maxRetries + 1; attempt++) {
73
+
74
+ data = await page.evaluate(async (t) => {
75
+ try {
76
+
77
+ let response = await fetch(t.url, {
78
+ method: 'POST',
79
+ headers: { 'Content-Type': 'application/json' },
80
+ body: JSON.stringify(t.payload),
81
+ })
82
+ if (response.status >= 500 || response.status === 403 || response.status === 429) {
83
+ return { error: `HTTP ${response.status}`, retryable: true }
84
+ }
85
+
86
+ let text = await response.text()
87
+ try {
88
+
89
+ let json = JSON.parse(text)
90
+
91
+ //驗證MOPS application-level狀態碼(HTTP 200不代表查詢成功)
92
+ //實測契約: code=200「查詢成功」、code=406「查無相符資料」(正常空結果)、其餘(如500)為application-level錯誤
93
+ let code = json && json.code
94
+ if (code !== 200 && code !== 406) {
95
+ //非成功且非「查無資料」→ application-level錯誤, 帶原始回應供除錯, 屬非暫時性故不重試
96
+ return { error: `MOPS code ${code}: ${(json && json.message) || ''}`, json, raw: text.substring(0, 500), retryable: false }
97
+ }
98
+
99
+ return { json, raw: text.substring(0, 500) }
100
+ }
101
+ catch (e) {
102
+ return { error: 'Parse Error', raw: text, retryable: false }
103
+ }
104
+
105
+ }
106
+ catch (err) {
107
+ return { error: err.toString(), retryable: true }
108
+ }
109
+ }, target)
110
+
111
+ if (!data.error) {
112
+ return data
113
+ }
114
+
115
+ let retryable = data.retryable !== false
116
+ let attemptsLeft = cfg.maxRetries + 1 - attempt
117
+ if (!retryable || attemptsLeft <= 0) {
118
+ return data //回傳錯誤結果, 由呼叫方處理
119
+ }
120
+
121
+ let ms = Math.min(cfg.baseDelayMs * attempt, cfg.maxDelayMs)
122
+ if (cfg.showLog) {
123
+ console.warn(`[${target.name}][Retry ${attempt}/${cfg.maxRetries}] ${data.error} — 等待 ${ms / 1000}s 後重試...`)
124
+ }
125
+ await delay(ms)
126
+
127
+ }
128
+
129
+ return data
130
+ }
131
+
132
+
133
+ /**
134
+ * 抓取公開資訊觀測站(MOPS)今日重大訊息
135
+ *
136
+ * 以playwright啟動本機Chrome(channel為'chrome', 可由環境變數CHROME_PATH指定執行檔)開啟MOPS重大訊息頁,
137
+ * 再於頁面context內逐一查詢上市、上櫃、興櫃與公開發行之今日重大訊息;
138
+ * 單一市場別失敗時記錄於該筆之error並續抓其餘市場別, 由hasError告知是否有失敗
139
+ *
140
+ * @param {Object} [opt={}] 輸入設定物件,預設{}
141
+ * @param {String} [opt.pageUrl='https://mops.twse.com.tw/mops/#/web/t146sb10'] 輸入MOPS重大訊息頁網址字串,供測試或改指向鏡像時覆寫
142
+ * @param {String} [opt.apiUrl='https://mops.twse.com.tw/mops/api/t146sb10'] 輸入MOPS查詢API網址字串,供測試或改指向鏡像時覆寫
143
+ * @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設10
144
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值,預設true
145
+ * @returns {Promise} 回傳Promise,resolve回傳結果物件{results,hasError},results各筆為{market,marketKind,data,error,timestamp},瀏覽器啟動失敗時reject回傳錯誤物件
146
+ * @example
147
+ *
148
+ * import fetchMops from './src/fetchMops.mjs'
149
+ *
150
+ * let test = async () => {
151
+ *
152
+ * let r = await fetchMops()
153
+ * console.log(r.hasError, r.results.length, r.results[0].market)
154
+ * // => false 4 '上市'
155
+ *
156
+ * }
157
+ * await test()
158
+ * .catch((err) => {
159
+ * console.log(err)
160
+ * })
161
+ *
162
+ */
163
+ async function fetchMops(opt = {}) {
164
+
165
+ //pageUrl
166
+ let pageUrl = get(opt, 'pageUrl')
167
+ if (!isestr(pageUrl)) {
168
+ pageUrl = PAGE_URL
169
+ }
170
+
171
+ //apiUrl
172
+ let apiUrl = get(opt, 'apiUrl')
173
+ if (!isestr(apiUrl)) {
174
+ apiUrl = API_URL
175
+ }
176
+
177
+ //maxRetries
178
+ let maxRetries = get(opt, 'maxRetries')
179
+ if (!isp0int(maxRetries)) {
180
+ maxRetries = DEFAULT_MAX_RETRIES
181
+ }
182
+ else {
183
+ maxRetries = cint(maxRetries)
184
+ }
185
+
186
+ //showLog
187
+ let showLog = get(opt, 'showLog')
188
+ if (!isbol(showLog)) {
189
+ showLog = true
190
+ }
191
+
192
+ //cfg
193
+ let cfg = {
194
+ maxRetries,
195
+ baseDelayMs: DEFAULT_BASE_DELAY_MS,
196
+ maxDelayMs: DEFAULT_MAX_DELAY_MS,
197
+ showLog,
198
+ }
199
+
200
+ //targets
201
+ let targets = MARKET_KINDS.map((v) => {
202
+ return {
203
+ name: v.name,
204
+ marketKind: v.marketKind,
205
+ url: apiUrl,
206
+ payload: getPayload(v.marketKind),
207
+ }
208
+ })
209
+
210
+ if (showLog) {
211
+ console.log('啟動瀏覽器...')
212
+ }
213
+
214
+ //browser, 帶重試之瀏覽器啟動
215
+ let browser = null
216
+ for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
217
+ try {
218
+ browser = await chromium.launch({
219
+ headless: true,
220
+ //設定環境變數CHROME_PATH則用該路徑啟動瀏覽器
221
+ //未設定時為undefined, playwright會忽略executablePath改走channel為'chrome', 行為不變
222
+ executablePath: process.env.CHROME_PATH || undefined,
223
+ channel: 'chrome',
224
+ args: ['--no-sandbox', '--disable-setuid-sandbox'],
225
+ timeout: LAUNCH_TIMEOUT_MS,
226
+ })
227
+ break
228
+ }
229
+ catch (err) {
230
+ let attemptsLeft = maxRetries + 1 - attempt
231
+ if (attemptsLeft <= 0) {
232
+ throw new Error(`瀏覽器啟動失敗(已重試 ${maxRetries} 次): ${err.message}`)
233
+ }
234
+ let ms = Math.min(DEFAULT_BASE_DELAY_MS * attempt, DEFAULT_MAX_DELAY_MS)
235
+ if (showLog) {
236
+ console.warn(`[browser.launch][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
237
+ }
238
+ await delay(ms)
239
+ }
240
+ }
241
+
242
+ try {
243
+
244
+ let page = await browser.newPage()
245
+
246
+ //帶重試之頁面導航
247
+ if (showLog) {
248
+ console.log('前往 MOPS 重大訊息頁面 (t146sb10)...')
249
+ }
250
+ for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
251
+ try {
252
+ await page.goto(pageUrl, { waitUntil: 'networkidle', timeout: GOTO_TIMEOUT_MS })
253
+ break
254
+ }
255
+ catch (err) {
256
+ let attemptsLeft = maxRetries + 1 - attempt
257
+ if (attemptsLeft <= 0) {
258
+ throw err
259
+ }
260
+ let ms = Math.min(DEFAULT_BASE_DELAY_MS * attempt, DEFAULT_MAX_DELAY_MS)
261
+ if (showLog) {
262
+ console.warn(`[page.goto][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
263
+ }
264
+ await delay(ms)
265
+ }
266
+ }
267
+
268
+ await page.waitForTimeout(2000)
269
+
270
+ //results
271
+ let results = []
272
+ for (let target of targets) {
273
+
274
+ if (showLog) {
275
+ console.log(`正在抓取 [${target.name}] 資料...`)
276
+ }
277
+
278
+ let data = await fetchTargetWithRetry(page, target, cfg)
279
+
280
+ results.push({
281
+ market: target.name,
282
+ marketKind: target.marketKind,
283
+ data: data.json || data,
284
+ ...(data.error && { error: data.error }),
285
+ timestamp: new Date().toISOString(),
286
+ })
287
+
288
+ await page.waitForTimeout(1000)
289
+ }
290
+
291
+ if (showLog) {
292
+ let summary = results.map((r) => {
293
+ let resultData = get(r, 'data.result', get(r, 'data'))
294
+ let count = Array.isArray(resultData) ? resultData.length : 0
295
+ return `${r.market}: 取得 ${count} 筆資料`
296
+ })
297
+ console.log('抓取完成。摘要:', summary)
298
+ }
299
+
300
+ //hasError
301
+ let hasError = results.some((r) => r.error)
302
+
303
+ return { results, hasError }
304
+
305
+ }
306
+ finally {
307
+ await browser.close()
308
+ }
309
+ }
310
+
311
+
312
+ export { MARKET_KINDS, getPayload }
313
+ export default fetchMops
@@ -0,0 +1,88 @@
1
+ import Parser from 'rss-parser'
2
+ import get from 'lodash-es/get.js'
3
+ import isestr from 'wsemi/src/isestr.mjs'
4
+ import isearr from 'wsemi/src/isearr.mjs'
5
+ import fetchWithRetry from './fetchWithRetry.mjs'
6
+ import getOptFetch from './getOptFetch.mjs'
7
+ import toDatetimeUTC8 from './toDatetimeUTC8.mjs'
8
+
9
+
10
+ //本函數之預設抓取設定, 因YouTube等站台之RSS會回傳暫時性404故將404納入重試範圍
11
+ let DFLT = {
12
+ responseType: 'text',
13
+ timeout: 30000,
14
+ maxRetries: 5,
15
+ baseDelayMs: 3000,
16
+ maxDelayMs: 15000,
17
+ retryStatus: [404],
18
+ label: 'fetch-rss',
19
+ }
20
+
21
+
22
+ /**
23
+ * 抓取RSS訂閱源
24
+ *
25
+ * 支援RSS與Atom, 並轉為統一格式, 時間欄位一律轉為UTC+8之'YYYY-MM-DD HH:mm:ss'字串;
26
+ * 因YouTube等站台之RSS會回傳暫時性404, 故404亦納入重試範圍
27
+ *
28
+ * @param {String} rssUrl 輸入RSS網址字串,須為http或https
29
+ * @param {Object} [opt={}] 輸入設定物件,預設{}
30
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數,預設30000
31
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設5
32
+ * @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數,預設3000
33
+ * @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數,預設15000
34
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值,預設true
35
+ * @returns {Promise} 回傳Promise,resolve回傳項目物件陣列,各物件為{url,time,title,description,from},網址無效或抓取失敗時reject回傳錯誤物件
36
+ * @example
37
+ *
38
+ * import fetchRSS from './src/fetchRSS.mjs'
39
+ *
40
+ * let test = async () => {
41
+ *
42
+ * let rs = await fetchRSS('https://www.ptt.cc/atom/Gossiping.xml')
43
+ * console.log(rs.length, rs[0])
44
+ * // => 20 { url: 'https://...', time: '2026-08-07 09:02:03', title: '...', description: '...', from: '...' }
45
+ *
46
+ * }
47
+ * await test()
48
+ * .catch((err) => {
49
+ * console.log(err)
50
+ * })
51
+ *
52
+ */
53
+ async function fetchRSS(rssUrl, opt = {}) {
54
+
55
+ //check
56
+ if (!isestr(rssUrl) || !/^https?:\/\//i.test(rssUrl)) {
57
+ throw new Error('rssUrl 須為有效的 http/https RSS 網址')
58
+ }
59
+
60
+ //xml
61
+ let xml = await fetchWithRetry(rssUrl, getOptFetch(opt, DFLT))
62
+
63
+ //feed
64
+ let parser = new Parser()
65
+ let feed = await parser.parseString(xml)
66
+
67
+ //feedFrom
68
+ let feedFrom = String(get(feed, 'title', '') || '').trim()
69
+
70
+ //items
71
+ let items = get(feed, 'items')
72
+ if (!isearr(items)) {
73
+ items = []
74
+ }
75
+
76
+ return items.map((item) => {
77
+ return {
78
+ url: String(get(item, 'link', '') || '').trim(),
79
+ time: toDatetimeUTC8(get(item, 'isoDate', '') || get(item, 'pubDate', '')),
80
+ title: String(get(item, 'title', '') || '').trim(),
81
+ description: String(get(item, 'contentSnippet', '') || get(item, 'summary', '') || '').trim(),
82
+ from: String(get(item, 'creator', '') || feedFrom).trim(),
83
+ }
84
+ })
85
+ }
86
+
87
+
88
+ export default fetchRSS