w-dwdata-hub 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/.editorconfig +9 -0
  2. package/.eslintignore +3 -0
  3. package/.eslintrc.js +55 -0
  4. package/.jsdoc +25 -0
  5. package/LICENSE +21 -0
  6. package/README.md +168 -0
  7. package/SECURITY.md +5 -0
  8. package/babel.config.js +16 -0
  9. package/dist/w-dwdata-hub.umd.js +7 -0
  10. package/dist/w-dwdata-hub.umd.js.map +1 -0
  11. package/docs/WDwdataHub.mjs.html +136 -0
  12. package/docs/decodeEntities.mjs.html +129 -0
  13. package/docs/extractDivContent.mjs.html +147 -0
  14. package/docs/fetchAiNewsAggregator.mjs.html +155 -0
  15. package/docs/fetchAisixiang.mjs.html +1038 -0
  16. package/docs/fetchCnyes.mjs.html +260 -0
  17. package/docs/fetchGuancha.mjs.html +1007 -0
  18. package/docs/fetchHackerNews.mjs.html +189 -0
  19. package/docs/fetchMoneydj.mjs.html +251 -0
  20. package/docs/fetchMops.mjs.html +385 -0
  21. package/docs/fetchRSS.mjs.html +160 -0
  22. package/docs/fetchStatementdog.mjs.html +213 -0
  23. package/docs/fetchTaifex.mjs.html +513 -0
  24. package/docs/fetchTpex3insti.mjs.html +278 -0
  25. package/docs/fetchTpexMargin.mjs.html +248 -0
  26. package/docs/fetchTpexStock.mjs.html +199 -0
  27. package/docs/fetchTwDataHoliday.mjs.html +236 -0
  28. package/docs/fetchTwseMargin.mjs.html +253 -0
  29. package/docs/fetchTwseStock.mjs.html +188 -0
  30. package/docs/fetchTwseT86.mjs.html +208 -0
  31. package/docs/fetchWithRetry.mjs.html +376 -0
  32. package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
  33. package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
  34. package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
  35. package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
  36. package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
  37. package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
  38. package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
  39. package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
  40. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
  41. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
  42. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
  43. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
  44. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
  45. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
  46. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
  47. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
  48. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
  49. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
  50. package/docs/getOptFetch.mjs.html +185 -0
  51. package/docs/global.html +22649 -0
  52. package/docs/htmlToMarkdown.mjs.html +168 -0
  53. package/docs/index.html +84 -0
  54. package/docs/isYmd.mjs.html +114 -0
  55. package/docs/parseIntComma.mjs.html +100 -0
  56. package/docs/safeFilename.mjs.html +102 -0
  57. package/docs/scripts/collapse.js +39 -0
  58. package/docs/scripts/commonNav.js +28 -0
  59. package/docs/scripts/linenumber.js +25 -0
  60. package/docs/scripts/nav.js +12 -0
  61. package/docs/scripts/polyfill.js +4 -0
  62. package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
  63. package/docs/scripts/prettify/lang-css.js +2 -0
  64. package/docs/scripts/prettify/prettify.js +28 -0
  65. package/docs/scripts/search.js +99 -0
  66. package/docs/styles/jsdoc.css +776 -0
  67. package/docs/styles/prettify.css +80 -0
  68. package/docs/toDatetimeUTC8.mjs.html +143 -0
  69. package/docs/toRocDate.mjs.html +107 -0
  70. package/g.mjs +73 -0
  71. package/package.json +31 -0
  72. package/script.txt +17 -0
  73. package/src/WDwdataHub.mjs +64 -0
  74. package/src/decodeEntities.mjs +57 -0
  75. package/src/extractDivContent.mjs +75 -0
  76. package/src/fetchAiNewsAggregator.mjs +83 -0
  77. package/src/fetchAisixiang.mjs +966 -0
  78. package/src/fetchCnyes.mjs +188 -0
  79. package/src/fetchGuancha.mjs +935 -0
  80. package/src/fetchHackerNews.mjs +117 -0
  81. package/src/fetchMoneydj.mjs +179 -0
  82. package/src/fetchMops.mjs +313 -0
  83. package/src/fetchRSS.mjs +88 -0
  84. package/src/fetchStatementdog.mjs +141 -0
  85. package/src/fetchTaifex.mjs +441 -0
  86. package/src/fetchTpex3insti.mjs +206 -0
  87. package/src/fetchTpexMargin.mjs +176 -0
  88. package/src/fetchTpexStock.mjs +127 -0
  89. package/src/fetchTwDataHoliday.mjs +164 -0
  90. package/src/fetchTwseMargin.mjs +181 -0
  91. package/src/fetchTwseStock.mjs +116 -0
  92. package/src/fetchTwseT86.mjs +136 -0
  93. package/src/fetchWithRetry.mjs +304 -0
  94. package/src/getOptFetch.mjs +113 -0
  95. package/src/htmlToMarkdown.mjs +96 -0
  96. package/src/isYmd.mjs +42 -0
  97. package/src/parseIntComma.mjs +28 -0
  98. package/src/safeFilename.mjs +30 -0
  99. package/src/toDatetimeUTC8.mjs +71 -0
  100. package/src/toRocDate.mjs +35 -0
  101. package/test/WDwdataHub.test.mjs +63 -0
  102. package/test/decodeEntities.test.mjs +42 -0
  103. package/test/extractDivContent.test.mjs +48 -0
  104. package/test/fetchAiNewsAggregator.test.mjs +79 -0
  105. package/test/fetchAisixiang.test.mjs +225 -0
  106. package/test/fetchCnyes.test.mjs +61 -0
  107. package/test/fetchGuancha.test.mjs +190 -0
  108. package/test/fetchHackerNews.test.mjs +78 -0
  109. package/test/fetchMoneydj.test.mjs +82 -0
  110. package/test/fetchMops.test.mjs +106 -0
  111. package/test/fetchRSS.test.mjs +85 -0
  112. package/test/fetchStatementdog.test.mjs +69 -0
  113. package/test/fetchTaifex.test.mjs +99 -0
  114. package/test/fetchTpex3insti.test.mjs +90 -0
  115. package/test/fetchTpexMargin.test.mjs +94 -0
  116. package/test/fetchTpexStock.test.mjs +68 -0
  117. package/test/fetchTwDataHoliday.test.mjs +100 -0
  118. package/test/fetchTwseMargin.test.mjs +81 -0
  119. package/test/fetchTwseStock.test.mjs +80 -0
  120. package/test/fetchTwseT86.test.mjs +76 -0
  121. package/test/fetchWithRetry.test.mjs +174 -0
  122. package/test/htmlToMarkdown.test.mjs +71 -0
  123. package/test/isYmd.test.mjs +56 -0
  124. package/test/parseIntComma.test.mjs +38 -0
  125. package/test/safeFilename.test.mjs +35 -0
  126. package/test/toDatetimeUTC8.test.mjs +65 -0
  127. package/test/toRocDate.test.mjs +38 -0
  128. package/test/tools/encodeBig5.mjs +80 -0
  129. package/test/tools/fixtures.mjs +539 -0
  130. package/test/tools/hasChrome.mjs +39 -0
  131. package/test/tools/serverForTest.mjs +397 -0
  132. package/toolg/addVersion.mjs +4 -0
  133. package/toolg/cleanFolder.mjs +5 -0
  134. package/toolg/gDistRollup.mjs +49 -0
  135. package/toolg/modifyReadme.mjs +4 -0
package/docs/fetchMops.mjs.html ADDED
@@ -0,0 +1,385 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchMops.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchMops.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import { chromium } from 'playwright'
49
+ import get from 'lodash-es/get.js'
50
+ import isbol from 'wsemi/src/isbol.mjs'
51
+ import isestr from 'wsemi/src/isestr.mjs'
52
+ import isp0int from 'wsemi/src/isp0int.mjs'
53
+ import cint from 'wsemi/src/cint.mjs'
54
+ import delay from 'wsemi/src/delay.mjs'
55
+
56
+
57
+ //公開資訊觀測站重大訊息頁與其查詢API
58
+ let PAGE_URL = 'https://mops.twse.com.tw/mops/#/web/t146sb10'
59
+ let API_URL = 'https://mops.twse.com.tw/mops/api/t146sb10'
60
+
61
+
62
+ //預設值
63
+ let DEFAULT_MAX_RETRIES = 10
64
+ let DEFAULT_BASE_DELAY_MS = 5000
65
+ let DEFAULT_MAX_DELAY_MS = 30000
66
+ let LAUNCH_TIMEOUT_MS = 360000
67
+ let GOTO_TIMEOUT_MS = 60000
68
+
69
+
70
+ //目標市場別, marketKind為API查詢參數
71
+ let MARKET_KINDS = [
72
+ { name: '上市', marketKind: 'sii' },
73
+ { name: '上櫃', marketKind: 'otc' },
74
+ { name: '興櫃', marketKind: 'rotc' },
75
+ { name: '公開發行', marketKind: 'pub' },
76
+ ]
77
+
78
+
79
+ /**
80
+ * 組出單一市場別之API查詢payload
81
+ *
82
+ * @param {String} marketKind 輸入市場別字串，可為'sii'、'otc'、'rotc'、'pub'
83
+ * @returns {Object} 回傳API查詢payload物件
84
+ */
85
+ function getPayload(marketKind) {
86
+ return {
87
+ scopeType: '2',
88
+ companyId: '',
89
+ dateType: '2',
90
+ firstDate: '',
91
+ lastDate: '',
92
+ marketKind,
93
+ announcementBasis: '0',
94
+ dateRangeType: '1',
95
+ announcementType: '1',
96
+ sort: '1',
97
+ encodeURIComponent: 1,
98
+ step: 1,
99
+ firstin: 1,
100
+ off: 1,
101
+ }
102
+ }
103
+
104
+
105
+ /**
106
+ * 於瀏覽器頁面內以fetch呼叫MOPS查詢API並自動重試
107
+ *
108
+ * 需於MOPS頁面context內發送請求(同源且帶站方cookie), 故採page.evaluate而非node端直接請求;
109
+ * 重試條件為HTTP 5xx、403、429或網路錯誤, application-level錯誤(如MOPS code 500)屬非暫時性故不重試
110
+ *
111
+ * @param {Object} page 輸入playwright之page物件
112
+ * @param {Object} target 輸入目標物件，內含name、marketKind、url與payload
113
+ * @param {Object} cfg 輸入已正規化之設定物件
114
+ * @returns {Promise} 回傳Promise，resolve回傳{json,raw}或{error,retryable}
115
+ */
116
+ async function fetchTargetWithRetry(page, target, cfg) {
117
+
118
+ let data = null
119
+ for (let attempt = 1; attempt &lt;= cfg.maxRetries + 1; attempt++) {
120
+
121
+ data = await page.evaluate(async (t) => {
122
+ try {
123
+
124
+ let response = await fetch(t.url, {
125
+ method: 'POST',
126
+ headers: { 'Content-Type': 'application/json' },
127
+ body: JSON.stringify(t.payload),
128
+ })
129
+ if (response.status >= 500 || response.status === 403 || response.status === 429) {
130
+ return { error: `HTTP ${response.status}`, retryable: true }
131
+ }
132
+
133
+ let text = await response.text()
134
+ try {
135
+
136
+ let json = JSON.parse(text)
137
+
138
+ //驗證MOPS application-level狀態碼(HTTP 200不代表查詢成功)
139
+ //實測契約: code=200「查詢成功」、code=406「查無相符資料」(正常空結果)、其餘(如500)為application-level錯誤
140
+ let code = json &amp;&amp; json.code
141
+ if (code !== 200 &amp;&amp; code !== 406) {
142
+ //非成功且非「查無資料」→ application-level錯誤, 帶原始回應供除錯, 屬非暫時性故不重試
143
+ return { error: `MOPS code ${code}: ${(json &amp;&amp; json.message) || ''}`, json, raw: text.substring(0, 500), retryable: false }
144
+ }
145
+
146
+ return { json, raw: text.substring(0, 500) }
147
+ }
148
+ catch (e) {
149
+ return { error: 'Parse Error', raw: text, retryable: false }
150
+ }
151
+
152
+ }
153
+ catch (err) {
154
+ return { error: err.toString(), retryable: true }
155
+ }
156
+ }, target)
157
+
158
+ if (!data.error) {
159
+ return data
160
+ }
161
+
162
+ let retryable = data.retryable !== false
163
+ let attemptsLeft = cfg.maxRetries + 1 - attempt
164
+ if (!retryable || attemptsLeft &lt;= 0) {
165
+ return data //回傳錯誤結果, 由呼叫方處理
166
+ }
167
+
168
+ let ms = Math.min(cfg.baseDelayMs * attempt, cfg.maxDelayMs)
169
+ if (cfg.showLog) {
170
+ console.warn(`[${target.name}][Retry ${attempt}/${cfg.maxRetries}] ${data.error} — 等待 ${ms / 1000}s 後重試...`)
171
+ }
172
+ await delay(ms)
173
+
174
+ }
175
+
176
+ return data
177
+ }
178
+
179
+
180
+ /**
181
+ * 抓取公開資訊觀測站(MOPS)今日重大訊息
182
+ *
183
+ * 以playwright啟動本機Chrome(channel為'chrome', 可由環境變數CHROME_PATH指定執行檔)開啟MOPS重大訊息頁,
184
+ * 再於頁面context內逐一查詢上市、上櫃、興櫃與公開發行之今日重大訊息;
185
+ * 單一市場別失敗時記錄於該筆之error並續抓其餘市場別, 由hasError告知是否有失敗
186
+ *
187
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
188
+ * @param {String} [opt.pageUrl='https://mops.twse.com.tw/mops/#/web/t146sb10'] 輸入MOPS重大訊息頁網址字串，供測試或改指向鏡像時覆寫
189
+ * @param {String} [opt.apiUrl='https://mops.twse.com.tw/mops/api/t146sb10'] 輸入MOPS查詢API網址字串，供測試或改指向鏡像時覆寫
190
+ * @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數，含初始共執行maxRetries+1次，預設10
191
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
192
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{results,hasError}，results各筆為{market,marketKind,data,error,timestamp}，瀏覽器啟動失敗時reject回傳錯誤物件
193
+ * @example
194
+ *
195
+ * import fetchMops from './src/fetchMops.mjs'
196
+ *
197
+ * let test = async () => {
198
+ *
199
+ * let r = await fetchMops()
200
+ * console.log(r.hasError, r.results.length, r.results[0].market)
201
+ * // => false 4 '上市'
202
+ *
203
+ * }
204
+ * await test()
205
+ * .catch((err) => {
206
+ * console.log(err)
207
+ * })
208
+ *
209
+ */
210
+ async function fetchMops(opt = {}) {
211
+
212
+ //pageUrl
213
+ let pageUrl = get(opt, 'pageUrl')
214
+ if (!isestr(pageUrl)) {
215
+ pageUrl = PAGE_URL
216
+ }
217
+
218
+ //apiUrl
219
+ let apiUrl = get(opt, 'apiUrl')
220
+ if (!isestr(apiUrl)) {
221
+ apiUrl = API_URL
222
+ }
223
+
224
+ //maxRetries
225
+ let maxRetries = get(opt, 'maxRetries')
226
+ if (!isp0int(maxRetries)) {
227
+ maxRetries = DEFAULT_MAX_RETRIES
228
+ }
229
+ else {
230
+ maxRetries = cint(maxRetries)
231
+ }
232
+
233
+ //showLog
234
+ let showLog = get(opt, 'showLog')
235
+ if (!isbol(showLog)) {
236
+ showLog = true
237
+ }
238
+
239
+ //cfg
240
+ let cfg = {
241
+ maxRetries,
242
+ baseDelayMs: DEFAULT_BASE_DELAY_MS,
243
+ maxDelayMs: DEFAULT_MAX_DELAY_MS,
244
+ showLog,
245
+ }
246
+
247
+ //targets
248
+ let targets = MARKET_KINDS.map((v) => {
249
+ return {
250
+ name: v.name,
251
+ marketKind: v.marketKind,
252
+ url: apiUrl,
253
+ payload: getPayload(v.marketKind),
254
+ }
255
+ })
256
+
257
+ if (showLog) {
258
+ console.log('啟動瀏覽器...')
259
+ }
260
+
261
+ //browser, 帶重試之瀏覽器啟動
262
+ let browser = null
263
+ for (let attempt = 1; attempt &lt;= maxRetries + 1; attempt++) {
264
+ try {
265
+ browser = await chromium.launch({
266
+ headless: true,
267
+ //設定環境變數CHROME_PATH則用該路徑啟動瀏覽器
268
+ //未設定時為undefined, playwright會忽略executablePath改走channel為'chrome', 行為不變
269
+ executablePath: process.env.CHROME_PATH || undefined,
270
+ channel: 'chrome',
271
+ args: ['--no-sandbox', '--disable-setuid-sandbox'],
272
+ timeout: LAUNCH_TIMEOUT_MS,
273
+ })
274
+ break
275
+ }
276
+ catch (err) {
277
+ let attemptsLeft = maxRetries + 1 - attempt
278
+ if (attemptsLeft &lt;= 0) {
279
+ throw new Error(`瀏覽器啟動失敗（已重試 ${maxRetries} 次）: ${err.message}`)
280
+ }
281
+ let ms = Math.min(DEFAULT_BASE_DELAY_MS * attempt, DEFAULT_MAX_DELAY_MS)
282
+ if (showLog) {
283
+ console.warn(`[browser.launch][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
284
+ }
285
+ await delay(ms)
286
+ }
287
+ }
288
+
289
+ try {
290
+
291
+ let page = await browser.newPage()
292
+
293
+ //帶重試之頁面導航
294
+ if (showLog) {
295
+ console.log('前往 MOPS 重大訊息頁面 (t146sb10)...')
296
+ }
297
+ for (let attempt = 1; attempt &lt;= maxRetries + 1; attempt++) {
298
+ try {
299
+ await page.goto(pageUrl, { waitUntil: 'networkidle', timeout: GOTO_TIMEOUT_MS })
300
+ break
301
+ }
302
+ catch (err) {
303
+ let attemptsLeft = maxRetries + 1 - attempt
304
+ if (attemptsLeft &lt;= 0) {
305
+ throw err
306
+ }
307
+ let ms = Math.min(DEFAULT_BASE_DELAY_MS * attempt, DEFAULT_MAX_DELAY_MS)
308
+ if (showLog) {
309
+ console.warn(`[page.goto][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
310
+ }
311
+ await delay(ms)
312
+ }
313
+ }
314
+
315
+ await page.waitForTimeout(2000)
316
+
317
+ //results
318
+ let results = []
319
+ for (let target of targets) {
320
+
321
+ if (showLog) {
322
+ console.log(`正在抓取 [${target.name}] 資料...`)
323
+ }
324
+
325
+ let data = await fetchTargetWithRetry(page, target, cfg)
326
+
327
+ results.push({
328
+ market: target.name,
329
+ marketKind: target.marketKind,
330
+ data: data.json || data,
331
+ ...(data.error &amp;&amp; { error: data.error }),
332
+ timestamp: new Date().toISOString(),
333
+ })
334
+
335
+ await page.waitForTimeout(1000)
336
+ }
337
+
338
+ if (showLog) {
339
+ let summary = results.map((r) => {
340
+ let resultData = get(r, 'data.result', get(r, 'data'))
341
+ let count = Array.isArray(resultData) ? resultData.length : 0
342
+ return `${r.market}: 取得 ${count} 筆資料`
343
+ })
344
+ console.log('抓取完成。摘要:', summary)
345
+ }
346
+
347
+ //hasError
348
+ let hasError = results.some((r) => r.error)
349
+
350
+ return { results, hasError }
351
+
352
+ }
353
+ finally {
354
+ await browser.close()
355
+ }
356
+ }
357
+
358
+
359
+ export { MARKET_KINDS, getPayload }
360
+ export default fetchMops
361
+ </code></pre>
362
+ </article>
363
+ </section>
364
+
365
+
366
+
367
+
368
+
369
+
370
+ </div>
371
+
372
+ <br class="clear">
373
+
374
+ <footer>
375
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
376
+ </footer>
377
+
378
+ <script>prettyPrint();</script>
379
+ <script src="scripts/polyfill.js"></script>
380
+ <script src="scripts/linenumber.js"></script>
381
+
382
+
383
+
384
+ </body>
385
+ </html>
package/docs/fetchRSS.mjs.html ADDED
@@ -0,0 +1,160 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchRSS.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchRSS.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import Parser from 'rss-parser'
49
+ import get from 'lodash-es/get.js'
50
+ import isestr from 'wsemi/src/isestr.mjs'
51
+ import isearr from 'wsemi/src/isearr.mjs'
52
+ import fetchWithRetry from './fetchWithRetry.mjs'
53
+ import getOptFetch from './getOptFetch.mjs'
54
+ import toDatetimeUTC8 from './toDatetimeUTC8.mjs'
55
+
56
+
57
+ //本函數之預設抓取設定, 因YouTube等站台之RSS會回傳暫時性404故將404納入重試範圍
58
+ let DFLT = {
59
+ responseType: 'text',
60
+ timeout: 30000,
61
+ maxRetries: 5,
62
+ baseDelayMs: 3000,
63
+ maxDelayMs: 15000,
64
+ retryStatus: [404],
65
+ label: 'fetch-rss',
66
+ }
67
+
68
+
69
+ /**
70
+ * 抓取RSS訂閱源
71
+ *
72
+ * 支援RSS與Atom, 並轉為統一格式, 時間欄位一律轉為UTC+8之'YYYY-MM-DD HH:mm:ss'字串;
73
+ * 因YouTube等站台之RSS會回傳暫時性404, 故404亦納入重試範圍
74
+ *
75
+ * @param {String} rssUrl 輸入RSS網址字串，須為http或https
76
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
77
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
78
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，含初始共執行maxRetries+1次，預設5
79
+ * @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數，預設3000
80
+ * @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數，預設15000
81
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值，預設true
82
+ * @returns {Promise} 回傳Promise，resolve回傳項目物件陣列，各物件為{url,time,title,description,from}，網址無效或抓取失敗時reject回傳錯誤物件
83
+ * @example
84
+ *
85
+ * import fetchRSS from './src/fetchRSS.mjs'
86
+ *
87
+ * let test = async () => {
88
+ *
89
+ * let rs = await fetchRSS('https://www.ptt.cc/atom/Gossiping.xml')
90
+ * console.log(rs.length, rs[0])
91
+ * // => 20 { url: 'https://...', time: '2026-08-07 09:02:03', title: '...', description: '...', from: '...' }
92
+ *
93
+ * }
94
+ * await test()
95
+ * .catch((err) => {
96
+ * console.log(err)
97
+ * })
98
+ *
99
+ */
100
+ async function fetchRSS(rssUrl, opt = {}) {
101
+
102
+ //check
103
+ if (!isestr(rssUrl) || !/^https?:\/\//i.test(rssUrl)) {
104
+ throw new Error('rssUrl 須為有效的 http/https RSS 網址')
105
+ }
106
+
107
+ //xml
108
+ let xml = await fetchWithRetry(rssUrl, getOptFetch(opt, DFLT))
109
+
110
+ //feed
111
+ let parser = new Parser()
112
+ let feed = await parser.parseString(xml)
113
+
114
+ //feedFrom
115
+ let feedFrom = String(get(feed, 'title', '') || '').trim()
116
+
117
+ //items
118
+ let items = get(feed, 'items')
119
+ if (!isearr(items)) {
120
+ items = []
121
+ }
122
+
123
+ return items.map((item) => {
124
+ return {
125
+ url: String(get(item, 'link', '') || '').trim(),
126
+ time: toDatetimeUTC8(get(item, 'isoDate', '') || get(item, 'pubDate', '')),
127
+ title: String(get(item, 'title', '') || '').trim(),
128
+ description: String(get(item, 'contentSnippet', '') || get(item, 'summary', '') || '').trim(),
129
+ from: String(get(item, 'creator', '') || feedFrom).trim(),
130
+ }
131
+ })
132
+ }
133
+
134
+
135
+ export default fetchRSS
136
+ </code></pre>
137
+ </article>
138
+ </section>
139
+
140
+
141
+
142
+
143
+
144
+
145
+ </div>
146
+
147
+ <br class="clear">
148
+
149
+ <footer>
150
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
151
+ </footer>
152
+
153
+ <script>prettyPrint();</script>
154
+ <script src="scripts/polyfill.js"></script>
155
+ <script src="scripts/linenumber.js"></script>
156
+
157
+
158
+
159
+ </body>
160
+ </html>