w-dwdata-hub 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/.editorconfig +9 -0
  2. package/.eslintignore +3 -0
  3. package/.eslintrc.js +55 -0
  4. package/.jsdoc +25 -0
  5. package/LICENSE +21 -0
  6. package/README.md +168 -0
  7. package/SECURITY.md +5 -0
  8. package/babel.config.js +16 -0
  9. package/dist/w-dwdata-hub.umd.js +7 -0
  10. package/dist/w-dwdata-hub.umd.js.map +1 -0
  11. package/docs/WDwdataHub.mjs.html +136 -0
  12. package/docs/decodeEntities.mjs.html +129 -0
  13. package/docs/extractDivContent.mjs.html +147 -0
  14. package/docs/fetchAiNewsAggregator.mjs.html +155 -0
  15. package/docs/fetchAisixiang.mjs.html +1038 -0
  16. package/docs/fetchCnyes.mjs.html +260 -0
  17. package/docs/fetchGuancha.mjs.html +1007 -0
  18. package/docs/fetchHackerNews.mjs.html +189 -0
  19. package/docs/fetchMoneydj.mjs.html +251 -0
  20. package/docs/fetchMops.mjs.html +385 -0
  21. package/docs/fetchRSS.mjs.html +160 -0
  22. package/docs/fetchStatementdog.mjs.html +213 -0
  23. package/docs/fetchTaifex.mjs.html +513 -0
  24. package/docs/fetchTpex3insti.mjs.html +278 -0
  25. package/docs/fetchTpexMargin.mjs.html +248 -0
  26. package/docs/fetchTpexStock.mjs.html +199 -0
  27. package/docs/fetchTwDataHoliday.mjs.html +236 -0
  28. package/docs/fetchTwseMargin.mjs.html +253 -0
  29. package/docs/fetchTwseStock.mjs.html +188 -0
  30. package/docs/fetchTwseT86.mjs.html +208 -0
  31. package/docs/fetchWithRetry.mjs.html +376 -0
  32. package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
  33. package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
  34. package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
  35. package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
  36. package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
  37. package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
  38. package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
  39. package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
  40. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
  41. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
  42. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
  43. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
  44. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
  45. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
  46. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
  47. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
  48. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
  49. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
  50. package/docs/getOptFetch.mjs.html +185 -0
  51. package/docs/global.html +22649 -0
  52. package/docs/htmlToMarkdown.mjs.html +168 -0
  53. package/docs/index.html +84 -0
  54. package/docs/isYmd.mjs.html +114 -0
  55. package/docs/parseIntComma.mjs.html +100 -0
  56. package/docs/safeFilename.mjs.html +102 -0
  57. package/docs/scripts/collapse.js +39 -0
  58. package/docs/scripts/commonNav.js +28 -0
  59. package/docs/scripts/linenumber.js +25 -0
  60. package/docs/scripts/nav.js +12 -0
  61. package/docs/scripts/polyfill.js +4 -0
  62. package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
  63. package/docs/scripts/prettify/lang-css.js +2 -0
  64. package/docs/scripts/prettify/prettify.js +28 -0
  65. package/docs/scripts/search.js +99 -0
  66. package/docs/styles/jsdoc.css +776 -0
  67. package/docs/styles/prettify.css +80 -0
  68. package/docs/toDatetimeUTC8.mjs.html +143 -0
  69. package/docs/toRocDate.mjs.html +107 -0
  70. package/g.mjs +73 -0
  71. package/package.json +31 -0
  72. package/script.txt +17 -0
  73. package/src/WDwdataHub.mjs +64 -0
  74. package/src/decodeEntities.mjs +57 -0
  75. package/src/extractDivContent.mjs +75 -0
  76. package/src/fetchAiNewsAggregator.mjs +83 -0
  77. package/src/fetchAisixiang.mjs +966 -0
  78. package/src/fetchCnyes.mjs +188 -0
  79. package/src/fetchGuancha.mjs +935 -0
  80. package/src/fetchHackerNews.mjs +117 -0
  81. package/src/fetchMoneydj.mjs +179 -0
  82. package/src/fetchMops.mjs +313 -0
  83. package/src/fetchRSS.mjs +88 -0
  84. package/src/fetchStatementdog.mjs +141 -0
  85. package/src/fetchTaifex.mjs +441 -0
  86. package/src/fetchTpex3insti.mjs +206 -0
  87. package/src/fetchTpexMargin.mjs +176 -0
  88. package/src/fetchTpexStock.mjs +127 -0
  89. package/src/fetchTwDataHoliday.mjs +164 -0
  90. package/src/fetchTwseMargin.mjs +181 -0
  91. package/src/fetchTwseStock.mjs +116 -0
  92. package/src/fetchTwseT86.mjs +136 -0
  93. package/src/fetchWithRetry.mjs +304 -0
  94. package/src/getOptFetch.mjs +113 -0
  95. package/src/htmlToMarkdown.mjs +96 -0
  96. package/src/isYmd.mjs +42 -0
  97. package/src/parseIntComma.mjs +28 -0
  98. package/src/safeFilename.mjs +30 -0
  99. package/src/toDatetimeUTC8.mjs +71 -0
  100. package/src/toRocDate.mjs +35 -0
  101. package/test/WDwdataHub.test.mjs +63 -0
  102. package/test/decodeEntities.test.mjs +42 -0
  103. package/test/extractDivContent.test.mjs +48 -0
  104. package/test/fetchAiNewsAggregator.test.mjs +79 -0
  105. package/test/fetchAisixiang.test.mjs +225 -0
  106. package/test/fetchCnyes.test.mjs +61 -0
  107. package/test/fetchGuancha.test.mjs +190 -0
  108. package/test/fetchHackerNews.test.mjs +78 -0
  109. package/test/fetchMoneydj.test.mjs +82 -0
  110. package/test/fetchMops.test.mjs +106 -0
  111. package/test/fetchRSS.test.mjs +85 -0
  112. package/test/fetchStatementdog.test.mjs +69 -0
  113. package/test/fetchTaifex.test.mjs +99 -0
  114. package/test/fetchTpex3insti.test.mjs +90 -0
  115. package/test/fetchTpexMargin.test.mjs +94 -0
  116. package/test/fetchTpexStock.test.mjs +68 -0
  117. package/test/fetchTwDataHoliday.test.mjs +100 -0
  118. package/test/fetchTwseMargin.test.mjs +81 -0
  119. package/test/fetchTwseStock.test.mjs +80 -0
  120. package/test/fetchTwseT86.test.mjs +76 -0
  121. package/test/fetchWithRetry.test.mjs +174 -0
  122. package/test/htmlToMarkdown.test.mjs +71 -0
  123. package/test/isYmd.test.mjs +56 -0
  124. package/test/parseIntComma.test.mjs +38 -0
  125. package/test/safeFilename.test.mjs +35 -0
  126. package/test/toDatetimeUTC8.test.mjs +65 -0
  127. package/test/toRocDate.test.mjs +38 -0
  128. package/test/tools/encodeBig5.mjs +80 -0
  129. package/test/tools/fixtures.mjs +539 -0
  130. package/test/tools/hasChrome.mjs +39 -0
  131. package/test/tools/serverForTest.mjs +397 -0
  132. package/toolg/addVersion.mjs +4 -0
  133. package/toolg/cleanFolder.mjs +5 -0
  134. package/toolg/gDistRollup.mjs +49 -0
  135. package/toolg/modifyReadme.mjs +4 -0
package/docs/fetchTwseT86.mjs.html ADDED
@@ -0,0 +1,208 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchTwseT86.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchTwseT86.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import get from 'lodash-es/get.js'
49
+ import isestr from 'wsemi/src/isestr.mjs'
50
+ import isearr from 'wsemi/src/isearr.mjs'
51
+ import fetchWithRetry from './fetchWithRetry.mjs'
52
+ import getOptFetch from './getOptFetch.mjs'
53
+ import isYmd from './isYmd.mjs'
54
+
55
+
56
+ //證交所網站根網址
57
+ let BASE_URL = 'https://www.twse.com.tw'
58
+
59
+
60
+ //本函數之預設抓取設定
61
+ let DFLT = {
62
+ timeout: 15000,
63
+ maxRetries: 10,
64
+ baseDelayMs: 5000,
65
+ maxDelayMs: 30000,
66
+ label: 'fetch-twse-t86',
67
+ }
68
+
69
+
70
+ /**
71
+ * 抓取證交所(TWSE)上市三大法人買賣超
72
+ *
73
+ * 抓取指定日之上市三大法人買賣超(T86), 以API回傳之fields為key轉為物件陣列, 可指定股票代號陣列過濾
74
+ *
75
+ * @param {String} dateStr 輸入日期YYYYMMDD字串
76
+ * @param {Array} [stockCodes] 輸入股票代號字串陣列，省略或空陣列表示全市場
77
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
78
+ * @param {String} [opt.baseUrl='https://www.twse.com.tw'] 輸入證交所根網址字串，供測試或改指向鏡像時覆寫
79
+ * @param {Integer} [opt.timeout=15000] 輸入單次請求逾時毫秒整數，預設15000
80
+ * @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數，含初始共執行maxRetries+1次，預設10
81
+ * @param {Integer} [opt.baseDelayMs=5000] 輸入重試之線性退避基礎毫秒整數，預設5000
82
+ * @param {Integer} [opt.maxDelayMs=30000] 輸入重試之線性退避上限毫秒整數，預設30000
83
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
84
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{source,date,data}，日期無效或API錯誤時reject回傳錯誤物件
85
+ * @example
86
+ *
87
+ * import fetchTwseT86 from './src/fetchTwseT86.mjs'
88
+ *
89
+ * let test = async () => {
90
+ *
91
+ * let r = await fetchTwseT86('20260807', ['2330'])
92
+ * console.log(r.source, r.data.length, r.data[0]['證券代號'])
93
+ * // => 'twse' 1 '2330'
94
+ *
95
+ * }
96
+ * await test()
97
+ * .catch((err) => {
98
+ * console.log(err)
99
+ * })
100
+ *
101
+ */
102
+ async function fetchTwseT86(dateStr, stockCodes, opt = {}) {
103
+
104
+ //check
105
+ if (!isestr(dateStr) || !/^\d{8}$/.test(dateStr)) {
106
+ throw new Error(`dateStr must be YYYYMMDD, got: ${dateStr}`)
107
+ }
108
+ if (!isYmd(dateStr)) {
109
+ //合法性驗證, 例如20260230雖然符合8碼但日期不存在
110
+ throw new Error(`dateStr 不是合法日期: ${dateStr}`)
111
+ }
112
+
113
+ //baseUrl
114
+ let baseUrl = get(opt, 'baseUrl')
115
+ if (!isestr(baseUrl)) {
116
+ baseUrl = BASE_URL
117
+ }
118
+ baseUrl = baseUrl.replace(/\/+$/, '')
119
+
120
+ //optFetch
121
+ let optFetch = getOptFetch(opt, DFLT)
122
+ let showLog = optFetch.showLog
123
+
124
+ //targetCodes
125
+ let targetCodes = isearr(stockCodes) ? stockCodes : []
126
+
127
+ //url
128
+ let url = `${baseUrl}/rwd/zh/fund/T86?response=json&amp;date=${dateStr}&amp;selectType=ALL`
129
+
130
+ if (showLog) {
131
+ console.log(`Fetching from: ${url}`)
132
+ console.log(`Target: ${targetCodes.length === 0 ? 'All Market' : targetCodes.join(', ')}`)
133
+ }
134
+
135
+ //data
136
+ let data = await fetchWithRetry(url, optFetch)
137
+
138
+ //check
139
+ if (get(data, 'stat') !== 'OK') {
140
+ throw new Error(`TWSE T86 API returned: ${get(data, 'stat')}`)
141
+ }
142
+
143
+ //fields, rawData
144
+ let fields = get(data, 'fields')
145
+ let rawData = get(data, 'data')
146
+ if (!Array.isArray(rawData)) {
147
+ throw new Error('TWSE T86: data not found in response.')
148
+ }
149
+ if (!Array.isArray(fields)) {
150
+ throw new Error('TWSE T86: fields not found in response.')
151
+ }
152
+
153
+ //parsedData, 以API回傳之fields為key
154
+ let parsedData = rawData.map((row) => {
155
+ let obj = {}
156
+ fields.forEach((field, index) => {
157
+ let value = row[index]
158
+ if (typeof value === 'string') {
159
+ value = value.trim()
160
+ }
161
+ obj[field] = value
162
+ })
163
+ return obj
164
+ })
165
+
166
+ //過濾指定個股
167
+ if (targetCodes.length > 0) {
168
+ let codeField = fields.find((f) => String(f).includes('證券代號'))
169
+ if (!codeField) {
170
+ throw new Error('無法篩選個股：API 回應中找不到證券代號欄位')
171
+ }
172
+ parsedData = parsedData.filter((item) => targetCodes.includes(item[codeField]))
173
+ }
174
+
175
+ if (showLog) {
176
+ console.log(`Fetched ${parsedData.length} records.`)
177
+ }
178
+
179
+ return { source: 'twse', date: dateStr, data: parsedData }
180
+ }
181
+
182
+
183
+ export default fetchTwseT86
184
+ </code></pre>
185
+ </article>
186
+ </section>
187
+
188
+
189
+
190
+
191
+
192
+
193
+ </div>
194
+
195
+ <br class="clear">
196
+
197
+ <footer>
198
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
199
+ </footer>
200
+
201
+ <script>prettyPrint();</script>
202
+ <script src="scripts/polyfill.js"></script>
203
+ <script src="scripts/linenumber.js"></script>
204
+
205
+
206
+
207
+ </body>
208
+ </html>
package/docs/fetchWithRetry.mjs.html ADDED
@@ -0,0 +1,376 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchWithRetry.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchWithRetry.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import get from 'lodash-es/get.js'
49
+ import isbol from 'wsemi/src/isbol.mjs'
50
+ import isestr from 'wsemi/src/isestr.mjs'
51
+ import isearr from 'wsemi/src/isearr.mjs'
52
+ import isobj from 'wsemi/src/isobj.mjs'
53
+ import ispint from 'wsemi/src/ispint.mjs'
54
+ import isp0int from 'wsemi/src/isp0int.mjs'
55
+ import cint from 'wsemi/src/cint.mjs'
56
+ import delay from 'wsemi/src/delay.mjs'
57
+
58
+
59
+ //預設User-Agent, 各資料來源網站多會擋無User-Agent之請求
60
+ let DEFAULT_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
61
+
62
+
63
+ //預設值
64
+ let DEFAULT_TIMEOUT_MS = 30000
65
+ let DEFAULT_MAX_RETRIES = 10
66
+ let DEFAULT_BASE_DELAY_MS = 5000
67
+ let DEFAULT_MAX_DELAY_MS = 30000
68
+
69
+
70
+ //視為暫時性故障之網路層錯誤碼, node內建fetch(undici)會把底層錯誤掛於err.cause
71
+ let NETWORK_ERROR_CODES = ['ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED', 'ECONNABORTED', 'EAI_AGAIN', 'EPIPE']
72
+
73
+
74
+ /**
75
+ * 判斷錯誤是否值得重試
76
+ *
77
+ * 有statusCode(即HTTP層錯誤)時, 5xx與429視為暫時性, 其餘由retryStatus指定;
78
+ * 無statusCode(即網路層錯誤)時, 由錯誤碼、AbortError與undici暫時性錯誤判斷
79
+ *
80
+ * @param {Error} err 輸入錯誤物件
81
+ * @param {Array} retryStatus 輸入額外視為可重試之HTTP狀態碼陣列
82
+ * @returns {Boolean} 回傳是否可重試布林值
83
+ */
84
+ function isRetryableError(err, retryStatus) {
85
+
86
+ //statusCode, 由fetchOnce於HTTP非2xx時掛入
87
+ let statusCode = get(err, 'statusCode', null)
88
+ if (ispint(statusCode)) {
89
+ return statusCode >= 500 || statusCode === 429 || retryStatus.includes(statusCode)
90
+ }
91
+
92
+ //code, undici把底層錯誤掛在err.cause
93
+ let code = get(err, 'cause.code', null) || get(err, 'code', null)
94
+ if (isestr(code) &amp;&amp; NETWORK_ERROR_CODES.includes(code)) {
95
+ return true
96
+ }
97
+
98
+ //AbortController觸發之逾時
99
+ let name = get(err, 'name', '')
100
+ if (name === 'AbortError' || name === 'TimeoutError') {
101
+ return true
102
+ }
103
+
104
+ //undici之socket與逾時類暫時性錯誤
105
+ let ms = String(get(err, 'cause.code', '') || get(err, 'message', ''))
106
+ if (/UND_ERR_(SOCKET|HEADERS_TIMEOUT|BODY_TIMEOUT|CONNECT_TIMEOUT)/.test(ms)) {
107
+ return true
108
+ }
109
+
110
+ return false
111
+ }
112
+
113
+
114
+ /**
115
+ * 單次抓取, HTTP非2xx時拋出帶statusCode之錯誤
116
+ *
117
+ * @param {String} url 輸入待抓取網址字串
118
+ * @param {Object} cfg 輸入已正規化之設定物件
119
+ * @returns {Promise} 回傳Promise，resolve回傳解析後資料
120
+ */
121
+ async function fetchOnce(url, cfg) {
122
+
123
+ //ac, 以AbortController實作逾時, 避免socket卡住永不回應
124
+ let ac = new AbortController()
125
+ let idTimer = setTimeout(() => {
126
+ ac.abort()
127
+ }, cfg.timeout)
128
+
129
+ try {
130
+
131
+ //opt
132
+ let opt = {
133
+ method: cfg.method,
134
+ headers: cfg.headers,
135
+ signal: ac.signal,
136
+ }
137
+ if (cfg.body !== null) {
138
+ opt.body = cfg.body
139
+ }
140
+
141
+ //fetch
142
+ let res = await fetch(url, opt)
143
+
144
+ //check, node內建fetch不會因HTTP非2xx而拋錯, 需自行檢核
145
+ if (!res.ok) {
146
+ let err = new Error(`HTTP ${res.status} ${res.statusText}`.trim())
147
+ err.statusCode = res.status
148
+ err.url = url
149
+ throw err
150
+ }
151
+
152
+ //arrayBuffer
153
+ if (cfg.responseType === 'arrayBuffer') {
154
+ return await res.arrayBuffer()
155
+ }
156
+
157
+ //text
158
+ if (cfg.responseType === 'text') {
159
+ if (cfg.encoding !== '') {
160
+ //指定編碼(如big5)時需自行解碼, res.text()一律以utf-8解碼
161
+ let ab = await res.arrayBuffer()
162
+ return new TextDecoder(cfg.encoding).decode(ab)
163
+ }
164
+ return await res.text()
165
+ }
166
+
167
+ //json, 先取text再自行parse, 令維護頁面(HTTP 200+HTML)能產生可讀訊息而非裸SyntaxError
168
+ let text = await res.text()
169
+ try {
170
+ return JSON.parse(text)
171
+ }
172
+ catch (err) {
173
+ throw new Error(`回傳非JSON格式(可能為維護頁面或網路中介): ${text.slice(0, 100)}`)
174
+ }
175
+
176
+ }
177
+ finally {
178
+ clearTimeout(idTimer)
179
+ }
180
+ }
181
+
182
+
183
+ /**
184
+ * 以node內建fetch抓取資料並自動重試
185
+ *
186
+ * 特點：
187
+ * HTTP 5xx與429及網路層錯誤(含逾時)會自動重試，採線性退避baseDelayMs*attempt並以maxDelayMs為上限；
188
+ * HTTP 4xx(429除外)預設不重試，可由opt.retryStatus指定額外可重試狀態碼(如403、404)；
189
+ * responseType為'json'時先取原始文字再自行JSON.parse，令維護頁面能產生可讀錯誤訊息
190
+ *
191
+ * @param {String} url 輸入待抓取網址字串
192
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
193
+ * @param {String} [opt.method='GET'] 輸入HTTP方法字串，預設'GET'
194
+ * @param {Object} [opt.headers={}] 輸入額外HTTP標頭物件，未指定User-Agent時自動補預設值，預設{}
195
+ * @param {String} [opt.body=null] 輸入請求主體字串，預設null代表無主體
196
+ * @param {String} [opt.responseType='json'] 輸入回應解析方式字串，可為'json'、'text'、'arrayBuffer'，預設'json'
197
+ * @param {String} [opt.encoding=''] 輸入responseType為'text'時之解碼字串，例如'big5'，預設''代表utf-8
198
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
199
+ * @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數，含初始共執行maxRetries+1次，預設10
200
+ * @param {Integer} [opt.baseDelayMs=5000] 輸入線性退避之基礎毫秒整數，預設5000
201
+ * @param {Integer} [opt.maxDelayMs=30000] 輸入線性退避之上限毫秒整數，預設30000
202
+ * @param {Array} [opt.retryStatus=[]] 輸入額外視為可重試之HTTP狀態碼陣列，例如[403]，預設[]
203
+ * @param {String} [opt.label=''] 輸入重試訊息前綴字串，預設''
204
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值，預設true
205
+ * @returns {Promise} 回傳Promise，resolve回傳解析後資料，responseType為'json'時回物件、'text'時回字串、'arrayBuffer'時回ArrayBuffer，重試耗盡或遇不可重試錯誤時reject回傳錯誤物件
206
+ * @example
207
+ *
208
+ * import fetchWithRetry from './src/fetchWithRetry.mjs'
209
+ *
210
+ * let test = async () => {
211
+ *
212
+ * let r = await fetchWithRetry('https://hacker-news.firebaseio.com/v0/item/1.json')
213
+ * console.log(r.type, r.by)
214
+ * // => 'story' 'pg'
215
+ *
216
+ * }
217
+ * await test()
218
+ * .catch((err) => {
219
+ * console.log(err)
220
+ * })
221
+ *
222
+ */
223
+ async function fetchWithRetry(url, opt = {}) {
224
+
225
+ //check
226
+ if (!isestr(url)) {
227
+ throw new Error(`url須為有效字串, 得到: ${url}`)
228
+ }
229
+
230
+ //method
231
+ let method = get(opt, 'method')
232
+ if (!isestr(method)) {
233
+ method = 'GET'
234
+ }
235
+
236
+ //headers
237
+ let headers = get(opt, 'headers')
238
+ if (!isobj(headers)) {
239
+ headers = {}
240
+ }
241
+ if (!isestr(get(headers, 'User-Agent'))) {
242
+ headers = { 'User-Agent': DEFAULT_UA, ...headers }
243
+ }
244
+
245
+ //body
246
+ let body = get(opt, 'body', null)
247
+ if (!isestr(body)) {
248
+ body = null
249
+ }
250
+
251
+ //responseType
252
+ let responseType = get(opt, 'responseType')
253
+ if (responseType !== 'text' &amp;&amp; responseType !== 'arrayBuffer') {
254
+ responseType = 'json'
255
+ }
256
+
257
+ //encoding
258
+ let encoding = get(opt, 'encoding')
259
+ if (!isestr(encoding)) {
260
+ encoding = ''
261
+ }
262
+
263
+ //timeout
264
+ let timeout = get(opt, 'timeout')
265
+ if (!ispint(timeout)) {
266
+ timeout = DEFAULT_TIMEOUT_MS
267
+ }
268
+ else {
269
+ timeout = cint(timeout)
270
+ }
271
+
272
+ //maxRetries
273
+ let maxRetries = get(opt, 'maxRetries')
274
+ if (!isp0int(maxRetries)) {
275
+ maxRetries = DEFAULT_MAX_RETRIES
276
+ }
277
+ else {
278
+ maxRetries = cint(maxRetries)
279
+ }
280
+
281
+ //baseDelayMs
282
+ let baseDelayMs = get(opt, 'baseDelayMs')
283
+ if (!ispint(baseDelayMs)) {
284
+ baseDelayMs = DEFAULT_BASE_DELAY_MS
285
+ }
286
+ else {
287
+ baseDelayMs = cint(baseDelayMs)
288
+ }
289
+
290
+ //maxDelayMs
291
+ let maxDelayMs = get(opt, 'maxDelayMs')
292
+ if (!ispint(maxDelayMs)) {
293
+ maxDelayMs = DEFAULT_MAX_DELAY_MS
294
+ }
295
+ else {
296
+ maxDelayMs = cint(maxDelayMs)
297
+ }
298
+
299
+ //retryStatus
300
+ let retryStatus = get(opt, 'retryStatus')
301
+ if (!isearr(retryStatus)) {
302
+ retryStatus = []
303
+ }
304
+
305
+ //label
306
+ let label = get(opt, 'label')
307
+ if (!isestr(label)) {
308
+ label = 'fetchWithRetry'
309
+ }
310
+
311
+ //showLog
312
+ let showLog = get(opt, 'showLog')
313
+ if (!isbol(showLog)) {
314
+ showLog = true
315
+ }
316
+
317
+ //cfg
318
+ let cfg = { method, headers, body, responseType, encoding, timeout }
319
+
320
+ let lastError = null
321
+ for (let attempt = 1; attempt &lt;= maxRetries + 1; attempt++) {
322
+
323
+ try {
324
+ return await fetchOnce(url, cfg)
325
+ }
326
+ catch (err) {
327
+ lastError = err
328
+
329
+ //check, 不可重試或已用盡次數則直接拋出
330
+ let attemptsLeft = maxRetries + 1 - attempt
331
+ if (!isRetryableError(err, retryStatus) || attemptsLeft &lt;= 0) {
332
+ throw err
333
+ }
334
+
335
+ //線性退避
336
+ let ms = Math.min(baseDelayMs * attempt, maxDelayMs)
337
+ if (showLog) {
338
+ console.warn(`[${label}][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
339
+ }
340
+ await delay(ms)
341
+
342
+ }
343
+
344
+ }
345
+
346
+ throw lastError
347
+ }
348
+
349
+
350
+ export { DEFAULT_UA, isRetryableError }
351
+ export default fetchWithRetry
352
+ </code></pre>
353
+ </article>
354
+ </section>
355
+
356
+
357
+
358
+
359
+
360
+
361
+ </div>
362
+
363
+ <br class="clear">
364
+
365
+ <footer>
366
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
367
+ </footer>
368
+
369
+ <script>prettyPrint();</script>
370
+ <script src="scripts/polyfill.js"></script>
371
+ <script src="scripts/linenumber.js"></script>
372
+
373
+
374
+
375
+ </body>
376
+ </html>