w-dwdata-hub 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +9 -0
- package/.eslintignore +3 -0
- package/.eslintrc.js +55 -0
- package/.jsdoc +25 -0
- package/LICENSE +21 -0
- package/README.md +168 -0
- package/SECURITY.md +5 -0
- package/babel.config.js +16 -0
- package/dist/w-dwdata-hub.umd.js +7 -0
- package/dist/w-dwdata-hub.umd.js.map +1 -0
- package/docs/WDwdataHub.mjs.html +136 -0
- package/docs/decodeEntities.mjs.html +129 -0
- package/docs/extractDivContent.mjs.html +147 -0
- package/docs/fetchAiNewsAggregator.mjs.html +155 -0
- package/docs/fetchAisixiang.mjs.html +1038 -0
- package/docs/fetchCnyes.mjs.html +260 -0
- package/docs/fetchGuancha.mjs.html +1007 -0
- package/docs/fetchHackerNews.mjs.html +189 -0
- package/docs/fetchMoneydj.mjs.html +251 -0
- package/docs/fetchMops.mjs.html +385 -0
- package/docs/fetchRSS.mjs.html +160 -0
- package/docs/fetchStatementdog.mjs.html +213 -0
- package/docs/fetchTaifex.mjs.html +513 -0
- package/docs/fetchTpex3insti.mjs.html +278 -0
- package/docs/fetchTpexMargin.mjs.html +248 -0
- package/docs/fetchTpexStock.mjs.html +199 -0
- package/docs/fetchTwDataHoliday.mjs.html +236 -0
- package/docs/fetchTwseMargin.mjs.html +253 -0
- package/docs/fetchTwseStock.mjs.html +188 -0
- package/docs/fetchTwseT86.mjs.html +208 -0
- package/docs/fetchWithRetry.mjs.html +376 -0
- package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
- package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
- package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
- package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
- package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
- package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
- package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
- package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
- package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
- package/docs/getOptFetch.mjs.html +185 -0
- package/docs/global.html +22649 -0
- package/docs/htmlToMarkdown.mjs.html +168 -0
- package/docs/index.html +84 -0
- package/docs/isYmd.mjs.html +114 -0
- package/docs/parseIntComma.mjs.html +100 -0
- package/docs/safeFilename.mjs.html +102 -0
- package/docs/scripts/collapse.js +39 -0
- package/docs/scripts/commonNav.js +28 -0
- package/docs/scripts/linenumber.js +25 -0
- package/docs/scripts/nav.js +12 -0
- package/docs/scripts/polyfill.js +4 -0
- package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
- package/docs/scripts/prettify/lang-css.js +2 -0
- package/docs/scripts/prettify/prettify.js +28 -0
- package/docs/scripts/search.js +99 -0
- package/docs/styles/jsdoc.css +776 -0
- package/docs/styles/prettify.css +80 -0
- package/docs/toDatetimeUTC8.mjs.html +143 -0
- package/docs/toRocDate.mjs.html +107 -0
- package/g.mjs +73 -0
- package/package.json +31 -0
- package/script.txt +17 -0
- package/src/WDwdataHub.mjs +64 -0
- package/src/decodeEntities.mjs +57 -0
- package/src/extractDivContent.mjs +75 -0
- package/src/fetchAiNewsAggregator.mjs +83 -0
- package/src/fetchAisixiang.mjs +966 -0
- package/src/fetchCnyes.mjs +188 -0
- package/src/fetchGuancha.mjs +935 -0
- package/src/fetchHackerNews.mjs +117 -0
- package/src/fetchMoneydj.mjs +179 -0
- package/src/fetchMops.mjs +313 -0
- package/src/fetchRSS.mjs +88 -0
- package/src/fetchStatementdog.mjs +141 -0
- package/src/fetchTaifex.mjs +441 -0
- package/src/fetchTpex3insti.mjs +206 -0
- package/src/fetchTpexMargin.mjs +176 -0
- package/src/fetchTpexStock.mjs +127 -0
- package/src/fetchTwDataHoliday.mjs +164 -0
- package/src/fetchTwseMargin.mjs +181 -0
- package/src/fetchTwseStock.mjs +116 -0
- package/src/fetchTwseT86.mjs +136 -0
- package/src/fetchWithRetry.mjs +304 -0
- package/src/getOptFetch.mjs +113 -0
- package/src/htmlToMarkdown.mjs +96 -0
- package/src/isYmd.mjs +42 -0
- package/src/parseIntComma.mjs +28 -0
- package/src/safeFilename.mjs +30 -0
- package/src/toDatetimeUTC8.mjs +71 -0
- package/src/toRocDate.mjs +35 -0
- package/test/WDwdataHub.test.mjs +63 -0
- package/test/decodeEntities.test.mjs +42 -0
- package/test/extractDivContent.test.mjs +48 -0
- package/test/fetchAiNewsAggregator.test.mjs +79 -0
- package/test/fetchAisixiang.test.mjs +225 -0
- package/test/fetchCnyes.test.mjs +61 -0
- package/test/fetchGuancha.test.mjs +190 -0
- package/test/fetchHackerNews.test.mjs +78 -0
- package/test/fetchMoneydj.test.mjs +82 -0
- package/test/fetchMops.test.mjs +106 -0
- package/test/fetchRSS.test.mjs +85 -0
- package/test/fetchStatementdog.test.mjs +69 -0
- package/test/fetchTaifex.test.mjs +99 -0
- package/test/fetchTpex3insti.test.mjs +90 -0
- package/test/fetchTpexMargin.test.mjs +94 -0
- package/test/fetchTpexStock.test.mjs +68 -0
- package/test/fetchTwDataHoliday.test.mjs +100 -0
- package/test/fetchTwseMargin.test.mjs +81 -0
- package/test/fetchTwseStock.test.mjs +80 -0
- package/test/fetchTwseT86.test.mjs +76 -0
- package/test/fetchWithRetry.test.mjs +174 -0
- package/test/htmlToMarkdown.test.mjs +71 -0
- package/test/isYmd.test.mjs +56 -0
- package/test/parseIntComma.test.mjs +38 -0
- package/test/safeFilename.test.mjs +35 -0
- package/test/toDatetimeUTC8.test.mjs +65 -0
- package/test/toRocDate.test.mjs +38 -0
- package/test/tools/encodeBig5.mjs +80 -0
- package/test/tools/fixtures.mjs +539 -0
- package/test/tools/hasChrome.mjs +39 -0
- package/test/tools/serverForTest.mjs +397 -0
- package/toolg/addVersion.mjs +4 -0
- package/toolg/cleanFolder.mjs +5 -0
- package/toolg/gDistRollup.mjs +49 -0
- package/toolg/modifyReadme.mjs +4 -0
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import get from 'lodash-es/get.js'
|
|
2
|
+
import isbol from 'wsemi/src/isbol.mjs'
|
|
3
|
+
import isestr from 'wsemi/src/isestr.mjs'
|
|
4
|
+
import isearr from 'wsemi/src/isearr.mjs'
|
|
5
|
+
import isobj from 'wsemi/src/isobj.mjs'
|
|
6
|
+
import ispint from 'wsemi/src/ispint.mjs'
|
|
7
|
+
import isp0int from 'wsemi/src/isp0int.mjs'
|
|
8
|
+
import cint from 'wsemi/src/cint.mjs'
|
|
9
|
+
import delay from 'wsemi/src/delay.mjs'
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
//預設User-Agent, 各資料來源網站多會擋無User-Agent之請求
|
|
13
|
+
let DEFAULT_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
//預設值
|
|
17
|
+
let DEFAULT_TIMEOUT_MS = 30000
|
|
18
|
+
let DEFAULT_MAX_RETRIES = 10
|
|
19
|
+
let DEFAULT_BASE_DELAY_MS = 5000
|
|
20
|
+
let DEFAULT_MAX_DELAY_MS = 30000
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
//視為暫時性故障之網路層錯誤碼, node內建fetch(undici)會把底層錯誤掛於err.cause
|
|
24
|
+
let NETWORK_ERROR_CODES = ['ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED', 'ECONNABORTED', 'EAI_AGAIN', 'EPIPE']
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* 判斷錯誤是否值得重試
|
|
29
|
+
*
|
|
30
|
+
* 有statusCode(即HTTP層錯誤)時, 5xx與429視為暫時性, 其餘由retryStatus指定;
|
|
31
|
+
* 無statusCode(即網路層錯誤)時, 由錯誤碼、AbortError與undici暫時性錯誤判斷
|
|
32
|
+
*
|
|
33
|
+
* @param {Error} err 輸入錯誤物件
|
|
34
|
+
* @param {Array} retryStatus 輸入額外視為可重試之HTTP狀態碼陣列
|
|
35
|
+
* @returns {Boolean} 回傳是否可重試布林值
|
|
36
|
+
*/
|
|
37
|
+
function isRetryableError(err, retryStatus) {
|
|
38
|
+
|
|
39
|
+
//statusCode, 由fetchOnce於HTTP非2xx時掛入
|
|
40
|
+
let statusCode = get(err, 'statusCode', null)
|
|
41
|
+
if (ispint(statusCode)) {
|
|
42
|
+
return statusCode >= 500 || statusCode === 429 || retryStatus.includes(statusCode)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
//code, undici把底層錯誤掛在err.cause
|
|
46
|
+
let code = get(err, 'cause.code', null) || get(err, 'code', null)
|
|
47
|
+
if (isestr(code) && NETWORK_ERROR_CODES.includes(code)) {
|
|
48
|
+
return true
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
//AbortController觸發之逾時
|
|
52
|
+
let name = get(err, 'name', '')
|
|
53
|
+
if (name === 'AbortError' || name === 'TimeoutError') {
|
|
54
|
+
return true
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
//undici之socket與逾時類暫時性錯誤
|
|
58
|
+
let ms = String(get(err, 'cause.code', '') || get(err, 'message', ''))
|
|
59
|
+
if (/UND_ERR_(SOCKET|HEADERS_TIMEOUT|BODY_TIMEOUT|CONNECT_TIMEOUT)/.test(ms)) {
|
|
60
|
+
return true
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return false
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* 單次抓取, HTTP非2xx時拋出帶statusCode之錯誤
|
|
69
|
+
*
|
|
70
|
+
* @param {String} url 輸入待抓取網址字串
|
|
71
|
+
* @param {Object} cfg 輸入已正規化之設定物件
|
|
72
|
+
* @returns {Promise} 回傳Promise,resolve回傳解析後資料
|
|
73
|
+
*/
|
|
74
|
+
async function fetchOnce(url, cfg) {
|
|
75
|
+
|
|
76
|
+
//ac, 以AbortController實作逾時, 避免socket卡住永不回應
|
|
77
|
+
let ac = new AbortController()
|
|
78
|
+
let idTimer = setTimeout(() => {
|
|
79
|
+
ac.abort()
|
|
80
|
+
}, cfg.timeout)
|
|
81
|
+
|
|
82
|
+
try {
|
|
83
|
+
|
|
84
|
+
//opt
|
|
85
|
+
let opt = {
|
|
86
|
+
method: cfg.method,
|
|
87
|
+
headers: cfg.headers,
|
|
88
|
+
signal: ac.signal,
|
|
89
|
+
}
|
|
90
|
+
if (cfg.body !== null) {
|
|
91
|
+
opt.body = cfg.body
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
//fetch
|
|
95
|
+
let res = await fetch(url, opt)
|
|
96
|
+
|
|
97
|
+
//check, node內建fetch不會因HTTP非2xx而拋錯, 需自行檢核
|
|
98
|
+
if (!res.ok) {
|
|
99
|
+
let err = new Error(`HTTP ${res.status} ${res.statusText}`.trim())
|
|
100
|
+
err.statusCode = res.status
|
|
101
|
+
err.url = url
|
|
102
|
+
throw err
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
//arrayBuffer
|
|
106
|
+
if (cfg.responseType === 'arrayBuffer') {
|
|
107
|
+
return await res.arrayBuffer()
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
//text
|
|
111
|
+
if (cfg.responseType === 'text') {
|
|
112
|
+
if (cfg.encoding !== '') {
|
|
113
|
+
//指定編碼(如big5)時需自行解碼, res.text()一律以utf-8解碼
|
|
114
|
+
let ab = await res.arrayBuffer()
|
|
115
|
+
return new TextDecoder(cfg.encoding).decode(ab)
|
|
116
|
+
}
|
|
117
|
+
return await res.text()
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
//json, 先取text再自行parse, 令維護頁面(HTTP 200+HTML)能產生可讀訊息而非裸SyntaxError
|
|
121
|
+
let text = await res.text()
|
|
122
|
+
try {
|
|
123
|
+
return JSON.parse(text)
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
throw new Error(`回傳非JSON格式(可能為維護頁面或網路中介): ${text.slice(0, 100)}`)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
}
|
|
130
|
+
finally {
|
|
131
|
+
clearTimeout(idTimer)
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* 以node內建fetch抓取資料並自動重試
|
|
138
|
+
*
|
|
139
|
+
* 特點:
|
|
140
|
+
* HTTP 5xx與429及網路層錯誤(含逾時)會自動重試,採線性退避baseDelayMs*attempt並以maxDelayMs為上限;
|
|
141
|
+
* HTTP 4xx(429除外)預設不重試,可由opt.retryStatus指定額外可重試狀態碼(如403、404);
|
|
142
|
+
* responseType為'json'時先取原始文字再自行JSON.parse,令維護頁面能產生可讀錯誤訊息
|
|
143
|
+
*
|
|
144
|
+
* @param {String} url 輸入待抓取網址字串
|
|
145
|
+
* @param {Object} [opt={}] 輸入設定物件,預設{}
|
|
146
|
+
* @param {String} [opt.method='GET'] 輸入HTTP方法字串,預設'GET'
|
|
147
|
+
* @param {Object} [opt.headers={}] 輸入額外HTTP標頭物件,未指定User-Agent時自動補預設值,預設{}
|
|
148
|
+
* @param {String} [opt.body=null] 輸入請求主體字串,預設null代表無主體
|
|
149
|
+
* @param {String} [opt.responseType='json'] 輸入回應解析方式字串,可為'json'、'text'、'arrayBuffer',預設'json'
|
|
150
|
+
* @param {String} [opt.encoding=''] 輸入responseType為'text'時之解碼字串,例如'big5',預設''代表utf-8
|
|
151
|
+
* @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數,預設30000
|
|
152
|
+
* @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設10
|
|
153
|
+
* @param {Integer} [opt.baseDelayMs=5000] 輸入線性退避之基礎毫秒整數,預設5000
|
|
154
|
+
* @param {Integer} [opt.maxDelayMs=30000] 輸入線性退避之上限毫秒整數,預設30000
|
|
155
|
+
* @param {Array} [opt.retryStatus=[]] 輸入額外視為可重試之HTTP狀態碼陣列,例如[403],預設[]
|
|
156
|
+
* @param {String} [opt.label=''] 輸入重試訊息前綴字串,預設''
|
|
157
|
+
* @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值,預設true
|
|
158
|
+
* @returns {Promise} 回傳Promise,resolve回傳解析後資料,responseType為'json'時回物件、'text'時回字串、'arrayBuffer'時回ArrayBuffer,重試耗盡或遇不可重試錯誤時reject回傳錯誤物件
|
|
159
|
+
* @example
|
|
160
|
+
*
|
|
161
|
+
* import fetchWithRetry from './src/fetchWithRetry.mjs'
|
|
162
|
+
*
|
|
163
|
+
* let test = async () => {
|
|
164
|
+
*
|
|
165
|
+
* let r = await fetchWithRetry('https://hacker-news.firebaseio.com/v0/item/1.json')
|
|
166
|
+
* console.log(r.type, r.by)
|
|
167
|
+
* // => 'story' 'pg'
|
|
168
|
+
*
|
|
169
|
+
* }
|
|
170
|
+
* await test()
|
|
171
|
+
* .catch((err) => {
|
|
172
|
+
* console.log(err)
|
|
173
|
+
* })
|
|
174
|
+
*
|
|
175
|
+
*/
|
|
176
|
+
async function fetchWithRetry(url, opt = {}) {
|
|
177
|
+
|
|
178
|
+
//check
|
|
179
|
+
if (!isestr(url)) {
|
|
180
|
+
throw new Error(`url須為有效字串, 得到: ${url}`)
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
//method
|
|
184
|
+
let method = get(opt, 'method')
|
|
185
|
+
if (!isestr(method)) {
|
|
186
|
+
method = 'GET'
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
//headers
|
|
190
|
+
let headers = get(opt, 'headers')
|
|
191
|
+
if (!isobj(headers)) {
|
|
192
|
+
headers = {}
|
|
193
|
+
}
|
|
194
|
+
if (!isestr(get(headers, 'User-Agent'))) {
|
|
195
|
+
headers = { 'User-Agent': DEFAULT_UA, ...headers }
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
//body
|
|
199
|
+
let body = get(opt, 'body', null)
|
|
200
|
+
if (!isestr(body)) {
|
|
201
|
+
body = null
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
//responseType
|
|
205
|
+
let responseType = get(opt, 'responseType')
|
|
206
|
+
if (responseType !== 'text' && responseType !== 'arrayBuffer') {
|
|
207
|
+
responseType = 'json'
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
//encoding
|
|
211
|
+
let encoding = get(opt, 'encoding')
|
|
212
|
+
if (!isestr(encoding)) {
|
|
213
|
+
encoding = ''
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
//timeout
|
|
217
|
+
let timeout = get(opt, 'timeout')
|
|
218
|
+
if (!ispint(timeout)) {
|
|
219
|
+
timeout = DEFAULT_TIMEOUT_MS
|
|
220
|
+
}
|
|
221
|
+
else {
|
|
222
|
+
timeout = cint(timeout)
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
//maxRetries
|
|
226
|
+
let maxRetries = get(opt, 'maxRetries')
|
|
227
|
+
if (!isp0int(maxRetries)) {
|
|
228
|
+
maxRetries = DEFAULT_MAX_RETRIES
|
|
229
|
+
}
|
|
230
|
+
else {
|
|
231
|
+
maxRetries = cint(maxRetries)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
//baseDelayMs
|
|
235
|
+
let baseDelayMs = get(opt, 'baseDelayMs')
|
|
236
|
+
if (!ispint(baseDelayMs)) {
|
|
237
|
+
baseDelayMs = DEFAULT_BASE_DELAY_MS
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
baseDelayMs = cint(baseDelayMs)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
//maxDelayMs
|
|
244
|
+
let maxDelayMs = get(opt, 'maxDelayMs')
|
|
245
|
+
if (!ispint(maxDelayMs)) {
|
|
246
|
+
maxDelayMs = DEFAULT_MAX_DELAY_MS
|
|
247
|
+
}
|
|
248
|
+
else {
|
|
249
|
+
maxDelayMs = cint(maxDelayMs)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
//retryStatus
|
|
253
|
+
let retryStatus = get(opt, 'retryStatus')
|
|
254
|
+
if (!isearr(retryStatus)) {
|
|
255
|
+
retryStatus = []
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
//label
|
|
259
|
+
let label = get(opt, 'label')
|
|
260
|
+
if (!isestr(label)) {
|
|
261
|
+
label = 'fetchWithRetry'
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
//showLog
|
|
265
|
+
let showLog = get(opt, 'showLog')
|
|
266
|
+
if (!isbol(showLog)) {
|
|
267
|
+
showLog = true
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
//cfg
|
|
271
|
+
let cfg = { method, headers, body, responseType, encoding, timeout }
|
|
272
|
+
|
|
273
|
+
let lastError = null
|
|
274
|
+
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
|
|
275
|
+
|
|
276
|
+
try {
|
|
277
|
+
return await fetchOnce(url, cfg)
|
|
278
|
+
}
|
|
279
|
+
catch (err) {
|
|
280
|
+
lastError = err
|
|
281
|
+
|
|
282
|
+
//check, 不可重試或已用盡次數則直接拋出
|
|
283
|
+
let attemptsLeft = maxRetries + 1 - attempt
|
|
284
|
+
if (!isRetryableError(err, retryStatus) || attemptsLeft <= 0) {
|
|
285
|
+
throw err
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
//線性退避
|
|
289
|
+
let ms = Math.min(baseDelayMs * attempt, maxDelayMs)
|
|
290
|
+
if (showLog) {
|
|
291
|
+
console.warn(`[${label}][Retry ${attempt}/${maxRetries}] ${err.message} — 等待 ${ms / 1000}s 後重試...`)
|
|
292
|
+
}
|
|
293
|
+
await delay(ms)
|
|
294
|
+
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
throw lastError
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
export { DEFAULT_UA, isRetryableError }
|
|
304
|
+
export default fetchWithRetry
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import get from 'lodash-es/get.js'
|
|
2
|
+
import isbol from 'wsemi/src/isbol.mjs'
|
|
3
|
+
import isobj from 'wsemi/src/isobj.mjs'
|
|
4
|
+
import isestr from 'wsemi/src/isestr.mjs'
|
|
5
|
+
import isearr from 'wsemi/src/isearr.mjs'
|
|
6
|
+
import ispint from 'wsemi/src/ispint.mjs'
|
|
7
|
+
import isp0int from 'wsemi/src/isp0int.mjs'
|
|
8
|
+
import cint from 'wsemi/src/cint.mjs'
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
//各抓取函數共用之預設值
|
|
12
|
+
let DEFAULT_TIMEOUT_MS = 30000
|
|
13
|
+
let DEFAULT_MAX_RETRIES = 10
|
|
14
|
+
let DEFAULT_BASE_DELAY_MS = 5000
|
|
15
|
+
let DEFAULT_MAX_DELAY_MS = 30000
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* 取得正整數設定值
|
|
20
|
+
*
|
|
21
|
+
* @param {Object} opt 輸入呼叫端設定物件
|
|
22
|
+
* @param {Object} dflt 輸入各抓取函數之預設值物件
|
|
23
|
+
* @param {String} key 輸入設定鍵名字串
|
|
24
|
+
* @param {Integer} fallback 輸入無設定時之預設整數
|
|
25
|
+
* @returns {Integer} 回傳正整數
|
|
26
|
+
*/
|
|
27
|
+
function pickPint(opt, dflt, key, fallback) {
|
|
28
|
+
|
|
29
|
+
let v = get(opt, key)
|
|
30
|
+
if (ispint(v)) {
|
|
31
|
+
return cint(v)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
v = get(dflt, key)
|
|
35
|
+
if (ispint(v)) {
|
|
36
|
+
return cint(v)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return fallback
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* 正規化抓取設定
|
|
45
|
+
*
|
|
46
|
+
* 各抓取函數之timeout、maxRetries、baseDelayMs、maxDelayMs與showLog皆可由呼叫端覆寫,
|
|
47
|
+
* 未給者取該函數之預設值, 再未給者取全域預設值;
|
|
48
|
+
* responseType、encoding、headers、retryStatus與label屬各函數固有特性, 僅由dflt指定不開放覆寫
|
|
49
|
+
*
|
|
50
|
+
* @param {Object} [opt={}] 輸入呼叫端設定物件,預設{}
|
|
51
|
+
* @param {Object} [dflt={}] 輸入各抓取函數之預設值物件,預設{}
|
|
52
|
+
* @returns {Object} 回傳可直接傳入fetchWithRetry之設定物件,另含showLog供呼叫端判斷是否輸出過程訊息
|
|
53
|
+
* @example
|
|
54
|
+
*
|
|
55
|
+
* import getOptFetch from './src/getOptFetch.mjs'
|
|
56
|
+
*
|
|
57
|
+
* console.log(getOptFetch({ maxRetries: 2 }, { timeout: 15000, label: 'abc' }))
|
|
58
|
+
* // => { timeout: 15000, maxRetries: 2, baseDelayMs: 5000, maxDelayMs: 30000, showLog: true, label: 'abc' }
|
|
59
|
+
*
|
|
60
|
+
*/
|
|
61
|
+
function getOptFetch(opt = {}, dflt = {}) {
|
|
62
|
+
|
|
63
|
+
//maxRetries, 可為0故另行處理
|
|
64
|
+
let maxRetries = get(opt, 'maxRetries')
|
|
65
|
+
if (!isp0int(maxRetries)) {
|
|
66
|
+
maxRetries = get(dflt, 'maxRetries')
|
|
67
|
+
}
|
|
68
|
+
if (!isp0int(maxRetries)) {
|
|
69
|
+
maxRetries = DEFAULT_MAX_RETRIES
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
maxRetries = cint(maxRetries)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
//showLog
|
|
76
|
+
let showLog = get(opt, 'showLog')
|
|
77
|
+
if (!isbol(showLog)) {
|
|
78
|
+
showLog = get(dflt, 'showLog')
|
|
79
|
+
}
|
|
80
|
+
if (!isbol(showLog)) {
|
|
81
|
+
showLog = true
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
let r = {
|
|
85
|
+
timeout: pickPint(opt, dflt, 'timeout', DEFAULT_TIMEOUT_MS),
|
|
86
|
+
maxRetries,
|
|
87
|
+
baseDelayMs: pickPint(opt, dflt, 'baseDelayMs', DEFAULT_BASE_DELAY_MS),
|
|
88
|
+
maxDelayMs: pickPint(opt, dflt, 'maxDelayMs', DEFAULT_MAX_DELAY_MS),
|
|
89
|
+
showLog,
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
//各函數固有特性, 不開放呼叫端覆寫
|
|
93
|
+
if (isestr(get(dflt, 'responseType'))) {
|
|
94
|
+
r.responseType = dflt.responseType
|
|
95
|
+
}
|
|
96
|
+
if (isestr(get(dflt, 'encoding'))) {
|
|
97
|
+
r.encoding = dflt.encoding
|
|
98
|
+
}
|
|
99
|
+
if (isobj(get(dflt, 'headers'))) {
|
|
100
|
+
r.headers = dflt.headers
|
|
101
|
+
}
|
|
102
|
+
if (isearr(get(dflt, 'retryStatus'))) {
|
|
103
|
+
r.retryStatus = dflt.retryStatus
|
|
104
|
+
}
|
|
105
|
+
if (isestr(get(dflt, 'label'))) {
|
|
106
|
+
r.label = dflt.label
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return r
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
export default getOptFetch
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import get from 'lodash-es/get.js'
|
|
2
|
+
import isbol from 'wsemi/src/isbol.mjs'
|
|
3
|
+
import isstr from 'wsemi/src/isstr.mjs'
|
|
4
|
+
import decodeEntities from './decodeEntities.mjs'
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* HTML轉Markdown
|
|
9
|
+
*
|
|
10
|
+
* 採正則式逐項轉換, 不引入DOM解析器, 適用於已切出之正文區塊;
|
|
11
|
+
* 先移除script、style與註解, 再轉換標題、粗體、斜體、換行、連結與段落, 最後清除殘留標籤並收斂空行
|
|
12
|
+
*
|
|
13
|
+
* @param {String} html 輸入HTML字串
|
|
14
|
+
* @param {Object} [opt={}] 輸入設定物件,預設{}
|
|
15
|
+
* @param {Boolean} [opt.image=false] 輸入是否將img標籤轉為Markdown圖片語法布林值,預設false代表直接移除
|
|
16
|
+
* @returns {String} 回傳Markdown字串,輸入非字串時回傳''
|
|
17
|
+
* @example
|
|
18
|
+
*
|
|
19
|
+
* import htmlToMarkdown from './src/htmlToMarkdown.mjs'
|
|
20
|
+
*
|
|
21
|
+
* console.log(htmlToMarkdown('<p>a<b>b</b></p><p>c</p>'))
|
|
22
|
+
* // => 'a**b**\n\nc'
|
|
23
|
+
*
|
|
24
|
+
* console.log(htmlToMarkdown('<p><img src="a.png" /></p>', { image: true }))
|
|
25
|
+
* // => ''
|
|
26
|
+
*
|
|
27
|
+
*/
|
|
28
|
+
function htmlToMarkdown(html, opt = {}) {
|
|
29
|
+
|
|
30
|
+
//check
|
|
31
|
+
if (!isstr(html)) {
|
|
32
|
+
return ''
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
//image
|
|
36
|
+
let image = get(opt, 'image')
|
|
37
|
+
if (!isbol(image)) {
|
|
38
|
+
image = false
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let s = html
|
|
42
|
+
|
|
43
|
+
//移除script、style與註解
|
|
44
|
+
s = s.replace(/<script[\s\S]*?<\/script>/gi, '')
|
|
45
|
+
s = s.replace(/<style[\s\S]*?<\/style>/gi, '')
|
|
46
|
+
s = s.replace(/<!--[\s\S]*?-->/g, '')
|
|
47
|
+
|
|
48
|
+
//img
|
|
49
|
+
if (image) {
|
|
50
|
+
s = s.replace(/<img[^>]*src="([^"]+)"[^>]*\/?>/gi, (m, url) => ``)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
//h1-h6轉粗體段落
|
|
54
|
+
s = s.replace(/<h[1-6][^>]*>([\s\S]*?)<\/h[1-6]>/gi, '\n\n**$1**\n\n')
|
|
55
|
+
|
|
56
|
+
//strong與b
|
|
57
|
+
//標籤名後須緊接'>'或空白, 否則<br>會被b規則吃掉而變成'**'(在br規則之前)
|
|
58
|
+
s = s.replace(/<\/?(?:strong|b)(?:\s[^>]*)?>/gi, '**')
|
|
59
|
+
|
|
60
|
+
//em與i
|
|
61
|
+
//同上, 標籤名後須緊接'>'或空白, 否則<img>會被i規則吃掉而變成'*'
|
|
62
|
+
s = s.replace(/<\/?(?:em|i)(?:\s[^>]*)?>/gi, '*')
|
|
63
|
+
|
|
64
|
+
//br
|
|
65
|
+
s = s.replace(/<br\s*\/?>/gi, '\n')
|
|
66
|
+
|
|
67
|
+
//a
|
|
68
|
+
s = s.replace(/<a\s+[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi, (m, url, text) => {
|
|
69
|
+
let t = text.replace(/<[^>]+>/g, '').trim()
|
|
70
|
+
return t ? `[${t}](${url})` : ''
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
//p
|
|
74
|
+
s = s.replace(/<p(?:\s[^>]*)?>/gi, '')
|
|
75
|
+
s = s.replace(/<\/p>/gi, '\n\n')
|
|
76
|
+
|
|
77
|
+
//雜項容器標籤
|
|
78
|
+
s = s.replace(/<\/?(?:div|span|section|article|font|u)(?:\s[^>]*)?>/gi, '')
|
|
79
|
+
|
|
80
|
+
//其餘殘留標籤
|
|
81
|
+
s = s.replace(/<[^>]+>/g, '')
|
|
82
|
+
|
|
83
|
+
//實體
|
|
84
|
+
s = decodeEntities(s)
|
|
85
|
+
|
|
86
|
+
//行內空白標準化
|
|
87
|
+
s = s.split('\n').map((l) => l.replace(/[ \t\u00A0]+/g, ' ').trimEnd()).join('\n')
|
|
88
|
+
|
|
89
|
+
//多重空行收斂
|
|
90
|
+
s = s.replace(/\n{3,}/g, '\n\n').replace(/\*\*\s*\*\*/g, '')
|
|
91
|
+
|
|
92
|
+
return s.trim()
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
export default htmlToMarkdown
|
package/src/isYmd.mjs
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import isestr from 'wsemi/src/isestr.mjs'
|
|
2
|
+
import cint from 'wsemi/src/cint.mjs'
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* 判斷是否為合法之YYYYMMDD日期字串
|
|
7
|
+
*
|
|
8
|
+
* 除檢核8碼數字格式外, 另檢核日曆合法性, 例如20260230雖為8碼數字但該日不存在
|
|
9
|
+
*
|
|
10
|
+
* @param {String} dateStr 輸入日期字串
|
|
11
|
+
* @returns {Boolean} 回傳是否為合法YYYYMMDD日期字串之布林值
|
|
12
|
+
* @example
|
|
13
|
+
*
|
|
14
|
+
* import isYmd from './src/isYmd.mjs'
|
|
15
|
+
*
|
|
16
|
+
* console.log(isYmd('20260807'), isYmd('20260230'), isYmd('2026-08-07'), isYmd(20260807))
|
|
17
|
+
* // => true false false false
|
|
18
|
+
*
|
|
19
|
+
*/
|
|
20
|
+
function isYmd(dateStr) {
|
|
21
|
+
|
|
22
|
+
//check
|
|
23
|
+
if (!isestr(dateStr)) {
|
|
24
|
+
return false
|
|
25
|
+
}
|
|
26
|
+
if (!/^\d{8}$/.test(dateStr)) {
|
|
27
|
+
return false
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
//y, m, d
|
|
31
|
+
let y = cint(dateStr.slice(0, 4))
|
|
32
|
+
let m = cint(dateStr.slice(4, 6))
|
|
33
|
+
let d = cint(dateStr.slice(6, 8))
|
|
34
|
+
|
|
35
|
+
//dt, 由Date回推以檢核日曆合法性
|
|
36
|
+
let dt = new Date(y, m - 1, d)
|
|
37
|
+
|
|
38
|
+
return dt.getFullYear() === y && dt.getMonth() === m - 1 && dt.getDate() === d
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
export default isYmd
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 含千分位逗號之字串轉整數
|
|
3
|
+
*
|
|
4
|
+
* 證交所與櫃買中心之數值欄位皆以逗號分隔千分位, 且缺值以'-'或空字串表示;
|
|
5
|
+
* 非字串者原樣回傳(該類API之數值欄位偶為number), 無法轉為整數者回傳0
|
|
6
|
+
*
|
|
7
|
+
* @param {String} str 輸入含千分位逗號之數值字串
|
|
8
|
+
* @returns {Number} 回傳整數,非字串輸入時原樣回傳,無法轉換時回傳0
|
|
9
|
+
* @example
|
|
10
|
+
*
|
|
11
|
+
* import parseIntComma from './src/parseIntComma.mjs'
|
|
12
|
+
*
|
|
13
|
+
* console.log(parseIntComma('1,234,567'), parseIntComma('-'), parseIntComma(''), parseIntComma(12))
|
|
14
|
+
* // => 1234567 0 0 12
|
|
15
|
+
*
|
|
16
|
+
*/
|
|
17
|
+
function parseIntComma(str) {
|
|
18
|
+
|
|
19
|
+
//check, 該類API之數值欄位偶為number, 原樣回傳
|
|
20
|
+
if (typeof str !== 'string') {
|
|
21
|
+
return str
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
return parseInt(str.replace(/,/g, ''), 10) || 0
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
export default parseIntComma
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import isstr from 'wsemi/src/isstr.mjs'
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* 字串轉安全檔名
|
|
6
|
+
*
|
|
7
|
+
* 將Windows與POSIX皆不允許之字元(\/:*?"<>|)替換為底線, 並截斷至200字元以避開檔名長度上限
|
|
8
|
+
*
|
|
9
|
+
* @param {String} name 輸入檔名字串
|
|
10
|
+
* @returns {String} 回傳安全檔名字串
|
|
11
|
+
* @example
|
|
12
|
+
*
|
|
13
|
+
* import safeFilename from './src/safeFilename.mjs'
|
|
14
|
+
*
|
|
15
|
+
* console.log(safeFilename('a/b:c*d?e'))
|
|
16
|
+
* // => 'a_b_c_d_e'
|
|
17
|
+
*
|
|
18
|
+
*/
|
|
19
|
+
function safeFilename(name) {
|
|
20
|
+
|
|
21
|
+
//check
|
|
22
|
+
if (!isstr(name)) {
|
|
23
|
+
name = String(name === null || name === undefined ? '' : name)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
return name.replace(/[\\/:*?"<>|]/g, '_').slice(0, 200)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
export default safeFilename
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import ot from 'dayjs'
|
|
2
|
+
import utc from 'dayjs/plugin/utc.js'
|
|
3
|
+
import isnum from 'wsemi/src/isnum.mjs'
|
|
4
|
+
import isestr from 'wsemi/src/isestr.mjs'
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
//utc外掛, dayjs核心之format()一律以執行環境時區輸出, 需掛utc外掛才能以utcOffset指定固定偏移
|
|
8
|
+
ot.extend(utc)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
//台灣時區偏移小時數, 台灣自1980年起無日光節約時間, 故固定為UTC+8
|
|
12
|
+
let TZ_OFFSET_HOUR = 8
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
//輸出格式
|
|
16
|
+
let FORMAT = 'YYYY-MM-DD HH:mm:ss'
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* 時間轉UTC+8之日期時間字串
|
|
21
|
+
*
|
|
22
|
+
* 可接受ISO時間字串、RFC822時間字串、unix秒數(非毫秒)或Date物件, 一律輸出台灣時間'YYYY-MM-DD HH:mm:ss';
|
|
23
|
+
* 以dayjs之utc外掛指定固定偏移+8, 不受執行環境時區影響(在UTC伺服器上結果相同)
|
|
24
|
+
*
|
|
25
|
+
* @param {String|Number|Date} value 輸入ISO或RFC822時間字串、unix秒數或Date物件
|
|
26
|
+
* @returns {String} 回傳UTC+8之日期時間字串,格式為'YYYY-MM-DD HH:mm:ss',輸入為空或無法解析時回傳''
|
|
27
|
+
* @example
|
|
28
|
+
*
|
|
29
|
+
* import toDatetimeUTC8 from './src/toDatetimeUTC8.mjs'
|
|
30
|
+
*
|
|
31
|
+
* console.log(toDatetimeUTC8('2026-08-07T01:02:03Z'))
|
|
32
|
+
* // => '2026-08-07 09:02:03'
|
|
33
|
+
*
|
|
34
|
+
* console.log(toDatetimeUTC8(1786000000))
|
|
35
|
+
* // => '2026-08-06 15:06:40'
|
|
36
|
+
*
|
|
37
|
+
* console.log(toDatetimeUTC8('abc'), toDatetimeUTC8(''), toDatetimeUTC8(null))
|
|
38
|
+
* // => '' '' ''
|
|
39
|
+
*
|
|
40
|
+
*/
|
|
41
|
+
function toDatetimeUTC8(value) {
|
|
42
|
+
|
|
43
|
+
let dt = null
|
|
44
|
+
|
|
45
|
+
//數值一律視為unix秒數, dayjs之ot(number)為毫秒故須改用ot.unix
|
|
46
|
+
if (isnum(value)) {
|
|
47
|
+
if (value === 0) {
|
|
48
|
+
return ''
|
|
49
|
+
}
|
|
50
|
+
dt = ot.unix(value)
|
|
51
|
+
}
|
|
52
|
+
else if (value instanceof Date) {
|
|
53
|
+
dt = ot(value)
|
|
54
|
+
}
|
|
55
|
+
else if (isestr(value)) {
|
|
56
|
+
dt = ot(value)
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
return ''
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
//check
|
|
63
|
+
if (!dt.isValid()) {
|
|
64
|
+
return ''
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return dt.utcOffset(TZ_OFFSET_HOUR).format(FORMAT)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
export default toDatetimeUTC8
|