w-dwdata-hub 1.0.1 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +16 -0
  2. package/dist/w-dwdata-hub.umd.js +2 -2
  3. package/dist/w-dwdata-hub.umd.js.map +1 -1
  4. package/docs/WDwdataHub.mjs.html +2 -2
  5. package/docs/decodeEntities.mjs.html +2 -2
  6. package/docs/extractDivContent.mjs.html +2 -2
  7. package/docs/fetchAiNewsAggregator.mjs.html +2 -2
  8. package/docs/fetchAisixiang.mjs.html +2 -2
  9. package/docs/fetchCnyes.mjs.html +2 -2
  10. package/docs/fetchGuancha.mjs.html +3 -7
  11. package/docs/fetchHackerNews.mjs.html +2 -2
  12. package/docs/fetchMoneydj.mjs.html +2 -2
  13. package/docs/fetchMops.mjs.html +2 -2
  14. package/docs/fetchRSS.mjs.html +106 -8
  15. package/docs/fetchStatementdog.mjs.html +2 -2
  16. package/docs/fetchTaifex.mjs.html +2 -2
  17. package/docs/fetchTpex3insti.mjs.html +2 -2
  18. package/docs/fetchTpexMargin.mjs.html +2 -2
  19. package/docs/fetchTpexStock.mjs.html +2 -2
  20. package/docs/fetchTwDataHoliday.mjs.html +2 -2
  21. package/docs/fetchTwseMargin.mjs.html +2 -2
  22. package/docs/fetchTwseStock.mjs.html +2 -2
  23. package/docs/fetchTwseT86.mjs.html +2 -2
  24. package/docs/fetchWithRetry.mjs.html +2 -2
  25. package/docs/getOptFetch.mjs.html +2 -2
  26. package/docs/global.html +807 -151
  27. package/docs/htmlToMarkdown.mjs.html +2 -2
  28. package/docs/index.html +2 -2
  29. package/docs/isYmd.mjs.html +2 -2
  30. package/docs/parseIntComma.mjs.html +2 -2
  31. package/docs/safeFilename.mjs.html +2 -2
  32. package/docs/stripHtml.mjs.html +130 -0
  33. package/docs/toDatetimeUTC8.mjs.html +26 -9
  34. package/docs/toRocDate.mjs.html +2 -2
  35. package/g.mjs +5 -0
  36. package/package.json +1 -1
  37. package/src/fetchGuancha.mjs +1 -5
  38. package/src/fetchRSS.mjs +104 -6
  39. package/src/stripHtml.mjs +58 -0
  40. package/src/toDatetimeUTC8.mjs +24 -7
  41. package/test/WDwdataHub.test.mjs +1 -1
  42. package/test/fetchRSS.test.mjs +78 -0
  43. package/test/stripHtml.test.mjs +58 -0
  44. package/test/toDatetimeUTC8.test.mjs +32 -0
  45. package/test/tools/fixtures.mjs +43 -0
  46. package/test/tools/serverForTest.mjs +12 -0
  47. package/toolg/gDistRollup.mjs +2 -2
package/src/fetchRSS.mjs CHANGED
@@ -1,10 +1,13 @@
1
1
  import Parser from 'rss-parser'
2
+ import fetchWebByCurl from 'w-fetch-web/src/fetchWebByCurl.mjs'
2
3
  import get from 'lodash-es/get.js'
4
+ import isbol from 'wsemi/src/isbol.mjs'
3
5
  import isestr from 'wsemi/src/isestr.mjs'
4
6
  import isearr from 'wsemi/src/isearr.mjs'
5
7
  import fetchWithRetry from './fetchWithRetry.mjs'
6
8
  import getOptFetch from './getOptFetch.mjs'
7
9
  import toDatetimeUTC8 from './toDatetimeUTC8.mjs'
10
+ import stripHtml from './stripHtml.mjs'
8
11
 
9
12
 
10
13
  //本函數之預設抓取設定, 因YouTube等站台之RSS會回傳暫時性404故將404納入重試範圍
@@ -19,20 +22,84 @@ let DFLT = {
19
22
  }
20
23
 
21
24
 
25
+ /**
26
+ * 以curl取回XML
27
+ *
28
+ * 委派w-fetch-web之fetchWebByCurl, 失敗時拋出帶statusCode之錯誤(與fetchWithRetry之錯誤形狀一致, 供上層統一處理)。
29
+ * 注意: fetchWebByCurl之參數名為timeoutMs(非timeout), 且其重試退避為內建之3至15秒線性,
30
+ * 故opt之baseDelayMs、maxDelayMs與retryStatus於curl模式不生效
31
+ *
32
+ * @param {String} rssUrl 輸入RSS網址字串
33
+ * @param {Object} cfg 輸入已正規化之抓取設定物件
34
+ * @returns {Promise} 回傳Promise,resolve回傳XML字串
35
+ */
36
+ async function getXmlByCurl(rssUrl, cfg) {
37
+
38
+ let r = await fetchWebByCurl(rssUrl, {
39
+ timeoutMs: cfg.timeout,
40
+ maxRetries: cfg.maxRetries,
41
+ })
42
+ if (r.status !== 'success') {
43
+ let err = new Error(r.message || 'curl fetch failed')
44
+ err.statusCode = r.httpCode
45
+ err.reason = r.reason
46
+ err.url = rssUrl
47
+ throw err
48
+ }
49
+
50
+ return r.html
51
+ }
52
+
53
+
54
+ /**
55
+ * 依method分派取回XML
56
+ *
57
+ * @param {String} rssUrl 輸入RSS網址字串
58
+ * @param {String} method 輸入抓取方式字串,可為'fetch'、'curl'、'auto'
59
+ * @param {Object} cfg 輸入已正規化之抓取設定物件
60
+ * @returns {Promise} 回傳Promise,resolve回傳XML字串
61
+ */
62
+ async function getXml(rssUrl, method, cfg) {
63
+
64
+ if (method === 'curl') {
65
+ return await getXmlByCurl(rssUrl, cfg)
66
+ }
67
+
68
+ if (method === 'auto') {
69
+ try {
70
+ return await getXmlByCurl(rssUrl, cfg)
71
+ }
72
+ catch (err) {
73
+ //curl不可用(系統無curl)或該站擋curl時退回fetch
74
+ return await fetchWithRetry(rssUrl, cfg)
75
+ }
76
+ }
77
+
78
+ return await fetchWithRetry(rssUrl, cfg)
79
+ }
80
+
81
+
22
82
  /**
23
83
  * 抓取RSS訂閱源
24
84
  *
25
85
  * 支援RSS與Atom, 並轉為統一格式, 時間欄位一律轉為UTC+8之'YYYY-MM-DD HH:mm:ss'字串;
26
- * 因YouTube等站台之RSS會回傳暫時性404, 故404亦納入重試範圍
86
+ * 因YouTube等站台之RSS會回傳暫時性404, 故404亦納入重試範圍(僅fetch模式)。
87
+ *
88
+ * 部分站台(如以TLS指紋阻擋非瀏覽器用戶端者)對node內建fetch回403、對curl回200,
89
+ * 此時可指定opt.method為'curl'改走系統curl, 或'auto'先curl失敗再退回fetch;
90
+ * curl模式需系統PATH內有curl, 且重試退避由w-fetch-web內建(3至15秒線性),
91
+ * opt之baseDelayMs、maxDelayMs與retryStatus於curl模式不生效, 極短之feed(小於100字元)會被curl模式判為empty-response
27
92
  *
28
93
  * @param {String} rssUrl 輸入RSS網址字串,須為http或https
29
94
  * @param {Object} [opt={}] 輸入設定物件,預設{}
95
+ * @param {String} [opt.method='fetch'] 輸入抓取方式字串,可為'fetch'(node內建fetch)、'curl'(系統curl)、'auto'(curl優先失敗退回fetch),預設'fetch'
96
+ * @param {Boolean} [opt.withContent=false] 輸入是否於各項目附加content欄布林值,true時依content:encoded、content、summary、description之優先序取feed內嵌全文並去除HTML標籤與實體,預設false
30
97
  * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數,預設30000
31
98
  * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數,含初始共執行maxRetries+1次,預設5
32
- * @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數,預設3000
33
- * @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數,預設15000
99
+ * @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數(僅fetch模式生效),預設3000
100
+ * @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數(僅fetch模式生效),預設15000
34
101
  * @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值,預設true
35
- * @returns {Promise} 回傳Promise,resolve回傳項目物件陣列,各物件為{url,time,title,description,from},網址無效或抓取失敗時reject回傳錯誤物件
102
+ * @returns {Promise} 回傳Promise,resolve回傳項目物件陣列,各物件為{url,time,title,description,from},withContent為true時另含content欄,網址無效或抓取失敗時reject回傳錯誤物件
36
103
  * @example
37
104
  *
38
105
  * import fetchRSS from './src/fetchRSS.mjs'
@@ -43,6 +110,11 @@ let DFLT = {
43
110
  * console.log(rs.length, rs[0])
44
111
  * // => 20 { url: 'https://...', time: '2026-08-07 09:02:03', title: '...', description: '...', from: '...' }
45
112
  *
113
+ * //擋node fetch之站台改走curl, 並取feed內嵌全文
114
+ * let rs2 = await fetchRSS('https://alphaarchitect.com/feed', { method: 'curl', withContent: true })
115
+ * console.log(rs2.length, rs2[0].content.length)
116
+ * // => 5 2093
117
+ *
46
118
  * }
47
119
  * await test()
48
120
  * .catch((err) => {
@@ -57,8 +129,20 @@ async function fetchRSS(rssUrl, opt = {}) {
57
129
  throw new Error('rssUrl 須為有效的 http/https RSS 網址')
58
130
  }
59
131
 
132
+ //method
133
+ let method = get(opt, 'method')
134
+ if (method !== 'curl' && method !== 'auto') {
135
+ method = 'fetch' //預設維持原行為
136
+ }
137
+
138
+ //withContent
139
+ let withContent = get(opt, 'withContent')
140
+ if (!isbol(withContent)) {
141
+ withContent = false
142
+ }
143
+
60
144
  //xml
61
- let xml = await fetchWithRetry(rssUrl, getOptFetch(opt, DFLT))
145
+ let xml = await getXml(rssUrl, method, getOptFetch(opt, DFLT))
62
146
 
63
147
  //feed
64
148
  let parser = new Parser()
@@ -74,13 +158,27 @@ async function fetchRSS(rssUrl, opt = {}) {
74
158
  }
75
159
 
76
160
  return items.map((item) => {
77
- return {
161
+
162
+ let r = {
78
163
  url: String(get(item, 'link', '') || '').trim(),
79
164
  time: toDatetimeUTC8(get(item, 'isoDate', '') || get(item, 'pubDate', '')),
80
165
  title: String(get(item, 'title', '') || '').trim(),
81
166
  description: String(get(item, 'contentSnippet', '') || get(item, 'summary', '') || '').trim(),
82
167
  from: String(get(item, 'creator', '') || feedFrom).trim(),
83
168
  }
169
+
170
+ //content, 取feed內嵌全文, 依content:encoded、content、summary、description優先序
171
+ //content:encoded為RSS content模組之全文欄(如WordPress系), arXiv系全文則在content(源自description)
172
+ if (withContent) {
173
+ r.content = stripHtml(
174
+ get(item, 'content:encoded', '') ||
175
+ get(item, 'content', '') ||
176
+ get(item, 'summary', '') ||
177
+ get(item, 'description', '')
178
+ )
179
+ }
180
+
181
+ return r
84
182
  })
85
183
  }
86
184
 
@@ -0,0 +1,58 @@
1
+ import isstr from 'wsemi/src/isstr.mjs'
2
+ import decodeEntities from './decodeEntities.mjs'
3
+
4
+
5
+ /**
6
+ * 去除HTML標籤與實體, 保留段落換行
7
+ *
8
+ * 與htmlToMarkdown不同, 本函數輸出純文字(不產生Markdown標記), 供feed內嵌全文等
9
+ * 「只要內文、不要格式」之場景使用;
10
+ * 先移除script與style, 再將段落類閉合標籤與br轉為換行, 最後清除殘留標籤並解碼實體
11
+ *
12
+ * @param {String} html 輸入HTML字串
13
+ * @returns {String} 回傳純文字字串,輸入非字串時回傳''
14
+ * @example
15
+ *
16
+ * import stripHtml from './src/stripHtml.mjs'
17
+ *
18
+ * console.log(stripHtml('<p>第一段<b>粗體</b></p><p>第二段&amp;實體</p>'))
19
+ * // => '第一段粗體\n第二段&實體'
20
+ *
21
+ */
22
+ function stripHtml(html) {
23
+
24
+ //check
25
+ if (!isstr(html)) {
26
+ return ''
27
+ }
28
+
29
+ let s = html
30
+
31
+ //移除script與style
32
+ s = s.replace(/<script[\s\S]*?<\/script>/gi, '')
33
+ s = s.replace(/<style[\s\S]*?<\/style>/gi, '')
34
+
35
+ //段落類閉合標籤與br轉為換行
36
+ s = s.replace(/<\/(?:p|div|li|h[1-6])>/gi, '\n')
37
+ s = s.replace(/<br\s*\/?>/gi, '\n')
38
+
39
+ //清除殘留標籤
40
+ s = s.replace(/<[^>]+>/g, '')
41
+
42
+ //解碼實體
43
+ s = decodeEntities(s)
44
+
45
+ //decodeEntities把&nbsp;解為U+00A0, 統一為半形空白(必須用顯式跳脫u00A0, 不可貼字面字元)
46
+ s = s.replace(/\u00A0/g, ' ')
47
+
48
+ //行內空白標準化
49
+ s = s.split('\n').map((l) => l.replace(/[ \t]+/g, ' ').trim()).join('\n')
50
+
51
+ //多重空行收斂
52
+ s = s.replace(/\n{3,}/g, '\n\n')
53
+
54
+ return s.trim()
55
+ }
56
+
57
+
58
+ export default stripHtml
@@ -1,6 +1,5 @@
1
1
  import ot from 'dayjs'
2
2
  import utc from 'dayjs/plugin/utc.js'
3
- import isnum from 'wsemi/src/isnum.mjs'
4
3
  import isestr from 'wsemi/src/isestr.mjs'
5
4
 
6
5
 
@@ -16,13 +15,22 @@ let TZ_OFFSET_HOUR = 8
16
15
  let FORMAT = 'YYYY-MM-DD HH:mm:ss'
17
16
 
18
17
 
18
+ //無時間部分之日期字串(如'2024'、'20260807'、'2026-08-07'), 即不含'T'與':'者
19
+ let REG_DATE_ONLY = /^[^T:]*$/
20
+
21
+
19
22
  /**
20
23
  * 時間轉UTC+8之日期時間字串
21
24
  *
22
- * 可接受ISO時間字串、RFC822時間字串、unix秒數(非毫秒)或Date物件, 一律輸出台灣時間'YYYY-MM-DD HH:mm:ss';
23
- * 以dayjs之utc外掛指定固定偏移+8, 不受執行環境時區影響(在UTC伺服器上結果相同)
25
+ * 可接受ISO時間字串、RFC822時間字串、純日期字串、unix秒數(Number, 非毫秒)或Date物件,
26
+ * 一律輸出台灣時間'YYYY-MM-DD HH:mm:ss', 且不受執行環境時區影響。
27
+ *
28
+ * 兩類輸入之時區語意不同, 故分開處理:
29
+ * 帶有時刻資訊者(ISO、RFC822、unix秒數、Date)代表一個絕對時點, 換算為UTC+8之當地時刻;
30
+ * 純日期字串本身不含時點, 直接視為UTC+8當日之00:00:00, 不因執行環境時區而位移日期
31
+ * (否則在UTC+14環境下'2024'會被解為2023-12-31)。
24
32
  *
25
- * @param {String|Number|Date} value 輸入ISO或RFC822時間字串、unix秒數或Date物件
33
+ * @param {String|Number|Date} value 輸入ISO或RFC822時間字串、純日期字串、unix秒數或Date物件
26
34
  * @returns {String} 回傳UTC+8之日期時間字串,格式為'YYYY-MM-DD HH:mm:ss',輸入為空或無法解析時回傳''
27
35
  * @example
28
36
  *
@@ -34,6 +42,9 @@ let FORMAT = 'YYYY-MM-DD HH:mm:ss'
34
42
  * console.log(toDatetimeUTC8(1786000000))
35
43
  * // => '2026-08-06 15:06:40'
36
44
  *
45
+ * console.log(toDatetimeUTC8('2024'), toDatetimeUTC8('20260807'))
46
+ * // => '2024-01-01 00:00:00' '2026-08-07 00:00:00'
47
+ *
37
48
  * console.log(toDatetimeUTC8('abc'), toDatetimeUTC8(''), toDatetimeUTC8(null))
38
49
  * // => '' '' ''
39
50
  *
@@ -42,8 +53,13 @@ function toDatetimeUTC8(value) {
42
53
 
43
54
  let dt = null
44
55
 
45
- //數值一律視為unix秒數, dayjs之ot(number)為毫秒故須改用ot.unix
46
- if (isnum(value)) {
56
+ //keepLocalTime, 為true時保留原始年月日時分秒僅改標時區偏移, 供純日期字串使用
57
+ let keepLocalTime = false
58
+
59
+ //Number型別一律視為unix秒數, dayjs之ot(number)為毫秒故須改用ot.unix
60
+ //型別分派不可用wsemi之isnum, 其對數值字串亦回true, 會使'2024'、'20260807'等日期字串
61
+ //被誤當unix秒數而解成1970年
62
+ if (typeof value === 'number') {
47
63
  if (value === 0) {
48
64
  return ''
49
65
  }
@@ -54,6 +70,7 @@ function toDatetimeUTC8(value) {
54
70
  }
55
71
  else if (isestr(value)) {
56
72
  dt = ot(value)
73
+ keepLocalTime = REG_DATE_ONLY.test(value)
57
74
  }
58
75
  else {
59
76
  return ''
@@ -64,7 +81,7 @@ function toDatetimeUTC8(value) {
64
81
  return ''
65
82
  }
66
83
 
67
- return dt.utcOffset(TZ_OFFSET_HOUR).format(FORMAT)
84
+ return dt.utcOffset(TZ_OFFSET_HOUR, keepLocalTime).format(FORMAT)
68
85
  }
69
86
 
70
87
 
@@ -54,7 +54,7 @@ describe('WDwdataHub', function() {
54
54
  })
55
55
 
56
56
  it('不匯出內部共用函數', function() {
57
- let ks = ['fetchWithRetry', 'getOptFetch', 'isYmd', 'toRocDate', 'toDatetimeUTC8', 'decodeEntities', 'htmlToMarkdown', 'extractDivContent', 'parseIntComma']
57
+ let ks = ['fetchWithRetry', 'getOptFetch', 'isYmd', 'toRocDate', 'toDatetimeUTC8', 'decodeEntities', 'htmlToMarkdown', 'stripHtml', 'extractDivContent', 'parseIntComma']
58
58
  let r = ks.map((k) => Object.prototype.hasOwnProperty.call(WDwdataHub, k))
59
59
  let rr = ks.map(() => false)
60
60
  assert.strict.deepEqual(r, rr)
@@ -82,4 +82,82 @@ describe('fetchRSS', function() {
82
82
  assert.strict.deepEqual(r, rr)
83
83
  })
84
84
 
85
+ it('預設不附content欄, 回傳形狀維持五欄不變', async function() {
86
+ let t = await fetchRSS(svr.url('/rss/feed-content.xml'), { showLog: false })
87
+ let r = Object.keys(t[0])
88
+ let rr = ['url', 'time', 'title', 'description', 'from']
89
+ assert.strict.deepEqual(r, rr)
90
+ })
91
+
92
+ it('withContent為true時附content欄, 取content:encoded去標籤與實體且顯著長於description', async function() {
93
+ let t = await fetchRSS(svr.url('/rss/feed-content.xml'), { withContent: true, showLog: false })
94
+ let r = [
95
+ Object.keys(t[0]),
96
+ t[0].description,
97
+ t[0].content,
98
+ t[0].content.length > t[0].description.length * 4,
99
+ t[0].content.includes('<'),
100
+ t[0].content.includes('bad()'),
101
+ ]
102
+ let rr = [
103
+ ['url', 'time', 'title', 'description', 'from', 'content'],
104
+ '摘要開頭',
105
+ '第一段完整全文,含粗體與&實體。\n第二段 內容,長度遠大於摘要,供斷言withContent顯著長於description。',
106
+ true,
107
+ false,
108
+ false,
109
+ ]
110
+ assert.strict.deepEqual(r, rr)
111
+ })
112
+
113
+ it('withContent對無content:encoded之feed(arXiv系)退取content, 全文與description一致', async function() {
114
+ let t = await fetchRSS(svr.url('/rss/feed-desc.xml'), { withContent: true, showLog: false })
115
+ let r = [t[0].content, t[0].content === t[0].description]
116
+ let rr = ['arXiv:2608.01234 Announce Type: new Abstract: 這是一段完整摘要,不含HTML標籤,長度足以代表全文內容本體。', true]
117
+ assert.strict.deepEqual(r, rr)
118
+ })
119
+
120
+ it('method為curl時走系統curl抓取, 解析結果與fetch相同', async function() {
121
+ let t = await fetchRSS(svr.url('/rss/feed.xml'), { method: 'curl', maxRetries: 0, showLog: false })
122
+ let r = [t.length, t[0].title, t[0].from, t[1].from]
123
+ let rr = [2, 'RSS標題一', '作者甲', '測試訂閱源']
124
+ assert.strict.deepEqual(r, rr)
125
+ })
126
+
127
+ it('curl模式對極短feed判empty-response而reject, 證明分派確實走curl路徑', async function() {
128
+ //同一網址fetch模式可正常解析(見下一測試), 故此reject即為curl路徑之行為證據
129
+ let r = null
130
+ await fetchRSS(svr.url('/rss/feed-tiny.xml'), { method: 'curl', maxRetries: 0, showLog: false })
131
+ .then(() => {
132
+ r = 'resolved'
133
+ })
134
+ .catch((err) => {
135
+ r = err.reason
136
+ })
137
+ let rr = 'empty-response'
138
+ assert.strict.deepEqual(r, rr)
139
+ })
140
+
141
+ it('fetch模式對極短feed正常解析為0項, 與curl模式行為對照', async function() {
142
+ let r = await fetchRSS(svr.url('/rss/feed-tiny.xml'), { maxRetries: 0, showLog: false })
143
+ let rr = []
144
+ assert.strict.deepEqual(r, rr)
145
+ })
146
+
147
+ it('auto模式於curl失敗時退回fetch, 極短feed不reject而回0項', async function() {
148
+ let r = await fetchRSS(svr.url('/rss/feed-tiny.xml'), { method: 'auto', maxRetries: 0, showLog: false })
149
+ let rr = []
150
+ assert.strict.deepEqual(r, rr)
151
+ })
152
+
153
+ it('method非法值一律退回預設fetch行為', async function() {
154
+ let r = []
155
+ for (let method of ['FETCH', 'wget', 123, null, {}]) {
156
+ let t = await fetchRSS(svr.url('/rss/feed.xml'), { method, showLog: false })
157
+ r.push(t.length)
158
+ }
159
+ let rr = [2, 2, 2, 2, 2]
160
+ assert.strict.deepEqual(r, rr)
161
+ })
162
+
85
163
  })
@@ -0,0 +1,58 @@
1
+ import assert from 'assert'
2
+ import stripHtml from '../src/stripHtml.mjs'
3
+
4
+
5
+ describe('stripHtml', function() {
6
+
7
+ it('去除標籤輸出純文字, 段落閉合轉為換行', function() {
8
+ let r = stripHtml('<p>第一段<b>粗體</b></p><p>第二段</p>')
9
+ let rr = '第一段粗體\n第二段'
10
+ assert.strict.deepEqual(r, rr)
11
+ })
12
+
13
+ it('與htmlToMarkdown不同, 不產生Markdown標記', function() {
14
+ let r = stripHtml('<h1>標題</h1><p><a href="https://a.example/">連結</a>與<em>斜體</em></p>')
15
+ let rr = '標題\n連結與斜體'
16
+ assert.strict.deepEqual(r, rr)
17
+ })
18
+
19
+ it('移除script與style', function() {
20
+ let r = stripHtml('<p>前</p><script>var a=1</script><style>.a{}</style><p>後</p>')
21
+ let rr = '前\n後'
22
+ assert.strict.deepEqual(r, rr)
23
+ })
24
+
25
+ it('br與li閉合轉為換行, 開標籤不產生換行', function() {
26
+ let r = [
27
+ stripHtml('一<br/>二<br>三'),
28
+ stripHtml('<ul><li>三</li><li>四</li></ul>'),
29
+ ]
30
+ let rr = ['一\n二\n三', '三\n四']
31
+ assert.strict.deepEqual(r, rr)
32
+ })
33
+
34
+ it('解碼實體且nbsp統一為半形空白', function() {
35
+ let t = stripHtml('<p>a&amp;b&nbsp;c&hellip;</p>')
36
+ let r = [t, t.includes('\u00A0')]
37
+ let rr = ['a&b c…', false]
38
+ assert.strict.deepEqual(r, rr)
39
+ })
40
+
41
+ it('多重空行收斂且前後修剪', function() {
42
+ let r = stripHtml('<p></p><p></p><p>內文</p><p></p>')
43
+ let rr = '內文'
44
+ assert.strict.deepEqual(r, rr)
45
+ })
46
+
47
+ it('非字串回傳空字串', function() {
48
+ let r = [
49
+ stripHtml(null),
50
+ stripHtml(undefined),
51
+ stripHtml(123),
52
+ stripHtml({}),
53
+ ]
54
+ let rr = ['', '', '', '']
55
+ assert.strict.deepEqual(r, rr)
56
+ })
57
+
58
+ })
@@ -48,6 +48,38 @@ describe('toDatetimeUTC8', function() {
48
48
  assert.strict.deepEqual(r, rr)
49
49
  })
50
50
 
51
+ it('數值字串為日期字串而非unix秒數, 不得解成1970年', function() {
52
+ //wsemi之isnum對數值字串亦回true, 若以其做型別分派會使日期字串被誤當unix秒數
53
+ let r = [
54
+ toDatetimeUTC8('2024'),
55
+ toDatetimeUTC8('20260807'),
56
+ toDatetimeUTC8('2026-08-07'),
57
+ ]
58
+ let rr = ['2024-01-01 00:00:00', '2026-08-07 00:00:00', '2026-08-07 00:00:00']
59
+ assert.strict.deepEqual(r, rr)
60
+ })
61
+
62
+ it('純日期字串視為UTC+8當日00:00:00, 不受執行環境時區位移日期', function() {
63
+ //在UTC+14環境下若以當地時區解析, '2024'會位移為2023-12-31而跳年
64
+ let r = [
65
+ toDatetimeUTC8('2024'),
66
+ toDatetimeUTC8('20260807'),
67
+ toDatetimeUTC8('2026/08/07'),
68
+ ]
69
+ let rr = ['2024-01-01 00:00:00', '2026-08-07 00:00:00', '2026-08-07 00:00:00']
70
+ assert.strict.deepEqual(r, rr)
71
+ })
72
+
73
+ it('帶時刻資訊者仍為絕對時點換算, 不套用保留當地時間', function() {
74
+ let r = [
75
+ toDatetimeUTC8('2026-08-07T01:02:03Z'),
76
+ toDatetimeUTC8('Fri, 07 Aug 2026 01:02:03 GMT'),
77
+ toDatetimeUTC8(1786000000),
78
+ ]
79
+ let rr = ['2026-08-07 09:02:03', '2026-08-07 09:02:03', '2026-08-06 15:06:40']
80
+ assert.strict.deepEqual(r, rr)
81
+ })
82
+
51
83
  it('無法解析或為空時回傳空字串', function() {
52
84
  let r = [
53
85
  toDatetimeUTC8('not-a-date'),
@@ -40,6 +40,46 @@ let rssXml = `<?xml version="1.0" encoding="UTF-8"?>
40
40
  </rss>`
41
41
 
42
42
 
43
+ //含content:encoded內嵌全文之feed(WordPress系), 供斷言withContent取全文且顯著長於description;
44
+ //全文內含粗體、實體、&nbsp;與script, 供斷言stripHtml各轉換
45
+ let rssContentXml = `<?xml version="1.0" encoding="UTF-8"?>
46
+ <rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
47
+ <channel>
48
+ <title>內容測試源</title>
49
+ <link>https://rss.example/</link>
50
+ <item>
51
+ <title>含全文項目</title>
52
+ <link>https://rss.example/c1</link>
53
+ <pubDate>Fri, 07 Aug 2026 01:02:03 GMT</pubDate>
54
+ <description>摘要開頭</description>
55
+ <content:encoded><![CDATA[<p>第一段完整全文,含<b>粗體</b>與&amp;實體。</p><p>第二段&nbsp;內容,長度遠大於摘要,供斷言withContent顯著長於description。</p><script>bad()</script>]]></content:encoded>
56
+ </item>
57
+ </channel>
58
+ </rss>`
59
+
60
+
61
+ //全文位於description之feed(arXiv系), rss-parser將其映為item.content,
62
+ //供斷言withContent之content fallback(無content:encoded時取content)
63
+ let rssDescOnlyXml = `<?xml version="1.0" encoding="UTF-8"?>
64
+ <rss version="2.0">
65
+ <channel>
66
+ <title>arXiv式測試源</title>
67
+ <link>https://rss.example/</link>
68
+ <item>
69
+ <title>論文一</title>
70
+ <link>https://rss.example/p1</link>
71
+ <pubDate>Fri, 07 Aug 2026 01:02:03 GMT</pubDate>
72
+ <description>arXiv:2608.01234 Announce Type: new Abstract: 這是一段完整摘要,不含HTML標籤,長度足以代表全文內容本體。</description>
73
+ </item>
74
+ </channel>
75
+ </rss>`
76
+
77
+
78
+ //合法但小於100字元之feed, 低於fetchWebByCurl之MIN_HTML_LENGTH,
79
+ //供斷言curl模式判empty-response、auto模式退回fetch
80
+ let rssTinyXml = '<rss version="2.0"><channel><title>t</title></channel></rss>'
81
+
82
+
43
83
  //===== Hacker News =====
44
84
 
45
85
 
@@ -494,6 +534,9 @@ let guanchaArticleHtml = `<!DOCTYPE html><html><head><title>OPEC+宣布增产_
494
534
  export {
495
535
  aiNews,
496
536
  rssXml,
537
+ rssContentXml,
538
+ rssDescOnlyXml,
539
+ rssTinyXml,
497
540
  hnNewStories,
498
541
  hnItems,
499
542
  cnyesPage1,
@@ -126,6 +126,18 @@ function serverForTest() {
126
126
  sendText(200, fx.rssXml, 'application/rss+xml; charset=utf-8')
127
127
  return
128
128
  }
129
+ if (pathname === '/rss/feed-content.xml') {
130
+ sendText(200, fx.rssContentXml, 'application/rss+xml; charset=utf-8')
131
+ return
132
+ }
133
+ if (pathname === '/rss/feed-desc.xml') {
134
+ sendText(200, fx.rssDescOnlyXml, 'application/rss+xml; charset=utf-8')
135
+ return
136
+ }
137
+ if (pathname === '/rss/feed-tiny.xml') {
138
+ sendText(200, fx.rssTinyXml, 'application/rss+xml; charset=utf-8')
139
+ return
140
+ }
129
141
 
130
142
  //===== Hacker News =====
131
143
 
@@ -12,6 +12,7 @@ rollupFiles({
12
12
  fdTar,
13
13
  nameDistType: 'kebabCase', //直接由hookNameDist給予
14
14
  globals: {
15
+ 'child_process': 'child_process',
15
16
  'util': 'util',
16
17
  'path': 'path',
17
18
  'module': 'module',
@@ -24,11 +25,11 @@ rollupFiles({
24
25
  'cheerio': 'cheerio',
25
26
  'rss-parser': 'rss-parser',
26
27
  'playwright': 'playwright',
27
- 'w-fetch-web': 'w-fetch-web',
28
28
  '@mozilla/readability': '@mozilla/readability',
29
29
  'jsdom': 'jsdom',
30
30
  },
31
31
  external: [
32
+ 'child_process',
32
33
  'util',
33
34
  'path',
34
35
  'module',
@@ -41,7 +42,6 @@ rollupFiles({
41
42
  'cheerio',
42
43
  'rss-parser',
43
44
  'playwright',
44
- 'w-fetch-web',
45
45
  '@mozilla/readability',
46
46
  'jsdom',
47
47
  ],