w-dwdata-hub 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/.editorconfig +9 -0
  2. package/.eslintignore +3 -0
  3. package/.eslintrc.js +55 -0
  4. package/.jsdoc +25 -0
  5. package/LICENSE +21 -0
  6. package/README.md +168 -0
  7. package/SECURITY.md +5 -0
  8. package/babel.config.js +16 -0
  9. package/dist/w-dwdata-hub.umd.js +7 -0
  10. package/dist/w-dwdata-hub.umd.js.map +1 -0
  11. package/docs/WDwdataHub.mjs.html +136 -0
  12. package/docs/decodeEntities.mjs.html +129 -0
  13. package/docs/extractDivContent.mjs.html +147 -0
  14. package/docs/fetchAiNewsAggregator.mjs.html +155 -0
  15. package/docs/fetchAisixiang.mjs.html +1038 -0
  16. package/docs/fetchCnyes.mjs.html +260 -0
  17. package/docs/fetchGuancha.mjs.html +1007 -0
  18. package/docs/fetchHackerNews.mjs.html +189 -0
  19. package/docs/fetchMoneydj.mjs.html +251 -0
  20. package/docs/fetchMops.mjs.html +385 -0
  21. package/docs/fetchRSS.mjs.html +160 -0
  22. package/docs/fetchStatementdog.mjs.html +213 -0
  23. package/docs/fetchTaifex.mjs.html +513 -0
  24. package/docs/fetchTpex3insti.mjs.html +278 -0
  25. package/docs/fetchTpexMargin.mjs.html +248 -0
  26. package/docs/fetchTpexStock.mjs.html +199 -0
  27. package/docs/fetchTwDataHoliday.mjs.html +236 -0
  28. package/docs/fetchTwseMargin.mjs.html +253 -0
  29. package/docs/fetchTwseStock.mjs.html +188 -0
  30. package/docs/fetchTwseT86.mjs.html +208 -0
  31. package/docs/fetchWithRetry.mjs.html +376 -0
  32. package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
  33. package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
  34. package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
  35. package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
  36. package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
  37. package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
  38. package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
  39. package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
  40. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
  41. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
  42. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
  43. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
  44. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
  45. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
  46. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
  47. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
  48. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
  49. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
  50. package/docs/getOptFetch.mjs.html +185 -0
  51. package/docs/global.html +22649 -0
  52. package/docs/htmlToMarkdown.mjs.html +168 -0
  53. package/docs/index.html +84 -0
  54. package/docs/isYmd.mjs.html +114 -0
  55. package/docs/parseIntComma.mjs.html +100 -0
  56. package/docs/safeFilename.mjs.html +102 -0
  57. package/docs/scripts/collapse.js +39 -0
  58. package/docs/scripts/commonNav.js +28 -0
  59. package/docs/scripts/linenumber.js +25 -0
  60. package/docs/scripts/nav.js +12 -0
  61. package/docs/scripts/polyfill.js +4 -0
  62. package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
  63. package/docs/scripts/prettify/lang-css.js +2 -0
  64. package/docs/scripts/prettify/prettify.js +28 -0
  65. package/docs/scripts/search.js +99 -0
  66. package/docs/styles/jsdoc.css +776 -0
  67. package/docs/styles/prettify.css +80 -0
  68. package/docs/toDatetimeUTC8.mjs.html +143 -0
  69. package/docs/toRocDate.mjs.html +107 -0
  70. package/g.mjs +73 -0
  71. package/package.json +31 -0
  72. package/script.txt +17 -0
  73. package/src/WDwdataHub.mjs +64 -0
  74. package/src/decodeEntities.mjs +57 -0
  75. package/src/extractDivContent.mjs +75 -0
  76. package/src/fetchAiNewsAggregator.mjs +83 -0
  77. package/src/fetchAisixiang.mjs +966 -0
  78. package/src/fetchCnyes.mjs +188 -0
  79. package/src/fetchGuancha.mjs +935 -0
  80. package/src/fetchHackerNews.mjs +117 -0
  81. package/src/fetchMoneydj.mjs +179 -0
  82. package/src/fetchMops.mjs +313 -0
  83. package/src/fetchRSS.mjs +88 -0
  84. package/src/fetchStatementdog.mjs +141 -0
  85. package/src/fetchTaifex.mjs +441 -0
  86. package/src/fetchTpex3insti.mjs +206 -0
  87. package/src/fetchTpexMargin.mjs +176 -0
  88. package/src/fetchTpexStock.mjs +127 -0
  89. package/src/fetchTwDataHoliday.mjs +164 -0
  90. package/src/fetchTwseMargin.mjs +181 -0
  91. package/src/fetchTwseStock.mjs +116 -0
  92. package/src/fetchTwseT86.mjs +136 -0
  93. package/src/fetchWithRetry.mjs +304 -0
  94. package/src/getOptFetch.mjs +113 -0
  95. package/src/htmlToMarkdown.mjs +96 -0
  96. package/src/isYmd.mjs +42 -0
  97. package/src/parseIntComma.mjs +28 -0
  98. package/src/safeFilename.mjs +30 -0
  99. package/src/toDatetimeUTC8.mjs +71 -0
  100. package/src/toRocDate.mjs +35 -0
  101. package/test/WDwdataHub.test.mjs +63 -0
  102. package/test/decodeEntities.test.mjs +42 -0
  103. package/test/extractDivContent.test.mjs +48 -0
  104. package/test/fetchAiNewsAggregator.test.mjs +79 -0
  105. package/test/fetchAisixiang.test.mjs +225 -0
  106. package/test/fetchCnyes.test.mjs +61 -0
  107. package/test/fetchGuancha.test.mjs +190 -0
  108. package/test/fetchHackerNews.test.mjs +78 -0
  109. package/test/fetchMoneydj.test.mjs +82 -0
  110. package/test/fetchMops.test.mjs +106 -0
  111. package/test/fetchRSS.test.mjs +85 -0
  112. package/test/fetchStatementdog.test.mjs +69 -0
  113. package/test/fetchTaifex.test.mjs +99 -0
  114. package/test/fetchTpex3insti.test.mjs +90 -0
  115. package/test/fetchTpexMargin.test.mjs +94 -0
  116. package/test/fetchTpexStock.test.mjs +68 -0
  117. package/test/fetchTwDataHoliday.test.mjs +100 -0
  118. package/test/fetchTwseMargin.test.mjs +81 -0
  119. package/test/fetchTwseStock.test.mjs +80 -0
  120. package/test/fetchTwseT86.test.mjs +76 -0
  121. package/test/fetchWithRetry.test.mjs +174 -0
  122. package/test/htmlToMarkdown.test.mjs +71 -0
  123. package/test/isYmd.test.mjs +56 -0
  124. package/test/parseIntComma.test.mjs +38 -0
  125. package/test/safeFilename.test.mjs +35 -0
  126. package/test/toDatetimeUTC8.test.mjs +65 -0
  127. package/test/toRocDate.test.mjs +38 -0
  128. package/test/tools/encodeBig5.mjs +80 -0
  129. package/test/tools/fixtures.mjs +539 -0
  130. package/test/tools/hasChrome.mjs +39 -0
  131. package/test/tools/serverForTest.mjs +397 -0
  132. package/toolg/addVersion.mjs +4 -0
  133. package/toolg/cleanFolder.mjs +5 -0
  134. package/toolg/gDistRollup.mjs +49 -0
  135. package/toolg/modifyReadme.mjs +4 -0
package/docs/fetchHackerNews.mjs.html ADDED
@@ -0,0 +1,189 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchHackerNews.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchHackerNews.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import get from 'lodash-es/get.js'
49
+ import isestr from 'wsemi/src/isestr.mjs'
50
+ import isearr from 'wsemi/src/isearr.mjs'
51
+ import ispint from 'wsemi/src/ispint.mjs'
52
+ import cint from 'wsemi/src/cint.mjs'
53
+ import fetchWithRetry from './fetchWithRetry.mjs'
54
+ import getOptFetch from './getOptFetch.mjs'
55
+ import toDatetimeUTC8 from './toDatetimeUTC8.mjs'
56
+
57
+
58
+ //Hacker News官方Firebase API
59
+ let API_BASE = 'https://hacker-news.firebaseio.com/v0'
60
+
61
+
62
+ //本函數之預設抓取設定
63
+ let DFLT = {
64
+ timeout: 30000,
65
+ maxRetries: 5,
66
+ baseDelayMs: 3000,
67
+ maxDelayMs: 15000,
68
+ label: 'fetch-hacker-news',
69
+ }
70
+
71
+
72
+ //預設取回篇數與取詳情之併發數
73
+ let DEFAULT_LIMIT = 30
74
+ let CONCURRENCY = 10
75
+
76
+
77
+ /**
78
+ * 抓取Hacker News最新文章
79
+ *
80
+ * 先取newstories.json之ID清單, 再分批(每批10筆)取各篇詳情並轉為統一格式;
81
+ * 分批採Promise.allSettled, 單篇重試耗盡不會拖垮同批其他已成功文章;
82
+ * 僅保留type為story者, 無外部連結時以Hacker News站內討論頁網址替代
83
+ *
84
+ * @param {Integer} [limit=30] 輸入取回文章數量正整數，非正整數時改用預設值，預設30
85
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
86
+ * @param {String} [opt.apiBase='https://hacker-news.firebaseio.com/v0'] 輸入API根網址字串，供測試或改指向鏡像時覆寫
87
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
88
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，含初始共執行maxRetries+1次，預設5
89
+ * @param {Integer} [opt.baseDelayMs=3000] 輸入重試之線性退避基礎毫秒整數，預設3000
90
+ * @param {Integer} [opt.maxDelayMs=15000] 輸入重試之線性退避上限毫秒整數，預設15000
91
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示重試訊息布林值，預設true
92
+ * @returns {Promise} 回傳Promise，resolve回傳文章物件陣列，各物件為{url,time,title,description,from}，抓取ID清單失敗時reject回傳錯誤物件
93
+ * @example
94
+ *
95
+ * import fetchHackerNews from './src/fetchHackerNews.mjs'
96
+ *
97
+ * let test = async () => {
98
+ *
99
+ * let rs = await fetchHackerNews(5)
100
+ * console.log(rs.length, rs[0])
101
+ * // => 5 { url: 'https://...', time: '2026-08-07 09:02:03', title: '...', description: '', from: 'Hacker News' }
102
+ *
103
+ * }
104
+ * await test()
105
+ * .catch((err) => {
106
+ * console.log(err)
107
+ * })
108
+ *
109
+ */
110
+ async function fetchHackerNews(limit = DEFAULT_LIMIT, opt = {}) {
111
+
112
+ //limit, 非正整數(如NaN或&lt;=0)時改用預設值, 避免slice(0,NaN)靜默回傳空陣列
113
+ limit = ispint(limit) ? cint(limit) : DEFAULT_LIMIT
114
+
115
+ //apiBase
116
+ let apiBase = get(opt, 'apiBase')
117
+ if (!isestr(apiBase)) {
118
+ apiBase = API_BASE
119
+ }
120
+ apiBase = apiBase.replace(/\/+$/, '')
121
+
122
+ //optFetch
123
+ let optFetch = getOptFetch(opt, DFLT)
124
+
125
+ //ids, 取得最新文章ID列表
126
+ let ids = await fetchWithRetry(`${apiBase}/newstories.json`, optFetch)
127
+ if (!isearr(ids)) {
128
+ ids = []
129
+ }
130
+ let selected = ids.slice(0, limit)
131
+
132
+ //items, 批次取得文章詳情並控制併發數
133
+ let items = []
134
+ for (let i = 0; i &lt; selected.length; i += CONCURRENCY) {
135
+ let batch = selected.slice(i, i + CONCURRENCY)
136
+
137
+ //allSettled, 單一item重試耗盡而reject時不丟失整批其他已成功之文章
138
+ let rs = await Promise.allSettled(batch.map((id) => {
139
+ return fetchWithRetry(`${apiBase}/item/${id}.json`, optFetch)
140
+ }))
141
+ for (let r of rs) {
142
+ if (r.status === 'fulfilled') {
143
+ items.push(r.value)
144
+ }
145
+ }
146
+
147
+ }
148
+
149
+ //轉換為統一格式
150
+ return items
151
+ .filter((item) => item &amp;&amp; item.type === 'story')
152
+ .map((item) => {
153
+ return {
154
+ url: String(get(item, 'url', '') || `https://news.ycombinator.com/item?id=${get(item, 'id', '')}`).trim(),
155
+ time: toDatetimeUTC8(get(item, 'time', 0)),
156
+ title: String(get(item, 'title', '') || '').trim(),
157
+ description: '',
158
+ from: 'Hacker News',
159
+ }
160
+ })
161
+ }
162
+
163
+
164
+ export default fetchHackerNews
165
+ </code></pre>
166
+ </article>
167
+ </section>
168
+
169
+
170
+
171
+
172
+
173
+
174
+ </div>
175
+
176
+ <br class="clear">
177
+
178
+ <footer>
179
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
180
+ </footer>
181
+
182
+ <script>prettyPrint();</script>
183
+ <script src="scripts/polyfill.js"></script>
184
+ <script src="scripts/linenumber.js"></script>
185
+
186
+
187
+
188
+ </body>
189
+ </html>
package/docs/fetchMoneydj.mjs.html ADDED
@@ -0,0 +1,251 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchMoneydj.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchMoneydj.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import * as cheerio from 'cheerio'
49
+ import get from 'lodash-es/get.js'
50
+ import isfun from 'wsemi/src/isfun.mjs'
51
+ import isestr from 'wsemi/src/isestr.mjs'
52
+ import ispint from 'wsemi/src/ispint.mjs'
53
+ import cint from 'wsemi/src/cint.mjs'
54
+ import delay from 'wsemi/src/delay.mjs'
55
+ import fetchWithRetry from './fetchWithRetry.mjs'
56
+ import getOptFetch from './getOptFetch.mjs'
57
+
58
+
59
+ //MoneyDJ台股新聞(MB06)列表頁, 頁碼接於index1之後
60
+ let BASE_URL = 'https://www.moneydj.com/kmdj/news/newsreallist.aspx?a=mb06&amp;index1='
61
+
62
+
63
+ //本函數之預設抓取設定
64
+ let DFLT = {
65
+ responseType: 'text',
66
+ timeout: 30000,
67
+ maxRetries: 10,
68
+ baseDelayMs: 5000,
69
+ maxDelayMs: 30000,
70
+ retryStatus: [403],
71
+ label: 'fetch-moneydj',
72
+ }
73
+
74
+
75
+ //預設抓取頁數
76
+ let DEFAULT_TOTAL_PAGES = 50
77
+
78
+
79
+ //頁間隨機延遲之下限與變動範圍毫秒
80
+ let PAGE_DELAY_MIN_MS = 1000
81
+ let PAGE_DELAY_RANGE_MS = 2000
82
+
83
+
84
+ //列表頁時間欄位格式, 例如'08/07 09:02'、'09:02'、'昨 09:02'
85
+ let REG_TIME = /^(\d{2}\/\d{2}\s+\d{2}:\d{2}|\d{2}:\d{2}|昨\s*\d{2}:\d{2})$/
86
+
87
+
88
+ /**
89
+ * 抓取MoneyDJ台股新聞
90
+ *
91
+ * 逐頁抓取MoneyDJ台股新聞(MB06)列表, 頁間隨機延遲1至3秒以降低被封鎖風險;
92
+ * 單頁抓取失敗(重試耗盡)時記錄並跳過該頁, 不中斷整體抓取; 全部頁面皆0筆時拋出錯誤
93
+ *
94
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
95
+ * @param {String} [opt.baseUrl='https://www.moneydj.com/kmdj/news/newsreallist.aspx?a=mb06&amp;index1='] 輸入列表頁網址前綴字串，頁碼會直接串接於後，供測試或改指向鏡像時覆寫
96
+ * @param {Integer} [opt.totalPages=50] 輸入要抓取的頁數正整數，預設50
97
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
98
+ * @param {Integer} [opt.maxRetries=10] 輸入每頁最大重試次數整數，含初始共執行maxRetries+1次，預設10
99
+ * @param {Integer} [opt.baseDelayMs=5000] 輸入重試基礎延遲毫秒整數，預設5000
100
+ * @param {Integer} [opt.maxDelayMs=30000] 輸入重試最大延遲毫秒整數，預設30000
101
+ * @param {Integer} [opt.pageDelayMs] 輸入頁間延遲毫秒整數，未給時採隨機1000至3000毫秒
102
+ * @param {Function} [opt.onPageDone] 輸入每頁完成時之回呼函數，傳入(pageIndex,itemCount,totalPages)
103
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
104
+ * @returns {Promise} 回傳Promise，resolve回傳新聞物件陣列，各物件為{time,title,link}，抓取到0筆時reject回傳錯誤物件
105
+ * @example
106
+ *
107
+ * import fetchMoneydj from './src/fetchMoneydj.mjs'
108
+ *
109
+ * let test = async () => {
110
+ *
111
+ * let rs = await fetchMoneydj({ totalPages: 2 })
112
+ * console.log(rs.length, rs[0])
113
+ * // => 60 { time: '08/07 09:02', title: '...', link: 'https://www.moneydj.com/kmdj/news/...' }
114
+ *
115
+ * }
116
+ * await test()
117
+ * .catch((err) => {
118
+ * console.log(err)
119
+ * })
120
+ *
121
+ */
122
+ async function fetchMoneydj(opt = {}) {
123
+
124
+ //baseUrl
125
+ let baseUrl = get(opt, 'baseUrl')
126
+ if (!isestr(baseUrl)) {
127
+ baseUrl = BASE_URL
128
+ }
129
+
130
+ //totalPages
131
+ let totalPages = get(opt, 'totalPages')
132
+ if (!ispint(totalPages)) {
133
+ totalPages = DEFAULT_TOTAL_PAGES
134
+ }
135
+ else {
136
+ totalPages = cint(totalPages)
137
+ }
138
+
139
+ //onPageDone
140
+ let onPageDone = get(opt, 'onPageDone')
141
+ if (!isfun(onPageDone)) {
142
+ onPageDone = () => {}
143
+ }
144
+
145
+ //optFetch
146
+ let optFetch = getOptFetch(opt, DFLT)
147
+ let showLog = optFetch.showLog
148
+
149
+ //pageDelayMs, 頁間隨機延遲毫秒
150
+ let pageDelayMs = get(opt, 'pageDelayMs')
151
+ if (!ispint(pageDelayMs) &amp;&amp; pageDelayMs !== 0) {
152
+ pageDelayMs = null
153
+ }
154
+ else {
155
+ pageDelayMs = cint(pageDelayMs)
156
+ }
157
+
158
+ //domain, 由列表頁網址推得, 供補齊相對路徑連結
159
+ let domain = new URL(baseUrl).origin
160
+
161
+ //fetchPage
162
+ let fetchPage = async (pageIndex) => {
163
+
164
+ //html, MoneyDJ已確認使用UTF-8編碼(多次實測結果正確), 無需Big5解碼
165
+ let html = await fetchWithRetry(`${baseUrl}${pageIndex}`, { ...optFetch, label: `${DFLT.label} Page ${pageIndex}` })
166
+
167
+ //$
168
+ let $ = cheerio.load(html)
169
+
170
+ //newsItems
171
+ let newsItems = []
172
+ $('tr').each((i, el) => {
173
+
174
+ let $row = $(el)
175
+ let timeText = $row.find('td').eq(0).text().trim()
176
+ let $link = $row.find('td').eq(1).find('a')
177
+
178
+ if (timeText &amp;&amp; REG_TIME.test(timeText) &amp;&amp; $link.length > 0) {
179
+ let title = $link.attr('title') || $link.text().trim()
180
+ let linkRel = $link.attr('href')
181
+ if (linkRel) {
182
+ let link = linkRel.startsWith('http') ? linkRel : domain + linkRel
183
+ newsItems.push({ time: timeText, title, link })
184
+ }
185
+ }
186
+
187
+ })
188
+
189
+ return newsItems
190
+ }
191
+
192
+ let allNewsItems = []
193
+ for (let i = 1; i &lt;= totalPages; i++) {
194
+
195
+ //items, 單頁失敗記錄並跳過, 不中斷整體抓取
196
+ let items = null
197
+ try {
198
+ items = await fetchPage(i)
199
+ }
200
+ catch (err) {
201
+ if (showLog) {
202
+ console.warn(`[Page ${i}] 抓取失敗，已跳過：${err.message}`)
203
+ }
204
+ continue
205
+ }
206
+ allNewsItems = allNewsItems.concat(items)
207
+
208
+ onPageDone(i, items.length, totalPages)
209
+
210
+ //頁間延遲, 未指定pageDelayMs時採隨機1至3秒以降低被封鎖風險
211
+ if (i &lt; totalPages) {
212
+ await delay(pageDelayMs === null ? Math.floor(Math.random() * PAGE_DELAY_RANGE_MS) + PAGE_DELAY_MIN_MS : pageDelayMs)
213
+ }
214
+
215
+ }
216
+
217
+ //check, 0筆代表版型改變或selector失效
218
+ if (allNewsItems.length === 0) {
219
+ throw new Error('抓取到 0 筆新聞，可能是頁面結構改變或 selector 失效，請確認 MoneyDJ 頁面是否正常。')
220
+ }
221
+
222
+ return allNewsItems
223
+ }
224
+
225
+
226
+ export default fetchMoneydj
227
+ </code></pre>
228
+ </article>
229
+ </section>
230
+
231
+
232
+
233
+
234
+
235
+
236
+ </div>
237
+
238
+ <br class="clear">
239
+
240
+ <footer>
241
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
242
+ </footer>
243
+
244
+ <script>prettyPrint();</script>
245
+ <script src="scripts/polyfill.js"></script>
246
+ <script src="scripts/linenumber.js"></script>
247
+
248
+
249
+
250
+ </body>
251
+ </html>