w-dwdata-hub 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/.editorconfig +9 -0
  2. package/.eslintignore +3 -0
  3. package/.eslintrc.js +55 -0
  4. package/.jsdoc +25 -0
  5. package/LICENSE +21 -0
  6. package/README.md +168 -0
  7. package/SECURITY.md +5 -0
  8. package/babel.config.js +16 -0
  9. package/dist/w-dwdata-hub.umd.js +7 -0
  10. package/dist/w-dwdata-hub.umd.js.map +1 -0
  11. package/docs/WDwdataHub.mjs.html +136 -0
  12. package/docs/decodeEntities.mjs.html +129 -0
  13. package/docs/extractDivContent.mjs.html +147 -0
  14. package/docs/fetchAiNewsAggregator.mjs.html +155 -0
  15. package/docs/fetchAisixiang.mjs.html +1038 -0
  16. package/docs/fetchCnyes.mjs.html +260 -0
  17. package/docs/fetchGuancha.mjs.html +1007 -0
  18. package/docs/fetchHackerNews.mjs.html +189 -0
  19. package/docs/fetchMoneydj.mjs.html +251 -0
  20. package/docs/fetchMops.mjs.html +385 -0
  21. package/docs/fetchRSS.mjs.html +160 -0
  22. package/docs/fetchStatementdog.mjs.html +213 -0
  23. package/docs/fetchTaifex.mjs.html +513 -0
  24. package/docs/fetchTpex3insti.mjs.html +278 -0
  25. package/docs/fetchTpexMargin.mjs.html +248 -0
  26. package/docs/fetchTpexStock.mjs.html +199 -0
  27. package/docs/fetchTwDataHoliday.mjs.html +236 -0
  28. package/docs/fetchTwseMargin.mjs.html +253 -0
  29. package/docs/fetchTwseStock.mjs.html +188 -0
  30. package/docs/fetchTwseT86.mjs.html +208 -0
  31. package/docs/fetchWithRetry.mjs.html +376 -0
  32. package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
  33. package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
  34. package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
  35. package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
  36. package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
  37. package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
  38. package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
  39. package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
  40. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
  41. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
  42. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
  43. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
  44. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
  45. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
  46. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
  47. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
  48. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
  49. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
  50. package/docs/getOptFetch.mjs.html +185 -0
  51. package/docs/global.html +22649 -0
  52. package/docs/htmlToMarkdown.mjs.html +168 -0
  53. package/docs/index.html +84 -0
  54. package/docs/isYmd.mjs.html +114 -0
  55. package/docs/parseIntComma.mjs.html +100 -0
  56. package/docs/safeFilename.mjs.html +102 -0
  57. package/docs/scripts/collapse.js +39 -0
  58. package/docs/scripts/commonNav.js +28 -0
  59. package/docs/scripts/linenumber.js +25 -0
  60. package/docs/scripts/nav.js +12 -0
  61. package/docs/scripts/polyfill.js +4 -0
  62. package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
  63. package/docs/scripts/prettify/lang-css.js +2 -0
  64. package/docs/scripts/prettify/prettify.js +28 -0
  65. package/docs/scripts/search.js +99 -0
  66. package/docs/styles/jsdoc.css +776 -0
  67. package/docs/styles/prettify.css +80 -0
  68. package/docs/toDatetimeUTC8.mjs.html +143 -0
  69. package/docs/toRocDate.mjs.html +107 -0
  70. package/g.mjs +73 -0
  71. package/package.json +31 -0
  72. package/script.txt +17 -0
  73. package/src/WDwdataHub.mjs +64 -0
  74. package/src/decodeEntities.mjs +57 -0
  75. package/src/extractDivContent.mjs +75 -0
  76. package/src/fetchAiNewsAggregator.mjs +83 -0
  77. package/src/fetchAisixiang.mjs +966 -0
  78. package/src/fetchCnyes.mjs +188 -0
  79. package/src/fetchGuancha.mjs +935 -0
  80. package/src/fetchHackerNews.mjs +117 -0
  81. package/src/fetchMoneydj.mjs +179 -0
  82. package/src/fetchMops.mjs +313 -0
  83. package/src/fetchRSS.mjs +88 -0
  84. package/src/fetchStatementdog.mjs +141 -0
  85. package/src/fetchTaifex.mjs +441 -0
  86. package/src/fetchTpex3insti.mjs +206 -0
  87. package/src/fetchTpexMargin.mjs +176 -0
  88. package/src/fetchTpexStock.mjs +127 -0
  89. package/src/fetchTwDataHoliday.mjs +164 -0
  90. package/src/fetchTwseMargin.mjs +181 -0
  91. package/src/fetchTwseStock.mjs +116 -0
  92. package/src/fetchTwseT86.mjs +136 -0
  93. package/src/fetchWithRetry.mjs +304 -0
  94. package/src/getOptFetch.mjs +113 -0
  95. package/src/htmlToMarkdown.mjs +96 -0
  96. package/src/isYmd.mjs +42 -0
  97. package/src/parseIntComma.mjs +28 -0
  98. package/src/safeFilename.mjs +30 -0
  99. package/src/toDatetimeUTC8.mjs +71 -0
  100. package/src/toRocDate.mjs +35 -0
  101. package/test/WDwdataHub.test.mjs +63 -0
  102. package/test/decodeEntities.test.mjs +42 -0
  103. package/test/extractDivContent.test.mjs +48 -0
  104. package/test/fetchAiNewsAggregator.test.mjs +79 -0
  105. package/test/fetchAisixiang.test.mjs +225 -0
  106. package/test/fetchCnyes.test.mjs +61 -0
  107. package/test/fetchGuancha.test.mjs +190 -0
  108. package/test/fetchHackerNews.test.mjs +78 -0
  109. package/test/fetchMoneydj.test.mjs +82 -0
  110. package/test/fetchMops.test.mjs +106 -0
  111. package/test/fetchRSS.test.mjs +85 -0
  112. package/test/fetchStatementdog.test.mjs +69 -0
  113. package/test/fetchTaifex.test.mjs +99 -0
  114. package/test/fetchTpex3insti.test.mjs +90 -0
  115. package/test/fetchTpexMargin.test.mjs +94 -0
  116. package/test/fetchTpexStock.test.mjs +68 -0
  117. package/test/fetchTwDataHoliday.test.mjs +100 -0
  118. package/test/fetchTwseMargin.test.mjs +81 -0
  119. package/test/fetchTwseStock.test.mjs +80 -0
  120. package/test/fetchTwseT86.test.mjs +76 -0
  121. package/test/fetchWithRetry.test.mjs +174 -0
  122. package/test/htmlToMarkdown.test.mjs +71 -0
  123. package/test/isYmd.test.mjs +56 -0
  124. package/test/parseIntComma.test.mjs +38 -0
  125. package/test/safeFilename.test.mjs +35 -0
  126. package/test/toDatetimeUTC8.test.mjs +65 -0
  127. package/test/toRocDate.test.mjs +38 -0
  128. package/test/tools/encodeBig5.mjs +80 -0
  129. package/test/tools/fixtures.mjs +539 -0
  130. package/test/tools/hasChrome.mjs +39 -0
  131. package/test/tools/serverForTest.mjs +397 -0
  132. package/toolg/addVersion.mjs +4 -0
  133. package/toolg/cleanFolder.mjs +5 -0
  134. package/toolg/gDistRollup.mjs +49 -0
  135. package/toolg/modifyReadme.mjs +4 -0
package/docs/fetchCnyes.mjs.html ADDED
@@ -0,0 +1,260 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchCnyes.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchCnyes.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import get from 'lodash-es/get.js'
49
+ import isestr from 'wsemi/src/isestr.mjs'
50
+ import isearr from 'wsemi/src/isearr.mjs'
51
+ import ispint from 'wsemi/src/ispint.mjs'
52
+ import isp0int from 'wsemi/src/isp0int.mjs'
53
+ import cint from 'wsemi/src/cint.mjs'
54
+ import delay from 'wsemi/src/delay.mjs'
55
+ import fetchWithRetry from './fetchWithRetry.mjs'
56
+ import getOptFetch from './getOptFetch.mjs'
57
+ import toDatetimeUTC8 from './toDatetimeUTC8.mjs'
58
+
59
+
60
+ //鉅亨網(Anue)台股新聞列表API
61
+ let API_URL = 'https://api.cnyes.com/media/api/v1/newslist/category/tw_stock'
62
+
63
+
64
+ //鉅亨網新聞內容頁網址前綴
65
+ let NEWS_URL = 'https://news.cnyes.com/news/id'
66
+
67
+
68
+ //本函數之預設抓取設定
69
+ let DFLT = {
70
+ timeout: 30000,
71
+ maxRetries: 10,
72
+ baseDelayMs: 5000,
73
+ maxDelayMs: 30000,
74
+ retryStatus: [403],
75
+ label: 'fetch-cnyes',
76
+ }
77
+
78
+
79
+ //預設值
80
+ let DEFAULT_PAGE_DELAY_MS = 500
81
+ let DEFAULT_TARGET_TOTAL = 100
82
+ let DEFAULT_PAGE_LIMIT = 30
83
+ let DEFAULT_DAYS_BACK = 10
84
+
85
+
86
+ /**
87
+ * 抓取鉅亨網台股新聞
88
+ *
89
+ * 逐頁抓取鉅亨網tw_stock分類新聞直至達到targetTotal或無更多資料, 頁間預設間隔500毫秒以降低限流風險;
90
+ * 時間欄位一律轉為UTC+8之'YYYY-MM-DD HH:mm:ss'字串, 單筆publishAt缺漏或異常時該筆time退回空字串並記錄, 不影響其餘新聞
91
+ *
92
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
93
+ * @param {String} [opt.url='https://api.cnyes.com/media/api/v1/newslist/category/tw_stock'] 輸入新聞列表API網址字串，供測試或改指向鏡像時覆寫
94
+ * @param {Integer} [opt.targetTotal=100] 輸入目標新聞則數正整數，預設100
95
+ * @param {Integer} [opt.pageLimit=30] 輸入單頁請求則數正整數，預設30
96
+ * @param {Integer} [opt.daysBack=10] 輸入回溯天數正整數，預設10
97
+ * @param {Integer} [opt.pageDelayMs=500] 輸入頁間延遲毫秒整數，預設500
98
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
99
+ * @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數，含初始共執行maxRetries+1次，預設10
100
+ * @param {Integer} [opt.baseDelayMs=5000] 輸入重試之線性退避基礎毫秒整數，預設5000
101
+ * @param {Integer} [opt.maxDelayMs=30000] 輸入重試之線性退避上限毫秒整數，預設30000
102
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
103
+ * @returns {Promise} 回傳Promise，resolve回傳新聞物件陣列，各物件為{time,title,link}，抓取失敗時reject回傳錯誤物件
104
+ * @example
105
+ *
106
+ * import fetchCnyes from './src/fetchCnyes.mjs'
107
+ *
108
+ * let test = async () => {
109
+ *
110
+ * let rs = await fetchCnyes({ targetTotal: 30 })
111
+ * console.log(rs.length, rs[0])
112
+ * // => 30 { time: '2026-08-07 09:02:03', title: '...', link: 'https://news.cnyes.com/news/id/1234567' }
113
+ *
114
+ * }
115
+ * await test()
116
+ * .catch((err) => {
117
+ * console.log(err)
118
+ * })
119
+ *
120
+ */
121
+ async function fetchCnyes(opt = {}) {
122
+
123
+ //url
124
+ let url = get(opt, 'url')
125
+ if (!isestr(url)) {
126
+ url = API_URL
127
+ }
128
+
129
+ //targetTotal
130
+ let targetTotal = get(opt, 'targetTotal')
131
+ if (!ispint(targetTotal)) {
132
+ targetTotal = DEFAULT_TARGET_TOTAL
133
+ }
134
+ else {
135
+ targetTotal = cint(targetTotal)
136
+ }
137
+
138
+ //pageLimit
139
+ let pageLimit = get(opt, 'pageLimit')
140
+ if (!ispint(pageLimit)) {
141
+ pageLimit = DEFAULT_PAGE_LIMIT
142
+ }
143
+ else {
144
+ pageLimit = cint(pageLimit)
145
+ }
146
+
147
+ //daysBack
148
+ let daysBack = get(opt, 'daysBack')
149
+ if (!ispint(daysBack)) {
150
+ daysBack = DEFAULT_DAYS_BACK
151
+ }
152
+ else {
153
+ daysBack = cint(daysBack)
154
+ }
155
+
156
+ //pageDelayMs
157
+ let pageDelayMs = get(opt, 'pageDelayMs')
158
+ if (!isp0int(pageDelayMs)) {
159
+ pageDelayMs = DEFAULT_PAGE_DELAY_MS
160
+ }
161
+ else {
162
+ pageDelayMs = cint(pageDelayMs)
163
+ }
164
+
165
+ //optFetch
166
+ let optFetch = getOptFetch(opt, DFLT)
167
+ let showLog = optFetch.showLog
168
+
169
+ //now, startAt
170
+ let now = Math.floor(Date.now() / 1000)
171
+ let startAt = now - 86400 * daysBack
172
+
173
+ if (showLog) {
174
+ console.log('Starting to fetch Anue (tw_stock) news...')
175
+ }
176
+
177
+ let allItems = []
178
+ let page = 1
179
+ while (allItems.length &lt; targetTotal) {
180
+
181
+ //qs
182
+ let qs = new URLSearchParams({
183
+ page: String(page),
184
+ limit: String(pageLimit),
185
+ isCategoryHeadline: '1',
186
+ startAt: String(startAt),
187
+ endAt: String(now),
188
+ })
189
+
190
+ //data
191
+ let data = await fetchWithRetry(`${url}?${qs.toString()}`, optFetch)
192
+
193
+ //items
194
+ let items = get(data, 'items.data')
195
+ if (!isearr(items)) {
196
+ if (showLog) {
197
+ console.log('No more items found.')
198
+ }
199
+ break
200
+ }
201
+
202
+ allItems = allItems.concat(items)
203
+ if (showLog) {
204
+ console.log(`Page ${page}: Fetched ${items.length} items. Total so far: ${allItems.length}`)
205
+ }
206
+ page++
207
+
208
+ await delay(pageDelayMs)
209
+
210
+ }
211
+
212
+ //finalItems
213
+ let finalItems = allItems.slice(0, targetTotal)
214
+ if (showLog) {
215
+ console.log(`Total items collected: ${finalItems.length}`)
216
+ }
217
+
218
+ return finalItems.map((item) => {
219
+
220
+ //time, 單筆publishAt缺漏或異常時退回空字串並記錄, 保留title與link避免拖垮其餘正常新聞
221
+ let time = toDatetimeUTC8(get(item, 'publishAt', 0))
222
+ if (time === '' &amp;&amp; showLog) {
223
+ console.warn(`[fetch-cnyes] 時間格式化失敗(newsId=${get(item, 'newsId', '')}, publishAt=${get(item, 'publishAt', '')}) — 該筆time退回空字串`)
224
+ }
225
+
226
+ return {
227
+ time,
228
+ title: get(item, 'title', ''),
229
+ link: `${NEWS_URL}/${get(item, 'newsId', '')}`,
230
+ }
231
+ })
232
+ }
233
+
234
+
235
+ export default fetchCnyes
236
+ </code></pre>
237
+ </article>
238
+ </section>
239
+
240
+
241
+
242
+
243
+
244
+
245
+ </div>
246
+
247
+ <br class="clear">
248
+
249
+ <footer>
250
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
251
+ </footer>
252
+
253
+ <script>prettyPrint();</script>
254
+ <script src="scripts/polyfill.js"></script>
255
+ <script src="scripts/linenumber.js"></script>
256
+
257
+
258
+
259
+ </body>
260
+ </html>