w-dwdata-hub 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/.editorconfig +9 -0
  2. package/.eslintignore +3 -0
  3. package/.eslintrc.js +55 -0
  4. package/.jsdoc +25 -0
  5. package/LICENSE +21 -0
  6. package/README.md +168 -0
  7. package/SECURITY.md +5 -0
  8. package/babel.config.js +16 -0
  9. package/dist/w-dwdata-hub.umd.js +7 -0
  10. package/dist/w-dwdata-hub.umd.js.map +1 -0
  11. package/docs/WDwdataHub.mjs.html +136 -0
  12. package/docs/decodeEntities.mjs.html +129 -0
  13. package/docs/extractDivContent.mjs.html +147 -0
  14. package/docs/fetchAiNewsAggregator.mjs.html +155 -0
  15. package/docs/fetchAisixiang.mjs.html +1038 -0
  16. package/docs/fetchCnyes.mjs.html +260 -0
  17. package/docs/fetchGuancha.mjs.html +1007 -0
  18. package/docs/fetchHackerNews.mjs.html +189 -0
  19. package/docs/fetchMoneydj.mjs.html +251 -0
  20. package/docs/fetchMops.mjs.html +385 -0
  21. package/docs/fetchRSS.mjs.html +160 -0
  22. package/docs/fetchStatementdog.mjs.html +213 -0
  23. package/docs/fetchTaifex.mjs.html +513 -0
  24. package/docs/fetchTpex3insti.mjs.html +278 -0
  25. package/docs/fetchTpexMargin.mjs.html +248 -0
  26. package/docs/fetchTpexStock.mjs.html +199 -0
  27. package/docs/fetchTwDataHoliday.mjs.html +236 -0
  28. package/docs/fetchTwseMargin.mjs.html +253 -0
  29. package/docs/fetchTwseStock.mjs.html +188 -0
  30. package/docs/fetchTwseT86.mjs.html +208 -0
  31. package/docs/fetchWithRetry.mjs.html +376 -0
  32. package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
  33. package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
  34. package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
  35. package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
  36. package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
  37. package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
  38. package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
  39. package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
  40. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
  41. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
  42. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
  43. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
  44. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
  45. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
  46. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
  47. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
  48. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
  49. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
  50. package/docs/getOptFetch.mjs.html +185 -0
  51. package/docs/global.html +22649 -0
  52. package/docs/htmlToMarkdown.mjs.html +168 -0
  53. package/docs/index.html +84 -0
  54. package/docs/isYmd.mjs.html +114 -0
  55. package/docs/parseIntComma.mjs.html +100 -0
  56. package/docs/safeFilename.mjs.html +102 -0
  57. package/docs/scripts/collapse.js +39 -0
  58. package/docs/scripts/commonNav.js +28 -0
  59. package/docs/scripts/linenumber.js +25 -0
  60. package/docs/scripts/nav.js +12 -0
  61. package/docs/scripts/polyfill.js +4 -0
  62. package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
  63. package/docs/scripts/prettify/lang-css.js +2 -0
  64. package/docs/scripts/prettify/prettify.js +28 -0
  65. package/docs/scripts/search.js +99 -0
  66. package/docs/styles/jsdoc.css +776 -0
  67. package/docs/styles/prettify.css +80 -0
  68. package/docs/toDatetimeUTC8.mjs.html +143 -0
  69. package/docs/toRocDate.mjs.html +107 -0
  70. package/g.mjs +73 -0
  71. package/package.json +31 -0
  72. package/script.txt +17 -0
  73. package/src/WDwdataHub.mjs +64 -0
  74. package/src/decodeEntities.mjs +57 -0
  75. package/src/extractDivContent.mjs +75 -0
  76. package/src/fetchAiNewsAggregator.mjs +83 -0
  77. package/src/fetchAisixiang.mjs +966 -0
  78. package/src/fetchCnyes.mjs +188 -0
  79. package/src/fetchGuancha.mjs +935 -0
  80. package/src/fetchHackerNews.mjs +117 -0
  81. package/src/fetchMoneydj.mjs +179 -0
  82. package/src/fetchMops.mjs +313 -0
  83. package/src/fetchRSS.mjs +88 -0
  84. package/src/fetchStatementdog.mjs +141 -0
  85. package/src/fetchTaifex.mjs +441 -0
  86. package/src/fetchTpex3insti.mjs +206 -0
  87. package/src/fetchTpexMargin.mjs +176 -0
  88. package/src/fetchTpexStock.mjs +127 -0
  89. package/src/fetchTwDataHoliday.mjs +164 -0
  90. package/src/fetchTwseMargin.mjs +181 -0
  91. package/src/fetchTwseStock.mjs +116 -0
  92. package/src/fetchTwseT86.mjs +136 -0
  93. package/src/fetchWithRetry.mjs +304 -0
  94. package/src/getOptFetch.mjs +113 -0
  95. package/src/htmlToMarkdown.mjs +96 -0
  96. package/src/isYmd.mjs +42 -0
  97. package/src/parseIntComma.mjs +28 -0
  98. package/src/safeFilename.mjs +30 -0
  99. package/src/toDatetimeUTC8.mjs +71 -0
  100. package/src/toRocDate.mjs +35 -0
  101. package/test/WDwdataHub.test.mjs +63 -0
  102. package/test/decodeEntities.test.mjs +42 -0
  103. package/test/extractDivContent.test.mjs +48 -0
  104. package/test/fetchAiNewsAggregator.test.mjs +79 -0
  105. package/test/fetchAisixiang.test.mjs +225 -0
  106. package/test/fetchCnyes.test.mjs +61 -0
  107. package/test/fetchGuancha.test.mjs +190 -0
  108. package/test/fetchHackerNews.test.mjs +78 -0
  109. package/test/fetchMoneydj.test.mjs +82 -0
  110. package/test/fetchMops.test.mjs +106 -0
  111. package/test/fetchRSS.test.mjs +85 -0
  112. package/test/fetchStatementdog.test.mjs +69 -0
  113. package/test/fetchTaifex.test.mjs +99 -0
  114. package/test/fetchTpex3insti.test.mjs +90 -0
  115. package/test/fetchTpexMargin.test.mjs +94 -0
  116. package/test/fetchTpexStock.test.mjs +68 -0
  117. package/test/fetchTwDataHoliday.test.mjs +100 -0
  118. package/test/fetchTwseMargin.test.mjs +81 -0
  119. package/test/fetchTwseStock.test.mjs +80 -0
  120. package/test/fetchTwseT86.test.mjs +76 -0
  121. package/test/fetchWithRetry.test.mjs +174 -0
  122. package/test/htmlToMarkdown.test.mjs +71 -0
  123. package/test/isYmd.test.mjs +56 -0
  124. package/test/parseIntComma.test.mjs +38 -0
  125. package/test/safeFilename.test.mjs +35 -0
  126. package/test/toDatetimeUTC8.test.mjs +65 -0
  127. package/test/toRocDate.test.mjs +38 -0
  128. package/test/tools/encodeBig5.mjs +80 -0
  129. package/test/tools/fixtures.mjs +539 -0
  130. package/test/tools/hasChrome.mjs +39 -0
  131. package/test/tools/serverForTest.mjs +397 -0
  132. package/toolg/addVersion.mjs +4 -0
  133. package/toolg/cleanFolder.mjs +5 -0
  134. package/toolg/gDistRollup.mjs +49 -0
  135. package/toolg/modifyReadme.mjs +4 -0
package/docs/fetchStatementdog.mjs.html ADDED
@@ -0,0 +1,213 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchStatementdog.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchStatementdog.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import * as cheerio from 'cheerio'
49
+ import get from 'lodash-es/get.js'
50
+ import isestr from 'wsemi/src/isestr.mjs'
51
+ import fetchWithRetry from './fetchWithRetry.mjs'
52
+ import getOptFetch from './getOptFetch.mjs'
53
+
54
+
55
+ //財報狗最新新聞頁
56
+ let PAGE_URL = 'https://statementdog.com/news/latest'
57
+
58
+
59
+ //本函數之預設抓取設定
60
+ let DFLT = {
61
+ responseType: 'text',
62
+ timeout: 30000,
63
+ maxRetries: 10,
64
+ baseDelayMs: 5000,
65
+ maxDelayMs: 30000,
66
+ retryStatus: [403],
67
+ label: 'fetch-statementdog',
68
+ }
69
+
70
+
71
+ //主要selector, 對應財報狗現行版型
72
+ let PRIMARY_ITEM = '.statementdog-news-list-item'
73
+ let PRIMARY_TITLE = '.statementdog-news-list-item-title'
74
+ let PRIMARY_LINK = '.statementdog-news-list-item-link'
75
+ let PRIMARY_DATE = '.statementdog-news-list-item-date'
76
+
77
+
78
+ //備援selector, 財報狗改版導致主要selector失效時改用之通用結構
79
+ let FALLBACK_ITEM = 'article, .news-item, [class*="news"][class*="item"]'
80
+ let FALLBACK_TITLE = 'h2, h3, [class*="title"]'
81
+ let FALLBACK_LINK = 'a[href]'
82
+ let FALLBACK_DATE = 'time, [class*="date"], [class*="time"]'
83
+
84
+
85
+ /**
86
+ * 抓取財報狗最新新聞
87
+ *
88
+ * 以cheerio解析財報狗最新新聞頁, 主要selector失效時自動改用通用結構備援selector;
89
+ * 相對路徑連結會補上來源網址之origin, 抓取到0筆時視為版型改變而拋出錯誤(不重試)
90
+ *
91
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
92
+ * @param {String} [opt.url='https://statementdog.com/news/latest'] 輸入新聞頁網址字串，供測試或改指向鏡像時覆寫
93
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
94
+ * @param {Integer} [opt.maxRetries=10] 輸入最大重試次數整數，含初始共執行maxRetries+1次，預設10
95
+ * @param {Integer} [opt.baseDelayMs=5000] 輸入重試之線性退避基礎毫秒整數，預設5000
96
+ * @param {Integer} [opt.maxDelayMs=30000] 輸入重試之線性退避上限毫秒整數，預設30000
97
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
98
+ * @returns {Promise} 回傳Promise，resolve回傳新聞物件陣列，各物件為{time,title,link}，抓取失敗或解析到0筆時reject回傳錯誤物件
99
+ * @example
100
+ *
101
+ * import fetchStatementdog from './src/fetchStatementdog.mjs'
102
+ *
103
+ * let test = async () => {
104
+ *
105
+ * let rs = await fetchStatementdog()
106
+ * console.log(rs.length, rs[0])
107
+ * // => 30 { time: '2026-08-07', title: '...', link: 'https://statementdog.com/news/...' }
108
+ *
109
+ * }
110
+ * await test()
111
+ * .catch((err) => {
112
+ * console.log(err)
113
+ * })
114
+ *
115
+ */
116
+ async function fetchStatementdog(opt = {}) {
117
+
118
+ //url
119
+ let url = get(opt, 'url')
120
+ if (!isestr(url)) {
121
+ url = PAGE_URL
122
+ }
123
+
124
+ //optFetch
125
+ let optFetch = getOptFetch(opt, DFLT)
126
+ let showLog = optFetch.showLog
127
+
128
+ if (showLog) {
129
+ console.log(`Fetching ${url}...`)
130
+ }
131
+
132
+ //html
133
+ let html = await fetchWithRetry(url, optFetch)
134
+
135
+ //origin, 由來源網址推得, 供補齊相對路徑連結
136
+ let origin = new URL(url).origin
137
+
138
+ //$
139
+ let $ = cheerio.load(html)
140
+
141
+ //usePrimary
142
+ let usePrimary = $(PRIMARY_ITEM).length > 0
143
+ let itemSel = usePrimary ? PRIMARY_ITEM : FALLBACK_ITEM
144
+ let titleSel = usePrimary ? PRIMARY_TITLE : FALLBACK_TITLE
145
+ let linkSel = usePrimary ? PRIMARY_LINK : FALLBACK_LINK
146
+ let dateSel = usePrimary ? PRIMARY_DATE : FALLBACK_DATE
147
+ if (!usePrimary &amp;&amp; showLog) {
148
+ console.warn('主要 CSS selector 未匹配，嘗試 fallback selector...')
149
+ }
150
+
151
+ //newsItems
152
+ let newsItems = []
153
+ $(itemSel).each((index, element) => {
154
+
155
+ let titleElement = $(element).find(titleSel)
156
+ let linkElement = usePrimary ? $(element).find(linkSel) : $(element).find(linkSel).first()
157
+ let timeElement = $(element).find(dateSel)
158
+
159
+ let title = titleElement.text().trim()
160
+ let link = linkElement.attr('href')
161
+ let time = timeElement.text().trim()
162
+
163
+ if (!title || !link) {
164
+ return
165
+ }
166
+
167
+ //補齊相對路徑連結
168
+ if (!link.startsWith('http')) {
169
+ link = `${origin}${link.startsWith('/') ? '' : '/'}${link}`
170
+ }
171
+
172
+ newsItems.push({ time, title, link })
173
+ })
174
+
175
+ if (showLog) {
176
+ console.log(`Extracted News Items: ${newsItems.length}`)
177
+ }
178
+
179
+ //check, 0筆代表版型改變或selector失效, 屬非暫時性錯誤故不重試
180
+ if (newsItems.length === 0) {
181
+ throw new Error('抓取到 0 筆新聞，可能是頁面結構改變或 selector 失效，請確認財報狗頁面是否正常。')
182
+ }
183
+
184
+ return newsItems
185
+ }
186
+
187
+
188
+ export default fetchStatementdog
189
+ </code></pre>
190
+ </article>
191
+ </section>
192
+
193
+
194
+
195
+
196
+
197
+
198
+ </div>
199
+
200
+ <br class="clear">
201
+
202
+ <footer>
203
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
204
+ </footer>
205
+
206
+ <script>prettyPrint();</script>
207
+ <script src="scripts/polyfill.js"></script>
208
+ <script src="scripts/linenumber.js"></script>
209
+
210
+
211
+
212
+ </body>
213
+ </html>