w-dwdata-hub 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/.editorconfig +9 -0
  2. package/.eslintignore +3 -0
  3. package/.eslintrc.js +55 -0
  4. package/.jsdoc +25 -0
  5. package/LICENSE +21 -0
  6. package/README.md +168 -0
  7. package/SECURITY.md +5 -0
  8. package/babel.config.js +16 -0
  9. package/dist/w-dwdata-hub.umd.js +7 -0
  10. package/dist/w-dwdata-hub.umd.js.map +1 -0
  11. package/docs/WDwdataHub.mjs.html +136 -0
  12. package/docs/decodeEntities.mjs.html +129 -0
  13. package/docs/extractDivContent.mjs.html +147 -0
  14. package/docs/fetchAiNewsAggregator.mjs.html +155 -0
  15. package/docs/fetchAisixiang.mjs.html +1038 -0
  16. package/docs/fetchCnyes.mjs.html +260 -0
  17. package/docs/fetchGuancha.mjs.html +1007 -0
  18. package/docs/fetchHackerNews.mjs.html +189 -0
  19. package/docs/fetchMoneydj.mjs.html +251 -0
  20. package/docs/fetchMops.mjs.html +385 -0
  21. package/docs/fetchRSS.mjs.html +160 -0
  22. package/docs/fetchStatementdog.mjs.html +213 -0
  23. package/docs/fetchTaifex.mjs.html +513 -0
  24. package/docs/fetchTpex3insti.mjs.html +278 -0
  25. package/docs/fetchTpexMargin.mjs.html +248 -0
  26. package/docs/fetchTpexStock.mjs.html +199 -0
  27. package/docs/fetchTwDataHoliday.mjs.html +236 -0
  28. package/docs/fetchTwseMargin.mjs.html +253 -0
  29. package/docs/fetchTwseStock.mjs.html +188 -0
  30. package/docs/fetchTwseT86.mjs.html +208 -0
  31. package/docs/fetchWithRetry.mjs.html +376 -0
  32. package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
  33. package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
  34. package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
  35. package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
  36. package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
  37. package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
  38. package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
  39. package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
  40. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
  41. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
  42. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
  43. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
  44. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
  45. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
  46. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
  47. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
  48. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
  49. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
  50. package/docs/getOptFetch.mjs.html +185 -0
  51. package/docs/global.html +22649 -0
  52. package/docs/htmlToMarkdown.mjs.html +168 -0
  53. package/docs/index.html +84 -0
  54. package/docs/isYmd.mjs.html +114 -0
  55. package/docs/parseIntComma.mjs.html +100 -0
  56. package/docs/safeFilename.mjs.html +102 -0
  57. package/docs/scripts/collapse.js +39 -0
  58. package/docs/scripts/commonNav.js +28 -0
  59. package/docs/scripts/linenumber.js +25 -0
  60. package/docs/scripts/nav.js +12 -0
  61. package/docs/scripts/polyfill.js +4 -0
  62. package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
  63. package/docs/scripts/prettify/lang-css.js +2 -0
  64. package/docs/scripts/prettify/prettify.js +28 -0
  65. package/docs/scripts/search.js +99 -0
  66. package/docs/styles/jsdoc.css +776 -0
  67. package/docs/styles/prettify.css +80 -0
  68. package/docs/toDatetimeUTC8.mjs.html +143 -0
  69. package/docs/toRocDate.mjs.html +107 -0
  70. package/g.mjs +73 -0
  71. package/package.json +31 -0
  72. package/script.txt +17 -0
  73. package/src/WDwdataHub.mjs +64 -0
  74. package/src/decodeEntities.mjs +57 -0
  75. package/src/extractDivContent.mjs +75 -0
  76. package/src/fetchAiNewsAggregator.mjs +83 -0
  77. package/src/fetchAisixiang.mjs +966 -0
  78. package/src/fetchCnyes.mjs +188 -0
  79. package/src/fetchGuancha.mjs +935 -0
  80. package/src/fetchHackerNews.mjs +117 -0
  81. package/src/fetchMoneydj.mjs +179 -0
  82. package/src/fetchMops.mjs +313 -0
  83. package/src/fetchRSS.mjs +88 -0
  84. package/src/fetchStatementdog.mjs +141 -0
  85. package/src/fetchTaifex.mjs +441 -0
  86. package/src/fetchTpex3insti.mjs +206 -0
  87. package/src/fetchTpexMargin.mjs +176 -0
  88. package/src/fetchTpexStock.mjs +127 -0
  89. package/src/fetchTwDataHoliday.mjs +164 -0
  90. package/src/fetchTwseMargin.mjs +181 -0
  91. package/src/fetchTwseStock.mjs +116 -0
  92. package/src/fetchTwseT86.mjs +136 -0
  93. package/src/fetchWithRetry.mjs +304 -0
  94. package/src/getOptFetch.mjs +113 -0
  95. package/src/htmlToMarkdown.mjs +96 -0
  96. package/src/isYmd.mjs +42 -0
  97. package/src/parseIntComma.mjs +28 -0
  98. package/src/safeFilename.mjs +30 -0
  99. package/src/toDatetimeUTC8.mjs +71 -0
  100. package/src/toRocDate.mjs +35 -0
  101. package/test/WDwdataHub.test.mjs +63 -0
  102. package/test/decodeEntities.test.mjs +42 -0
  103. package/test/extractDivContent.test.mjs +48 -0
  104. package/test/fetchAiNewsAggregator.test.mjs +79 -0
  105. package/test/fetchAisixiang.test.mjs +225 -0
  106. package/test/fetchCnyes.test.mjs +61 -0
  107. package/test/fetchGuancha.test.mjs +190 -0
  108. package/test/fetchHackerNews.test.mjs +78 -0
  109. package/test/fetchMoneydj.test.mjs +82 -0
  110. package/test/fetchMops.test.mjs +106 -0
  111. package/test/fetchRSS.test.mjs +85 -0
  112. package/test/fetchStatementdog.test.mjs +69 -0
  113. package/test/fetchTaifex.test.mjs +99 -0
  114. package/test/fetchTpex3insti.test.mjs +90 -0
  115. package/test/fetchTpexMargin.test.mjs +94 -0
  116. package/test/fetchTpexStock.test.mjs +68 -0
  117. package/test/fetchTwDataHoliday.test.mjs +100 -0
  118. package/test/fetchTwseMargin.test.mjs +81 -0
  119. package/test/fetchTwseStock.test.mjs +80 -0
  120. package/test/fetchTwseT86.test.mjs +76 -0
  121. package/test/fetchWithRetry.test.mjs +174 -0
  122. package/test/htmlToMarkdown.test.mjs +71 -0
  123. package/test/isYmd.test.mjs +56 -0
  124. package/test/parseIntComma.test.mjs +38 -0
  125. package/test/safeFilename.test.mjs +35 -0
  126. package/test/toDatetimeUTC8.test.mjs +65 -0
  127. package/test/toRocDate.test.mjs +38 -0
  128. package/test/tools/encodeBig5.mjs +80 -0
  129. package/test/tools/fixtures.mjs +539 -0
  130. package/test/tools/hasChrome.mjs +39 -0
  131. package/test/tools/serverForTest.mjs +397 -0
  132. package/toolg/addVersion.mjs +4 -0
  133. package/toolg/cleanFolder.mjs +5 -0
  134. package/toolg/gDistRollup.mjs +49 -0
  135. package/toolg/modifyReadme.mjs +4 -0
package/docs/fetchAisixiang.mjs.html ADDED
@@ -0,0 +1,1038 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchAisixiang.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchAisixiang.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import get from 'lodash-es/get.js'
49
+ import isestr from 'wsemi/src/isestr.mjs'
50
+ import ispint from 'wsemi/src/ispint.mjs'
51
+ import isp0int from 'wsemi/src/isp0int.mjs'
52
+ import cint from 'wsemi/src/cint.mjs'
53
+ import delay from 'wsemi/src/delay.mjs'
54
+ import fetchWithRetry from './fetchWithRetry.mjs'
55
+ import getOptFetch from './getOptFetch.mjs'
56
+ import decodeEntities from './decodeEntities.mjs'
57
+ import htmlToMarkdown from './htmlToMarkdown.mjs'
58
+ import extractDivContent from './extractDivContent.mjs'
59
+ import safeFilename from './safeFilename.mjs'
60
+
61
+
62
+ //愛思想網站根網址
63
+ let BASE_URL = 'https://www.aisixiang.com'
64
+
65
+
66
+ //愛思想對User-Agent較敏感, 故固定送出桌面瀏覽器標頭
67
+ let USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
68
+ '(KHTML, like Gecko) Chrome/120.0 Safari/537.36'
69
+
70
+
71
+ //頁間延遲毫秒, 降低被封鎖機率
72
+ let PAGE_DELAY_MS = 1000
73
+
74
+
75
+ //翻頁安全上限, 50頁×30筆=1500筆
76
+ let MAX_PAGES = 50
77
+
78
+
79
+ //本模組之預設抓取設定
80
+ let DFLT = {
81
+ responseType: 'text',
82
+ headers: { 'User-Agent': USER_AGENT, 'Accept-Language': 'zh-CN,zh;q=0.9' },
83
+ timeout: 30000,
84
+ maxRetries: 5,
85
+ baseDelayMs: 5000,
86
+ maxDelayMs: 30000,
87
+ label: 'fetch-aisixiang',
88
+ }
89
+
90
+
91
+ /**
92
+ * 正規化本模組共用設定
93
+ *
94
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
95
+ * @returns {Object} 回傳已正規化之設定物件，內含baseUrl、showLog與optFetch
96
+ */
97
+ function getCfg(opt = {}) {
98
+
99
+ //baseUrl
100
+ let baseUrl = get(opt, 'baseUrl')
101
+ if (!isestr(baseUrl)) {
102
+ baseUrl = BASE_URL
103
+ }
104
+ baseUrl = baseUrl.replace(/\/+$/, '')
105
+
106
+ //optFetch
107
+ let optFetch = getOptFetch(opt, DFLT)
108
+
109
+ //pageDelayMs
110
+ let pageDelayMs = get(opt, 'pageDelayMs')
111
+ if (!isp0int(pageDelayMs)) {
112
+ pageDelayMs = PAGE_DELAY_MS
113
+ }
114
+ else {
115
+ pageDelayMs = cint(pageDelayMs)
116
+ }
117
+
118
+ //maxPages
119
+ let maxPages = get(opt, 'maxPages')
120
+ if (!ispint(maxPages)) {
121
+ maxPages = MAX_PAGES
122
+ }
123
+ else {
124
+ maxPages = Math.min(cint(maxPages), MAX_PAGES)
125
+ }
126
+
127
+ return { baseUrl, showLog: optFetch.showLog, pageDelayMs, maxPages, optFetch }
128
+ }
129
+
130
+
131
+ /**
132
+ * 由HTML抽出title標籤內容
133
+ *
134
+ * @param {String} html 輸入HTML字串
135
+ * @returns {String|null} 回傳標題字串，無title標籤時回傳null
136
+ */
137
+ function extractTitle(html) {
138
+ let m = html.match(/&lt;title>([\s\S]*?)&lt;\/title>/i)
139
+ return m ? decodeEntities(m[1].trim()) : null
140
+ }
141
+
142
+
143
+ /**
144
+ * 解析作者專欄頁之文章清單
145
+ *
146
+ * 專欄頁以h3標記分類(论文、时评、随笔等), 分類區塊內為/data/&lt;aid>.html之文章連結
147
+ *
148
+ * @param {String} html 輸入作者專欄頁HTML字串
149
+ * @param {String} baseUrl 輸入網站根網址字串
150
+ * @returns {Array} 回傳文章物件陣列，各物件為{aid,url,title,category}
151
+ */
152
+ function parseAuthorColumnPage(html, baseUrl) {
153
+
154
+ let cats = ['论文', '时评', '随笔', '著作', '演讲', '读书', '访谈', '未分类']
155
+ let positions = []
156
+ for (let cat of cats) {
157
+ let re = new RegExp(`&lt;h3>${cat}&lt;\\/h3>`, 'g')
158
+ let m
159
+ while ((m = re.exec(html)) !== null) {
160
+ positions.push({ cat, pos: m.index })
161
+ }
162
+ }
163
+ positions.sort((a, b) => a.pos - b.pos)
164
+ positions.push({ cat: null, pos: html.length })
165
+
166
+ let items = []
167
+ for (let i = 0; i &lt; positions.length - 1; i++) {
168
+ let { cat, pos } = positions[i]
169
+ let block = html.slice(pos, positions[i + 1].pos)
170
+ let linkRe = /&lt;a\s+href="\/data\/(\d+)\.html"[^>]*>([\s\S]*?)&lt;\/a>/g
171
+ let m
172
+ while ((m = linkRe.exec(block)) !== null) {
173
+ let title = m[2].replace(/&lt;[^>]+>/g, '').trim()
174
+ if (!title) {
175
+ continue
176
+ }
177
+ items.push({
178
+ aid: m[1],
179
+ url: `${baseUrl}/data/${m[1]}.html`,
180
+ title,
181
+ category: cat,
182
+ })
183
+ }
184
+ }
185
+
186
+ return items
187
+ }
188
+
189
+
190
+ /**
191
+ * 解析搜尋結果頁之文章清單
192
+ *
193
+ * 精確切出search_list區塊再解析, 避免0結果時誤抓側邊欄連結
194
+ *
195
+ * @param {String} html 輸入搜尋結果頁HTML字串
196
+ * @param {String} baseUrl 輸入網站根網址字串
197
+ * @returns {Array} 回傳文章物件陣列，各物件為{aid,url,title,author}
198
+ */
199
+ function parseSearchResults(html, baseUrl) {
200
+
201
+ let block = extractDivContent(html, 'search_list')
202
+ if (block === null) {
203
+ return []
204
+ }
205
+
206
+ let items = []
207
+ let seen = new Set()
208
+ let re = /href="\/data\/(\d+)\.html"[^>]*title="([^"]+)"/g
209
+ let m
210
+ while ((m = re.exec(block)) !== null) {
211
+ if (seen.has(m[1])) {
212
+ continue
213
+ }
214
+ seen.add(m[1])
215
+ let raw = m[2]
216
+ let idx = raw.indexOf('：')
217
+ items.push({
218
+ aid: m[1],
219
+ url: `${baseUrl}/data/${m[1]}.html`,
220
+ title: idx > 0 ? raw.slice(idx + 1).trim() : raw.trim(),
221
+ author: idx > 0 ? raw.slice(0, idx).trim() : '',
222
+ })
223
+ }
224
+
225
+ return items
226
+ }
227
+
228
+
229
+ /**
230
+ * 解析策展主題頁之文章清單
231
+ *
232
+ * 主題頁之連結無title屬性, 標題在anchor內文, 格式為&lt;a>作者：標題&lt;/a>
233
+ *
234
+ * @param {String} html 輸入主題頁HTML字串
235
+ * @param {String} baseUrl 輸入網站根網址字串
236
+ * @returns {Array} 回傳文章物件陣列，各物件為{aid,url,title,author}
237
+ */
238
+ function parseZhuantiArticles(html, baseUrl) {
239
+
240
+ let m = html.match(/([\s\S]+?)&lt;div class="list_page">/)
241
+ let block = m ? m[1] : html
242
+
243
+ let items = []
244
+ let seen = new Set()
245
+ let re = /&lt;a\s+href="\/data\/(\d+)\.html"[^>]*>([^&lt;]+)&lt;\/a>/g
246
+ let mm
247
+ while ((mm = re.exec(block)) !== null) {
248
+ if (seen.has(mm[1])) {
249
+ continue
250
+ }
251
+ seen.add(mm[1])
252
+ let raw = mm[2].trim()
253
+ let idx = raw.indexOf('：')
254
+ items.push({
255
+ aid: mm[1],
256
+ url: `${baseUrl}/data/${mm[1]}.html`,
257
+ title: idx > 0 ? raw.slice(idx + 1).trim() : raw,
258
+ author: idx > 0 ? raw.slice(0, idx).trim() : '',
259
+ })
260
+ }
261
+
262
+ return items
263
+ }
264
+
265
+
266
+ /**
267
+ * 由list_page區塊解析最大頁碼
268
+ *
269
+ * @param {String} html 輸入列表頁HTML字串
270
+ * @returns {Integer} 回傳總頁數整數，無分頁區塊時回傳1
271
+ */
272
+ function parseSearchTotalPages(html) {
273
+
274
+ let m = html.match(/&lt;div class="list_page">([\s\S]+?)&lt;\/div>/)
275
+ if (!m) {
276
+ return 1
277
+ }
278
+
279
+ let nums = [...m[1].matchAll(/page=(\d+)/g)].map((x) => Number(x[1]))
280
+
281
+ return nums.length ? Math.max(...nums) : 1
282
+ }
283
+
284
+
285
+ /**
286
+ * 自動翻頁抓取全部列表頁
287
+ *
288
+ * 由第1頁解析總頁數後逐頁抓取, 頁間延遲1秒; 該頁無新項目即停止翻頁, 並以MAX_PAGES為安全上限
289
+ *
290
+ * @param {Function} buildUrl 輸入回傳指定頁碼網址之函數
291
+ * @param {Function} parser 輸入解析單頁HTML為項目陣列之函數
292
+ * @param {Object} cfg 輸入已正規化之設定物件
293
+ * @returns {Promise} 回傳Promise，resolve回傳{items,total_pages,pages_fetched}
294
+ */
295
+ async function fetchAllSearchPages(buildUrl, parser, cfg) {
296
+
297
+ let all = []
298
+ let seen = new Set()
299
+ let totalPages = 1
300
+ let pagesFetched = 0
301
+
302
+ for (let page = 1; page &lt;= cfg.maxPages; page++) {
303
+
304
+ let url = buildUrl(page)
305
+ if (cfg.showLog) {
306
+ process.stderr.write(`[info] fetching page ${page}${totalPages > 1 ? `/${totalPages}` : ''} ...\n`)
307
+ }
308
+ let html = await fetchWithRetry(url, cfg.optFetch)
309
+ pagesFetched++
310
+
311
+ if (page === 1) {
312
+ totalPages = parseSearchTotalPages(html)
313
+ }
314
+
315
+ let items = parser(html, cfg.baseUrl)
316
+ let fresh = items.filter((it) => !seen.has(it.aid))
317
+ if (fresh.length === 0) {
318
+ if (cfg.showLog) {
319
+ process.stderr.write(`[info] page ${page} 無新項目，停止翻頁\n`)
320
+ }
321
+ break
322
+ }
323
+
324
+ for (let it of fresh) {
325
+ seen.add(it.aid)
326
+ }
327
+ all.push(...fresh)
328
+
329
+ if (page >= totalPages) {
330
+ break
331
+ }
332
+ await delay(cfg.pageDelayMs)
333
+
334
+ }
335
+
336
+ return { items: all, total_pages: totalPages, pages_fetched: pagesFetched }
337
+ }
338
+
339
+
340
+ /**
341
+ * 抓取愛思想專欄作者清單
342
+ *
343
+ * 即時解析/thinktank/頁面, 回傳約960位專欄作者之slug與中文名
344
+ *
345
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
346
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
347
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
348
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
349
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
350
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
351
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
352
+ * @returns {Promise} 回傳Promise，resolve回傳作者物件陣列，各物件為{slug,name}
353
+ * @example
354
+ *
355
+ * import { fetchAuthorsList } from './src/fetchAisixiang.mjs'
356
+ *
357
+ * let test = async () => {
358
+ *
359
+ * let rs = await fetchAuthorsList()
360
+ * console.log(rs.length, rs[0])
361
+ * // => 963 { slug: 'gezhaoguang', name: '葛兆光' }
362
+ *
363
+ * }
364
+ * await test()
365
+ * .catch((err) => {
366
+ * console.log(err)
367
+ * })
368
+ *
369
+ */
370
+ async function fetchAuthorsList(opt = {}) {
371
+
372
+ let cfg = getCfg(opt)
373
+
374
+ let html = await fetchWithRetry(`${cfg.baseUrl}/thinktank/`, cfg.optFetch)
375
+
376
+ let matches = [...html.matchAll(/href="\/thinktank\/([^"]+?)\.html"[^>]*>([^&lt;]+)&lt;/g)]
377
+ let uniq = new Map()
378
+ for (let [, slug, rawName] of matches) {
379
+ let key = slug.toLowerCase()
380
+ if (uniq.has(key)) {
381
+ continue
382
+ }
383
+ let name = rawName.split(/[，、]/)[0].trim()
384
+ if (!name || name.length > 30) {
385
+ continue
386
+ }
387
+ uniq.set(key, { slug, name })
388
+ }
389
+
390
+ return [...uniq.values()]
391
+ }
392
+
393
+
394
+ /**
395
+ * 由作者清單查找指定中文名之作者
396
+ *
397
+ * 本函數不轉繁簡, 呼叫端須自行使用站方登錄字形(通常為簡體)
398
+ *
399
+ * @param {Array} authors 輸入作者物件陣列
400
+ * @param {String} name 輸入作者中文名字串
401
+ * @returns {Object|null} 回傳作者物件，查無時回傳null
402
+ * @example
403
+ *
404
+ * import { lookupAuthor } from './src/fetchAisixiang.mjs'
405
+ *
406
+ * console.log(lookupAuthor([{ slug: 'gezhaoguang', name: '葛兆光' }], '葛兆光'))
407
+ * // => { slug: 'gezhaoguang', name: '葛兆光' }
408
+ *
409
+ */
410
+ function lookupAuthor(authors, name) {
411
+
412
+ if (!Array.isArray(authors)) {
413
+ return null
414
+ }
415
+
416
+ return authors.find((a) => a.name === name) || null
417
+ }
418
+
419
+
420
+ /**
421
+ * 抓取愛思想策展主題清單
422
+ *
423
+ * 即時解析/zhuanti/頁面, 回傳約800個策展主題之id、名稱與分類(学科、事件、人物)
424
+ *
425
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
426
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
427
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
428
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
429
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
430
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
431
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
432
+ * @returns {Promise} 回傳Promise，resolve回傳主題物件陣列，各物件為{id,name,category}
433
+ * @example
434
+ *
435
+ * import { fetchTopicsList } from './src/fetchAisixiang.mjs'
436
+ *
437
+ * let test = async () => {
438
+ *
439
+ * let rs = await fetchTopicsList()
440
+ * console.log(rs.length, rs[0])
441
+ * // => 803 { id: '301', name: '大数据', category: '学科' }
442
+ *
443
+ * }
444
+ * await test()
445
+ * .catch((err) => {
446
+ * console.log(err)
447
+ * })
448
+ *
449
+ */
450
+ async function fetchTopicsList(opt = {}) {
451
+
452
+ let cfg = getCfg(opt)
453
+
454
+ let html = await fetchWithRetry(`${cfg.baseUrl}/zhuanti/`, cfg.optFetch)
455
+
456
+ let cats = [
457
+ { name: '学科', start: html.indexOf('&lt;h3>学科关键词&lt;/h3>') },
458
+ { name: '事件', start: html.indexOf('&lt;h3>事件关键词&lt;/h3>') },
459
+ { name: '人物', start: html.indexOf('&lt;h3>人物关键词&lt;/h3>') },
460
+ ].filter((c) => c.start >= 0).sort((a, b) => a.start - b.start)
461
+ cats.push({ name: null, start: html.length })
462
+
463
+ let seen = new Set()
464
+ let topics = []
465
+ for (let i = 0; i &lt; cats.length - 1; i++) {
466
+ let block = html.slice(cats[i].start, cats[i + 1].start)
467
+ let re = /href="\/zhuanti\/(\d+)\.html"[^>]*>([^&lt;]+)&lt;/g
468
+ let m
469
+ while ((m = re.exec(block)) !== null) {
470
+ if (seen.has(m[1])) {
471
+ continue
472
+ }
473
+ seen.add(m[1])
474
+ let name = m[2].trim().replace(/^\[|\]$/g, '').trim()
475
+ if (!name || name.length > 50) {
476
+ continue
477
+ }
478
+ topics.push({ id: m[1], name, category: cats[i].name })
479
+ }
480
+ }
481
+
482
+ return topics
483
+ }
484
+
485
+
486
+ /**
487
+ * 由主題清單查找指定名稱之主題
488
+ *
489
+ * @param {Array} topics 輸入主題物件陣列
490
+ * @param {String} name 輸入主題名稱字串
491
+ * @returns {Object|null} 回傳主題物件，查無時回傳null
492
+ * @example
493
+ *
494
+ * import { lookupTopic } from './src/fetchAisixiang.mjs'
495
+ *
496
+ * console.log(lookupTopic([{ id: '301', name: '大数据' }], '大数据'))
497
+ * // => { id: '301', name: '大数据' }
498
+ *
499
+ */
500
+ function lookupTopic(topics, name) {
501
+
502
+ if (!Array.isArray(topics)) {
503
+ return null
504
+ }
505
+
506
+ return topics.find((t) => t.name === name) || null
507
+ }
508
+
509
+
510
+ /**
511
+ * 抓取愛思想指定作者之文章清單
512
+ *
513
+ * 給name時先抓站方作者清單解析出slug再抓專欄頁(兩跳), 給slug則直接抓專欄頁;
514
+ * 查無此作者或該欄頁0篇時仍回status為'success'且count為0, 並以message說明, 以與抓取失敗區分
515
+ *
516
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
517
+ * @param {String} [opt.name] 輸入作者中文名字串，與slug擇一必填
518
+ * @param {String} [opt.slug] 輸入作者slug字串，與name擇一必填
519
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
520
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
521
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
522
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
523
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
524
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
525
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，name與slug皆未給時reject回傳錯誤物件
526
+ * @example
527
+ *
528
+ * import { fetchAuthorArticles } from './src/fetchAisixiang.mjs'
529
+ *
530
+ * let test = async () => {
531
+ *
532
+ * let r = await fetchAuthorArticles({ name: '葛兆光' })
533
+ * console.log(r.status, r.count, r.resolved.slug)
534
+ * // => 'success' 123 'gezhaoguang'
535
+ *
536
+ * }
537
+ * await test()
538
+ * .catch((err) => {
539
+ * console.log(err)
540
+ * })
541
+ *
542
+ */
543
+ async function fetchAuthorArticles(opt = {}) {
544
+
545
+ let cfg = getCfg(opt)
546
+
547
+ //「有提供才檢」, 傳了但非有效字串視為未提供, 讓下方「至少一個」檢查接手
548
+ let name = get(opt, 'name')
549
+ if (!isestr(name)) {
550
+ name = ''
551
+ }
552
+ let slug = get(opt, 'slug')
553
+ if (!isestr(slug)) {
554
+ slug = ''
555
+ }
556
+ if (!name &amp;&amp; !slug) {
557
+ throw new Error('需要 name 或 slug')
558
+ }
559
+
560
+ let resolved = null
561
+ let chineseName = name
562
+ let usedSlug = slug
563
+
564
+ if (!usedSlug) {
565
+
566
+ if (cfg.showLog) {
567
+ process.stderr.write('[info] fetching authors list from /thinktank/ ...\n')
568
+ }
569
+ let authors = await fetchAuthorsList(opt)
570
+ let author = lookupAuthor(authors, name)
571
+ if (!author) {
572
+ //抓到真實作者清單且確定無此作者, 屬成功查詢、結果0筆(非抓取失敗)
573
+ return {
574
+ status: 'success',
575
+ site: 'aisixiang',
576
+ mode: 'author',
577
+ query: name,
578
+ fetched_at: new Date().toISOString(),
579
+ authors_count: authors.length,
580
+ count: 0,
581
+ items: [],
582
+ message:
583
+ `"${name}" 不在愛思想專欄作者清單中（共 ${authors.length} 位）。` +
584
+ `尚無此作者文章（此為查詢成功的真實結果、0 筆，非抓取失敗）。` +
585
+ `提醒：本函數不轉簡繁，呼叫端負責用站方登錄字形（通常是簡體）。`,
586
+ }
587
+ }
588
+
589
+ usedSlug = author.slug
590
+ chineseName = chineseName || author.name
591
+ resolved = { slug: usedSlug, url: `${cfg.baseUrl}/thinktank/${usedSlug}.html`, name: author.name }
592
+ }
593
+ else {
594
+ resolved = { slug: usedSlug, url: `${cfg.baseUrl}/thinktank/${usedSlug}.html` }
595
+ }
596
+
597
+ let html = await fetchWithRetry(resolved.url, cfg.optFetch)
598
+ let items = parseAuthorColumnPage(html, cfg.baseUrl)
599
+
600
+ let payload = {
601
+ status: 'success',
602
+ site: 'aisixiang',
603
+ mode: 'author',
604
+ query: chineseName || usedSlug,
605
+ resolved,
606
+ fetched_at: new Date().toISOString(),
607
+ count: items.length,
608
+ items,
609
+ }
610
+
611
+ //命中作者欄頁(含slug捷徑)但該欄頁實際0篇文章, 加message讓呼叫端能區分「真的查清楚、就是0篇」與「抓取失敗」
612
+ if (items.length === 0) {
613
+ payload.message =
614
+ `已命中愛思想作者欄頁（${resolved.url}）但該欄頁目前 0 篇文章。` +
615
+ `此為查詢成功的真實結果（非抓取失敗）；若 slug 為手動指定，請確認 slug 是否正確。`
616
+ }
617
+
618
+ return payload
619
+ }
620
+
621
+
622
+ /**
623
+ * 抓取愛思想指定關鍵字tag之文章清單
624
+ *
625
+ * 走站方keywords搜尋並自動翻頁全抓; 站方搜尋只認簡體, 呼叫端負責簡體化
626
+ *
627
+ * @param {String} keyword 輸入關鍵字字串
628
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
629
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
630
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
631
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
632
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
633
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
634
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
635
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，keyword非有效字串時reject回傳錯誤物件
636
+ * @example
637
+ *
638
+ * import { fetchKeywordArticles } from './src/fetchAisixiang.mjs'
639
+ *
640
+ * let test = async () => {
641
+ *
642
+ * let r = await fetchKeywordArticles('老庄')
643
+ * console.log(r.status, r.count)
644
+ * // => 'success' 42
645
+ *
646
+ * }
647
+ * await test()
648
+ * .catch((err) => {
649
+ * console.log(err)
650
+ * })
651
+ *
652
+ */
653
+ async function fetchKeywordArticles(keyword, opt = {}) {
654
+
655
+ //check
656
+ if (!isestr(keyword)) {
657
+ throw new Error('需要 keyword')
658
+ }
659
+
660
+ let cfg = getCfg(opt)
661
+
662
+ let buildUrl = (page) => `${cfg.baseUrl}/data/search?searchfield=keywords&amp;keywords=${encodeURIComponent(keyword)}&amp;page=${page}`
663
+ let { items, total_pages, pages_fetched } = await fetchAllSearchPages(buildUrl, parseSearchResults, cfg)
664
+
665
+ if (items.length === 0) {
666
+ return {
667
+ status: 'success',
668
+ site: 'aisixiang',
669
+ mode: 'keyword',
670
+ query: keyword,
671
+ resolved: { search_url: buildUrl(1) },
672
+ fetched_at: new Date().toISOString(),
673
+ count: 0,
674
+ items: [],
675
+ message:
676
+ `關鍵字 "${keyword}" 在愛思想無相關文章。` +
677
+ `提醒：站方搜尋只認簡體，呼叫端負責簡體化；若已是簡體仍 0 筆，該主題可能無 tag 索引。`,
678
+ }
679
+ }
680
+
681
+ return {
682
+ status: 'success',
683
+ site: 'aisixiang',
684
+ mode: 'keyword',
685
+ query: keyword,
686
+ resolved: { search_url: buildUrl(1), total_pages, pages_fetched },
687
+ fetched_at: new Date().toISOString(),
688
+ count: items.length,
689
+ items,
690
+ }
691
+ }
692
+
693
+
694
+ /**
695
+ * 抓取愛思想指定標題關鍵字之文章清單
696
+ *
697
+ * 走站方title搜尋並自動翻頁全抓; 站方搜尋只認簡體, 呼叫端負責簡體化
698
+ *
699
+ * @param {String} keyword 輸入標題關鍵字字串
700
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
701
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
702
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
703
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
704
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
705
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
706
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
707
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，keyword非有效字串時reject回傳錯誤物件
708
+ * @example
709
+ *
710
+ * import { fetchTitleArticles } from './src/fetchAisixiang.mjs'
711
+ *
712
+ * let test = async () => {
713
+ *
714
+ * let r = await fetchTitleArticles('禅宗与中国文化')
715
+ * console.log(r.status, r.count)
716
+ * // => 'success' 3
717
+ *
718
+ * }
719
+ * await test()
720
+ * .catch((err) => {
721
+ * console.log(err)
722
+ * })
723
+ *
724
+ */
725
+ async function fetchTitleArticles(keyword, opt = {}) {
726
+
727
+ //check
728
+ if (!isestr(keyword)) {
729
+ throw new Error('需要 keyword')
730
+ }
731
+
732
+ let cfg = getCfg(opt)
733
+
734
+ let buildUrl = (page) => `${cfg.baseUrl}/data/search?searchfield=title&amp;keywords=${encodeURIComponent(keyword)}&amp;page=${page}`
735
+ let { items, total_pages, pages_fetched } = await fetchAllSearchPages(buildUrl, parseSearchResults, cfg)
736
+
737
+ if (items.length === 0) {
738
+ return {
739
+ status: 'success',
740
+ site: 'aisixiang',
741
+ mode: 'title',
742
+ query: keyword,
743
+ resolved: { search_url: buildUrl(1) },
744
+ fetched_at: new Date().toISOString(),
745
+ count: 0,
746
+ items: [],
747
+ message:
748
+ `標題關鍵字 "${keyword}" 在愛思想無相關文章。` +
749
+ `提醒：站方搜尋只認簡體，呼叫端負責簡體化。`,
750
+ }
751
+ }
752
+
753
+ return {
754
+ status: 'success',
755
+ site: 'aisixiang',
756
+ mode: 'title',
757
+ query: keyword,
758
+ resolved: { search_url: buildUrl(1), total_pages, pages_fetched },
759
+ fetched_at: new Date().toISOString(),
760
+ count: items.length,
761
+ items,
762
+ }
763
+ }
764
+
765
+
766
+ /**
767
+ * 抓取愛思想指定策展主題之文章清單
768
+ *
769
+ * 給keyword時先抓站方主題清單解析出id再抓主題頁(兩跳), 給id則直接抓主題頁並自動翻頁全抓;
770
+ * 查無此主題時仍回status為'success'且count為0, 並以message建議改用fetchKeywordArticles(本函數不自動轉向)
771
+ *
772
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
773
+ * @param {String} [opt.keyword] 輸入主題名稱字串，與id擇一必填
774
+ * @param {String} [opt.id] 輸入主題id字串，與keyword擇一必填
775
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
776
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
777
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
778
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
779
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
780
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
781
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，keyword與id皆未給時reject回傳錯誤物件
782
+ * @example
783
+ *
784
+ * import { fetchTopicArticles } from './src/fetchAisixiang.mjs'
785
+ *
786
+ * let test = async () => {
787
+ *
788
+ * let r = await fetchTopicArticles({ keyword: '大数据' })
789
+ * console.log(r.status, r.count, r.resolved.id)
790
+ * // => 'success' 60 '301'
791
+ *
792
+ * }
793
+ * await test()
794
+ * .catch((err) => {
795
+ * console.log(err)
796
+ * })
797
+ *
798
+ */
799
+ async function fetchTopicArticles(opt = {}) {
800
+
801
+ let cfg = getCfg(opt)
802
+
803
+ //「有提供才檢」, 傳了但非有效字串視為未提供, 讓下方「至少一個」檢查接手
804
+ let keyword = get(opt, 'keyword')
805
+ if (!isestr(keyword)) {
806
+ keyword = ''
807
+ }
808
+ let id = get(opt, 'id')
809
+ if (!isestr(id)) {
810
+ id = ''
811
+ }
812
+ if (!keyword &amp;&amp; !id) {
813
+ throw new Error('需要 keyword 或 id')
814
+ }
815
+
816
+ let topicId = id
817
+ let topicName = keyword
818
+ let resolved = null
819
+
820
+ if (!topicId) {
821
+
822
+ if (cfg.showLog) {
823
+ process.stderr.write('[info] fetching topics list from /zhuanti/ ...\n')
824
+ }
825
+ let topics = await fetchTopicsList(opt)
826
+ let hit = lookupTopic(topics, keyword)
827
+ if (!hit) {
828
+ //抓到真實主題清單且確定不是策展主題, 屬成功查詢、結果0筆(非抓取失敗)
829
+ return {
830
+ status: 'success',
831
+ site: 'aisixiang',
832
+ mode: 'topic',
833
+ query: keyword,
834
+ fetched_at: new Date().toISOString(),
835
+ topics_count: topics.length,
836
+ count: 0,
837
+ items: [],
838
+ message:
839
+ `"${keyword}" 不在愛思想策展主題清單中（共 ${topics.length} 個主題）。` +
840
+ `建議改用 fetchKeywordArticles('${keyword}') 查 keyword tag 結果。` +
841
+ `本函數不自動轉向，請呼叫端決定是否重試。`,
842
+ }
843
+ }
844
+
845
+ topicId = hit.id
846
+ topicName = hit.name
847
+ resolved = {
848
+ id: topicId,
849
+ name: hit.name,
850
+ category: hit.category,
851
+ url: `${cfg.baseUrl}/zhuanti/${topicId}.html`,
852
+ }
853
+ }
854
+ else {
855
+ resolved = { id: topicId, url: `${cfg.baseUrl}/zhuanti/${topicId}.html` }
856
+ }
857
+
858
+ let buildUrl = (page) => `${cfg.baseUrl}/zhuanti/${topicId}.html?page=${page}`
859
+ let { items, total_pages, pages_fetched } = await fetchAllSearchPages(buildUrl, parseZhuantiArticles, cfg)
860
+
861
+ let payload = {
862
+ status: 'success',
863
+ site: 'aisixiang',
864
+ mode: 'topic',
865
+ query: topicName || topicId,
866
+ resolved: { ...resolved, total_pages, pages_fetched },
867
+ fetched_at: new Date().toISOString(),
868
+ count: items.length,
869
+ items,
870
+ }
871
+
872
+ //命中主題(含id捷徑)但該主題頁實際0篇文章, 加message讓呼叫端能區分「真的查清楚、就是0篇」與「抓取失敗」
873
+ if (items.length === 0) {
874
+ payload.message =
875
+ `已命中愛思想策展主題（${resolved.url}）但該主題頁目前 0 篇文章。` +
876
+ `此為查詢成功的真實結果（非抓取失敗）；若需更多結果可改用 fetchKeywordArticles 查 keyword tag。`
877
+ }
878
+
879
+ return payload
880
+ }
881
+
882
+
883
+ /**
884
+ * 抓取愛思想單篇文章並轉為Markdown
885
+ *
886
+ * 由article-content區塊切出正文轉Markdown, 並產生含title、source與created之YAML frontmatter;
887
+ * 標題會去除站名後綴, 例如「葛兆光：禅宗与中国文化_爱思想」轉為「葛兆光：禅宗与中国文化」
888
+ *
889
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
890
+ * @param {String} [opt.aid] 輸入文章aid字串，與url擇一必填
891
+ * @param {String} [opt.url] 輸入文章網址字串，與aid擇一必填
892
+ * @param {String} [opt.baseUrl='https://www.aisixiang.com'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
893
+ * @param {Integer} [opt.timeout=30000] 輸入單次請求逾時毫秒整數，預設30000
894
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
895
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
896
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
897
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
898
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,url,title,chars,markdown}，aid與url皆未給或解析失敗時reject回傳錯誤物件
899
+ * @example
900
+ *
901
+ * import { fetchArticle } from './src/fetchAisixiang.mjs'
902
+ *
903
+ * let test = async () => {
904
+ *
905
+ * let r = await fetchArticle({ aid: '146669' })
906
+ * console.log(r.status, r.title, r.chars)
907
+ * // => 'success' '葛兆光：禅宗与中国文化' 12345
908
+ *
909
+ * }
910
+ * await test()
911
+ * .catch((err) => {
912
+ * console.log(err)
913
+ * })
914
+ *
915
+ */
916
+ async function fetchArticle(opt = {}) {
917
+
918
+ let cfg = getCfg(opt)
919
+
920
+ //「有提供才檢」, 傳了但非有效字串視為未提供, 讓下方「至少一個」檢查接手
921
+ let aid = get(opt, 'aid')
922
+ if (!isestr(aid)) {
923
+ aid = ''
924
+ }
925
+ let url = get(opt, 'url')
926
+ if (!isestr(url)) {
927
+ url = ''
928
+ }
929
+ let targetUrl = aid ? `${cfg.baseUrl}/data/${aid}.html` : url
930
+ if (!targetUrl) {
931
+ throw new Error('需要 aid 或 url')
932
+ }
933
+
934
+ let html = await fetchWithRetry(targetUrl, cfg.optFetch)
935
+ let titleRaw = extractTitle(html)
936
+ let contentHtml = extractDivContent(html, 'article-content')
937
+ if (!titleRaw || !contentHtml) {
938
+ throw new Error(`解析失敗：title=${!!titleRaw}, content=${!!contentHtml}`)
939
+ }
940
+
941
+ let body = htmlToMarkdown(contentHtml)
942
+
943
+ //去掉站名後綴
944
+ let title = titleRaw.replace(/[_\-\s]*(?:_)?(?:爱思想|愛思想)\s*$/u, '').trim()
945
+ let today = new Date().toISOString().slice(0, 10)
946
+
947
+ let markdown =
948
+ `---
949
+ title: "${title}"
950
+ source: "${targetUrl}"
951
+ author:
952
+ published:
953
+ created: ${today}
954
+ description:
955
+ ---
956
+ ${body}
957
+ `
958
+
959
+ return {
960
+ status: 'success',
961
+ site: 'aisixiang',
962
+ mode: 'fetch',
963
+ url: targetUrl,
964
+ title,
965
+ chars: body.length,
966
+ markdown,
967
+ }
968
+ }
969
+
970
+
971
+ /**
972
+ * 抓取愛思想(aisixiang.com)
973
+ *
974
+ * @returns {Object} 回傳物件，其內可呼叫fetchAuthorsList、lookupAuthor、fetchTopicsList、lookupTopic、fetchAuthorArticles、fetchKeywordArticles、fetchTitleArticles、fetchTopicArticles、fetchArticle
975
+ * @example
976
+ *
977
+ * 詳見fetchAuthorArticles、fetchKeywordArticles、fetchTitleArticles、fetchTopicArticles、fetchArticle範例
978
+ *
979
+ */
980
+ let fetchAisixiang = {
981
+ BASE_URL,
982
+ USER_AGENT,
983
+ MAX_PAGES,
984
+ safeFilename,
985
+ fetchAuthorsList,
986
+ lookupAuthor,
987
+ fetchTopicsList,
988
+ lookupTopic,
989
+ fetchAuthorArticles,
990
+ fetchKeywordArticles,
991
+ fetchTitleArticles,
992
+ fetchTopicArticles,
993
+ fetchArticle,
994
+ }
995
+
996
+
997
+ export {
998
+ BASE_URL,
999
+ USER_AGENT,
1000
+ PAGE_DELAY_MS,
1001
+ MAX_PAGES,
1002
+ safeFilename,
1003
+ fetchAuthorsList,
1004
+ lookupAuthor,
1005
+ fetchTopicsList,
1006
+ lookupTopic,
1007
+ fetchAuthorArticles,
1008
+ fetchKeywordArticles,
1009
+ fetchTitleArticles,
1010
+ fetchTopicArticles,
1011
+ fetchArticle
1012
+ }
1013
+ export default fetchAisixiang
1014
+ </code></pre>
1015
+ </article>
1016
+ </section>
1017
+
1018
+
1019
+
1020
+
1021
+
1022
+
1023
+ </div>
1024
+
1025
+ <br class="clear">
1026
+
1027
+ <footer>
1028
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
1029
+ </footer>
1030
+
1031
+ <script>prettyPrint();</script>
1032
+ <script src="scripts/polyfill.js"></script>
1033
+ <script src="scripts/linenumber.js"></script>
1034
+
1035
+
1036
+
1037
+ </body>
1038
+ </html>