w-dwdata-hub 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/.editorconfig +9 -0
  2. package/.eslintignore +3 -0
  3. package/.eslintrc.js +55 -0
  4. package/.jsdoc +25 -0
  5. package/LICENSE +21 -0
  6. package/README.md +168 -0
  7. package/SECURITY.md +5 -0
  8. package/babel.config.js +16 -0
  9. package/dist/w-dwdata-hub.umd.js +7 -0
  10. package/dist/w-dwdata-hub.umd.js.map +1 -0
  11. package/docs/WDwdataHub.mjs.html +136 -0
  12. package/docs/decodeEntities.mjs.html +129 -0
  13. package/docs/extractDivContent.mjs.html +147 -0
  14. package/docs/fetchAiNewsAggregator.mjs.html +155 -0
  15. package/docs/fetchAisixiang.mjs.html +1038 -0
  16. package/docs/fetchCnyes.mjs.html +260 -0
  17. package/docs/fetchGuancha.mjs.html +1007 -0
  18. package/docs/fetchHackerNews.mjs.html +189 -0
  19. package/docs/fetchMoneydj.mjs.html +251 -0
  20. package/docs/fetchMops.mjs.html +385 -0
  21. package/docs/fetchRSS.mjs.html +160 -0
  22. package/docs/fetchStatementdog.mjs.html +213 -0
  23. package/docs/fetchTaifex.mjs.html +513 -0
  24. package/docs/fetchTpex3insti.mjs.html +278 -0
  25. package/docs/fetchTpexMargin.mjs.html +248 -0
  26. package/docs/fetchTpexStock.mjs.html +199 -0
  27. package/docs/fetchTwDataHoliday.mjs.html +236 -0
  28. package/docs/fetchTwseMargin.mjs.html +253 -0
  29. package/docs/fetchTwseStock.mjs.html +188 -0
  30. package/docs/fetchTwseT86.mjs.html +208 -0
  31. package/docs/fetchWithRetry.mjs.html +376 -0
  32. package/docs/fonts/Montserrat/Montserrat-Bold.eot +0 -0
  33. package/docs/fonts/Montserrat/Montserrat-Bold.ttf +0 -0
  34. package/docs/fonts/Montserrat/Montserrat-Bold.woff +0 -0
  35. package/docs/fonts/Montserrat/Montserrat-Bold.woff2 +0 -0
  36. package/docs/fonts/Montserrat/Montserrat-Regular.eot +0 -0
  37. package/docs/fonts/Montserrat/Montserrat-Regular.ttf +0 -0
  38. package/docs/fonts/Montserrat/Montserrat-Regular.woff +0 -0
  39. package/docs/fonts/Montserrat/Montserrat-Regular.woff2 +0 -0
  40. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.eot +0 -0
  41. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.svg +978 -0
  42. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.ttf +0 -0
  43. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff +0 -0
  44. package/docs/fonts/Source-Sans-Pro/sourcesanspro-light-webfont.woff2 +0 -0
  45. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.eot +0 -0
  46. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.svg +1049 -0
  47. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.ttf +0 -0
  48. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff +0 -0
  49. package/docs/fonts/Source-Sans-Pro/sourcesanspro-regular-webfont.woff2 +0 -0
  50. package/docs/getOptFetch.mjs.html +185 -0
  51. package/docs/global.html +22649 -0
  52. package/docs/htmlToMarkdown.mjs.html +168 -0
  53. package/docs/index.html +84 -0
  54. package/docs/isYmd.mjs.html +114 -0
  55. package/docs/parseIntComma.mjs.html +100 -0
  56. package/docs/safeFilename.mjs.html +102 -0
  57. package/docs/scripts/collapse.js +39 -0
  58. package/docs/scripts/commonNav.js +28 -0
  59. package/docs/scripts/linenumber.js +25 -0
  60. package/docs/scripts/nav.js +12 -0
  61. package/docs/scripts/polyfill.js +4 -0
  62. package/docs/scripts/prettify/Apache-License-2.0.txt +202 -0
  63. package/docs/scripts/prettify/lang-css.js +2 -0
  64. package/docs/scripts/prettify/prettify.js +28 -0
  65. package/docs/scripts/search.js +99 -0
  66. package/docs/styles/jsdoc.css +776 -0
  67. package/docs/styles/prettify.css +80 -0
  68. package/docs/toDatetimeUTC8.mjs.html +143 -0
  69. package/docs/toRocDate.mjs.html +107 -0
  70. package/g.mjs +73 -0
  71. package/package.json +31 -0
  72. package/script.txt +17 -0
  73. package/src/WDwdataHub.mjs +64 -0
  74. package/src/decodeEntities.mjs +57 -0
  75. package/src/extractDivContent.mjs +75 -0
  76. package/src/fetchAiNewsAggregator.mjs +83 -0
  77. package/src/fetchAisixiang.mjs +966 -0
  78. package/src/fetchCnyes.mjs +188 -0
  79. package/src/fetchGuancha.mjs +935 -0
  80. package/src/fetchHackerNews.mjs +117 -0
  81. package/src/fetchMoneydj.mjs +179 -0
  82. package/src/fetchMops.mjs +313 -0
  83. package/src/fetchRSS.mjs +88 -0
  84. package/src/fetchStatementdog.mjs +141 -0
  85. package/src/fetchTaifex.mjs +441 -0
  86. package/src/fetchTpex3insti.mjs +206 -0
  87. package/src/fetchTpexMargin.mjs +176 -0
  88. package/src/fetchTpexStock.mjs +127 -0
  89. package/src/fetchTwDataHoliday.mjs +164 -0
  90. package/src/fetchTwseMargin.mjs +181 -0
  91. package/src/fetchTwseStock.mjs +116 -0
  92. package/src/fetchTwseT86.mjs +136 -0
  93. package/src/fetchWithRetry.mjs +304 -0
  94. package/src/getOptFetch.mjs +113 -0
  95. package/src/htmlToMarkdown.mjs +96 -0
  96. package/src/isYmd.mjs +42 -0
  97. package/src/parseIntComma.mjs +28 -0
  98. package/src/safeFilename.mjs +30 -0
  99. package/src/toDatetimeUTC8.mjs +71 -0
  100. package/src/toRocDate.mjs +35 -0
  101. package/test/WDwdataHub.test.mjs +63 -0
  102. package/test/decodeEntities.test.mjs +42 -0
  103. package/test/extractDivContent.test.mjs +48 -0
  104. package/test/fetchAiNewsAggregator.test.mjs +79 -0
  105. package/test/fetchAisixiang.test.mjs +225 -0
  106. package/test/fetchCnyes.test.mjs +61 -0
  107. package/test/fetchGuancha.test.mjs +190 -0
  108. package/test/fetchHackerNews.test.mjs +78 -0
  109. package/test/fetchMoneydj.test.mjs +82 -0
  110. package/test/fetchMops.test.mjs +106 -0
  111. package/test/fetchRSS.test.mjs +85 -0
  112. package/test/fetchStatementdog.test.mjs +69 -0
  113. package/test/fetchTaifex.test.mjs +99 -0
  114. package/test/fetchTpex3insti.test.mjs +90 -0
  115. package/test/fetchTpexMargin.test.mjs +94 -0
  116. package/test/fetchTpexStock.test.mjs +68 -0
  117. package/test/fetchTwDataHoliday.test.mjs +100 -0
  118. package/test/fetchTwseMargin.test.mjs +81 -0
  119. package/test/fetchTwseStock.test.mjs +80 -0
  120. package/test/fetchTwseT86.test.mjs +76 -0
  121. package/test/fetchWithRetry.test.mjs +174 -0
  122. package/test/htmlToMarkdown.test.mjs +71 -0
  123. package/test/isYmd.test.mjs +56 -0
  124. package/test/parseIntComma.test.mjs +38 -0
  125. package/test/safeFilename.test.mjs +35 -0
  126. package/test/toDatetimeUTC8.test.mjs +65 -0
  127. package/test/toRocDate.test.mjs +38 -0
  128. package/test/tools/encodeBig5.mjs +80 -0
  129. package/test/tools/fixtures.mjs +539 -0
  130. package/test/tools/hasChrome.mjs +39 -0
  131. package/test/tools/serverForTest.mjs +397 -0
  132. package/toolg/addVersion.mjs +4 -0
  133. package/toolg/cleanFolder.mjs +5 -0
  134. package/toolg/gDistRollup.mjs +49 -0
  135. package/toolg/modifyReadme.mjs +4 -0
package/docs/fetchGuancha.mjs.html ADDED
@@ -0,0 +1,1007 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+
5
+ <meta charset="utf-8">
6
+ <title>fetchGuancha.mjs - Documentation</title>
7
+
8
+
9
+ <script src="scripts/prettify/prettify.js"></script>
10
+ <script src="scripts/prettify/lang-css.js"></script>
11
+ <!--[if lt IE 9]>
12
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
13
+ <![endif]-->
14
+ <link type="text/css" rel="stylesheet" href="styles/prettify.css">
15
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
16
+ <script src="scripts/nav.js" defer></script>
17
+
18
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
19
+ </head>
20
+ <body>
21
+
22
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
23
+ <label for="nav-trigger" class="navicon-button x">
24
+ <div class="navicon"></div>
25
+ </label>
26
+
27
+ <label for="nav-trigger" class="overlay"></label>
28
+
29
+ <nav >
30
+
31
+
32
+ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#KNOWN_TOPICS">KNOWN_TOPICS</a></li><li><a href="global.html#WDwdataHub">WDwdataHub</a></li><li><a href="global.html#assertTpexFieldShape">assertTpexFieldShape</a></li><li><a href="global.html#decodeEntities">decodeEntities</a></li><li><a href="global.html#extractAuthorFromContent">extractAuthorFromContent</a></li><li><a href="global.html#extractDivContent">extractDivContent</a></li><li><a href="global.html#extractHtmlTitle">extractHtmlTitle</a></li><li><a href="global.html#extractPubTime">extractPubTime</a></li><li><a href="global.html#extractTitle">extractTitle</a></li><li><a href="global.html#fetchAiNewsAggregator">fetchAiNewsAggregator</a></li><li><a href="global.html#fetchAisixiang">fetchAisixiang</a></li><li><a href="global.html#fetchAllListPages">fetchAllListPages</a></li><li><a href="global.html#fetchAllSearchPages">fetchAllSearchPages</a></li><li><a href="global.html#fetchArticle">fetchArticle</a></li><li><a href="global.html#fetchAuthorArticles">fetchAuthorArticles</a></li><li><a href="global.html#fetchAuthorsList">fetchAuthorsList</a></li><li><a href="global.html#fetchCnyes">fetchCnyes</a></li><li><a href="global.html#fetchFuturesData">fetchFuturesData</a></li><li><a href="global.html#fetchGuancha">fetchGuancha</a></li><li><a href="global.html#fetchHackerNews">fetchHackerNews</a></li><li><a href="global.html#fetchHtml">fetchHtml</a></li><li><a href="global.html#fetchInstitutionalData">fetchInstitutionalData</a></li><li><a href="global.html#fetchKeywordArticles">fetchKeywordArticles</a></li><li><a href="global.html#fetchMoneydj">fetchMoneydj</a></li><li><a href="global.html#fetchMops">fetchMops</a></li><li><a href="global.html#fetchOnce">fetchOnce</a></li><li><a href="global.html#fetchPCRatio">fetchPCRatio</a></li><li><a href="global.html#fetchRSS">fetchRSS</a></li><li><a href="global.html#fetchStatementdog">fetchStatementdog</a></li><li><a href="global.html#fetchTaifex">fetchTaifex</a></li><li><a href="global.html#fetchTargetWithRetry">fetchTargetWithRetry</a></li><li><a href="global.html#fetchTitleArticles">fetchTitleArticles</a></li><li><a href="global.html#fetchTopicArticles">fetchTopicArticles</a></li><li><a href="global.html#fetchTopicsList">fetchTopicsList</a></li><li><a href="global.html#fetchTpex3insti">fetchTpex3insti</a></li><li><a href="global.html#fetchTpexMargin">fetchTpexMargin</a></li><li><a href="global.html#fetchTpexStock">fetchTpexStock</a></li><li><a href="global.html#fetchTwDataHoliday">fetchTwDataHoliday</a></li><li><a href="global.html#fetchTwseMargin">fetchTwseMargin</a></li><li><a href="global.html#fetchTwseStock">fetchTwseStock</a></li><li><a href="global.html#fetchTwseT86">fetchTwseT86</a></li><li><a href="global.html#fetchWithRetry">fetchWithRetry</a></li><li><a href="global.html#getCfg">getCfg</a></li><li><a href="global.html#getOptFetch">getOptFetch</a></li><li><a href="global.html#getPayload">getPayload</a></li><li><a href="global.html#htmlToMarkdown">htmlToMarkdown</a></li><li><a href="global.html#isHomepageHtml">isHomepageHtml</a></li><li><a href="global.html#isNonHolidayEntry">isNonHolidayEntry</a></li><li><a href="global.html#isRetryableError">isRetryableError</a></li><li><a href="global.html#isYmd">isYmd</a></li><li><a href="global.html#lookupAuthor">lookupAuthor</a></li><li><a href="global.html#lookupTopic">lookupTopic</a></li><li><a href="global.html#parseAuthorColumnPage">parseAuthorColumnPage</a></li><li><a href="global.html#parseAuthorIndex">parseAuthorIndex</a></li><li><a href="global.html#parseCSV">parseCSV</a></li><li><a href="global.html#parseCSVLine">parseCSVLine</a></li><li><a href="global.html#parseIntComma">parseIntComma</a></li><li><a href="global.html#parseListPage">parseListPage</a></li><li><a href="global.html#parseNum">parseNum</a></li><li><a href="global.html#parseSearchResults">parseSearchResults</a></li><li><a href="global.html#parseSearchTotalPages">parseSearchTotalPages</a></li><li><a href="global.html#parseZhuantiArticles">parseZhuantiArticles</a></li><li><a href="global.html#pickPint">pickPint</a></li><li><a href="global.html#rocToWestern">rocToWestern</a></li><li><a href="global.html#safeFilename">safeFilename</a></li><li><a href="global.html#toDatetimeUTC8">toDatetimeUTC8</a></li><li><a href="global.html#toRocDate">toRocDate</a></li></ul>
33
+
34
+ </nav>
35
+
36
+ <div id="main">
37
+
38
+ <h1 class="page-title">fetchGuancha.mjs</h1>
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+ <section>
47
+ <article>
48
+ <pre class="prettyprint source linenums"><code>import wfw from 'w-fetch-web'
49
+ import get from 'lodash-es/get.js'
50
+ import isbol from 'wsemi/src/isbol.mjs'
51
+ import isestr from 'wsemi/src/isestr.mjs'
52
+ import ispint from 'wsemi/src/ispint.mjs'
53
+ import isp0int from 'wsemi/src/isp0int.mjs'
54
+ import cint from 'wsemi/src/cint.mjs'
55
+ import delay from 'wsemi/src/delay.mjs'
56
+ import decodeEntities from './decodeEntities.mjs'
57
+ import htmlToMarkdown from './htmlToMarkdown.mjs'
58
+ import extractDivContent from './extractDivContent.mjs'
59
+ import safeFilename from './safeFilename.mjs'
60
+
61
+
62
+ //w-fetch-web為UMD套件, 只能default import, named import取不到函數
63
+ let { fetchWebByCurl } = wfw
64
+
65
+
66
+ //觀察者網網站根網址
67
+ let BASE_URL = 'https://www.guancha.cn'
68
+
69
+
70
+ //觀察者網對User-Agent較敏感, 故固定送出桌面瀏覽器標頭
71
+ let USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
72
+ '(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
73
+
74
+
75
+ //頁間延遲毫秒
76
+ let PAGE_DELAY_MS = 1000
77
+
78
+
79
+ //翻頁安全上限, 50頁×60筆=3000筆
80
+ let MAX_PAGES = 50
81
+
82
+
83
+ //預設值
84
+ let DEFAULT_MAX_RETRIES = 5
85
+
86
+
87
+ //偵測首頁之門檻, 首頁底部A-Z作者索引實測有約22個&lt;dt>[A-Z]&lt;/dt>, 真實list頁為0個
88
+ //取門檻5以容忍站方版型微調, 同時遠離0
89
+ let HOMEPAGE_DT_THRESHOLD = 5
90
+
91
+
92
+ /**
93
+ * 已知主題對照表
94
+ *
95
+ * 觀察者網無公開主題清單頁, 本表為手工整理(由首頁專題標籤與大頻道nav), 未涵蓋者請直接傳slug。
96
+ * 注意：同一中文名有多個slug時(如「财经」對應大頻道/economy與分頁欄目/CaiJing),
97
+ * 分頁欄目slug必須排在前面, 因lookupTopic採first-match;
98
+ * /economy等大頻道是「精選首頁」不分頁, /CaiJing等欄目才能以list_N.shtml翻頁全抓
99
+ */
100
+ let KNOWN_TOPICS = [
101
+ //中文名重複者, 分頁版slug在前, 大頻道版slug用獨立中文名標記避免覆蓋
102
+ { slug: 'CaiJing', name: '财经' }, //分頁版(首選)
103
+ { slug: 'economy', name: '财经-大頻道' }, //大頻道精選首頁, 不分頁
104
+ { slug: 'JunShi', name: '军事' }, //分頁版(首選)
105
+ { slug: 'military-affairs', name: '军事-大頻道' }, //大頻道精選首頁, 不分頁
106
+ { slug: 'ZhengZhi', name: '政治' },
107
+ { slug: 'WenHua', name: '文化' },
108
+ { slug: 'chanjing', name: '产经' },
109
+ { slug: 'qiche', name: '观出行' },
110
+ { slug: 'gongye-keji', name: '科技' },
111
+ { slug: 'ChengShi', name: '城事' },
112
+ { slug: 'GuanJinRong', name: '观金融' },
113
+ { slug: 'XinShiDai', name: '新时代' },
114
+ { slug: 'ChaoJiGongCheng', name: '超级工程' },
115
+ { slug: 'NengYuanZhanLue', name: '能源战略' },
116
+ { slug: 'RenGongZhiNeng', name: '人工智能' },
117
+ { slug: 'XinZhiGuanChaSuoNews', name: '心智观察所' },
118
+ { slug: 'YiLangJuShi', name: '伊朗局势' },
119
+ { slug: 'MeiGuoMeng', name: '美国一梦' },
120
+ { slug: 'MeiGuoJingJi', name: '美国经济' },
121
+ { slug: 'ELuoSiZhiSheng', name: '俄罗斯之声' },
122
+ { slug: 'lianganyuanzhuopai', name: '两岸圆桌派' },
123
+ { slug: 'ZheJiuShiZhongGuo', name: '这就是中国' },
124
+ { slug: 'YiZhouJunQingGuanCha', name: '一周军事观察' },
125
+ { slug: 'feizhoushangkou', name: '非洲之窗' },
126
+ { slug: 'toutiao', name: '观察者头条' },
127
+ { slug: 'gushi', name: '股市' },
128
+ { slug: 'guanwangwenyu', name: '新潮观鱼' },
129
+ { slug: 'jingtiriben', name: '冲破战后秩序 日本想干什么' },
130
+ { slug: 'DaoGuoDianAVI', name: '日本' },
131
+ ]
132
+
133
+
134
+ /**
135
+ * 正規化本模組共用設定
136
+ *
137
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
138
+ * @returns {Object} 回傳已正規化之設定物件，內含baseUrl、showLog與optFetch
139
+ */
140
+ function getCfg(opt = {}) {
141
+
142
+ //baseUrl
143
+ let baseUrl = get(opt, 'baseUrl')
144
+ if (!isestr(baseUrl)) {
145
+ baseUrl = BASE_URL
146
+ }
147
+ baseUrl = baseUrl.replace(/\/+$/, '')
148
+
149
+ //maxRetries
150
+ let maxRetries = get(opt, 'maxRetries')
151
+ if (!isp0int(maxRetries)) {
152
+ maxRetries = DEFAULT_MAX_RETRIES
153
+ }
154
+ else {
155
+ maxRetries = cint(maxRetries)
156
+ }
157
+
158
+ //timeoutMs
159
+ let timeoutMs = get(opt, 'timeout')
160
+ if (!ispint(timeoutMs)) {
161
+ timeoutMs = null
162
+ }
163
+ else {
164
+ timeoutMs = cint(timeoutMs)
165
+ }
166
+
167
+ //showLog
168
+ let showLog = get(opt, 'showLog')
169
+ if (!isbol(showLog)) {
170
+ showLog = true
171
+ }
172
+
173
+ //optFetch
174
+ let optFetch = { userAgent: USER_AGENT, referer: baseUrl, maxRetries }
175
+ if (timeoutMs !== null) {
176
+ optFetch.timeoutMs = timeoutMs
177
+ }
178
+
179
+ //pageDelayMs
180
+ let pageDelayMs = get(opt, 'pageDelayMs')
181
+ if (!isp0int(pageDelayMs)) {
182
+ pageDelayMs = PAGE_DELAY_MS
183
+ }
184
+ else {
185
+ pageDelayMs = cint(pageDelayMs)
186
+ }
187
+
188
+ //maxPages
189
+ let maxPages = get(opt, 'maxPages')
190
+ if (!ispint(maxPages)) {
191
+ maxPages = MAX_PAGES
192
+ }
193
+ else {
194
+ maxPages = Math.min(cint(maxPages), MAX_PAGES)
195
+ }
196
+
197
+ return { baseUrl, showLog, pageDelayMs, maxPages, optFetch }
198
+ }
199
+
200
+
201
+ /**
202
+ * 抓取網頁原始HTML
203
+ *
204
+ * 委派w-fetch-web之fetchWebByCurl(其本身內建重試與線性退避), 失敗時拋出帶reason與httpCode之錯誤
205
+ *
206
+ * @param {String} url 輸入待抓取網址字串
207
+ * @param {Object} cfg 輸入已正規化之設定物件
208
+ * @returns {Promise} 回傳Promise，resolve回傳原始HTML字串
209
+ */
210
+ async function fetchHtml(url, cfg) {
211
+
212
+ let r = await fetchWebByCurl(url, cfg.optFetch)
213
+ if (r.status !== 'success') {
214
+ let err = new Error(r.message || 'fetch failed')
215
+ err.reason = r.reason
216
+ err.url = url
217
+ err.httpCode = r.httpCode
218
+ throw err
219
+ }
220
+
221
+ //注意：觀察者網對「不存在的slug／已下架文章」會302跳首頁(curl -L跟隨後httpCode仍為200)
222
+ //fetchWebByCurl不回傳effective URL無法靠URL比對, list頁與首頁之&lt;title>又同為「观察者网」故title也無法區分
223
+ //改以結構特徵偵測(見isHomepageHtml), 由各caller視情境處理
224
+ return r.html
225
+ }
226
+
227
+
228
+ /**
229
+ * 偵測HTML是否為觀察者網首頁
230
+ *
231
+ * 判準為首頁底部含A-Z作者索引之&lt;dt>[A-Z]&lt;/dt>標記(真正的作者欄頁、欄目頁與主題集頁皆無此區塊),
232
+ * 用以識別「請求被302導向首頁」之情形
233
+ *
234
+ * @param {String} html 輸入HTML字串
235
+ * @returns {Boolean} 回傳是否為首頁之布林值
236
+ */
237
+ function isHomepageHtml(html) {
238
+ let m = html.match(/&lt;dt>[A-Z]&lt;\/dt>/g)
239
+ return !!m &amp;&amp; m.length >= HOMEPAGE_DT_THRESHOLD
240
+ }
241
+
242
+
243
+ /**
244
+ * 由HTML抽出title標籤內容
245
+ *
246
+ * @param {String} html 輸入HTML字串
247
+ * @returns {String|null} 回傳標題字串，無title標籤時回傳null
248
+ */
249
+ function extractHtmlTitle(html) {
250
+ let m = html.match(/&lt;title>([\s\S]*?)&lt;\/title>/i)
251
+ return m ? decodeEntities(m[1].trim()) : null
252
+ }
253
+
254
+
255
+ /**
256
+ * 由正文第一段抽出作者署名
257
+ *
258
+ * 觀察者網正文首段常見【文/观察者网 XXX】或【文/观察者网专栏作者 XXX，翻译/ XXX】
259
+ *
260
+ * @param {String} contentHtml 輸入正文區塊HTML字串
261
+ * @returns {String|null} 回傳作者署名字串，無署名時回傳null
262
+ */
263
+ function extractAuthorFromContent(contentHtml) {
264
+ let m = contentHtml.match(/【([^】]{4,120})】/)
265
+ if (!m) {
266
+ return null
267
+ }
268
+ return m[1].replace(/&lt;[^>]+>/g, '').trim()
269
+ }
270
+
271
+
272
+ /**
273
+ * 由HTML抽出發布時間
274
+ *
275
+ * @param {String} html 輸入HTML字串
276
+ * @returns {String|null} 回傳發布時間字串，格式為'YYYY-MM-DD HH:mm:ss'，無發布時間時回傳null
277
+ */
278
+ function extractPubTime(html) {
279
+ let m = html.match(/&lt;span[^>]*>(20\d{2}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})&lt;\/span>/)
280
+ return m ? m[1] : null
281
+ }
282
+
283
+
284
+ /**
285
+ * 由首頁底部A-Z索引解析作者對照
286
+ *
287
+ * @param {String} html 輸入首頁HTML字串
288
+ * @returns {Array} 回傳作者物件陣列，各物件為{slug,name,letter}
289
+ */
290
+ function parseAuthorIndex(html) {
291
+
292
+ let flat = html.replace(/\n/g, ' ')
293
+ let items = []
294
+ let seen = new Set()
295
+
296
+ let dlBlocks = [...flat.matchAll(/&lt;dt>([A-Z])&lt;\/dt>\s*&lt;dd[^>]*>([\s\S]*?)&lt;\/dd>/g)]
297
+ for (let [, letter, body] of dlBlocks) {
298
+ let linkRe = /&lt;a[^>]+href="(?:\.\.\/|\/)?([A-Za-z][a-zA-Z0-9_-]+)\/list_1\.shtml"[^>]*>([^&lt;]+)&lt;\/a>/g
299
+ let m
300
+ while ((m = linkRe.exec(body)) !== null) {
301
+ let slug = m[1]
302
+ let name = decodeEntities(m[2]).trim()
303
+ if (!name || seen.has(slug)) {
304
+ continue
305
+ }
306
+ seen.add(slug)
307
+ items.push({ slug, name, letter })
308
+ }
309
+ }
310
+
311
+ return items
312
+ }
313
+
314
+
315
+ /**
316
+ * 解析列表頁之文章清單
317
+ *
318
+ * 列表頁slug與文章URL slug通常不同(作者頁同slug, 主題集頁用各自作者slug, 欄目頁用大頻道slug),
319
+ * 故本函數不限定文章URL之slug, 抽出所有符合/&lt;slug>/&lt;YYYY_MM_DD>_&lt;id>.shtml模式之連結
320
+ *
321
+ * @param {String} html 輸入列表頁HTML字串
322
+ * @param {String} baseUrl 輸入網站根網址字串
323
+ * @returns {Array} 回傳文章物件陣列，各物件為{url,title,slug}
324
+ */
325
+ function parseListPage(html, baseUrl) {
326
+
327
+ let flat = html.replace(/\n/g, ' ')
328
+ let items = []
329
+ let seen = new Set()
330
+
331
+ let linkRe = /&lt;a[^>]+href="\/([A-Za-z][a-zA-Z0-9_-]+)\/(\d{4}_\d{2}_\d{2}_\d+)\.shtml"(?:[^>]*title="([^"]+)")?[^>]*>([^&lt;]*)&lt;\/a>/g
332
+ let m
333
+ while ((m = linkRe.exec(flat)) !== null) {
334
+ let articleSlug = m[1]
335
+ let dateId = m[2]
336
+ let titleAttr = m[3] &amp;&amp; decodeEntities(m[3]).trim()
337
+ let inner = m[4] &amp;&amp; decodeEntities(m[4]).replace(/&lt;[^>]+>/g, '').trim()
338
+ let title = titleAttr || inner || ''
339
+ let key = `${articleSlug}/${dateId}`
340
+ if (seen.has(key) || !title) {
341
+ continue
342
+ }
343
+ seen.add(key)
344
+ items.push({ url: `${baseUrl}/${articleSlug}/${dateId}.shtml`, title, slug: articleSlug })
345
+ }
346
+
347
+ return items
348
+ }
349
+
350
+
351
+ /**
352
+ * 自動翻頁抓取全部列表頁
353
+ *
354
+ * 逐頁抓/&lt;slug>/list_N.shtml, 適用於作者頁、欄目頁與主題集頁;
355
+ * 第1頁被302導向首頁代表slug不存在故拋錯, 後續頁被導向首頁或無新項目則視為到底
356
+ *
357
+ * @param {String} slug 輸入欄目或作者之slug字串
358
+ * @param {Object} cfg 輸入已正規化之設定物件
359
+ * @returns {Promise} 回傳Promise，resolve回傳{items,pages_fetched}
360
+ */
361
+ async function fetchAllListPages(slug, cfg) {
362
+
363
+ let all = []
364
+ let seen = new Set()
365
+ let pagesFetched = 0
366
+
367
+ for (let page = 1; page &lt;= cfg.maxPages; page++) {
368
+
369
+ let url = `${cfg.baseUrl}/${slug}/list_${page}.shtml`
370
+ if (cfg.showLog) {
371
+ process.stderr.write(`[info] fetching ${slug} list_${page} ...\n`)
372
+ }
373
+
374
+ //html, 翻頁失敗時第1頁失敗即拋, 後續頁失敗視為到底
375
+ let html = null
376
+ try {
377
+ html = await fetchHtml(url, cfg)
378
+ }
379
+ catch (err) {
380
+ if (page === 1) {
381
+ throw err
382
+ }
383
+ if (cfg.showLog) {
384
+ process.stderr.write(`[info] page ${page} fetch error (assumed end of list): ${err.message}\n`)
385
+ }
386
+ break
387
+ }
388
+
389
+ //偵測「不存在的slug被302跳首頁」
390
+ if (isHomepageHtml(html)) {
391
+ if (page === 1) {
392
+ let err = new Error(`slug "${slug}" 不存在：請求 list 頁被觀察者網 302 導向首頁。`)
393
+ err.reason = 'redirected-to-homepage'
394
+ throw err
395
+ }
396
+ if (cfg.showLog) {
397
+ process.stderr.write(`[info] page ${page} 被導向首頁，視為到底，停止翻頁\n`)
398
+ }
399
+ break
400
+ }
401
+
402
+ pagesFetched++
403
+
404
+ let items = parseListPage(html, cfg.baseUrl)
405
+ let fresh = items.filter((it) => !seen.has(it.url))
406
+ if (fresh.length === 0) {
407
+ if (cfg.showLog) {
408
+ process.stderr.write(`[info] page ${page} 無新項目，停止翻頁\n`)
409
+ }
410
+ break
411
+ }
412
+
413
+ for (let it of fresh) {
414
+ seen.add(it.url)
415
+ }
416
+ all.push(...fresh)
417
+
418
+ if (page &lt; cfg.maxPages) {
419
+ await delay(cfg.pageDelayMs)
420
+ }
421
+
422
+ }
423
+
424
+ return { items: all, pages_fetched: pagesFetched }
425
+ }
426
+
427
+
428
+ /**
429
+ * 抓取觀察者網作者索引
430
+ *
431
+ * 由首頁底部A-Z索引解析作者中文名與slug對照, 注意首頁索引僅含部分作者
432
+ *
433
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
434
+ * @param {String} [opt.baseUrl='https://www.guancha.cn'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
435
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
436
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
437
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
438
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
439
+ * @returns {Promise} 回傳Promise，resolve回傳作者物件陣列，各物件為{slug,name,letter}
440
+ * @example
441
+ *
442
+ * import { fetchAuthorsList } from './src/fetchGuancha.mjs'
443
+ *
444
+ * let test = async () => {
445
+ *
446
+ * let rs = await fetchAuthorsList()
447
+ * console.log(rs.length, rs[0])
448
+ * // => 636 { slug: 'AnSheng', name: '安生', letter: 'A' }
449
+ *
450
+ * }
451
+ * await test()
452
+ * .catch((err) => {
453
+ * console.log(err)
454
+ * })
455
+ *
456
+ */
457
+ async function fetchAuthorsList(opt = {}) {
458
+
459
+ let cfg = getCfg(opt)
460
+
461
+ let html = await fetchHtml(`${cfg.baseUrl}/`, cfg)
462
+
463
+ return parseAuthorIndex(html)
464
+ }
465
+
466
+
467
+ /**
468
+ * 由作者清單查找指定中文名之作者
469
+ *
470
+ * 本函數不轉繁簡, 呼叫端須自行使用站方登錄字形(簡體)
471
+ *
472
+ * @param {Array} authors 輸入作者物件陣列
473
+ * @param {String} name 輸入作者中文名字串
474
+ * @returns {Object|null} 回傳作者物件，查無時回傳null
475
+ * @example
476
+ *
477
+ * import { lookupAuthor } from './src/fetchGuancha.mjs'
478
+ *
479
+ * console.log(lookupAuthor([{ slug: 'AnSheng', name: '安生' }], '安生'))
480
+ * // => { slug: 'AnSheng', name: '安生' }
481
+ *
482
+ */
483
+ function lookupAuthor(authors, name) {
484
+
485
+ if (!Array.isArray(authors) || !name) {
486
+ return null
487
+ }
488
+
489
+ return authors.find((a) => a.name === name) || null
490
+ }
491
+
492
+
493
+ /**
494
+ * 由已知主題對照表查找指定名稱之主題
495
+ *
496
+ * 採first-match, 故同名時分頁版slug優先(見KNOWN_TOPICS說明)
497
+ *
498
+ * @param {String} name 輸入主題名稱字串
499
+ * @returns {Object|null} 回傳主題物件，查無時回傳null
500
+ * @example
501
+ *
502
+ * import { lookupTopic } from './src/fetchGuancha.mjs'
503
+ *
504
+ * console.log(lookupTopic('财经'))
505
+ * // => { slug: 'CaiJing', name: '财经' }
506
+ *
507
+ */
508
+ function lookupTopic(name) {
509
+
510
+ if (!name) {
511
+ return null
512
+ }
513
+
514
+ return KNOWN_TOPICS.find((t) => t.name === name) || null
515
+ }
516
+
517
+
518
+ /**
519
+ * 抓取觀察者網指定作者之文章清單
520
+ *
521
+ * 給name時先抓首頁A-Z索引解析出slug再翻頁全抓(兩跳), 給slug則直接翻頁全抓;
522
+ * 查無此作者時仍回status為'success'且count為0, slug不存在(被302導向首頁)時回status為'error'
523
+ *
524
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
525
+ * @param {String} [opt.name] 輸入作者中文名字串，與slug擇一必填
526
+ * @param {String} [opt.slug] 輸入作者slug字串，與name擇一必填
527
+ * @param {String} [opt.baseUrl='https://www.guancha.cn'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
528
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
529
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
530
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
531
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
532
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，name與slug皆未給時reject回傳錯誤物件
533
+ * @example
534
+ *
535
+ * import { fetchAuthorArticles } from './src/fetchGuancha.mjs'
536
+ *
537
+ * let test = async () => {
538
+ *
539
+ * let r = await fetchAuthorArticles({ name: '安生' })
540
+ * console.log(r.status, r.count, r.resolved.slug)
541
+ * // => 'success' 120 'AnSheng'
542
+ *
543
+ * }
544
+ * await test()
545
+ * .catch((err) => {
546
+ * console.log(err)
547
+ * })
548
+ *
549
+ */
550
+ async function fetchAuthorArticles(opt = {}) {
551
+
552
+ let cfg = getCfg(opt)
553
+
554
+ //「有提供才檢」, 傳了但非有效字串視為未提供, 讓下方「至少一個」檢查接手
555
+ let name = get(opt, 'name')
556
+ if (!isestr(name)) {
557
+ name = ''
558
+ }
559
+ let slug = get(opt, 'slug')
560
+ if (!isestr(slug)) {
561
+ slug = ''
562
+ }
563
+ if (!name &amp;&amp; !slug) {
564
+ throw new Error('需要 name 或 slug')
565
+ }
566
+
567
+ let resolvedSlug = slug
568
+ let resolvedName = name
569
+
570
+ if (!resolvedSlug) {
571
+
572
+ if (cfg.showLog) {
573
+ process.stderr.write('[info] fetching authors index from / ...\n')
574
+ }
575
+ let authors = await fetchAuthorsList(opt)
576
+ let author = lookupAuthor(authors, name)
577
+ if (!author) {
578
+ //「真的沒這位作者」屬成功查詢、結果0筆
579
+ return {
580
+ status: 'success',
581
+ site: 'guancha',
582
+ mode: 'author',
583
+ query: name,
584
+ fetched_at: new Date().toISOString(),
585
+ authors_count: authors.length,
586
+ count: 0,
587
+ items: [],
588
+ message:
589
+ `"${name}" 不在觀察者網作者索引中（共 ${authors.length} 位）。尚無此作者文章。` +
590
+ `提醒：本函數不轉繁簡，呼叫端負責用站方登錄字形（簡體）。`,
591
+ }
592
+ }
593
+
594
+ resolvedSlug = author.slug
595
+ resolvedName = author.name
596
+ }
597
+
598
+ let url = `${cfg.baseUrl}/${resolvedSlug}`
599
+
600
+ let listResult = null
601
+ try {
602
+ listResult = await fetchAllListPages(resolvedSlug, cfg)
603
+ }
604
+ catch (err) {
605
+ return {
606
+ status: 'error',
607
+ site: 'guancha',
608
+ mode: 'author',
609
+ query: resolvedName || resolvedSlug,
610
+ resolved: { slug: resolvedSlug, url, name: resolvedName },
611
+ fetched_at: new Date().toISOString(),
612
+ reason: err.reason,
613
+ error: err.message,
614
+ }
615
+ }
616
+
617
+ return {
618
+ status: 'success',
619
+ site: 'guancha',
620
+ mode: 'author',
621
+ query: resolvedName || resolvedSlug,
622
+ resolved: { slug: resolvedSlug, url, name: resolvedName, pages_fetched: listResult.pages_fetched },
623
+ fetched_at: new Date().toISOString(),
624
+ count: listResult.items.length,
625
+ items: listResult.items,
626
+ }
627
+ }
628
+
629
+
630
+ /**
631
+ * 抓取觀察者網指定關鍵字之文章清單
632
+ *
633
+ * 站方搜尋API(s.guancha.cn/main/search-v2)由sojson.v4混淆並走MD5簽名, 純curl路徑無法支援,
634
+ * 故本函數一律回status為'error'且error為'unsupported-by-curl', 並於message建議改用作者或主題slug
635
+ *
636
+ * @param {String} keyword 輸入關鍵字字串
637
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
638
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,fetched_at,error,message}，keyword非有效字串時reject回傳錯誤物件
639
+ * @example
640
+ *
641
+ * import { fetchKeywordArticles } from './src/fetchGuancha.mjs'
642
+ *
643
+ * let test = async () => {
644
+ *
645
+ * let r = await fetchKeywordArticles('人工智能')
646
+ * console.log(r.status, r.error)
647
+ * // => 'error' 'unsupported-by-curl'
648
+ *
649
+ * }
650
+ * await test()
651
+ * .catch((err) => {
652
+ * console.log(err)
653
+ * })
654
+ *
655
+ */
656
+ async function fetchKeywordArticles(keyword, opt = {}) {
657
+
658
+ //check
659
+ if (!isestr(keyword)) {
660
+ throw new Error('需要 keyword')
661
+ }
662
+
663
+ return {
664
+ status: 'error',
665
+ site: 'guancha',
666
+ mode: 'keyword',
667
+ query: keyword,
668
+ fetched_at: new Date().toISOString(),
669
+ error: 'unsupported-by-curl',
670
+ message:
671
+ `觀察者網搜尋 API（s.guancha.cn/main/search-v2）由 sojson.v4 混淆並走 MD5 簽名，` +
672
+ `本函數（純 curl 路徑）無法支援。` +
673
+ `如已知作者拼音 slug，請改用 fetchAuthorArticles({ slug })；如已知主題 slug，請改用 fetchTopicArticles({ slug })。`,
674
+ }
675
+ }
676
+
677
+
678
+ /**
679
+ * 抓取觀察者網指定標題關鍵字之文章清單
680
+ *
681
+ * 站方無分「標題」與「全文」搜尋(兩者同源), 與fetchKeywordArticles同因簽名混淆而不可用,
682
+ * 故本函數一律回status為'error'且error為'unsupported-by-curl'
683
+ *
684
+ * @param {String} keyword 輸入標題關鍵字字串
685
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
686
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,fetched_at,error,message}，keyword非有效字串時reject回傳錯誤物件
687
+ * @example
688
+ *
689
+ * import { fetchTitleArticles } from './src/fetchGuancha.mjs'
690
+ *
691
+ * let test = async () => {
692
+ *
693
+ * let r = await fetchTitleArticles('人工智能')
694
+ * console.log(r.status, r.error)
695
+ * // => 'error' 'unsupported-by-curl'
696
+ *
697
+ * }
698
+ * await test()
699
+ * .catch((err) => {
700
+ * console.log(err)
701
+ * })
702
+ *
703
+ */
704
+ async function fetchTitleArticles(keyword, opt = {}) {
705
+
706
+ //check
707
+ if (!isestr(keyword)) {
708
+ throw new Error('需要 keyword')
709
+ }
710
+
711
+ return {
712
+ status: 'error',
713
+ site: 'guancha',
714
+ mode: 'title',
715
+ query: keyword,
716
+ fetched_at: new Date().toISOString(),
717
+ error: 'unsupported-by-curl',
718
+ message:
719
+ `觀察者網無分「標題」與「全文」搜尋（兩者同源），` +
720
+ `搜尋 API 由 sojson.v4 混淆並走 MD5 簽名，本函數（純 curl 路徑）無法支援。` +
721
+ `如已知作者拼音 slug，請改用 fetchAuthorArticles({ slug })。`,
722
+ }
723
+ }
724
+
725
+
726
+ /**
727
+ * 抓取觀察者網指定主題之文章清單
728
+ *
729
+ * 給name時先查KNOWN_TOPICS對照表解析出slug再翻頁全抓, 給slug則直接翻頁全抓;
730
+ * 主題不在對照表時fail-fast回status為'error'且error為'topic-not-in-table'
731
+ *
732
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
733
+ * @param {String} [opt.name] 輸入主題中文名字串，與slug擇一必填
734
+ * @param {String} [opt.slug] 輸入主題slug字串，與name擇一必填
735
+ * @param {String} [opt.baseUrl='https://www.guancha.cn'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
736
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
737
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
738
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
739
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
740
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,query,resolved,fetched_at,count,items}，name與slug皆未給時reject回傳錯誤物件
741
+ * @example
742
+ *
743
+ * import { fetchTopicArticles } from './src/fetchGuancha.mjs'
744
+ *
745
+ * let test = async () => {
746
+ *
747
+ * let r = await fetchTopicArticles({ name: '财经' })
748
+ * console.log(r.status, r.resolved.slug, r.count)
749
+ * // => 'success' 'CaiJing' 300
750
+ *
751
+ * }
752
+ * await test()
753
+ * .catch((err) => {
754
+ * console.log(err)
755
+ * })
756
+ *
757
+ */
758
+ async function fetchTopicArticles(opt = {}) {
759
+
760
+ let cfg = getCfg(opt)
761
+
762
+ //「有提供才檢」, 傳了但非有效字串視為未提供, 讓下方「至少一個」檢查接手
763
+ let name = get(opt, 'name')
764
+ if (!isestr(name)) {
765
+ name = ''
766
+ }
767
+ let slug = get(opt, 'slug')
768
+ if (!isestr(slug)) {
769
+ slug = ''
770
+ }
771
+ if (!name &amp;&amp; !slug) {
772
+ throw new Error('需要 name 或 slug')
773
+ }
774
+
775
+ let resolvedSlug = slug
776
+ let resolvedName = name
777
+
778
+ if (!resolvedSlug) {
779
+ let topic = lookupTopic(name)
780
+ if (!topic) {
781
+ //不命中對照表則fail-fast
782
+ return {
783
+ status: 'error',
784
+ site: 'guancha',
785
+ mode: 'topic',
786
+ query: name,
787
+ fetched_at: new Date().toISOString(),
788
+ topics_count: KNOWN_TOPICS.length,
789
+ error: 'topic-not-in-table',
790
+ message:
791
+ `主題 "${name}" 不在已知主題對照表中（共 ${KNOWN_TOPICS.length} 個）。` +
792
+ `觀察者網無公開主題清單頁，本函數採手工對照。建議直接傳 slug（拼音 slug）。` +
793
+ `本函數不自動轉向，請呼叫端決定是否重試。`,
794
+ }
795
+ }
796
+ resolvedSlug = topic.slug
797
+ resolvedName = topic.name
798
+ }
799
+
800
+ let url = `${cfg.baseUrl}/${resolvedSlug}`
801
+
802
+ let listResult = null
803
+ try {
804
+ listResult = await fetchAllListPages(resolvedSlug, cfg)
805
+ }
806
+ catch (err) {
807
+ return {
808
+ status: 'error',
809
+ site: 'guancha',
810
+ mode: 'topic',
811
+ query: resolvedName || resolvedSlug,
812
+ resolved: { slug: resolvedSlug, url, name: resolvedName },
813
+ fetched_at: new Date().toISOString(),
814
+ reason: err.reason,
815
+ error: err.message,
816
+ }
817
+ }
818
+
819
+ return {
820
+ status: 'success',
821
+ site: 'guancha',
822
+ mode: 'topic',
823
+ query: resolvedName || resolvedSlug,
824
+ resolved: { slug: resolvedSlug, url, name: resolvedName, pages_fetched: listResult.pages_fetched },
825
+ fetched_at: new Date().toISOString(),
826
+ count: listResult.items.length,
827
+ items: listResult.items,
828
+ }
829
+ }
830
+
831
+
832
+ /**
833
+ * 抓取觀察者網單篇文章並轉為Markdown
834
+ *
835
+ * 由content all-txt區塊切出正文轉Markdown(保留圖片), 並產生含title、source、author、published與created之YAML frontmatter;
836
+ * 文章已下架時站方會302跳首頁, 本函數偵測此模式並回status為'error'且error為'article-not-found'
837
+ *
838
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
839
+ * @param {String} opt.url 輸入文章網址字串
840
+ * @param {String} [opt.baseUrl='https://www.guancha.cn'] 輸入網站根網址字串，供測試或改指向鏡像時覆寫
841
+ * @param {Integer} [opt.maxRetries=5] 輸入最大重試次數整數，預設5
842
+ * @param {Integer} [opt.pageDelayMs=1000] 輸入翻頁之頁間延遲毫秒整數，預設1000
843
+ * @param {Integer} [opt.maxPages=50] 輸入翻頁上限正整數，上限為50，預設50
844
+ * @param {Boolean} [opt.showLog=true] 輸入是否顯示過程訊息布林值，預設true
845
+ * @returns {Promise} 回傳Promise，resolve回傳結果物件{status,site,mode,url,title,author,published,chars,markdown}，url非有效字串時reject回傳錯誤物件
846
+ * @example
847
+ *
848
+ * import { fetchArticle } from './src/fetchGuancha.mjs'
849
+ *
850
+ * let test = async () => {
851
+ *
852
+ * let r = await fetchArticle({ url: 'https://www.guancha.cn/internation/2026_04_29_815417.shtml' })
853
+ * console.log(r.status, r.title, r.chars)
854
+ * // => 'success' 'OPEC+宣布增產' 2345
855
+ *
856
+ * }
857
+ * await test()
858
+ * .catch((err) => {
859
+ * console.log(err)
860
+ * })
861
+ *
862
+ */
863
+ async function fetchArticle(opt = {}) {
864
+
865
+ let cfg = getCfg(opt)
866
+
867
+ //check
868
+ let url = get(opt, 'url')
869
+ if (!isestr(url)) {
870
+ throw new Error('需要 url')
871
+ }
872
+
873
+ let html = null
874
+ try {
875
+ html = await fetchHtml(url, cfg)
876
+ }
877
+ catch (err) {
878
+ return {
879
+ status: 'error',
880
+ site: 'guancha',
881
+ mode: 'fetch',
882
+ url,
883
+ fetched_at: new Date().toISOString(),
884
+ error: err.message || 'fetch failed',
885
+ }
886
+ }
887
+
888
+ //偵測「文章下架被302跳首頁」, 即title為單純「观察者网」且找不到content all-txt區塊
889
+ let titleRaw = extractHtmlTitle(html) || ''
890
+ let contentBlock = extractDivContent(html, 'content all-txt')
891
+ if (titleRaw === '观察者网' || titleRaw === '觀察者網' || !contentBlock) {
892
+ return {
893
+ status: 'error',
894
+ site: 'guancha',
895
+ mode: 'fetch',
896
+ url,
897
+ fetched_at: new Date().toISOString(),
898
+ error: 'article-not-found',
899
+ message: `無此文章（title="${titleRaw}"，可能已下架或 URL 不正確）。`,
900
+ }
901
+ }
902
+
903
+ let title = titleRaw.replace(/[_\-\s|｜]*(?:观察者网|觀察者網)\s*$/u, '').trim()
904
+ let author = extractAuthorFromContent(contentBlock) || ''
905
+ let pubTime = extractPubTime(html) || ''
906
+ let created = new Date().toISOString().slice(0, 10)
907
+ let body = htmlToMarkdown(contentBlock, { image: true })
908
+
909
+ //YAML frontmatter, 雙引號escape內含雙引號
910
+ let yamlEscape = (s) => String(s).replace(/"/g, '\\"')
911
+ let markdown =
912
+ `---
913
+ title: "${yamlEscape(title)}"
914
+ source: "${url}"
915
+ author: "${yamlEscape(author)}"
916
+ published: "${pubTime}"
917
+ created: ${created}
918
+ description:
919
+ ---
920
+ ${body}
921
+ `
922
+
923
+ return {
924
+ status: 'success',
925
+ site: 'guancha',
926
+ mode: 'fetch',
927
+ url,
928
+ title,
929
+ author,
930
+ published: pubTime,
931
+ chars: body.length,
932
+ markdown,
933
+ }
934
+ }
935
+
936
+
937
+ /**
938
+ * 抓取觀察者網(guancha.cn)
939
+ *
940
+ * @returns {Object} 回傳物件，其內可呼叫fetchAuthorsList、lookupAuthor、lookupTopic、fetchAuthorArticles、fetchKeywordArticles、fetchTitleArticles、fetchTopicArticles、fetchArticle
941
+ * @example
942
+ *
943
+ * 詳見fetchAuthorArticles、fetchTopicArticles、fetchArticle範例
944
+ *
945
+ */
946
+ let fetchGuancha = {
947
+ BASE_URL,
948
+ USER_AGENT,
949
+ MAX_PAGES,
950
+ KNOWN_TOPICS,
951
+ safeFilename,
952
+ fetchAuthorsList,
953
+ lookupAuthor,
954
+ lookupTopic,
955
+ fetchAuthorArticles,
956
+ fetchKeywordArticles,
957
+ fetchTitleArticles,
958
+ fetchTopicArticles,
959
+ fetchArticle,
960
+ }
961
+
962
+
963
+ export {
964
+ BASE_URL,
965
+ USER_AGENT,
966
+ PAGE_DELAY_MS,
967
+ MAX_PAGES,
968
+ KNOWN_TOPICS,
969
+ safeFilename,
970
+ isHomepageHtml,
971
+ parseAuthorIndex,
972
+ parseListPage,
973
+ fetchAuthorsList,
974
+ lookupAuthor,
975
+ lookupTopic,
976
+ fetchAuthorArticles,
977
+ fetchKeywordArticles,
978
+ fetchTitleArticles,
979
+ fetchTopicArticles,
980
+ fetchArticle
981
+ }
982
+ export default fetchGuancha
983
+ </code></pre>
984
+ </article>
985
+ </section>
986
+
987
+
988
+
989
+
990
+
991
+
992
+ </div>
993
+
994
+ <br class="clear">
995
+
996
+ <footer>
997
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.5</a> on Fri Aug 07 2026 12:04:51 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
998
+ </footer>
999
+
1000
+ <script>prettyPrint();</script>
1001
+ <script src="scripts/polyfill.js"></script>
1002
+ <script src="scripts/linenumber.js"></script>
1003
+
1004
+
1005
+
1006
+ </body>
1007
+ </html>