rsshub 1.0.0-master.e6d4e1e → 1.0.0-master.e6d53dc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/lib/api/radar/rules/all.ts +3 -6
  2. package/lib/api/radar/rules/one.ts +3 -6
  3. package/lib/config.ts +12 -22
  4. package/lib/errors/index.tsx +1 -1
  5. package/lib/index.ts +1 -1
  6. package/lib/middleware/anti-hotlink.test.ts +191 -6
  7. package/lib/middleware/anti-hotlink.ts +28 -0
  8. package/lib/middleware/parameter.test.ts +22 -5
  9. package/lib/middleware/parameter.ts +47 -17
  10. package/lib/registry.test.ts +8 -0
  11. package/lib/registry.ts +2 -0
  12. package/lib/router.js +0 -3
  13. package/lib/routes/141jav/index.ts +25 -21
  14. package/lib/routes/141ppv/index.ts +24 -20
  15. package/lib/routes/51read/article.ts +82 -0
  16. package/lib/routes/51read/namespace.ts +6 -0
  17. package/lib/routes/5eplay/utils.ts +5 -3
  18. package/lib/routes/agri/index.ts +307 -0
  19. package/lib/routes/agri/namespace.ts +8 -0
  20. package/lib/routes/agri/templates/description.art +21 -0
  21. package/lib/routes/aljazeera/index.ts +2 -2
  22. package/lib/routes/apnews/sitemap.ts +98 -0
  23. package/lib/routes/apnews/utils.ts +7 -5
  24. package/lib/routes/autocentre/index.ts +29 -0
  25. package/lib/routes/autocentre/namespace.ts +7 -0
  26. package/lib/routes/bilibili/api-interface.d.ts +1 -0
  27. package/lib/routes/bilibili/cache.ts +3 -1
  28. package/lib/routes/bilibili/dynamic.ts +7 -9
  29. package/lib/routes/binance/announcement.ts +168 -0
  30. package/lib/routes/binance/types.ts +26 -0
  31. package/lib/routes/bing/daily-wallpaper.ts +1 -1
  32. package/lib/routes/bjnews/cat.ts +43 -0
  33. package/lib/routes/bjnews/column.ts +43 -0
  34. package/lib/routes/bjnews/namespace.ts +6 -0
  35. package/lib/routes/bjnews/utils.ts +19 -0
  36. package/lib/routes/bjx/fd.ts +35 -26
  37. package/lib/routes/buaa/jiaowu.ts +1 -1
  38. package/lib/routes/buaa/lib/space/newbook.ts +171 -0
  39. package/lib/routes/buaa/lib/space/templates/newbook.art +44 -0
  40. package/lib/routes/cccfna/index.ts +2 -2
  41. package/lib/routes/cctv/category.ts +16 -12
  42. package/lib/routes/cctv/xwlb.ts +32 -10
  43. package/lib/routes/cdu/jwgg.ts +79 -0
  44. package/lib/routes/cdu/namespace.ts +6 -0
  45. package/lib/routes/cffex/announcement.ts +73 -0
  46. package/lib/routes/cffex/namespace.ts +6 -0
  47. package/lib/routes/chlinlearn/daily-blog.ts +55 -0
  48. package/lib/routes/chlinlearn/namespcae.ts +6 -0
  49. package/lib/routes/copymanga/comic.ts +3 -1
  50. package/lib/routes/devolverdigital/blog.ts +59 -42
  51. package/lib/routes/digitalcameraworld/news.ts +1 -1
  52. package/lib/routes/discord/channel.ts +10 -10
  53. package/lib/routes/discord/discord-api.ts +27 -32
  54. package/lib/routes/discord/templates/message.art +15 -0
  55. package/lib/routes/discourse/notifications.ts +2 -1
  56. package/lib/routes/dongqiudi/result.ts +1 -1
  57. package/lib/routes/eastmoney/report/index.ts +63 -15
  58. package/lib/routes/eastmoney/templates/stock_description.art +36 -0
  59. package/lib/routes/eastmoney/utils.ts +35 -0
  60. package/lib/routes/fediverse/timeline.ts +35 -18
  61. package/lib/routes/flashcat/blog.ts +61 -0
  62. package/lib/routes/flashcat/namespace.ts +6 -0
  63. package/lib/routes/ftm/index.ts +60 -0
  64. package/lib/routes/ftm/namespace.ts +6 -0
  65. package/lib/routes/galxe/index.ts +94 -0
  66. package/lib/routes/galxe/namespace.ts +9 -0
  67. package/lib/routes/gamersky/ent.ts +57 -0
  68. package/lib/routes/gamersky/namespace.ts +10 -0
  69. package/lib/routes/gamersky/news.ts +91 -0
  70. package/lib/routes/gamersky/review.ts +80 -0
  71. package/lib/routes/gamersky/utils.ts +94 -0
  72. package/lib/routes/gdut/oa-news.ts +12 -17
  73. package/lib/routes/gisreportsonline/index.ts +60 -0
  74. package/lib/routes/gisreportsonline/namespace.ts +6 -0
  75. package/lib/routes/github/activity.ts +4 -3
  76. package/lib/routes/github/search.ts +20 -18
  77. package/lib/routes/google/research.ts +45 -0
  78. package/lib/routes/gov/cbirc/index.ts +371 -0
  79. package/lib/routes/gov/chongqing/sydwgkzp.ts +4 -4
  80. package/lib/routes/gov/lswz/index.ts +289 -0
  81. package/lib/routes/gov/zj/ningbogzw-notice.ts +50 -0
  82. package/lib/routes/gov/zj/ningborsjnotice.ts +50 -0
  83. package/lib/routes/gq/news.ts +1 -1
  84. package/lib/routes/gs/developer/blog.ts +58 -0
  85. package/lib/routes/gs/namespace.ts +9 -0
  86. package/lib/routes/healthz.ts +8 -0
  87. package/lib/routes/hket/index.ts +80 -76
  88. package/lib/routes/hottoys/index.ts +63 -0
  89. package/lib/routes/hottoys/namespace.ts +6 -0
  90. package/lib/routes/hust/gs.ts +281 -0
  91. package/lib/routes/hust/namespace.ts +1 -1
  92. package/lib/routes/imhcg/blog.ts +47 -0
  93. package/lib/routes/imhcg/namespace.ts +7 -0
  94. package/lib/routes/imop/namespace.ts +10 -0
  95. package/lib/routes/imop/tianshu.ts +53 -0
  96. package/lib/routes/infzm/hot.ts +39 -0
  97. package/lib/routes/infzm/index.ts +3 -30
  98. package/lib/routes/infzm/utils.ts +34 -0
  99. package/lib/routes/inspirehep/author.ts +46 -0
  100. package/lib/routes/inspirehep/literature.ts +42 -0
  101. package/lib/routes/inspirehep/namespace.ts +7 -0
  102. package/lib/routes/inspirehep/types.ts +210 -0
  103. package/lib/routes/inspirehep/utils.ts +15 -0
  104. package/lib/routes/iwara/subscriptions.ts +1 -1
  105. package/lib/routes/javdb/actors.ts +6 -2
  106. package/lib/routes/joins/chinese.ts +218 -0
  107. package/lib/routes/joins/namespace.ts +8 -0
  108. package/lib/routes/joins/templates/description.art +17 -0
  109. package/lib/routes/juejin/dynamic.ts +114 -0
  110. package/lib/routes/kaopu/news.ts +3 -3
  111. package/lib/routes/kemono/index.ts +21 -0
  112. package/lib/routes/keylol/index.ts +56 -11
  113. package/lib/routes/kurogames/namespace.ts +7 -0
  114. package/lib/routes/kurogames/wutheringwaves/news.ts +59 -0
  115. package/lib/routes/link3/events.ts +86 -0
  116. package/lib/routes/link3/namespace.ts +6 -0
  117. package/lib/routes/link3/profile.ts +143 -0
  118. package/lib/routes/linkedin/cn/renderer.ts +1 -1
  119. package/lib/routes/lmu/jobs.ts +96 -0
  120. package/lib/routes/lmu/namespace.ts +17 -0
  121. package/lib/routes/lmu/templates/jobPosting.art +11 -0
  122. package/lib/routes/mcmod/index.ts +105 -0
  123. package/lib/routes/mcmod/namespace.ts +6 -0
  124. package/lib/routes/mcmod/templates/mod.art +13 -0
  125. package/lib/routes/misskey/user-timeline.ts +42 -0
  126. package/lib/routes/misskey/utils.ts +41 -1
  127. package/lib/routes/nationalgeographic/latest-stories.ts +1 -1
  128. package/lib/routes/nationalgeographic/templates/stories.art +1 -1
  129. package/lib/routes/newyorker/news.ts +24 -17
  130. package/lib/routes/oschina/news.ts +2 -0
  131. package/lib/routes/people/index.ts +4 -2
  132. package/lib/routes/picnob/user.ts +11 -3
  133. package/lib/routes/pixiv/search.ts +20 -1
  134. package/lib/routes/pixiv/utils.ts +3 -3
  135. package/lib/routes/psyche/topic.ts +1 -1
  136. package/lib/routes/psyche/type.ts +1 -1
  137. package/lib/routes/psyche/utils.ts +1 -1
  138. package/lib/routes/questn/community.ts +76 -0
  139. package/lib/routes/questn/events.ts +101 -0
  140. package/lib/routes/questn/namespace.ts +6 -0
  141. package/lib/routes/questn/util.ts +16 -0
  142. package/lib/routes/rebase/geekdaily.ts +35 -0
  143. package/lib/routes/rebase/namespace.ts +6 -0
  144. package/lib/routes/rss3/index.ts +22 -13
  145. package/lib/routes/rss3/interfaces/metadata.ts +67 -0
  146. package/lib/routes/rsshub/routes.ts +18 -12
  147. package/lib/routes/samd/news.ts +1 -1
  148. package/lib/routes/sara/index.ts +1 -1
  149. package/lib/routes/scut/gzic/media.ts +71 -0
  150. package/lib/routes/scut/gzic/news.ts +65 -0
  151. package/lib/routes/scut/gzic/notice.ts +88 -0
  152. package/lib/routes/shiep/config.ts +4 -10
  153. package/lib/routes/shiep/index.ts +9 -9
  154. package/lib/routes/showstart/artist.ts +5 -4
  155. package/lib/routes/showstart/brand.ts +5 -4
  156. package/lib/routes/showstart/event.ts +8 -7
  157. package/lib/routes/showstart/search.ts +43 -9
  158. package/lib/routes/showstart/service.ts +81 -53
  159. package/lib/routes/showstart/site.ts +41 -0
  160. package/lib/routes/showstart/utils.ts +9 -8
  161. package/lib/routes/sis001/author.ts +52 -0
  162. package/lib/routes/sis001/common.ts +38 -0
  163. package/lib/routes/sis001/forum.ts +2 -29
  164. package/lib/routes/sjtu/jwc.ts +4 -1
  165. package/lib/routes/sjtu/namespace.ts +1 -1
  166. package/lib/routes/sjtu/seiee/index.ts +97 -0
  167. package/lib/routes/smzdm/product.ts +86 -0
  168. package/lib/routes/spotify/show.ts +1 -1
  169. package/lib/routes/springer/journal.ts +10 -12
  170. package/lib/routes/sse/sselawsrules.ts +370 -0
  171. package/lib/routes/sspai/author.ts +1 -1
  172. package/lib/routes/sspai/column.ts +1 -1
  173. package/lib/routes/sspai/index.ts +1 -1
  174. package/lib/routes/sspai/matrix.ts +1 -1
  175. package/lib/routes/sspai/series-update.ts +1 -1
  176. package/lib/routes/sspai/tag.ts +1 -1
  177. package/lib/routes/sspai/topic.ts +1 -1
  178. package/lib/routes/sustainabilitymag/articles.ts +2 -0
  179. package/lib/routes/szse/rule.ts +315 -42
  180. package/lib/routes/t66y/index.ts +29 -7
  181. package/lib/routes/taoguba/blog.ts +1 -1
  182. package/lib/routes/taoguba/index.ts +1 -1
  183. package/lib/routes/techcrunch/news.ts +20 -29
  184. package/lib/routes/techcrunch/templates/description.art +7 -0
  185. package/lib/routes/telegram/channel.ts +34 -1
  186. package/lib/routes/test/index.ts +50 -43
  187. package/lib/routes/the/index.ts +210 -0
  188. package/lib/routes/the/namespace.ts +8 -0
  189. package/lib/routes/the/templates/description.art +27 -0
  190. package/lib/routes/the/util.ts +307 -0
  191. package/lib/routes/theblockbeats/index.ts +10 -5
  192. package/lib/routes/towardsdatascience/latest.ts +67 -0
  193. package/lib/routes/towardsdatascience/namespace.ts +6 -0
  194. package/lib/routes/tvb/news.ts +1 -0
  195. package/lib/routes/twitter/api/index.ts +2 -0
  196. package/lib/routes/twitter/api/web-api/api.ts +18 -0
  197. package/lib/routes/twitter/api/web-api/constants.ts +2 -0
  198. package/lib/routes/twitter/api/web-api/login.ts +66 -0
  199. package/lib/routes/twitter/api/web-api/utils.ts +53 -27
  200. package/lib/routes/twitter/home-latest.ts +60 -0
  201. package/lib/routes/twitter/home.ts +1 -1
  202. package/lib/routes/twitter/user.ts +1 -1
  203. package/lib/routes/twitter/utils.ts +4 -1
  204. package/lib/routes/uber/blog.ts +1 -1
  205. package/lib/routes/ustc/scms.ts +97 -0
  206. package/lib/routes/v2ex/xna.ts +55 -0
  207. package/lib/routes/wchscu/namespace.ts +6 -0
  208. package/lib/routes/wchscu/recruit.ts +73 -0
  209. package/lib/routes/xiaohongshu/notes.ts +10 -2
  210. package/lib/routes/xiaohongshu/user.ts +23 -2
  211. package/lib/routes/xidian/cs.ts +143 -0
  212. package/lib/routes/xidian/gr.ts +289 -0
  213. package/lib/routes/xidian/jwc.ts +1 -0
  214. package/lib/routes/xidian/namespace.ts +1 -1
  215. package/lib/routes/yahoo/news/index.ts +1 -0
  216. package/lib/routes/yahoo/news/listid.ts +67 -0
  217. package/lib/routes/yahoo/news/utils.ts +8 -2
  218. package/lib/routes/yicai/carousel.ts +48 -0
  219. package/lib/routes/yicai/utils.ts +23 -17
  220. package/lib/routes/youtube/subscriptions.ts +1 -0
  221. package/lib/routes/yystv/docs.ts +28 -19
  222. package/lib/routes/zhihu/answers.ts +5 -1
  223. package/lib/routes/zhihu/execlib/x-zse-96-v3.ts +2 -0
  224. package/lib/routes/zhihu/namespace.ts +3 -0
  225. package/lib/routes/zhihu/utils.ts +15 -2
  226. package/lib/routes/zsxq/types.ts +9 -0
  227. package/lib/routes/zsxq/utils.ts +4 -3
  228. package/lib/setup.test.ts +1 -1
  229. package/lib/types.ts +6 -0
  230. package/lib/utils/cache/index.ts +8 -0
  231. package/lib/utils/camelcase-keys.spec.ts +127 -0
  232. package/lib/utils/camelcase-keys.ts +27 -0
  233. package/lib/utils/common-utils.ts +1 -0
  234. package/lib/views/error.tsx +3 -3
  235. package/lib/views/index.tsx +4 -3
  236. package/lib/views/layout.tsx +1 -1
  237. package/package.json +46 -44
  238. package/lib/routes/sse/lawandrules.ts +0 -68
  239. package/lib/routes-deprecated/bjnews/news.js +0 -44
  240. package/lib/routes-deprecated/gamersky/ent.js +0 -89
  241. package/lib/routes-deprecated/gamersky/news.js +0 -34
@@ -0,0 +1,307 @@
1
+ import got from '@/utils/got';
2
+ import { load } from 'cheerio';
3
+
4
+ const apiSlug = 'wp-json/wp/v2';
5
+
6
+ interface Filter {
7
+ id: string;
8
+ name: string;
9
+ slug: string;
10
+ }
11
+
12
+ const filterKeys: Record<string, string> = {
13
+ search: 's',
14
+ };
15
+
16
+ const filterApiKeys: Record<string, string | undefined> = {
17
+ category: 'categories',
18
+ tag: 'tags',
19
+ search: undefined,
20
+ };
21
+
22
+ const filterApiKeysWithNoId = new Set(['search']);
23
+
24
+ /**
25
+ * Bake filter search parameters.
26
+ *
27
+ * @param filterPairs - The filter pairs object.
28
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
29
+ * @param pairKey - The filter pair key.
30
+ * e.g. `{ id: ..., name: ..., slug: ... }`.
31
+ * @param isApi - Indicates if the search parameters are for API.
32
+ * @returns The baked filter search parameters.
33
+ */
34
+ const bakeFilterSearchParams = (filterPairs: Record<string, Filter[] | string[]>, pairKey: string, isApi: boolean = false): URLSearchParams => {
35
+ /**
36
+ * Bake filters recursively.
37
+ *
38
+ * @param filterPairs - The filter pairs object.
39
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
40
+ * @param filterSearchParams - The filter search parameters.
41
+ * e.g. `category=a,b&tag=c`.
42
+ * @returns The baked filter search parameters.
43
+ * e.g. `category=a,b&tag=c`.
44
+ */
45
+ const bakeFilters = (filterPairs: Record<string, Filter[] | string[]>, filterSearchParams: URLSearchParams): URLSearchParams => {
46
+ const keys = Object.keys(filterPairs).filter((key) => filterPairs[key]?.length > 0 && (isApi ? Object.hasOwn(filterApiKeys, key) : Object.hasOwn(filterKeys, key)));
47
+
48
+ if (keys.length === 0) {
49
+ return filterSearchParams;
50
+ }
51
+
52
+ const key = keys[0];
53
+ const pairs = filterPairs[key];
54
+
55
+ const originalFilters = { ...filterPairs };
56
+ delete originalFilters[key];
57
+
58
+ const filterKey = getFilterKeyForSearchParams(key, isApi);
59
+ const pairValues = pairs.map((pair) => (Object.hasOwn(pair, pairKey) ? pair[pairKey] : pair));
60
+
61
+ if (filterKey) {
62
+ filterSearchParams.append(filterKey, pairValues.join(','));
63
+ }
64
+
65
+ return bakeFilters(originalFilters, filterSearchParams);
66
+ };
67
+
68
+ return bakeFilters(filterPairs, new URLSearchParams());
69
+ };
70
+
71
+ /**
72
+ * Bake filters with pair.
73
+ *
74
+ * @param filters - The filters object.
75
+ * e.g. `{ category: [ a, b ], tag: [ c ] }`.
76
+ * @returns The baked filters.
77
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
78
+ */
79
+ const bakeFiltersWithPair = async (filters: Record<string, string[]>, rootUrl: string) => {
80
+ /**
81
+ * Bake keywords recursively.
82
+ *
83
+ * @param key - The key.
84
+ * e.g. `category` or `tag`.
85
+ * @param keywords - The keywords.
86
+ * e.g. `[ a, b ]`.
87
+ * @returns The baked keywords.
88
+ * e.g. `[ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ]`.
89
+ */
90
+ const bakeKeywords = async (key: string, keywords: string[]) => {
91
+ if (keywords.length === 0) {
92
+ return [];
93
+ }
94
+
95
+ const [keyword, ...rest] = keywords;
96
+
97
+ const filter = await getFilterByKeyAndKeyword(key, keyword, rootUrl);
98
+
99
+ return [
100
+ ...(filter?.id && filter?.slug
101
+ ? [
102
+ {
103
+ id: filter.id,
104
+ name: filter.name,
105
+ slug: filter.slug,
106
+ },
107
+ ]
108
+ : []),
109
+ ...(await bakeKeywords(key, rest)),
110
+ ];
111
+ };
112
+
113
+ /**
114
+ * Bake filters recursively.
115
+ *
116
+ * @param filters - The filters object.
117
+ * e.g. `{ category: [ a, b ], tag: [ c ] }`.
118
+ * @param filtersWithPair - The filters with pairs.
119
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
120
+ * @returns The baked filters.
121
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
122
+ */
123
+ const bakeFilters = async (filters: Record<string, string[]>, filtersWithPair: Record<string, Filter[]>) => {
124
+ const keys = Object.keys(filters);
125
+
126
+ if (keys.length === 0) {
127
+ return filtersWithPair;
128
+ }
129
+
130
+ const key = keys[0];
131
+ const keywords = filters[key];
132
+
133
+ const originalFilters = { ...filters };
134
+ delete originalFilters[key];
135
+
136
+ return bakeFilters(originalFilters, {
137
+ ...filtersWithPair,
138
+ [key]: filterApiKeysWithNoId.has(key) ? keywords : await bakeKeywords(key, keywords),
139
+ });
140
+ };
141
+
142
+ return await bakeFilters(filters, {});
143
+ };
144
+
145
+ /**
146
+ * Bake URL with search parameters.
147
+ *
148
+ * @param url - The URL.
149
+ * @param rootUrl - The root URL.
150
+ * @param searchParams - The search parameters.
151
+ * @returns The baked URL.
152
+ */
153
+ const bakeUrl = (url: string, rootUrl: string, searchParams: URLSearchParams = new URLSearchParams()): string => {
154
+ const searchParamsStr = searchParams.toString();
155
+ const searchParamsSuffix = searchParamsStr ? `?${searchParamsStr}` : '';
156
+
157
+ return `${rootUrl}/${url}${searchParamsSuffix}`;
158
+ };
159
+
160
+ /**
161
+ * Fetch data from the specified URL.
162
+ *
163
+ * @param url - The URL to fetch data from.
164
+ * @param rootUrl - The root URL.
165
+ * @returns A promise that resolves to an object containing the fetched data to be added into `ctx.state.data`.
166
+ */
167
+ const fetchData = async (url: string, rootUrl: string): Promise<object> => {
168
+ /**
169
+ * Request URLs recursively.
170
+ *
171
+ * @param urls - The URLs to request.
172
+ * @returns A promise that resolves to the response data or undefined if no response is available.
173
+ */
174
+ const requestUrls = async (urls: string[]): Promise<string | undefined> => {
175
+ if (urls.length === 0) {
176
+ return;
177
+ }
178
+
179
+ const [currentUrl, ...remainingUrls] = urls;
180
+ try {
181
+ const { data: response } = await got.get(currentUrl);
182
+ return response;
183
+ } catch {
184
+ return requestUrls(remainingUrls);
185
+ }
186
+ };
187
+
188
+ const response = await requestUrls([url, rootUrl]);
189
+
190
+ if (!response) {
191
+ return {};
192
+ }
193
+
194
+ const $ = load(response);
195
+
196
+ const title = $('title').first().text();
197
+ const image = new URL('wp-content/uploads/site_logo.png', rootUrl).href;
198
+
199
+ return {
200
+ title,
201
+ description: $('meta[property="og:description"]').attr('content') || $('meta[name="description"]').attr('content'),
202
+ link: url,
203
+ allowEmpty: true,
204
+ image,
205
+ author: $('meta[property="og:site_name"]').attr('content'),
206
+ language: $('html').attr('lang'),
207
+ };
208
+ };
209
+
210
+ /**
211
+ * Get filter by key and keyword.
212
+ *
213
+ * @param key - The key.
214
+ * e.g. `category` or `tag`.
215
+ * @param keyword - The keywords.
216
+ * e.g. `keyword1`.
217
+ * @returns A promise that resolves to the filter object if found, or undefined if not found.
218
+ */
219
+ const getFilterByKeyAndKeyword = async (key: string, keyword: string, rootUrl: string): Promise<Filter | undefined> => {
220
+ const apiFilterUrl = `${rootUrl}/${apiSlug}/${getFilterKeyForSearchParams(key, true)}`;
221
+
222
+ const { data: response } = await got(apiFilterUrl, {
223
+ searchParams: {
224
+ search: keyword,
225
+ },
226
+ });
227
+
228
+ return response.length > 0 ? response[0] : undefined;
229
+ };
230
+
231
+ /**
232
+ * Get filter key for search parameters.
233
+ *
234
+ * @param key - The key. e.g. `category` or `tag`.
235
+ * @param isApi - Indicates whether the key is for the API.
236
+ * @returns The filter key for search parameters, or undefined if not found.
237
+ * e.g. `categories` or `tags`.
238
+ */
239
+ const getFilterKeyForSearchParams = (key: string, isApi: boolean = false): string | undefined => {
240
+ const keys = isApi ? filterApiKeys : filterKeys;
241
+
242
+ return Object.hasOwn(keys, key) ? (keys[key] ?? key) : undefined;
243
+ };
244
+
245
+ /**
246
+ * Get filter parameters for URL.
247
+ *
248
+ * @param filterPairs - The filter pairs object.
249
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
250
+ * @returns The filter parameters for the URL, or undefined if no filters are available.
251
+ */
252
+ const getFilterParamsForUrl = (filterPairs: Record<string, Filter[]>): string | undefined => {
253
+ const keys = Object.keys(filterPairs).filter((key) => filterPairs[key].length > 0 && !Object.hasOwn(filterKeys, key));
254
+
255
+ if (keys.length === 0) {
256
+ return;
257
+ }
258
+
259
+ const key = keys[0];
260
+
261
+ return filterPairs[key].map((pair) => pair.slug).join('/');
262
+ };
263
+
264
+ /**
265
+ * Parses a filter string into a filters object.
266
+ *
267
+ * @param filterStr - The filter string to parse.
268
+ * e.g. `category/a,b/tag/c`.
269
+ * @returns The parsed filters object.
270
+ * e.g. `{ category: [ 'a', 'b' ], tag: [ 'c' ] }`.
271
+ */
272
+ const parseFilterStr = (filterStr: string | undefined): Record<string, string[]> => {
273
+ /**
274
+ * Recursively parses a filter string.
275
+ *
276
+ * @param remainingStr - The remaining filter string to parse.
277
+ * e.g. `category/a,b/tag/c`.
278
+ * @param filters - The accumulated filters object.
279
+ * e.g. `{ category: [ a, b ], tag: [ c ] }`.
280
+ * @param currentKey - The current filter key.
281
+ * e.g. `category` or `tag`.
282
+ * @returns The parsed filters object.
283
+ */
284
+ const parseStr = (remainingStr: string | undefined, filters: Record<string, string[]> = {}, currentKey?: string): Record<string, string[]> => {
285
+ if (!remainingStr) {
286
+ return filters;
287
+ }
288
+
289
+ const [word, ...rest] = remainingStr.split(/\/|,/);
290
+
291
+ const isKey = Object.hasOwn(filterApiKeys, word);
292
+ const key = isKey ? word : currentKey;
293
+
294
+ const newFilters = key
295
+ ? {
296
+ ...filters,
297
+ [key]: [...(filters[key] || []), ...(isKey ? [] : [word])],
298
+ }
299
+ : filters;
300
+
301
+ return parseStr(rest.join('/'), newFilters, key);
302
+ };
303
+
304
+ return parseStr(filterStr, {});
305
+ };
306
+
307
+ export { apiSlug, bakeFilterSearchParams, bakeFiltersWithPair, bakeUrl, fetchData, getFilterParamsForUrl, parseFilterStr };
@@ -40,10 +40,10 @@ const channelMap = {
40
40
  };
41
41
 
42
42
  export const route: Route = {
43
- path: '/:channel?',
43
+ path: '/:channel?/:original?',
44
44
  categories: ['finance'],
45
45
  example: '/theblockbeats/newsflash',
46
- parameters: { channel: '类型,见下表,默认为快讯' },
46
+ parameters: { channel: '类型,见下表,默认为快讯', original: '文章类型,仅 `channel` 为 `article` 时有效,见下表,留空为全部' },
47
47
  name: '新闻快讯',
48
48
  maintainers: ['Fatpandac', 'jameshih'],
49
49
  handler,
@@ -61,15 +61,20 @@ export const route: Route = {
61
61
  ],
62
62
  description: `| 快讯 | 文章 |
63
63
  | :-------: | :-----: |
64
- | newsflash | article |`,
64
+ | newsflash | article |
65
+
66
+ | 全部 | 深度 | 精选 | 热点追踪 |
67
+ | :--: | :--: | :--: | :---: |
68
+ | | -2 | 1 | 2 |`,
65
69
  };
66
70
 
67
71
  async function handler(ctx) {
68
- const { channel = 'newsflash' } = ctx.req.param();
72
+ const { channel = 'newsflash', original } = ctx.req.param();
69
73
 
70
74
  const response = await ofetch(channelMap[channel].api, {
71
75
  query: {
72
76
  limit: ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 20,
77
+ original: channel === 'article' ? original : undefined,
73
78
  },
74
79
  });
75
80
 
@@ -86,7 +91,7 @@ async function handler(ctx) {
86
91
  if (channel !== 'newsflash') {
87
92
  list = await Promise.all(
88
93
  list.map((item) =>
89
- cache.tryGet(item.link, async () => {
94
+ cache.tryGet(`theblockbeats:${item.link}`, async () => {
90
95
  const response = await ofetch(item.link);
91
96
  const $ = load(response);
92
97
  item.description = render({
@@ -0,0 +1,67 @@
1
+ import { Route } from '@/types';
2
+ import ofetch from '@/utils/ofetch';
3
+ import { load } from 'cheerio';
4
+ import { parseDate } from '@/utils/parse-date';
5
+ import cache from '@/utils/cache';
6
+
7
+ export const route: Route = {
8
+ path: '/latest',
9
+ categories: ['blog'],
10
+ example: '/latest',
11
+ radar: [
12
+ {
13
+ source: ['towardsdatascience.com/'],
14
+ },
15
+ ],
16
+ name: 'Towards Data Science',
17
+ maintainers: ['mintyfrankie'],
18
+ url: 'towardsdatascience.com/latest',
19
+ handler,
20
+ };
21
+
22
+ async function handler() {
23
+ const baseUrl = 'https://towardsdatascience.com/latest';
24
+ const feedLang = 'en';
25
+ const feedDescription = 'Latest articles from Towards Data Science';
26
+
27
+ const response = await ofetch('https://medium.com/towards-data-science/latest?posts=true', {
28
+ headers: {
29
+ accept: 'application/json',
30
+ },
31
+ });
32
+ const data = JSON.parse(response.slice(16));
33
+
34
+ const list = data.payload.posts.map((item) => {
35
+ const title = item.title;
36
+ const link = `https://towardsdatascience.com/${item.uniqueSlug}`;
37
+ const freediumLink = `https://freedium.cfd/https://towardsdatascience.com/${item.uniqueSlug}`;
38
+ const author = data.payload.references.User[item.creatorId].name;
39
+ const pubDate = parseDate(item.createdAt);
40
+ return {
41
+ title,
42
+ link,
43
+ freediumLink,
44
+ author,
45
+ pubDate,
46
+ };
47
+ });
48
+
49
+ const items = await Promise.all(
50
+ list.map((item) =>
51
+ cache.tryGet(item.freediumLink, async () => {
52
+ const response = await ofetch(item.freediumLink);
53
+ const $ = load(response);
54
+ item.description = $('div.main-content').first().html();
55
+ return item;
56
+ })
57
+ )
58
+ );
59
+
60
+ return {
61
+ title: 'Towards Data Science - Latest',
62
+ language: feedLang,
63
+ description: feedDescription,
64
+ link: baseUrl,
65
+ item: items,
66
+ };
67
+ }
@@ -0,0 +1,6 @@
1
+ import type { Namespace } from '@/types';
2
+
3
+ export const namespace: Namespace = {
4
+ name: 'Towards Data Science',
5
+ url: 'towardsdatascience.com',
6
+ };
@@ -96,6 +96,7 @@ async function handler(ctx) {
96
96
  lang: language,
97
97
  page: 1,
98
98
  limit: ctx.req.query('limit') ?? 50,
99
+ country: 'HK',
99
100
  },
100
101
  });
101
102
 
@@ -18,6 +18,7 @@ let api: {
18
18
  getSearch: ApiItem;
19
19
  getList: ApiItem;
20
20
  getHomeTimeline: ApiItem;
21
+ getHomeLatestTimeline: ApiItem;
21
22
  } = {
22
23
  init: () => {
23
24
  throw new ConfigNotFoundError('Twitter API is not configured');
@@ -31,6 +32,7 @@ let api: {
31
32
  getSearch: () => null,
32
33
  getList: () => null,
33
34
  getHomeTimeline: () => null,
35
+ getHomeLatestTimeline: () => null,
34
36
  };
35
37
 
36
38
  if (enableWebApi) {
@@ -184,6 +184,23 @@ const getHomeTimeline = async (id: string, params?: Record<string, any>) =>
184
184
  )
185
185
  );
186
186
 
187
+ const getHomeLatestTimeline = async (id: string, params?: Record<string, any>) =>
188
+ gatherLegacyFromData(
189
+ await paginationTweets(
190
+ 'HomeLatestTimeline',
191
+ undefined,
192
+ {
193
+ ...params,
194
+ count: 20,
195
+ includePromotedContent: true,
196
+ latestControlAvailable: true,
197
+ requestContext: 'launch',
198
+ withCommunity: true,
199
+ },
200
+ ['home', 'home_timeline_urt']
201
+ )
202
+ );
203
+
187
204
  export default {
188
205
  getUser,
189
206
  getUserTweets,
@@ -194,5 +211,6 @@ export default {
194
211
  getSearch,
195
212
  getList,
196
213
  getHomeTimeline,
214
+ getHomeLatestTimeline,
197
215
  init: () => {},
198
216
  };
@@ -4,6 +4,7 @@ const graphQLEndpointsPlain = [
4
4
  '/graphql/eS7LO5Jy3xgmd3dbL044EA/UserTweets',
5
5
  '/graphql/k5XapwcSikNsEsILW5FvgA/UserByScreenName',
6
6
  '/graphql/k3YiLNE_MAy5J-NANLERdg/HomeTimeline',
7
+ '/graphql/DiTkXJgLqBBxCs7zaYsbtA/HomeLatestTimeline',
7
8
  '/graphql/3GeIaLmNhTm1YsUmxR57tg/UserTweetsAndReplies',
8
9
  '/graphql/TOU4gQw8wXIqpSzA4TYKgg/UserMedia',
9
10
  '/graphql/B8I_QCljDBVfin21TTWMqA/Likes',
@@ -87,6 +88,7 @@ const gqlFeatures = {
87
88
  SearchTimeline: gqlFeatureFeed,
88
89
  ListLatestTweetsTimeline: gqlFeatureFeed,
89
90
  HomeTimeline: gqlFeatureFeed,
91
+ HomeLatestTimeline: TweetDetailFeatures,
90
92
  TweetDetail: TweetDetailFeatures,
91
93
  Likes: gqlFeatureFeed,
92
94
  };
@@ -0,0 +1,66 @@
1
+ import { authenticator } from 'otplib';
2
+ import logger from '@/utils/logger';
3
+ import cache from '@/utils/cache';
4
+ import { RateLimiterMemory, RateLimiterRedis, RateLimiterQueue } from 'rate-limiter-flexible';
5
+ import puppeteer from '@/utils/puppeteer';
6
+ import { CookieJar } from 'tough-cookie';
7
+
8
+ const loginLimiter = cache.clients.redisClient
9
+ ? new RateLimiterRedis({
10
+ points: 1,
11
+ duration: 20,
12
+ execEvenly: true,
13
+ storeClient: cache.clients.redisClient,
14
+ })
15
+ : new RateLimiterMemory({
16
+ points: 1,
17
+ duration: 20,
18
+ execEvenly: true,
19
+ });
20
+
21
+ const loginLimiterQueue = new RateLimiterQueue(loginLimiter);
22
+
23
+ async function login({ username, password, authenticationSecret }) {
24
+ if (!username || !password || !authenticationSecret) {
25
+ return;
26
+ }
27
+ try {
28
+ await loginLimiterQueue.removeTokens(1);
29
+
30
+ const cookieJar = new CookieJar();
31
+ const browser = await puppeteer({
32
+ stealth: true,
33
+ });
34
+ const page = await browser.newPage();
35
+ await page.goto('https://x.com/i/flow/login');
36
+ await page.waitForSelector('input[autocomplete="username"]');
37
+ await page.type('input[autocomplete="username"]', username);
38
+ const buttons = await page.$$('button');
39
+ await buttons[3]?.click();
40
+ await page.waitForSelector('input[autocomplete="current-password"]');
41
+ await page.type('input[autocomplete="current-password"]', password);
42
+ (await page.waitForSelector('button[data-testid="LoginForm_Login_Button"]'))?.click();
43
+ await page.waitForSelector('input[inputmode="numeric"]');
44
+ const token = authenticator.generate(authenticationSecret);
45
+ await page.type('input[inputmode="numeric"]', token);
46
+ (await page.waitForSelector('button[data-testid="ocfEnterTextNextButton"]'))?.click();
47
+ const waitForRequest = new Promise<string>((resolve) => {
48
+ page.on('requestfinished', async (request) => {
49
+ if (request.url().includes('/HomeTimeline')) {
50
+ const cookies = await page.cookies();
51
+ for (const cookie of cookies) {
52
+ cookieJar.setCookieSync(`${cookie.name}=${cookie.value}`, 'https://x.com');
53
+ }
54
+ resolve(JSON.stringify(cookieJar.serializeSync()));
55
+ }
56
+ });
57
+ });
58
+ const cookieString = await waitForRequest;
59
+ await browser.close();
60
+ return cookieString;
61
+ } catch (error) {
62
+ logger.error(`Twitter username ${username} login failed:`, error);
63
+ }
64
+ }
65
+
66
+ export default login;