rsshub 1.0.0-master.b76ffa1 → 1.0.0-master.b77ea8a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/lib/config.ts +3 -0
  2. package/lib/middleware/access-control.ts +1 -1
  3. package/lib/routes/afdian/dynamic.ts +1 -1
  4. package/lib/routes/afdian/explore.ts +1 -1
  5. package/lib/routes/bilibili/bangumi.ts +45 -43
  6. package/lib/routes/bilibili/bilibili-recommend.ts +4 -4
  7. package/lib/routes/bilibili/coin.ts +4 -4
  8. package/lib/routes/bilibili/dynamic.ts +14 -14
  9. package/lib/routes/bilibili/fav.ts +4 -4
  10. package/lib/routes/bilibili/followings-dynamic.ts +25 -3
  11. package/lib/routes/bilibili/followings-video.ts +4 -4
  12. package/lib/routes/bilibili/like.ts +4 -4
  13. package/lib/routes/bilibili/page.ts +5 -4
  14. package/lib/routes/bilibili/partion-ranking.ts +4 -4
  15. package/lib/routes/bilibili/partion.ts +4 -4
  16. package/lib/routes/bilibili/popular.ts +6 -4
  17. package/lib/routes/bilibili/ranking.ts +4 -6
  18. package/lib/routes/bilibili/templates/description.art +12 -0
  19. package/lib/routes/bilibili/types.ts +52 -0
  20. package/lib/routes/bilibili/user-channel.ts +10 -17
  21. package/lib/routes/bilibili/user-collection.ts +10 -18
  22. package/lib/routes/bilibili/user-fav.ts +4 -4
  23. package/lib/routes/bilibili/utils.ts +111 -7
  24. package/lib/routes/bilibili/video-all.ts +8 -3
  25. package/lib/routes/bilibili/video.ts +5 -4
  26. package/lib/routes/bilibili/vsearch.ts +21 -5
  27. package/lib/routes/bilibili/watchlater.ts +4 -4
  28. package/lib/routes/bilibili/weekly-recommend.ts +4 -9
  29. package/lib/routes/bsky/posts.ts +2 -2
  30. package/lib/routes/bsky/templates/post.art +4 -2
  31. package/lib/routes/chinania/index.ts +240 -0
  32. package/lib/routes/chinania/namespace.ts +8 -0
  33. package/lib/routes/dedao/articles.ts +149 -0
  34. package/lib/routes/eshukan/academic.ts +134 -0
  35. package/lib/routes/eshukan/namespace.ts +8 -0
  36. package/lib/routes/eshukan/templates/description.art +7 -0
  37. package/lib/routes/follow/profile.ts +37 -3
  38. package/lib/routes/lovelive-anime/schedules.ts +4 -5
  39. package/lib/routes/lovelive-anime/topics.ts +88 -37
  40. package/lib/routes/mingpao/index.ts +7 -1
  41. package/lib/routes/nature/cover.ts +5 -3
  42. package/lib/routes/nature/research.ts +1 -1
  43. package/lib/routes/nature/siteindex.ts +25 -11
  44. package/lib/routes/nature/utils.ts +5 -7
  45. package/lib/routes/qq/pd/guild.ts +146 -0
  46. package/lib/routes/qq/pd/types.ts +82 -0
  47. package/lib/routes/qq/pd/utils.ts +119 -0
  48. package/lib/routes/reuters/common.ts +160 -159
  49. package/lib/routes/reuters/investigates.ts +3 -2
  50. package/lib/routes/rsshub/transform/html.ts +2 -2
  51. package/lib/routes/telegram/channel.ts +14 -5
  52. package/lib/routes/telegram/scripts/get-telegram-session.mjs +41 -0
  53. package/lib/routes/telegram/tglib/channel.ts +8 -4
  54. package/lib/routes/theinitium/app.ts +49 -27
  55. package/lib/routes/theinitium/templates/description.art +2 -2
  56. package/lib/routes/threads/index.ts +1 -1
  57. package/lib/routes/threads/utils.ts +19 -1
  58. package/lib/routes/twitter/api/mobile-api/login.ts +125 -127
  59. package/lib/routes/twitter/api/mobile-api/token.ts +2 -0
  60. package/lib/routes/twitter/home-latest.ts +4 -1
  61. package/lib/routes/twitter/home.ts +4 -1
  62. package/lib/routes/twitter/likes.ts +4 -1
  63. package/lib/routes/twitter/list.ts +4 -1
  64. package/lib/routes/twitter/namespace.ts +1 -0
  65. package/lib/routes/twitter/utils.ts +9 -3
  66. package/lib/routes/warthunder/news.ts +8 -8
  67. package/lib/routes/washingtonpost/app.ts +118 -0
  68. package/lib/routes/washingtonpost/namespace.ts +6 -0
  69. package/lib/routes/washingtonpost/templates/description.art +59 -0
  70. package/lib/routes/zhihu/daily.ts +30 -64
  71. package/package.json +12 -12
@@ -7,6 +7,7 @@ import NotFoundError from '@/errors/types/not-found';
7
7
 
8
8
  const profileUrl = (user: string) => `https://www.threads.net/@${user}`;
9
9
  const threadUrl = (code: string) => `https://www.threads.net/t/${code}`;
10
+ const instagramUrl = (user: string) => `https://i.instagram.com/api/v1/users/web_profile_info/?username=${user}`;
10
11
 
11
12
  const apiUrl = 'https://www.threads.net/api/graphql';
12
13
  // const PROFILE_QUERY = 23_996_318_473_300_828; // no longer works
@@ -49,6 +50,7 @@ const makeHeader = (user: string, lsd: string) => ({
49
50
  'Sec-Fetch-Site': 'same-origin',
50
51
  });
51
52
 
53
+ // the formal way always reachs the rate limit, so use instagram api to get user id instead
52
54
  const getUserId = (user: string, lsd: string): Promise<string> =>
53
55
  cache.tryGet(`threads:userId:${user}`, async () => {
54
56
  const pathName = `/@${user}`;
@@ -69,7 +71,23 @@ const getUserId = (user: string, lsd: string): Promise<string> =>
69
71
  parseResponse: (txt) => destr(txt.slice(9)), // remove "for (;;);"
70
72
  });
71
73
 
72
- const userId = response.payload.payloads[pathName].result.exports.rootView.props.user_id;
74
+ let userId = response.payload.payloads[pathName].result.exports.rootView.props.user_id;
75
+
76
+ if (!userId) {
77
+ const fallbackResponse = await ofetch(instagramUrl(user), {
78
+ headers: makeHeader(user, lsd),
79
+ });
80
+
81
+ if (!fallbackResponse?.data?.user) {
82
+ throw new NotFoundError('Instagram getUser API response is invalid');
83
+ }
84
+
85
+ userId = fallbackResponse.data.user.id;
86
+ if (!userId) {
87
+ throw new NotFoundError('User ID not found in Instagram getUser API response');
88
+ }
89
+ }
90
+
73
91
  return userId;
74
92
  });
75
93
 
@@ -8,11 +8,11 @@ import { v5 as uuidv5 } from 'uuid';
8
8
  import { authenticator } from 'otplib';
9
9
  import logger from '@/utils/logger';
10
10
  import cache from '@/utils/cache';
11
- import { RateLimiterMemory, RateLimiterRedis, RateLimiterQueue } from 'rate-limiter-flexible';
11
+ import { RateLimiterMemory, RateLimiterQueue, RateLimiterRedis } from 'rate-limiter-flexible';
12
12
 
13
- const NAMESPACE = 'd41d092b-b007-48f7-9129-e9538d2d8fe9';
13
+ const ENDPOINT = 'https://api.x.com/1.1/onboarding/task.json';
14
14
 
15
- let authentication = null;
15
+ const NAMESPACE = 'd41d092b-b007-48f7-9129-e9538d2d8fe9';
16
16
 
17
17
  const headers = {
18
18
  'User-Agent': 'TwitterAndroid/10.21.0-release.0 (310210000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)',
@@ -41,7 +41,65 @@ const loginLimiter = cache.clients.redisClient
41
41
 
42
42
  const loginLimiterQueue = new RateLimiterQueue(loginLimiter);
43
43
 
44
- async function login({ username, password, authenticationSecret }) {
44
+ const postTask = async (flowToken: string, subtaskId: string, subtaskInput: Record<string, unknown>) =>
45
+ await got.post(ENDPOINT, {
46
+ headers,
47
+ json: {
48
+ flow_token: flowToken,
49
+ subtask_inputs: [Object.assign({ subtask_id: subtaskId }, subtaskInput)],
50
+ },
51
+ });
52
+
53
+ // In the Twitter login flow, each task successfully requested will respond with a 'subtask_id' to determine what the next task is, and the execution sequence of the tasks is non-fixed.
54
+ // So abstract these tasks out into a map so that they can be dynamically executed during the login flow.
55
+ // If there are missing tasks in the future, simply add the implementation of that task to it.
56
+ const flowTasks = {
57
+ async LoginEnterUserIdentifier({ flowToken, username }) {
58
+ return await postTask(flowToken, 'LoginEnterUserIdentifier', {
59
+ enter_text: {
60
+ suggestion_id: null,
61
+ text: username,
62
+ link: 'next_link',
63
+ },
64
+ });
65
+ },
66
+ async LoginEnterPassword({ flowToken, password }) {
67
+ return await postTask(flowToken, 'LoginEnterPassword', {
68
+ enter_password: {
69
+ password,
70
+ link: 'next_link',
71
+ },
72
+ });
73
+ },
74
+ async LoginEnterAlternateIdentifierSubtask({ flowToken, phoneOrEmail }) {
75
+ return await postTask(flowToken, 'LoginEnterAlternateIdentifierSubtask', {
76
+ enter_text: {
77
+ suggestion_id: null,
78
+ text: phoneOrEmail,
79
+ link: 'next_link',
80
+ },
81
+ });
82
+ },
83
+ async AccountDuplicationCheck({ flowToken }) {
84
+ return await postTask(flowToken, 'AccountDuplicationCheck', {
85
+ check_logged_in_account: {
86
+ link: 'AccountDuplicationCheck_false',
87
+ },
88
+ });
89
+ },
90
+ async LoginTwoFactorAuthChallenge({ flowToken, authenticationSecret }) {
91
+ const token = authenticator.generate(authenticationSecret);
92
+ return await postTask(flowToken, 'LoginTwoFactorAuthChallenge', {
93
+ enter_text: {
94
+ suggestion_id: null,
95
+ text: token,
96
+ link: 'next_link',
97
+ },
98
+ });
99
+ },
100
+ };
101
+
102
+ async function login({ username, password, authenticationSecret, phoneOrEmail }) {
45
103
  return (await cache.tryGet(
46
104
  `twitter:authentication:${username}`,
47
105
  async () => {
@@ -49,8 +107,8 @@ async function login({ username, password, authenticationSecret }) {
49
107
  await loginLimiterQueue.removeTokens(1);
50
108
 
51
109
  logger.debug('Twitter login start.');
52
- const android_id = uuidv5(username, NAMESPACE);
53
- headers['X-Twitter-Client-DeviceID'] = android_id;
110
+
111
+ headers['X-Twitter-Client-DeviceID'] = uuidv5(username, NAMESPACE);
54
112
 
55
113
  const ct0 = crypto.randomUUID().replaceAll('-', '');
56
114
  const guestToken = await got(guestActivateUrl, {
@@ -61,140 +119,80 @@ async function login({ username, password, authenticationSecret }) {
61
119
  },
62
120
  method: 'POST',
63
121
  });
64
- logger.debug('Twitter login 1 finished: guest token.');
122
+ logger.debug('Twitter login: guest token');
65
123
 
66
124
  headers['x-guest-token'] = guestToken.data.guest_token;
67
125
 
68
- const task1 = await ofetch.raw(
69
- 'https://api.x.com/1.1/onboarding/task.json?' +
70
- new URLSearchParams({
71
- flow_name: 'login',
72
- api_version: '1',
73
- known_device_token: '',
74
- sim_country_code: 'us',
75
- }).toString(),
76
- {
77
- method: 'POST',
78
- headers,
79
- body: {
80
- flow_token: null,
81
- input_flow_data: {
82
- country_code: null,
83
- flow_context: {
84
- referrer_context: {
85
- referral_details: 'utm_source=google-play&utm_medium=organic',
86
- referrer_url: '',
87
- },
88
- start_location: {
89
- location: 'deeplink',
126
+ let task = await ofetch
127
+ .raw(
128
+ ENDPOINT +
129
+ '?' +
130
+ new URLSearchParams({
131
+ flow_name: 'login',
132
+ api_version: '1',
133
+ known_device_token: '',
134
+ sim_country_code: 'us',
135
+ }).toString(),
136
+ {
137
+ method: 'POST',
138
+ headers,
139
+ body: {
140
+ flow_token: null,
141
+ input_flow_data: {
142
+ country_code: null,
143
+ flow_context: {
144
+ referrer_context: {
145
+ referral_details: 'utm_source=google-play&utm_medium=organic',
146
+ referrer_url: '',
147
+ },
148
+ start_location: {
149
+ location: 'deeplink',
150
+ },
90
151
  },
152
+ requested_variant: null,
153
+ target_user_id: 0,
91
154
  },
92
- requested_variant: null,
93
- target_user_id: 0,
94
155
  },
95
- },
156
+ }
157
+ )
158
+ .then(({ headers: _headers, _data }) => {
159
+ headers.att = _headers.get('att');
160
+ return { data: _data };
161
+ });
162
+
163
+ logger.debug('Twitter login flow start.');
164
+ const runTask = async ({ data }) => {
165
+ const { subtask_id, open_account } = data.subtasks.shift();
166
+
167
+ // If `open_account` exists (and 'subtask_id' is `LoginSuccessSubtask`), it means the login was successful.
168
+ if (open_account) {
169
+ return open_account;
96
170
  }
97
- );
98
- logger.debug('Twitter login 2 finished: login flow.');
99
-
100
- headers.att = task1.headers.get('att');
101
-
102
- const task2 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
103
- headers,
104
- json: {
105
- flow_token: task1._data.flow_token,
106
- subtask_inputs: [
107
- {
108
- enter_text: {
109
- suggestion_id: null,
110
- text: username,
111
- link: 'next_link',
112
- },
113
- subtask_id: 'LoginEnterUserIdentifier',
114
- },
115
- ],
116
- },
117
- });
118
- logger.debug('Twitter login 3 finished: LoginEnterUserIdentifier.');
119
-
120
- const task3 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
121
- headers,
122
- json: {
123
- flow_token: task2.data.flow_token,
124
- subtask_inputs: [
125
- {
126
- enter_password: {
127
- password,
128
- link: 'next_link',
129
- },
130
- subtask_id: 'LoginEnterPassword',
131
- },
132
- ],
133
- },
134
- });
135
- logger.debug('Twitter login 4 finished: LoginEnterPassword.');
136
- for (const subtask of task3.data?.subtasks || []) {
137
- if (subtask.open_account) {
138
- return subtask.open_account;
171
+
172
+ // If task does not exist in `flowTasks`, we need to implement it.
173
+ if (!(subtask_id in flowTasks)) {
174
+ logger.error(`Twitter login flow task failed: unknown subtask: ${subtask_id}`);
175
+ return;
139
176
  }
140
- }
141
177
 
142
- const task4 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
143
- headers,
144
- json: {
145
- flow_token: task3.data.flow_token,
146
- subtask_inputs: [
147
- {
148
- check_logged_in_account: {
149
- link: 'AccountDuplicationCheck_false',
150
- },
151
- subtask_id: 'AccountDuplicationCheck',
152
- },
153
- ],
154
- },
155
- });
156
- logger.debug('Twitter login 5 finished: AccountDuplicationCheck.');
178
+ task = await flowTasks[subtask_id]({
179
+ flowToken: data.flow_token,
180
+ username,
181
+ password,
182
+ authenticationSecret,
183
+ phoneOrEmail,
184
+ });
185
+ logger.debug(`Twitter login flow task finished: subtask: ${subtask_id}.`);
157
186
 
158
- for await (const subtask of task4.data?.subtasks || []) {
159
- if (subtask.open_account) {
160
- authentication = subtask.open_account;
161
- break;
162
- } else if (authenticationSecret && subtask.subtask_id === 'LoginTwoFactorAuthChallenge') {
163
- const token = authenticator.generate(authenticationSecret);
187
+ return await runTask(task);
188
+ };
189
+ const authentication = await runTask(task);
190
+ logger.debug('Twitter login flow finished.');
164
191
 
165
- const task5 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
166
- headers,
167
- json: {
168
- flow_token: task4.data.flow_token,
169
- subtask_inputs: [
170
- {
171
- enter_text: {
172
- suggestion_id: null,
173
- text: token,
174
- link: 'next_link',
175
- },
176
- subtask_id: 'LoginTwoFactorAuthChallenge',
177
- },
178
- ],
179
- },
180
- });
181
- logger.debug('Twitter login 6 finished: LoginTwoFactorAuthChallenge.');
182
-
183
- for (const subtask of task5.data?.subtasks || []) {
184
- if (subtask.open_account) {
185
- authentication = subtask.open_account;
186
- break;
187
- }
188
- }
189
- break;
190
- } else {
191
- logger.error(`Twitter login 6 failed: unknown subtask: ${subtask.subtask_id}`);
192
- }
193
- }
194
192
  if (authentication) {
195
- logger.debug('Twitter login success.');
193
+ logger.debug('Twitter login success.', authentication);
196
194
  } else {
197
- logger.error(`Twitter login failed. ${JSON.stringify(task4.data?.subtasks, null, 2)}`);
195
+ logger.error(`Twitter login failed. ${JSON.stringify(task.data?.subtasks, null, 2)}`);
198
196
  }
199
197
 
200
198
  return authentication;
@@ -11,11 +11,13 @@ async function getToken() {
11
11
  const username = config.twitter.username[index];
12
12
  const password = config.twitter.password[index];
13
13
  const authenticationSecret = config.twitter.authenticationSecret?.[index];
14
+ const phoneOrEmail = config.twitter.phoneOrEmail?.[index];
14
15
  if (username && password) {
15
16
  const authentication = await login({
16
17
  username,
17
18
  password,
18
19
  authenticationSecret,
20
+ phoneOrEmail,
19
21
  });
20
22
  if (!authentication) {
21
23
  throw new ConfigNotFoundError(`Invalid twitter configs: ${username}`);
@@ -40,7 +40,7 @@ export const route: Route = {
40
40
 
41
41
  async function handler(ctx) {
42
42
  // For compatibility
43
- const { count, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
43
+ const { count, include_rts, only_media } = utils.parseRouteParams(ctx.req.param('routeParams'));
44
44
  const params = count ? { count } : {};
45
45
 
46
46
  await api.init();
@@ -48,6 +48,9 @@ async function handler(ctx) {
48
48
  if (!include_rts) {
49
49
  data = utils.excludeRetweet(data);
50
50
  }
51
+ if (only_media) {
52
+ data = utils.keepOnlyMedia(data);
53
+ }
51
54
 
52
55
  return {
53
56
  title: `Twitter following timeline`,
@@ -40,7 +40,7 @@ export const route: Route = {
40
40
 
41
41
  async function handler(ctx) {
42
42
  // For compatibility
43
- const { count, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
43
+ const { count, include_rts, only_media } = utils.parseRouteParams(ctx.req.param('routeParams'));
44
44
  const params = count ? { count } : {};
45
45
 
46
46
  await api.init();
@@ -48,6 +48,9 @@ async function handler(ctx) {
48
48
  if (!include_rts) {
49
49
  data = utils.excludeRetweet(data);
50
50
  }
51
+ if (only_media) {
52
+ data = utils.keepOnlyMedia(data);
53
+ }
51
54
 
52
55
  return {
53
56
  title: `Twitter following timeline`,
@@ -27,7 +27,7 @@ export const route: Route = {
27
27
 
28
28
  async function handler(ctx) {
29
29
  const id = ctx.req.param('id');
30
- const { count, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
30
+ const { count, include_rts, only_media } = utils.parseRouteParams(ctx.req.param('routeParams'));
31
31
  const params = count ? { count } : {};
32
32
 
33
33
  await api.init();
@@ -35,6 +35,9 @@ async function handler(ctx) {
35
35
  if (!include_rts) {
36
36
  data = utils.excludeRetweet(data);
37
37
  }
38
+ if (only_media) {
39
+ data = utils.keepOnlyMedia(data);
40
+ }
38
41
 
39
42
  return {
40
43
  title: `Twitter Likes - ${id}`,
@@ -33,7 +33,7 @@ export const route: Route = {
33
33
 
34
34
  async function handler(ctx) {
35
35
  const id = ctx.req.param('id');
36
- const { count, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
36
+ const { count, include_rts, only_media } = utils.parseRouteParams(ctx.req.param('routeParams'));
37
37
  const params = count ? { count } : {};
38
38
 
39
39
  await api.init();
@@ -41,6 +41,9 @@ async function handler(ctx) {
41
41
  if (!include_rts) {
42
42
  data = utils.excludeRetweet(data);
43
43
  }
44
+ if (only_media) {
45
+ data = utils.keepOnlyMedia(data);
46
+ }
44
47
 
45
48
  return {
46
49
  title: `Twitter List - ${id}`,
@@ -27,6 +27,7 @@ export const namespace: Namespace = {
27
27
  | \`includeRts\` | Include retweets, only available in \`/twitter/user\` | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` |
28
28
  | \`forceWebApi\` | Force using Web API even if Developer API is configured, only available in \`/twitter/user\` and \`/twitter/keyword\` | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` |
29
29
  | \`count\` | \`count\` parameter passed to Twitter API, only available in \`/twitter/user\` | Unspecified/Integer | Unspecified |
30
+ | \`onlyMedia\` | Only get tweets with a media | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` |
30
31
 
31
32
  Specify different option values than default values to improve readability. The URL
32
33
 
@@ -451,7 +451,7 @@ if (config.twitter.consumer_key && config.twitter.consumer_secret) {
451
451
  }
452
452
 
453
453
  const parseRouteParams = (routeParams) => {
454
- let count, exclude_replies, include_rts;
454
+ let count, exclude_replies, include_rts, only_media;
455
455
  let force_web_api = false;
456
456
  switch (routeParams) {
457
457
  case 'exclude_rts_replies':
@@ -479,9 +479,10 @@ const parseRouteParams = (routeParams) => {
479
479
  exclude_replies = fallback(undefined, queryToBoolean(parsed.get('excludeReplies')), false);
480
480
  include_rts = fallback(undefined, queryToBoolean(parsed.get('includeRts')), true);
481
481
  force_web_api = fallback(undefined, queryToBoolean(parsed.get('forceWebApi')), false);
482
+ only_media = fallback(undefined, queryToBoolean(parsed.get('onlyMedia')), false);
482
483
  }
483
484
  }
484
- return { count, exclude_replies, include_rts, force_web_api };
485
+ return { count, exclude_replies, include_rts, force_web_api, only_media };
485
486
  };
486
487
 
487
488
  export const excludeRetweet = function (tweets) {
@@ -495,4 +496,9 @@ export const excludeRetweet = function (tweets) {
495
496
  return excluded;
496
497
  };
497
498
 
498
- export default { ProcessFeed, getAppClient, parseRouteParams, excludeRetweet };
499
+ export const keepOnlyMedia = function (tweets) {
500
+ const excluded = tweets.filter((t) => t.extended_entities && t.extended_entities.media);
501
+ return excluded;
502
+ };
503
+
504
+ export default { ProcessFeed, getAppClient, parseRouteParams, excludeRetweet, keepOnlyMedia };
@@ -2,7 +2,7 @@ import { Route } from '@/types';
2
2
  import { getCurrentPath } from '@/utils/helpers';
3
3
  const __dirname = getCurrentPath(import.meta.url);
4
4
 
5
- import got from '@/utils/got';
5
+ import ofetch from '@/utils/ofetch';
6
6
  import { load } from 'cheerio';
7
7
  import path from 'node:path';
8
8
  import { art } from '@/utils/render';
@@ -33,20 +33,21 @@ export const route: Route = {
33
33
  handler,
34
34
  url: 'warthunder.com/en/news',
35
35
  description: `News data from [https://warthunder.com/en/news/](https://warthunder.com/en/news/)
36
- The year, month and day provided under UTC time zone are the same as the official website, so please ignore the specific time!!!`,
36
+ The \`pubDate\` provided under UTC time zone, so please ignore the specific time!!!`,
37
37
  };
38
38
 
39
39
  async function handler() {
40
40
  const rootUrl = 'https://warthunder.com/en/news/';
41
41
 
42
- const response = await got(rootUrl);
42
+ const response = await ofetch(rootUrl);
43
43
 
44
- const $ = load(response.data);
44
+ const $ = load(response);
45
45
 
46
46
  const pageFace = $('div.showcase__item.widget')
47
- .map((_, item) => {
47
+ .toArray()
48
+ .map((item) => {
48
49
  item = $(item);
49
- let pubDate = parseDate(item.find('div.widget__content > ul > li.widget-meta__item.widget-meta__item--right').text(), 'D MMMM YYYY');
50
+ let pubDate = parseDate(item.find('div.widget__content > ul > li.widget-meta__item.widget-meta__item--right').text(), 'D MMMM YYYY', 'en');
50
51
  pubDate = timezone(pubDate, 0);
51
52
  const category = [];
52
53
  if (item.find('div.widget__pin').length !== 0) {
@@ -68,8 +69,7 @@ async function handler() {
68
69
  }),
69
70
  category,
70
71
  };
71
- })
72
- .get();
72
+ });
73
73
 
74
74
  return {
75
75
  title: 'War Thunder News',
@@ -0,0 +1,118 @@
1
+ import { Route } from '@/types';
2
+ import cache from '@/utils/cache';
3
+ import got from '@/utils/got';
4
+ import { art } from '@/utils/render';
5
+ import path from 'node:path';
6
+ import { getCurrentPath } from '@/utils/helpers';
7
+ import { FetchError } from 'ofetch';
8
+ import dayjs from 'dayjs';
9
+ import utc from 'dayjs/plugin/utc';
10
+ import timezone from 'dayjs/plugin/timezone';
11
+ import advancedFormat from 'dayjs/plugin/advancedFormat';
12
+
13
+ export const route: Route = {
14
+ path: '/app/:category{.+}?',
15
+ categories: ['traditional-media'],
16
+ example: '/washingtonpost/app/national',
17
+ parameters: {
18
+ category: 'Category from the path of the URL of the corresponding site, see below',
19
+ },
20
+ features: {
21
+ requirePuppeteer: false,
22
+ antiCrawler: false,
23
+ supportBT: false,
24
+ supportPodcast: false,
25
+ supportScihub: false,
26
+ },
27
+ name: 'App',
28
+ maintainers: ['quiniapiezoelectricity'],
29
+ radar: [
30
+ {
31
+ source: ['www.washingtonpost.com/:category'],
32
+ target: '/app/:category',
33
+ },
34
+ ],
35
+ handler,
36
+ description: `:::tip
37
+ For example, the category for https://www.washingtonpost.com/national/investigations would be /national/investigations.
38
+ :::`,
39
+ };
40
+
41
+ function handleDuplicates(array) {
42
+ const objects = {};
43
+ for (const obj of array) {
44
+ objects[obj.id] = objects[obj.id] ? Object.assign(objects[obj.id], obj) : obj;
45
+ }
46
+ return Object.values(objects);
47
+ }
48
+
49
+ async function handler(ctx) {
50
+ const category = ctx.req.param('category') ?? '';
51
+ const __dirname = getCurrentPath(import.meta.url);
52
+ const headers = {
53
+ Accept: '*/*',
54
+ Connection: 'keep-alive',
55
+ 'User-Agent': 'Classic/6.70.0',
56
+ };
57
+ dayjs.extend(utc);
58
+ dayjs.extend(timezone);
59
+ dayjs.extend(advancedFormat);
60
+ art.defaults.imports.dayjs = dayjs;
61
+
62
+ const url = `https://jsonapp1.washingtonpost.com/fusion_prod/v2/${category}`;
63
+ const response = await got.get(url, { headers });
64
+ const title = response.data.tracking.page_title.includes('Washington Post') ? response.data.tracking.page_title : `The Washington Post - ${response.data.tracking.page_title}`;
65
+ const link = 'https://washingtonpost.com' + response.data.tracking.page_path;
66
+ const mains = response.data.regions[0].items.filter((item) => item.items);
67
+ const list = mains.flatMap((main) =>
68
+ main.items[0].items
69
+ .filter((item) => item.is_from_feed === true)
70
+ .map((item) => {
71
+ const object = {
72
+ id: item.id,
73
+ title: item.headline.text,
74
+ link: item.link.url,
75
+ pubDate: item.link.display_date,
76
+ updated: item.link.last_modified,
77
+ };
78
+ if (item.blurbs?.items[0]?.text) {
79
+ object.description = item.blurbs?.items[0]?.text;
80
+ }
81
+ return object;
82
+ })
83
+ );
84
+ const feed = handleDuplicates(list);
85
+ const items = await Promise.all(
86
+ feed.map((item) =>
87
+ cache.tryGet(item.link, async () => {
88
+ let response;
89
+ try {
90
+ response = await got(`https://rainbowapi-a.wpdigital.net/rainbow-data-service/rainbow/content-by-url.json?followLinks=false&url=${item.link}`, { headers });
91
+ } catch (error) {
92
+ if (error instanceof FetchError && error.statusCode === 415) {
93
+ // Interactive or podcast contents will return 415 Unsupported Media Type. Keep calm and carry on.
94
+ return item;
95
+ } else {
96
+ throw error;
97
+ }
98
+ }
99
+ item.title = response.data.title ?? item.title;
100
+ item.author =
101
+ response.data.items
102
+ .filter((entry) => entry.type === 'byline')
103
+ ?.flatMap((entry) => entry.authors.map((author) => author.name))
104
+ ?.join(', ') ?? '';
105
+ item.description = art(path.join(__dirname, 'templates/description.art'), {
106
+ content: response.data.items,
107
+ });
108
+ return item;
109
+ })
110
+ )
111
+ );
112
+
113
+ return {
114
+ title,
115
+ link,
116
+ item: items,
117
+ };
118
+ }
@@ -0,0 +1,6 @@
1
+ import type { Namespace } from '@/types';
2
+
3
+ export const namespace: Namespace = {
4
+ name: 'The Washington Post',
5
+ url: 'www.washingtonpost.com',
6
+ };