rsshub 1.0.0-master.b76ffa1 → 1.0.0-master.b77ea8a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.ts +3 -0
- package/lib/middleware/access-control.ts +1 -1
- package/lib/routes/afdian/dynamic.ts +1 -1
- package/lib/routes/afdian/explore.ts +1 -1
- package/lib/routes/bilibili/bangumi.ts +45 -43
- package/lib/routes/bilibili/bilibili-recommend.ts +4 -4
- package/lib/routes/bilibili/coin.ts +4 -4
- package/lib/routes/bilibili/dynamic.ts +14 -14
- package/lib/routes/bilibili/fav.ts +4 -4
- package/lib/routes/bilibili/followings-dynamic.ts +25 -3
- package/lib/routes/bilibili/followings-video.ts +4 -4
- package/lib/routes/bilibili/like.ts +4 -4
- package/lib/routes/bilibili/page.ts +5 -4
- package/lib/routes/bilibili/partion-ranking.ts +4 -4
- package/lib/routes/bilibili/partion.ts +4 -4
- package/lib/routes/bilibili/popular.ts +6 -4
- package/lib/routes/bilibili/ranking.ts +4 -6
- package/lib/routes/bilibili/templates/description.art +12 -0
- package/lib/routes/bilibili/types.ts +52 -0
- package/lib/routes/bilibili/user-channel.ts +10 -17
- package/lib/routes/bilibili/user-collection.ts +10 -18
- package/lib/routes/bilibili/user-fav.ts +4 -4
- package/lib/routes/bilibili/utils.ts +111 -7
- package/lib/routes/bilibili/video-all.ts +8 -3
- package/lib/routes/bilibili/video.ts +5 -4
- package/lib/routes/bilibili/vsearch.ts +21 -5
- package/lib/routes/bilibili/watchlater.ts +4 -4
- package/lib/routes/bilibili/weekly-recommend.ts +4 -9
- package/lib/routes/bsky/posts.ts +2 -2
- package/lib/routes/bsky/templates/post.art +4 -2
- package/lib/routes/chinania/index.ts +240 -0
- package/lib/routes/chinania/namespace.ts +8 -0
- package/lib/routes/dedao/articles.ts +149 -0
- package/lib/routes/eshukan/academic.ts +134 -0
- package/lib/routes/eshukan/namespace.ts +8 -0
- package/lib/routes/eshukan/templates/description.art +7 -0
- package/lib/routes/follow/profile.ts +37 -3
- package/lib/routes/lovelive-anime/schedules.ts +4 -5
- package/lib/routes/lovelive-anime/topics.ts +88 -37
- package/lib/routes/mingpao/index.ts +7 -1
- package/lib/routes/nature/cover.ts +5 -3
- package/lib/routes/nature/research.ts +1 -1
- package/lib/routes/nature/siteindex.ts +25 -11
- package/lib/routes/nature/utils.ts +5 -7
- package/lib/routes/qq/pd/guild.ts +146 -0
- package/lib/routes/qq/pd/types.ts +82 -0
- package/lib/routes/qq/pd/utils.ts +119 -0
- package/lib/routes/reuters/common.ts +160 -159
- package/lib/routes/reuters/investigates.ts +3 -2
- package/lib/routes/rsshub/transform/html.ts +2 -2
- package/lib/routes/telegram/channel.ts +14 -5
- package/lib/routes/telegram/scripts/get-telegram-session.mjs +41 -0
- package/lib/routes/telegram/tglib/channel.ts +8 -4
- package/lib/routes/theinitium/app.ts +49 -27
- package/lib/routes/theinitium/templates/description.art +2 -2
- package/lib/routes/threads/index.ts +1 -1
- package/lib/routes/threads/utils.ts +19 -1
- package/lib/routes/twitter/api/mobile-api/login.ts +125 -127
- package/lib/routes/twitter/api/mobile-api/token.ts +2 -0
- package/lib/routes/twitter/home-latest.ts +4 -1
- package/lib/routes/twitter/home.ts +4 -1
- package/lib/routes/twitter/likes.ts +4 -1
- package/lib/routes/twitter/list.ts +4 -1
- package/lib/routes/twitter/namespace.ts +1 -0
- package/lib/routes/twitter/utils.ts +9 -3
- package/lib/routes/warthunder/news.ts +8 -8
- package/lib/routes/washingtonpost/app.ts +118 -0
- package/lib/routes/washingtonpost/namespace.ts +6 -0
- package/lib/routes/washingtonpost/templates/description.art +59 -0
- package/lib/routes/zhihu/daily.ts +30 -64
- package/package.json +12 -12
|
@@ -7,6 +7,7 @@ import NotFoundError from '@/errors/types/not-found';
|
|
|
7
7
|
|
|
8
8
|
const profileUrl = (user: string) => `https://www.threads.net/@${user}`;
|
|
9
9
|
const threadUrl = (code: string) => `https://www.threads.net/t/${code}`;
|
|
10
|
+
const instagramUrl = (user: string) => `https://i.instagram.com/api/v1/users/web_profile_info/?username=${user}`;
|
|
10
11
|
|
|
11
12
|
const apiUrl = 'https://www.threads.net/api/graphql';
|
|
12
13
|
// const PROFILE_QUERY = 23_996_318_473_300_828; // no longer works
|
|
@@ -49,6 +50,7 @@ const makeHeader = (user: string, lsd: string) => ({
|
|
|
49
50
|
'Sec-Fetch-Site': 'same-origin',
|
|
50
51
|
});
|
|
51
52
|
|
|
53
|
+
// the formal way always reachs the rate limit, so use instagram api to get user id instead
|
|
52
54
|
const getUserId = (user: string, lsd: string): Promise<string> =>
|
|
53
55
|
cache.tryGet(`threads:userId:${user}`, async () => {
|
|
54
56
|
const pathName = `/@${user}`;
|
|
@@ -69,7 +71,23 @@ const getUserId = (user: string, lsd: string): Promise<string> =>
|
|
|
69
71
|
parseResponse: (txt) => destr(txt.slice(9)), // remove "for (;;);"
|
|
70
72
|
});
|
|
71
73
|
|
|
72
|
-
|
|
74
|
+
let userId = response.payload.payloads[pathName].result.exports.rootView.props.user_id;
|
|
75
|
+
|
|
76
|
+
if (!userId) {
|
|
77
|
+
const fallbackResponse = await ofetch(instagramUrl(user), {
|
|
78
|
+
headers: makeHeader(user, lsd),
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
if (!fallbackResponse?.data?.user) {
|
|
82
|
+
throw new NotFoundError('Instagram getUser API response is invalid');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
userId = fallbackResponse.data.user.id;
|
|
86
|
+
if (!userId) {
|
|
87
|
+
throw new NotFoundError('User ID not found in Instagram getUser API response');
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
73
91
|
return userId;
|
|
74
92
|
});
|
|
75
93
|
|
|
@@ -8,11 +8,11 @@ import { v5 as uuidv5 } from 'uuid';
|
|
|
8
8
|
import { authenticator } from 'otplib';
|
|
9
9
|
import logger from '@/utils/logger';
|
|
10
10
|
import cache from '@/utils/cache';
|
|
11
|
-
import { RateLimiterMemory,
|
|
11
|
+
import { RateLimiterMemory, RateLimiterQueue, RateLimiterRedis } from 'rate-limiter-flexible';
|
|
12
12
|
|
|
13
|
-
const
|
|
13
|
+
const ENDPOINT = 'https://api.x.com/1.1/onboarding/task.json';
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
const NAMESPACE = 'd41d092b-b007-48f7-9129-e9538d2d8fe9';
|
|
16
16
|
|
|
17
17
|
const headers = {
|
|
18
18
|
'User-Agent': 'TwitterAndroid/10.21.0-release.0 (310210000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)',
|
|
@@ -41,7 +41,65 @@ const loginLimiter = cache.clients.redisClient
|
|
|
41
41
|
|
|
42
42
|
const loginLimiterQueue = new RateLimiterQueue(loginLimiter);
|
|
43
43
|
|
|
44
|
-
async
|
|
44
|
+
const postTask = async (flowToken: string, subtaskId: string, subtaskInput: Record<string, unknown>) =>
|
|
45
|
+
await got.post(ENDPOINT, {
|
|
46
|
+
headers,
|
|
47
|
+
json: {
|
|
48
|
+
flow_token: flowToken,
|
|
49
|
+
subtask_inputs: [Object.assign({ subtask_id: subtaskId }, subtaskInput)],
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
// In the Twitter login flow, each task successfully requested will respond with a 'subtask_id' to determine what the next task is, and the execution sequence of the tasks is non-fixed.
|
|
54
|
+
// So abstract these tasks out into a map so that they can be dynamically executed during the login flow.
|
|
55
|
+
// If there are missing tasks in the future, simply add the implementation of that task to it.
|
|
56
|
+
const flowTasks = {
|
|
57
|
+
async LoginEnterUserIdentifier({ flowToken, username }) {
|
|
58
|
+
return await postTask(flowToken, 'LoginEnterUserIdentifier', {
|
|
59
|
+
enter_text: {
|
|
60
|
+
suggestion_id: null,
|
|
61
|
+
text: username,
|
|
62
|
+
link: 'next_link',
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
},
|
|
66
|
+
async LoginEnterPassword({ flowToken, password }) {
|
|
67
|
+
return await postTask(flowToken, 'LoginEnterPassword', {
|
|
68
|
+
enter_password: {
|
|
69
|
+
password,
|
|
70
|
+
link: 'next_link',
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
},
|
|
74
|
+
async LoginEnterAlternateIdentifierSubtask({ flowToken, phoneOrEmail }) {
|
|
75
|
+
return await postTask(flowToken, 'LoginEnterAlternateIdentifierSubtask', {
|
|
76
|
+
enter_text: {
|
|
77
|
+
suggestion_id: null,
|
|
78
|
+
text: phoneOrEmail,
|
|
79
|
+
link: 'next_link',
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
},
|
|
83
|
+
async AccountDuplicationCheck({ flowToken }) {
|
|
84
|
+
return await postTask(flowToken, 'AccountDuplicationCheck', {
|
|
85
|
+
check_logged_in_account: {
|
|
86
|
+
link: 'AccountDuplicationCheck_false',
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
},
|
|
90
|
+
async LoginTwoFactorAuthChallenge({ flowToken, authenticationSecret }) {
|
|
91
|
+
const token = authenticator.generate(authenticationSecret);
|
|
92
|
+
return await postTask(flowToken, 'LoginTwoFactorAuthChallenge', {
|
|
93
|
+
enter_text: {
|
|
94
|
+
suggestion_id: null,
|
|
95
|
+
text: token,
|
|
96
|
+
link: 'next_link',
|
|
97
|
+
},
|
|
98
|
+
});
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
async function login({ username, password, authenticationSecret, phoneOrEmail }) {
|
|
45
103
|
return (await cache.tryGet(
|
|
46
104
|
`twitter:authentication:${username}`,
|
|
47
105
|
async () => {
|
|
@@ -49,8 +107,8 @@ async function login({ username, password, authenticationSecret }) {
|
|
|
49
107
|
await loginLimiterQueue.removeTokens(1);
|
|
50
108
|
|
|
51
109
|
logger.debug('Twitter login start.');
|
|
52
|
-
|
|
53
|
-
headers['X-Twitter-Client-DeviceID'] =
|
|
110
|
+
|
|
111
|
+
headers['X-Twitter-Client-DeviceID'] = uuidv5(username, NAMESPACE);
|
|
54
112
|
|
|
55
113
|
const ct0 = crypto.randomUUID().replaceAll('-', '');
|
|
56
114
|
const guestToken = await got(guestActivateUrl, {
|
|
@@ -61,140 +119,80 @@ async function login({ username, password, authenticationSecret }) {
|
|
|
61
119
|
},
|
|
62
120
|
method: 'POST',
|
|
63
121
|
});
|
|
64
|
-
logger.debug('Twitter login
|
|
122
|
+
logger.debug('Twitter login: guest token');
|
|
65
123
|
|
|
66
124
|
headers['x-guest-token'] = guestToken.data.guest_token;
|
|
67
125
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
126
|
+
let task = await ofetch
|
|
127
|
+
.raw(
|
|
128
|
+
ENDPOINT +
|
|
129
|
+
'?' +
|
|
130
|
+
new URLSearchParams({
|
|
131
|
+
flow_name: 'login',
|
|
132
|
+
api_version: '1',
|
|
133
|
+
known_device_token: '',
|
|
134
|
+
sim_country_code: 'us',
|
|
135
|
+
}).toString(),
|
|
136
|
+
{
|
|
137
|
+
method: 'POST',
|
|
138
|
+
headers,
|
|
139
|
+
body: {
|
|
140
|
+
flow_token: null,
|
|
141
|
+
input_flow_data: {
|
|
142
|
+
country_code: null,
|
|
143
|
+
flow_context: {
|
|
144
|
+
referrer_context: {
|
|
145
|
+
referral_details: 'utm_source=google-play&utm_medium=organic',
|
|
146
|
+
referrer_url: '',
|
|
147
|
+
},
|
|
148
|
+
start_location: {
|
|
149
|
+
location: 'deeplink',
|
|
150
|
+
},
|
|
90
151
|
},
|
|
152
|
+
requested_variant: null,
|
|
153
|
+
target_user_id: 0,
|
|
91
154
|
},
|
|
92
|
-
requested_variant: null,
|
|
93
|
-
target_user_id: 0,
|
|
94
155
|
},
|
|
95
|
-
}
|
|
156
|
+
}
|
|
157
|
+
)
|
|
158
|
+
.then(({ headers: _headers, _data }) => {
|
|
159
|
+
headers.att = _headers.get('att');
|
|
160
|
+
return { data: _data };
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
logger.debug('Twitter login flow start.');
|
|
164
|
+
const runTask = async ({ data }) => {
|
|
165
|
+
const { subtask_id, open_account } = data.subtasks.shift();
|
|
166
|
+
|
|
167
|
+
// If `open_account` exists (and 'subtask_id' is `LoginSuccessSubtask`), it means the login was successful.
|
|
168
|
+
if (open_account) {
|
|
169
|
+
return open_account;
|
|
96
170
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
const task2 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
|
|
103
|
-
headers,
|
|
104
|
-
json: {
|
|
105
|
-
flow_token: task1._data.flow_token,
|
|
106
|
-
subtask_inputs: [
|
|
107
|
-
{
|
|
108
|
-
enter_text: {
|
|
109
|
-
suggestion_id: null,
|
|
110
|
-
text: username,
|
|
111
|
-
link: 'next_link',
|
|
112
|
-
},
|
|
113
|
-
subtask_id: 'LoginEnterUserIdentifier',
|
|
114
|
-
},
|
|
115
|
-
],
|
|
116
|
-
},
|
|
117
|
-
});
|
|
118
|
-
logger.debug('Twitter login 3 finished: LoginEnterUserIdentifier.');
|
|
119
|
-
|
|
120
|
-
const task3 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
|
|
121
|
-
headers,
|
|
122
|
-
json: {
|
|
123
|
-
flow_token: task2.data.flow_token,
|
|
124
|
-
subtask_inputs: [
|
|
125
|
-
{
|
|
126
|
-
enter_password: {
|
|
127
|
-
password,
|
|
128
|
-
link: 'next_link',
|
|
129
|
-
},
|
|
130
|
-
subtask_id: 'LoginEnterPassword',
|
|
131
|
-
},
|
|
132
|
-
],
|
|
133
|
-
},
|
|
134
|
-
});
|
|
135
|
-
logger.debug('Twitter login 4 finished: LoginEnterPassword.');
|
|
136
|
-
for (const subtask of task3.data?.subtasks || []) {
|
|
137
|
-
if (subtask.open_account) {
|
|
138
|
-
return subtask.open_account;
|
|
171
|
+
|
|
172
|
+
// If task does not exist in `flowTasks`, we need to implement it.
|
|
173
|
+
if (!(subtask_id in flowTasks)) {
|
|
174
|
+
logger.error(`Twitter login flow task failed: unknown subtask: ${subtask_id}`);
|
|
175
|
+
return;
|
|
139
176
|
}
|
|
140
|
-
}
|
|
141
177
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
},
|
|
151
|
-
subtask_id: 'AccountDuplicationCheck',
|
|
152
|
-
},
|
|
153
|
-
],
|
|
154
|
-
},
|
|
155
|
-
});
|
|
156
|
-
logger.debug('Twitter login 5 finished: AccountDuplicationCheck.');
|
|
178
|
+
task = await flowTasks[subtask_id]({
|
|
179
|
+
flowToken: data.flow_token,
|
|
180
|
+
username,
|
|
181
|
+
password,
|
|
182
|
+
authenticationSecret,
|
|
183
|
+
phoneOrEmail,
|
|
184
|
+
});
|
|
185
|
+
logger.debug(`Twitter login flow task finished: subtask: ${subtask_id}.`);
|
|
157
186
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
} else if (authenticationSecret && subtask.subtask_id === 'LoginTwoFactorAuthChallenge') {
|
|
163
|
-
const token = authenticator.generate(authenticationSecret);
|
|
187
|
+
return await runTask(task);
|
|
188
|
+
};
|
|
189
|
+
const authentication = await runTask(task);
|
|
190
|
+
logger.debug('Twitter login flow finished.');
|
|
164
191
|
|
|
165
|
-
const task5 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
|
|
166
|
-
headers,
|
|
167
|
-
json: {
|
|
168
|
-
flow_token: task4.data.flow_token,
|
|
169
|
-
subtask_inputs: [
|
|
170
|
-
{
|
|
171
|
-
enter_text: {
|
|
172
|
-
suggestion_id: null,
|
|
173
|
-
text: token,
|
|
174
|
-
link: 'next_link',
|
|
175
|
-
},
|
|
176
|
-
subtask_id: 'LoginTwoFactorAuthChallenge',
|
|
177
|
-
},
|
|
178
|
-
],
|
|
179
|
-
},
|
|
180
|
-
});
|
|
181
|
-
logger.debug('Twitter login 6 finished: LoginTwoFactorAuthChallenge.');
|
|
182
|
-
|
|
183
|
-
for (const subtask of task5.data?.subtasks || []) {
|
|
184
|
-
if (subtask.open_account) {
|
|
185
|
-
authentication = subtask.open_account;
|
|
186
|
-
break;
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
break;
|
|
190
|
-
} else {
|
|
191
|
-
logger.error(`Twitter login 6 failed: unknown subtask: ${subtask.subtask_id}`);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
192
|
if (authentication) {
|
|
195
|
-
logger.debug('Twitter login success.');
|
|
193
|
+
logger.debug('Twitter login success.', authentication);
|
|
196
194
|
} else {
|
|
197
|
-
logger.error(`Twitter login failed. ${JSON.stringify(
|
|
195
|
+
logger.error(`Twitter login failed. ${JSON.stringify(task.data?.subtasks, null, 2)}`);
|
|
198
196
|
}
|
|
199
197
|
|
|
200
198
|
return authentication;
|
|
@@ -11,11 +11,13 @@ async function getToken() {
|
|
|
11
11
|
const username = config.twitter.username[index];
|
|
12
12
|
const password = config.twitter.password[index];
|
|
13
13
|
const authenticationSecret = config.twitter.authenticationSecret?.[index];
|
|
14
|
+
const phoneOrEmail = config.twitter.phoneOrEmail?.[index];
|
|
14
15
|
if (username && password) {
|
|
15
16
|
const authentication = await login({
|
|
16
17
|
username,
|
|
17
18
|
password,
|
|
18
19
|
authenticationSecret,
|
|
20
|
+
phoneOrEmail,
|
|
19
21
|
});
|
|
20
22
|
if (!authentication) {
|
|
21
23
|
throw new ConfigNotFoundError(`Invalid twitter configs: ${username}`);
|
|
@@ -40,7 +40,7 @@ export const route: Route = {
|
|
|
40
40
|
|
|
41
41
|
async function handler(ctx) {
|
|
42
42
|
// For compatibility
|
|
43
|
-
const { count, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
|
|
43
|
+
const { count, include_rts, only_media } = utils.parseRouteParams(ctx.req.param('routeParams'));
|
|
44
44
|
const params = count ? { count } : {};
|
|
45
45
|
|
|
46
46
|
await api.init();
|
|
@@ -48,6 +48,9 @@ async function handler(ctx) {
|
|
|
48
48
|
if (!include_rts) {
|
|
49
49
|
data = utils.excludeRetweet(data);
|
|
50
50
|
}
|
|
51
|
+
if (only_media) {
|
|
52
|
+
data = utils.keepOnlyMedia(data);
|
|
53
|
+
}
|
|
51
54
|
|
|
52
55
|
return {
|
|
53
56
|
title: `Twitter following timeline`,
|
|
@@ -40,7 +40,7 @@ export const route: Route = {
|
|
|
40
40
|
|
|
41
41
|
async function handler(ctx) {
|
|
42
42
|
// For compatibility
|
|
43
|
-
const { count, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
|
|
43
|
+
const { count, include_rts, only_media } = utils.parseRouteParams(ctx.req.param('routeParams'));
|
|
44
44
|
const params = count ? { count } : {};
|
|
45
45
|
|
|
46
46
|
await api.init();
|
|
@@ -48,6 +48,9 @@ async function handler(ctx) {
|
|
|
48
48
|
if (!include_rts) {
|
|
49
49
|
data = utils.excludeRetweet(data);
|
|
50
50
|
}
|
|
51
|
+
if (only_media) {
|
|
52
|
+
data = utils.keepOnlyMedia(data);
|
|
53
|
+
}
|
|
51
54
|
|
|
52
55
|
return {
|
|
53
56
|
title: `Twitter following timeline`,
|
|
@@ -27,7 +27,7 @@ export const route: Route = {
|
|
|
27
27
|
|
|
28
28
|
async function handler(ctx) {
|
|
29
29
|
const id = ctx.req.param('id');
|
|
30
|
-
const { count, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
|
|
30
|
+
const { count, include_rts, only_media } = utils.parseRouteParams(ctx.req.param('routeParams'));
|
|
31
31
|
const params = count ? { count } : {};
|
|
32
32
|
|
|
33
33
|
await api.init();
|
|
@@ -35,6 +35,9 @@ async function handler(ctx) {
|
|
|
35
35
|
if (!include_rts) {
|
|
36
36
|
data = utils.excludeRetweet(data);
|
|
37
37
|
}
|
|
38
|
+
if (only_media) {
|
|
39
|
+
data = utils.keepOnlyMedia(data);
|
|
40
|
+
}
|
|
38
41
|
|
|
39
42
|
return {
|
|
40
43
|
title: `Twitter Likes - ${id}`,
|
|
@@ -33,7 +33,7 @@ export const route: Route = {
|
|
|
33
33
|
|
|
34
34
|
async function handler(ctx) {
|
|
35
35
|
const id = ctx.req.param('id');
|
|
36
|
-
const { count, include_rts } = utils.parseRouteParams(ctx.req.param('routeParams'));
|
|
36
|
+
const { count, include_rts, only_media } = utils.parseRouteParams(ctx.req.param('routeParams'));
|
|
37
37
|
const params = count ? { count } : {};
|
|
38
38
|
|
|
39
39
|
await api.init();
|
|
@@ -41,6 +41,9 @@ async function handler(ctx) {
|
|
|
41
41
|
if (!include_rts) {
|
|
42
42
|
data = utils.excludeRetweet(data);
|
|
43
43
|
}
|
|
44
|
+
if (only_media) {
|
|
45
|
+
data = utils.keepOnlyMedia(data);
|
|
46
|
+
}
|
|
44
47
|
|
|
45
48
|
return {
|
|
46
49
|
title: `Twitter List - ${id}`,
|
|
@@ -27,6 +27,7 @@ export const namespace: Namespace = {
|
|
|
27
27
|
| \`includeRts\` | Include retweets, only available in \`/twitter/user\` | \`0\`/\`1\`/\`true\`/\`false\` | \`true\` |
|
|
28
28
|
| \`forceWebApi\` | Force using Web API even if Developer API is configured, only available in \`/twitter/user\` and \`/twitter/keyword\` | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` |
|
|
29
29
|
| \`count\` | \`count\` parameter passed to Twitter API, only available in \`/twitter/user\` | Unspecified/Integer | Unspecified |
|
|
30
|
+
| \`onlyMedia\` | Only get tweets with a media | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` |
|
|
30
31
|
|
|
31
32
|
Specify different option values than default values to improve readability. The URL
|
|
32
33
|
|
|
@@ -451,7 +451,7 @@ if (config.twitter.consumer_key && config.twitter.consumer_secret) {
|
|
|
451
451
|
}
|
|
452
452
|
|
|
453
453
|
const parseRouteParams = (routeParams) => {
|
|
454
|
-
let count, exclude_replies, include_rts;
|
|
454
|
+
let count, exclude_replies, include_rts, only_media;
|
|
455
455
|
let force_web_api = false;
|
|
456
456
|
switch (routeParams) {
|
|
457
457
|
case 'exclude_rts_replies':
|
|
@@ -479,9 +479,10 @@ const parseRouteParams = (routeParams) => {
|
|
|
479
479
|
exclude_replies = fallback(undefined, queryToBoolean(parsed.get('excludeReplies')), false);
|
|
480
480
|
include_rts = fallback(undefined, queryToBoolean(parsed.get('includeRts')), true);
|
|
481
481
|
force_web_api = fallback(undefined, queryToBoolean(parsed.get('forceWebApi')), false);
|
|
482
|
+
only_media = fallback(undefined, queryToBoolean(parsed.get('onlyMedia')), false);
|
|
482
483
|
}
|
|
483
484
|
}
|
|
484
|
-
return { count, exclude_replies, include_rts, force_web_api };
|
|
485
|
+
return { count, exclude_replies, include_rts, force_web_api, only_media };
|
|
485
486
|
};
|
|
486
487
|
|
|
487
488
|
export const excludeRetweet = function (tweets) {
|
|
@@ -495,4 +496,9 @@ export const excludeRetweet = function (tweets) {
|
|
|
495
496
|
return excluded;
|
|
496
497
|
};
|
|
497
498
|
|
|
498
|
-
export
|
|
499
|
+
export const keepOnlyMedia = function (tweets) {
|
|
500
|
+
const excluded = tweets.filter((t) => t.extended_entities && t.extended_entities.media);
|
|
501
|
+
return excluded;
|
|
502
|
+
};
|
|
503
|
+
|
|
504
|
+
export default { ProcessFeed, getAppClient, parseRouteParams, excludeRetweet, keepOnlyMedia };
|
|
@@ -2,7 +2,7 @@ import { Route } from '@/types';
|
|
|
2
2
|
import { getCurrentPath } from '@/utils/helpers';
|
|
3
3
|
const __dirname = getCurrentPath(import.meta.url);
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import ofetch from '@/utils/ofetch';
|
|
6
6
|
import { load } from 'cheerio';
|
|
7
7
|
import path from 'node:path';
|
|
8
8
|
import { art } from '@/utils/render';
|
|
@@ -33,20 +33,21 @@ export const route: Route = {
|
|
|
33
33
|
handler,
|
|
34
34
|
url: 'warthunder.com/en/news',
|
|
35
35
|
description: `News data from [https://warthunder.com/en/news/](https://warthunder.com/en/news/)
|
|
36
|
-
The
|
|
36
|
+
The \`pubDate\` provided under UTC time zone, so please ignore the specific time!!!`,
|
|
37
37
|
};
|
|
38
38
|
|
|
39
39
|
async function handler() {
|
|
40
40
|
const rootUrl = 'https://warthunder.com/en/news/';
|
|
41
41
|
|
|
42
|
-
const response = await
|
|
42
|
+
const response = await ofetch(rootUrl);
|
|
43
43
|
|
|
44
|
-
const $ = load(response
|
|
44
|
+
const $ = load(response);
|
|
45
45
|
|
|
46
46
|
const pageFace = $('div.showcase__item.widget')
|
|
47
|
-
.
|
|
47
|
+
.toArray()
|
|
48
|
+
.map((item) => {
|
|
48
49
|
item = $(item);
|
|
49
|
-
let pubDate = parseDate(item.find('div.widget__content > ul > li.widget-meta__item.widget-meta__item--right').text(), 'D MMMM YYYY');
|
|
50
|
+
let pubDate = parseDate(item.find('div.widget__content > ul > li.widget-meta__item.widget-meta__item--right').text(), 'D MMMM YYYY', 'en');
|
|
50
51
|
pubDate = timezone(pubDate, 0);
|
|
51
52
|
const category = [];
|
|
52
53
|
if (item.find('div.widget__pin').length !== 0) {
|
|
@@ -68,8 +69,7 @@ async function handler() {
|
|
|
68
69
|
}),
|
|
69
70
|
category,
|
|
70
71
|
};
|
|
71
|
-
})
|
|
72
|
-
.get();
|
|
72
|
+
});
|
|
73
73
|
|
|
74
74
|
return {
|
|
75
75
|
title: 'War Thunder News',
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { Route } from '@/types';
|
|
2
|
+
import cache from '@/utils/cache';
|
|
3
|
+
import got from '@/utils/got';
|
|
4
|
+
import { art } from '@/utils/render';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { getCurrentPath } from '@/utils/helpers';
|
|
7
|
+
import { FetchError } from 'ofetch';
|
|
8
|
+
import dayjs from 'dayjs';
|
|
9
|
+
import utc from 'dayjs/plugin/utc';
|
|
10
|
+
import timezone from 'dayjs/plugin/timezone';
|
|
11
|
+
import advancedFormat from 'dayjs/plugin/advancedFormat';
|
|
12
|
+
|
|
13
|
+
export const route: Route = {
|
|
14
|
+
path: '/app/:category{.+}?',
|
|
15
|
+
categories: ['traditional-media'],
|
|
16
|
+
example: '/washingtonpost/app/national',
|
|
17
|
+
parameters: {
|
|
18
|
+
category: 'Category from the path of the URL of the corresponding site, see below',
|
|
19
|
+
},
|
|
20
|
+
features: {
|
|
21
|
+
requirePuppeteer: false,
|
|
22
|
+
antiCrawler: false,
|
|
23
|
+
supportBT: false,
|
|
24
|
+
supportPodcast: false,
|
|
25
|
+
supportScihub: false,
|
|
26
|
+
},
|
|
27
|
+
name: 'App',
|
|
28
|
+
maintainers: ['quiniapiezoelectricity'],
|
|
29
|
+
radar: [
|
|
30
|
+
{
|
|
31
|
+
source: ['www.washingtonpost.com/:category'],
|
|
32
|
+
target: '/app/:category',
|
|
33
|
+
},
|
|
34
|
+
],
|
|
35
|
+
handler,
|
|
36
|
+
description: `:::tip
|
|
37
|
+
For example, the category for https://www.washingtonpost.com/national/investigations would be /national/investigations.
|
|
38
|
+
:::`,
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
function handleDuplicates(array) {
|
|
42
|
+
const objects = {};
|
|
43
|
+
for (const obj of array) {
|
|
44
|
+
objects[obj.id] = objects[obj.id] ? Object.assign(objects[obj.id], obj) : obj;
|
|
45
|
+
}
|
|
46
|
+
return Object.values(objects);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function handler(ctx) {
|
|
50
|
+
const category = ctx.req.param('category') ?? '';
|
|
51
|
+
const __dirname = getCurrentPath(import.meta.url);
|
|
52
|
+
const headers = {
|
|
53
|
+
Accept: '*/*',
|
|
54
|
+
Connection: 'keep-alive',
|
|
55
|
+
'User-Agent': 'Classic/6.70.0',
|
|
56
|
+
};
|
|
57
|
+
dayjs.extend(utc);
|
|
58
|
+
dayjs.extend(timezone);
|
|
59
|
+
dayjs.extend(advancedFormat);
|
|
60
|
+
art.defaults.imports.dayjs = dayjs;
|
|
61
|
+
|
|
62
|
+
const url = `https://jsonapp1.washingtonpost.com/fusion_prod/v2/${category}`;
|
|
63
|
+
const response = await got.get(url, { headers });
|
|
64
|
+
const title = response.data.tracking.page_title.includes('Washington Post') ? response.data.tracking.page_title : `The Washington Post - ${response.data.tracking.page_title}`;
|
|
65
|
+
const link = 'https://washingtonpost.com' + response.data.tracking.page_path;
|
|
66
|
+
const mains = response.data.regions[0].items.filter((item) => item.items);
|
|
67
|
+
const list = mains.flatMap((main) =>
|
|
68
|
+
main.items[0].items
|
|
69
|
+
.filter((item) => item.is_from_feed === true)
|
|
70
|
+
.map((item) => {
|
|
71
|
+
const object = {
|
|
72
|
+
id: item.id,
|
|
73
|
+
title: item.headline.text,
|
|
74
|
+
link: item.link.url,
|
|
75
|
+
pubDate: item.link.display_date,
|
|
76
|
+
updated: item.link.last_modified,
|
|
77
|
+
};
|
|
78
|
+
if (item.blurbs?.items[0]?.text) {
|
|
79
|
+
object.description = item.blurbs?.items[0]?.text;
|
|
80
|
+
}
|
|
81
|
+
return object;
|
|
82
|
+
})
|
|
83
|
+
);
|
|
84
|
+
const feed = handleDuplicates(list);
|
|
85
|
+
const items = await Promise.all(
|
|
86
|
+
feed.map((item) =>
|
|
87
|
+
cache.tryGet(item.link, async () => {
|
|
88
|
+
let response;
|
|
89
|
+
try {
|
|
90
|
+
response = await got(`https://rainbowapi-a.wpdigital.net/rainbow-data-service/rainbow/content-by-url.json?followLinks=false&url=${item.link}`, { headers });
|
|
91
|
+
} catch (error) {
|
|
92
|
+
if (error instanceof FetchError && error.statusCode === 415) {
|
|
93
|
+
// Interactive or podcast contents will return 415 Unsupported Media Type. Keep calm and carry on.
|
|
94
|
+
return item;
|
|
95
|
+
} else {
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
item.title = response.data.title ?? item.title;
|
|
100
|
+
item.author =
|
|
101
|
+
response.data.items
|
|
102
|
+
.filter((entry) => entry.type === 'byline')
|
|
103
|
+
?.flatMap((entry) => entry.authors.map((author) => author.name))
|
|
104
|
+
?.join(', ') ?? '';
|
|
105
|
+
item.description = art(path.join(__dirname, 'templates/description.art'), {
|
|
106
|
+
content: response.data.items,
|
|
107
|
+
});
|
|
108
|
+
return item;
|
|
109
|
+
})
|
|
110
|
+
)
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
return {
|
|
114
|
+
title,
|
|
115
|
+
link,
|
|
116
|
+
item: items,
|
|
117
|
+
};
|
|
118
|
+
}
|