rsshub 1.0.0-master.f72af1b → 1.0.0-master.f7347d9

package/lib/routes/szftedu/gonggao.ts

@@ -0,0 +1,62 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+const host = 'https://ylxx.szftedu.cn/xx_5828/xygg_5832/';
+
+export const route: Route = {
+    path: '/gonggao',
+    categories: ['university'],
+    example: '/szftedu/gonggao',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '公告',
+    maintainers: ['valuex'],
+    handler,
+    description: '',
+};
+
+async function handler() {
+    const link = host;
+    const response = await got(link);
+    const $ = load(response.data);
+
+    const lists = $('div.pagenews04 div ul li')
+        .toArray()
+        .map((el) => ({
+            title: $('a', el).text().trim(),
+            link: $('a', el).attr('href'),
+            pubDate: timezone(parseDate($('span[class=canedit]', el).text()), 8),
+        }));
+
+    const items = await Promise.all(
+        lists.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const thisUrl = item.link;
+                const trueLink = thisUrl.includes('http') ? thisUrl : host + thisUrl.substring(1);
+                const response = await got(trueLink);
+                const $ = load(response.data);
+                item.description = thisUrl.includes('http') ? $('#page-content').html() : $('div.TRS_Editor').html();
+                item.pubDate = timezone(parseDate($('.item').first().text().replace('发布时间：', '')), 8);
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: '园岭小学公告',
+        link: host,
+        description: '园岭小学公告',
+        item: items,
+    };
+}

package/lib/routes/szftedu/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '园岭小学',
+    url: 'ylxx.szftedu.cn',
+};

package/lib/routes/the/index.ts

@@ -13,7 +13,7 @@ import timezone from '@/utils/timezone';
 
 export const handler = async (ctx) => {
     const { filter } = ctx.req.param();
-    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50;
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 40;
 
     const rootUrl = 'https://the.bi/s';
     const filters = parseFilterStr(filter);

package/lib/routes/tkww/index.ts

@@ -0,0 +1,83 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { config } from '@/config';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+
+export const route: Route = {
+    path: '/:column{.+}?',
+    categories: ['traditional-media'],
+    example: '/tkww/hong_kong',
+    parameters: {
+        column: '欄目，默認為 home (首頁)',
+    },
+    features: {
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '新聞',
+    maintainers: ['quiniapiezoelectricity'],
+    radar: [
+        {
+            source: ['www.tkww.hk/:column'],
+            target: '/:column',
+        },
+    ],
+    handler,
+    description: `
+:::tip
+欄目可用\`名稱\`或對應網頁的\`path\`，
+如 \`https://www.tkww.hk/hong_kong\` 的欄目可以填\`香港\`或是\`hong_kong\`
+而 \`https://www.tkww.hk/china/shanghai\` 的欄目則需填\`china/shanghai\`
+:::`,
+};
+
+async function handler(ctx) {
+    const column = ctx.req.param('column') ?? 'home';
+
+    const columns = await cache.tryGet('https://www.tkww.hk/columns.json', async () => await got('https://www.tkww.hk/columns.json'), config.cache.routeExpire, false);
+
+    let metadata;
+    let scope = columns.data.data;
+    for (const segment of column.split('/').filter((item) => typeof item === 'string')) {
+        metadata = scope.find((item) => item.name === segment || item.dirname === segment);
+        scope = metadata?.children ?? [];
+    }
+
+    if (metadata === undefined) {
+        throw new InvalidParameterError(`Invalid Column: ${column}`);
+    }
+
+    const stories = await got(`https://www.tkww.hk/columns/${metadata.uuid}/tkww/app/stories.json`);
+
+    const items = await Promise.all(
+        stories.data.data.stories.map((item) =>
+            cache.tryGet(item.url, async () => {
+                item.link = item.url;
+                item.description = item.summary;
+                item.pubDate = item.publishTime;
+                item.category = [];
+                if (item.keywords) {
+                    item.category = [...item.category, ...item.keywords];
+                }
+                if (item.tags) {
+                    item.category = [...item.category, ...item.tags];
+                }
+                item.category = [...new Set(item.category)];
+                const response = await got(item.jsonUrl);
+                item.description = response.data.data.content;
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: metadata.seoTitle,
+        description: metadata.seoDescription,
+        link: metadata.url,
+        item: items,
+    };
+}

package/lib/routes/tkww/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '大公文匯網',
+    url: 'www.tkww.hk',
+};

package/lib/routes/ttv/index.ts

@@ -34,7 +34,7 @@ async function handler(ctx) {
     const $ = load(response.data);
 
     let items = $('div.news-list li')
-        .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.query.limit) : 30)
+        .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 30)
         .toArray()
         .map((item) => {
             item = $(item);

package/lib/routes/tvb/news.ts

@@ -85,6 +85,7 @@ async function handler(ctx) {
     const language = ctx.req.param('language') ?? 'tc';
 
     const rootUrl = 'https://inews-api.tvb.com';
+    const linkRootUrl = 'https://news.tvb.com';
     const apiUrl = `${rootUrl}/news/entry/category`;
     const currentUrl = `${rootUrl}/${language}/${category}`;
 
@@ -102,7 +103,7 @@ async function handler(ctx) {
 
     const items = response.data.content.map((item) => ({
         title: item.title,
-        link: `${rootUrl}/${language}/${category}/${item.id}`,
+        link: `${linkRootUrl}/${language}/${category}/${item.id}`,
         pubDate: parseDate(item.publish_datetime),
         category: [...item.category.map((c) => c.title), ...item.tags],
         description: art(path.join(__dirname, 'templates/description.art'), {

package/lib/routes/twitter/api/mobile-api/login.ts

@@ -133,6 +133,11 @@ async function login({ username, password, authenticationSecret }) {
                     },
                 });
                 logger.debug('Twitter login 4 finished: LoginEnterPassword.');
+                for (const subtask of task3.data?.subtasks || []) {
+                    if (subtask.open_account) {
+                        return subtask.open_account;
+                    }
+                }
 
                 const task4 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
                     headers,

package/lib/routes/twitter/api/web-api/api.ts

@@ -18,16 +18,22 @@ const getUserData = (id) =>
                 }),
             });
         }
-        return twitterGot(`${baseUrl}${gqlMap.UserByScreenName}`, {
-            variables: JSON.stringify({
-                screen_name: id,
-                withSafetyModeUserFields: true,
-            }),
-            features: JSON.stringify(gqlFeatures.UserByScreenName),
-            fieldToggles: JSON.stringify({
-                withAuxiliaryUserLabels: false,
-            }),
-        });
+        return twitterGot(
+            `${baseUrl}${gqlMap.UserByScreenName}`,
+            {
+                variables: JSON.stringify({
+                    screen_name: id,
+                    withSafetyModeUserFields: true,
+                }),
+                features: JSON.stringify(gqlFeatures.UserByScreenName),
+                fieldToggles: JSON.stringify({
+                    withAuxiliaryUserLabels: false,
+                }),
+            },
+            {
+                allowNoAuth: true,
+            }
+        );
     });
 
 const cacheTryGet = async (_id, params, func) => {

package/lib/routes/twitter/api/web-api/constants.ts

@@ -7,7 +7,7 @@ const graphQLEndpointsPlain = [
     '/graphql/DiTkXJgLqBBxCs7zaYsbtA/HomeLatestTimeline',
     '/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies',
     '/graphql/dexO_2tohK86JDudXXG3Yw/UserMedia',
-    '/graphql/tD8zKvQzwY3kdx5yz6YmOw/UserByRestId',
+    '/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId',
     '/graphql/UN1i3zUiCWa-6r-Uaho4fw/SearchTimeline',
     '/graphql/Pa45JvqZuKcW1plybfgBlQ/ListLatestTweetsTimeline',
     '/graphql/QuBlQ6SxNAQCt6-kBiCXCQ/TweetDetail',

package/lib/routes/twitter/api/web-api/utils.ts

@@ -24,9 +24,19 @@ const token2Cookie = (token) =>
                       uri: proxy.proxyUri,
                   })
                 : new CookieAgent({ cookies: { jar } });
-            await ofetch('https://x.com', {
-                dispatcher: agent,
-            });
+            if (token) {
+                await ofetch('https://x.com', {
+                    dispatcher: agent,
+                });
+            } else {
+                const data = await ofetch('https://x.com/narendramodi?mx=2', {
+                    dispatcher: agent,
+                });
+                const gt = data.match(/document\.cookie="gt=(\d+)/)?.[1];
+                if (gt) {
+                    jar.setCookieSync(`gt=${gt}`, 'https://x.com');
+                }
+            }
             return JSON.stringify(jar.serializeSync());
         } catch {
             // ignore
@@ -34,17 +44,19 @@ const token2Cookie = (token) =>
         }
     });
 
+const lockPrefix = 'twitter:lock-token1:';
+
 const getAuth = async (retry: number) => {
     if (config.twitter.authToken && retry > 0) {
         const index = authTokenIndex++ % config.twitter.authToken.length;
         const token = config.twitter.authToken[index];
-        const lock = await cache.get(`twitter:lock-token:${token}`);
+        const lock = await cache.get(`${lockPrefix}${token}`, false);
         if (lock) {
             await new Promise((resolve) => setTimeout(resolve, Math.random() * 500 + 500));
             return await getAuth(retry - 1);
         } else {
             logger.debug(`twitter debug: lock twitter cookie for token ${token}`);
-            await cache.set(`twitter:lock-token:${token}`, '1', 20);
+            await cache.set(`${lockPrefix}${token}`, '1', 20);
             return {
                 token,
                 username: config.twitter.username?.[index],
@@ -55,17 +67,23 @@ const getAuth = async (retry: number) => {
     }
 };
 
-export const twitterGot = async (url, params) => {
+export const twitterGot = async (
+    url,
+    params,
+    options?: {
+        allowNoAuth?: boolean;
+    }
+) => {
     const auth = await getAuth(30);
 
-    if (!auth) {
+    if (!auth && !options?.allowNoAuth) {
         throw new ConfigNotFoundError('No valid Twitter token found');
     }
 
     const requestUrl = `${url}?${queryString.stringify(params)}`;
 
-    let cookie: string | Record<string, any> | null | undefined = await token2Cookie(auth.token);
-    if (!cookie) {
+    let cookie: string | Record<string, any> | null | undefined = await token2Cookie(auth?.token);
+    if (!cookie && auth) {
         cookie = await login({
             username: auth.username,
             password: auth.password,
@@ -79,7 +97,7 @@ export const twitterGot = async (url, params) => {
           }
         | undefined;
     if (cookie) {
-        logger.debug(`twitter debug: got twitter cookie for token ${auth.token}`);
+        logger.debug(`twitter debug: got twitter cookie for token ${auth?.token}`);
         if (typeof cookie === 'string') {
             cookie = JSON.parse(cookie);
         }
@@ -97,16 +115,18 @@ export const twitterGot = async (url, params) => {
             jar,
             agent,
         };
-    } else {
-        throw new ConfigNotFoundError(`Twitter cookie for token ${auth.token} is not valid`);
+    } else if (auth) {
+        throw new ConfigNotFoundError(`Twitter cookie for token ${auth?.token?.replace(/(\w{8})(\w+)/, (_, v1, v2) => v1 + '*'.repeat(v2.length))} is not valid`);
     }
-    const jsonCookie = Object.fromEntries(
-        dispatchers.jar
-            .getCookieStringSync(url)
-            .split(';')
-            .map((c) => Cookie.parse(c)?.toJSON())
-            .map((c) => [c?.key, c?.value])
-    );
+    const jsonCookie = dispatchers
+        ? Object.fromEntries(
+              dispatchers.jar
+                  .getCookieStringSync(url)
+                  .split(';')
+                  .map((c) => Cookie.parse(c)?.toJSON())
+                  .map((c) => [c?.key, c?.value])
+          )
+        : {};
 
     const response = await ofetch.raw(requestUrl, {
         retry: 0,
@@ -119,57 +139,75 @@ export const twitterGot = async (url, params) => {
             'content-type': 'application/json',
             dnt: '1',
             pragma: 'no-cache',
-            referer: 'https://x.com/narendramodi',
+            referer: 'https://x.com/',
             'x-twitter-active-user': 'yes',
-            'x-twitter-auth-type': 'OAuth2Session',
             'x-twitter-client-language': 'en',
             'x-csrf-token': jsonCookie.ct0,
+            ...(auth?.token
+                ? {
+                      'x-twitter-auth-type': 'OAuth2Session',
+                  }
+                : {
+                      'x-guest-token': jsonCookie.gt,
+                  }),
         },
-        dispatcher: dispatchers.agent,
+        dispatcher: dispatchers?.agent,
         onResponse: async ({ response }) => {
-            if (response.status === 429) {
-                logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token}`);
-                await cache.set(`twitter:lock-token:${auth.token}`, '1', 1000);
-            } else if (response.status === 403 || response.status === 401 || JSON.stringify(response._data?.data) === '{"user":{}}') {
-                const newCookie = await login({
-                    username: auth.username,
-                    password: auth.password,
-                    authenticationSecret: auth.authenticationSecret,
-                });
-                if (newCookie) {
-                    logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`);
-                    await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire);
-                    logger.debug(`twitter debug: unlock twitter cookie with error1 for token ${auth.token}`);
-                    await cache.set(`twitter:lock-token:${auth.token}`, '', 1);
-                } else {
-                    const tokenIndex = config.twitter.authToken?.indexOf(auth.token);
-                    if (tokenIndex !== undefined && tokenIndex !== -1) {
-                        config.twitter.authToken?.splice(tokenIndex, 1);
-                    }
-                    if (auth.username) {
-                        const usernameIndex = config.twitter.username?.indexOf(auth.username);
-                        if (usernameIndex !== undefined && usernameIndex !== -1) {
-                            config.twitter.username?.splice(usernameIndex, 1);
+            const remaining = response.headers.get('x-rate-limit-remaining');
+            const remainingInt = Number.parseInt(remaining || '0');
+            const reset = response.headers.get('x-rate-limit-reset');
+            logger.debug(`twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}`);
+            if (auth) {
+                if (remaining && remainingInt < 2 && reset) {
+                    const resetTime = new Date(Number.parseInt(reset) * 1000);
+                    const delay = (resetTime.getTime() - Date.now()) / 1000;
+                    logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`);
+                    await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay));
+                } else if (response.status === 429 || JSON.stringify(response._data?.data) === '{"user":{}}') {
+                    logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`);
+                    await cache.set(`${lockPrefix}${auth.token}`, '1', 2000);
+                } else if (response.status === 403 || response.status === 401) {
+                    const newCookie = await login({
+                        username: auth.username,
+                        password: auth.password,
+                        authenticationSecret: auth.authenticationSecret,
+                    });
+                    if (newCookie) {
+                        logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`);
+                        await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire);
+                        logger.debug(`twitter debug: unlock twitter cookie for token ${auth.token} with error1`);
+                        await cache.set(`${lockPrefix}${auth.token}`, '', 1);
+                    } else {
+                        const tokenIndex = config.twitter.authToken?.indexOf(auth.token);
+                        if (tokenIndex !== undefined && tokenIndex !== -1) {
+                            config.twitter.authToken?.splice(tokenIndex, 1);
                         }
-                    }
-                    if (auth.password) {
-                        const passwordIndex = config.twitter.password?.indexOf(auth.password);
-                        if (passwordIndex !== undefined && passwordIndex !== -1) {
-                            config.twitter.password?.splice(passwordIndex, 1);
+                        if (auth.username) {
+                            const usernameIndex = config.twitter.username?.indexOf(auth.username);
+                            if (usernameIndex !== undefined && usernameIndex !== -1) {
+                                config.twitter.username?.splice(usernameIndex, 1);
+                            }
+                        }
+                        if (auth.password) {
+                            const passwordIndex = config.twitter.password?.indexOf(auth.password);
+                            if (passwordIndex !== undefined && passwordIndex !== -1) {
+                                config.twitter.password?.splice(passwordIndex, 1);
+                            }
                         }
+                        logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`);
+                        await cache.set(`${lockPrefix}${auth.token}`, '1', 86400);
                     }
-                    logger.debug(`twitter debug: delete twitter cookie for token ${auth.token}, remaining tokens: ${config.twitter.authToken?.length}`);
-                    await cache.set(`twitter:lock-token:${auth.token}`, '1', 86400);
+                } else {
+                    logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`);
+                    await cache.set(`${lockPrefix}${auth.token}`, '', 1);
                 }
             }
         },
     });
 
-    if (auth.token) {
+    if (auth?.token) {
         logger.debug(`twitter debug: update twitter cookie for token ${auth.token}`);
-        await cache.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers.jar.serializeSync()), config.cache.contentExpire);
-        logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`);
-        await cache.set(`twitter:lock-token:${auth.token}`, '', 1);
+        await cache.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers?.jar.serializeSync()), config.cache.contentExpire);
     }
 
     return response._data;

package/lib/routes/twitter/list.ts

@@ -4,19 +4,11 @@ import utils from './utils';
 
 export const route: Route = {
     path: '/list/:id/:routeParams?',
-    categories: ['social-media'],
-    example: '/twitter/list/ladyleet/javascript',
-    parameters: { id: 'username', name: 'list name', routeParams: 'extra parameters, see the table above' },
+    categories: ['social-media', 'popular'],
+    example: '/twitter/list/1502570462752219136',
+    parameters: { id: 'list id, get from url', routeParams: 'extra parameters, see the table above' },
     features: {
         requireConfig: [
-            {
-                name: 'TWITTER_USERNAME',
-                description: 'Please see above for details.',
-            },
-            {
-                name: 'TWITTER_PASSWORD',
-                description: 'Please see above for details.',
-            },
             {
                 name: 'TWITTER_AUTH_TOKEN',
                 description: 'Please see above for details.',
@@ -29,7 +21,7 @@ export const route: Route = {
         supportScihub: false,
     },
     name: 'List timeline',
-    maintainers: ['DIYgod', 'xyqfer'],
+    maintainers: ['DIYgod', 'xyqfer', 'pseudoyu'],
     handler,
     radar: [
         {

package/lib/routes/twitter/media.ts

@@ -1,6 +1,7 @@
 import { Route, ViewType } from '@/types';
 import api from './api';
 import utils from './utils';
+import logger from '@/utils/logger';
 
 export const route: Route = {
     path: '/media/:id/:routeParams?',
@@ -47,7 +48,12 @@ async function handler(ctx) {
 
     await api.init();
     const userInfo = await api.getUser(id);
-    const data = await api.getUserMedia(id, params);
+    let data;
+    try {
+        data = await api.getUserMedia(id, params);
+    } catch (error) {
+        logger.error(error);
+    }
     const profileImageUrl = userInfo?.profile_image_url || userInfo?.profile_image_url_https;
 
     return {
@@ -55,8 +61,11 @@ async function handler(ctx) {
         link: `https://x.com/${userInfo?.screen_name}/media`,
         image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
         description: userInfo?.description,
-        item: utils.ProcessFeed(ctx, {
-            data,
-        }),
+        item:
+            data &&
+            utils.ProcessFeed(ctx, {
+                data,
+            }),
+        allowEmpty: true,
     };
 }

package/lib/routes/twitter/user.ts

@@ -1,6 +1,7 @@
 import { Route, ViewType } from '@/types';
 import utils from './utils';
 import api from './api';
+import logger from '@/utils/logger';
 
 export const route: Route = {
     path: '/user/:id/:routeParams?',
@@ -58,9 +59,14 @@ async function handler(ctx) {
 
     await api.init();
     const userInfo = await api.getUser(id);
-    let data = await (exclude_replies ? api.getUserTweets(id, params) : api.getUserTweetsAndReplies(id, params));
-    if (!include_rts) {
-        data = utils.excludeRetweet(data);
+    let data;
+    try {
+        data = await (exclude_replies ? api.getUserTweets(id, params) : api.getUserTweetsAndReplies(id, params));
+        if (!include_rts) {
+            data = utils.excludeRetweet(data);
+        }
+    } catch (error) {
+        logger.error(error);
     }
 
     const profileImageUrl = userInfo?.profile_image_url || userInfo?.profile_image_url_https;
@@ -70,8 +76,11 @@ async function handler(ctx) {
         link: `https://x.com/${userInfo?.screen_name}`,
         image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
         description: userInfo?.description,
-        item: utils.ProcessFeed(ctx, {
-            data,
-        }),
+        item:
+            data &&
+            utils.ProcessFeed(ctx, {
+                data,
+            }),
+        allowEmpty: true,
     };
 }

package/lib/routes/udn/breaking-news.ts

@@ -87,11 +87,11 @@ async function handler(ctx) {
 
                 if (data.publisher.name === '轉角國際 udn Global') {
                     // 轉角24小時
-                    description += $('.story_body_content')
+                    description = $('.story_body_content')
                         .html()
                         .split(/<!--\d+?-->/g)
                         .slice(1, -1)
-                        .join(',');
+                        .join('');
                 }
 
                 return {