rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa1dcc5

package/lib/routes/toutiao/types.ts

@@ -0,0 +1,443 @@
+export interface Feed {
+    abstract: string;
+    aggr_type: number;
+    article_sub_type: number;
+    article_type: number;
+    article_url: string;
+    article_version: number;
+    ban_comment: boolean;
+    behot_time: number;
+    bury_count: number;
+    bury_style_show: number;
+    cell_ctrls: CellCtrls;
+    cell_flag: number;
+    cell_layout_style: number;
+    /**
+     * 0: video
+     * 32: text (w)
+     * 49: video
+     * 60: article
+     */
+    cell_type: number;
+    comment_count: number;
+    common_raw_data: string;
+    /**
+     * Appears only if cell_type is 32
+     */
+    content: string;
+    content_decoration: string;
+    control_meta: ControlMeta;
+    cursor: number;
+    data_type: number;
+    digg_count: number;
+    display_url: string;
+    forum_extra_data: string;
+    forward_info: ForwardInfo;
+    gallary_image_count: number;
+    group_flags: number;
+    group_id: string;
+    group_source: number;
+    group_type: number;
+    has_image: boolean;
+    has_m3u8_video: boolean;
+    has_mp4_video: boolean;
+    has_video: boolean;
+    hot: number;
+    id: string;
+    is_original: boolean;
+    itemCell: ItemCell;
+    itemCellDebug: null;
+    item_id: string;
+    item_id_str: string;
+    item_version: number;
+    label_style: number;
+    level: number;
+    like_count: number;
+    log_pb: LogPb;
+    lynx_server: LynxServer;
+    natant_level: number;
+    preload_web: number;
+    publish_time: number;
+    /**
+     * Appears only if cell_type is 32
+     */
+    rich_content: string;
+    read_count: number;
+    reback_flag: number;
+    repin_count: number;
+    repin_time: number;
+    req_id: string;
+    share_url: string;
+    show_more: ShowMore;
+    source: string;
+    subject_group_id: number;
+    tag_id: number;
+    tip: number;
+    title: string;
+    url: string;
+    /**
+     * Appears only if cell_type is 32, 49(0)
+     */
+    user: UserInfoCell32 | UserInfoCell49;
+    user_bury: number;
+    user_digg: number;
+    /**
+     * Appears only if cell_type is 0, 60
+     */
+    user_info: UserInfoCell60;
+    user_like: number;
+    user_repin: number;
+    user_repin_time: number;
+    /**
+     * Appears only if cell_type is 0, 49
+     */
+    video: Video;
+    video_duration: number;
+    video_style: number;
+    image_list: ImageListItem[];
+    large_image_list: LargeImageListItem[];
+    middle_image: MiddleImage;
+    video_detail_info: VideoDetailInfo;
+}
+
+interface CellCtrls {
+    cell_flag: number;
+    cell_height: number;
+    cell_layout_style: number;
+}
+
+interface ControlMeta {
+    modify: Modify;
+    remove: Remove;
+    share: Share;
+}
+
+interface Modify {
+    hide: boolean;
+    name: string;
+    permission: boolean;
+    tips: string;
+}
+
+interface Remove {
+    hide: boolean;
+    name: string;
+    permission: boolean;
+    tips: string;
+}
+
+interface Share {
+    hide: boolean;
+    name: string;
+    permission: boolean;
+    tips: string;
+}
+
+interface ForwardInfo {
+    forward_count: number;
+}
+
+interface ItemCell {
+    actionCtrl: ActionCtrl;
+    articleBase: ArticleBase;
+    articleClassification: ArticleClassification;
+    cellCtrl: CellCtrl;
+    extra: Extra;
+    imageList: ImageList;
+    itemCounter: ItemCounter;
+    locationInfo: LocationInfo;
+    shareInfo: ShareInfo;
+    tagInfo: TagInfo;
+    userInteraction: UserInteraction;
+    videoInfo: VideoInfo;
+}
+
+interface ActionCtrl {
+    actionBar: ActionBar;
+    banBury: boolean;
+    banComment: boolean;
+    banDigg: boolean;
+    controlMeta: ActionControlMeta;
+}
+
+interface ActionBar {
+    actionSettingList: ActionSetting[];
+}
+
+interface ActionSetting {
+    actionType: number;
+    styleSetting: StyleSetting;
+}
+
+interface StyleSetting {
+    iconKey: string;
+    layoutDirection: number;
+    text: string;
+}
+
+interface ActionControlMeta {
+    modify: ActionModify;
+    remove: ActionRemove;
+    share: ActionShare;
+}
+
+interface ActionModify {
+    permission: boolean;
+    tips: string;
+}
+
+interface ActionRemove {
+    permission: boolean;
+    tips: string;
+}
+
+interface ActionShare {
+    permission: boolean;
+    tips: string;
+}
+
+interface ArticleBase {
+    gidStr: string;
+    itemStatus: number;
+}
+
+interface ArticleClassification {
+    aggrType: number;
+    articleSubType: number;
+    articleType: number;
+    bizID: number;
+    bizTag: number;
+    groupSource: number;
+    isForAudioPlaylist: boolean;
+    isOriginal: boolean;
+    isSubject: boolean;
+    level: number;
+}
+
+interface CellCtrl {
+    buryStyleShow: number;
+    cellFlag: number;
+    cellLayoutStyle: number;
+    cellType: number;
+    cellUIType: string;
+    groupFlags: number;
+}
+
+interface Extra {
+    ping: string;
+}
+
+type ImageList = unknown;
+
+interface ItemCounter {
+    commentCount: number;
+    diggCount: number;
+    forwardCount: number;
+    readCount: number;
+    repinCount: number;
+    shareCount: number;
+    showCount: number;
+    textCount: number;
+    videoWatchCount: number;
+    buryCount: number;
+}
+
+interface LocationInfo {
+    publishLocInfo: string;
+}
+
+interface ShareInfo {
+    shareURL: string;
+    shareControl: ShareControl;
+}
+
+interface ShareControl {
+    isHighQuality: boolean;
+}
+
+type TagInfo = unknown;
+
+interface UserInteraction {
+    userDigg: boolean;
+    userRepin: boolean;
+}
+
+type VideoInfo = unknown;
+
+interface Video {
+    bitrate: number;
+    codec_type: string;
+    definition: string;
+    download_addr: DownloadAddr;
+    duration: number;
+    encode_user_tag: string;
+    file_hash: string;
+    height: number;
+    origin_cover: OriginCover;
+    play_addr: PlayAddr;
+    play_addr_list: PlayAddrListItem[];
+    ratio: string;
+    size: number;
+    url_expire: number;
+    video_id: string;
+    volume: Volume;
+    vtype: string;
+    width: number;
+}
+
+interface DownloadAddr {
+    uri: string;
+    url_list: string[];
+}
+
+interface OriginCover {
+    uri: string;
+    url_list: string[];
+}
+
+interface PlayAddr {
+    uri: string;
+    url_list: string[];
+}
+
+interface PlayAddrListItem {
+    bitrate: number;
+    codec_type: string;
+    definition: string;
+    encode_user_tag: string;
+    file_hash: string;
+    play_url_list: string[];
+    quality: string;
+    size: number;
+    url_expire: number;
+    video_quality: number;
+    volume: Volume;
+    vtype: string;
+}
+
+interface Volume {
+    Loudness: number;
+    Peak: number;
+}
+
+interface LogPb {
+    cell_layout_style: string;
+    group_id_str: string;
+    group_source: string;
+    impr_id: string;
+    is_following: string;
+    is_yaowen: string;
+}
+
+type LynxServer = unknown;
+
+interface ShowMore {
+    title: string;
+    url: string;
+}
+
+interface UserInfoCell32 {
+    avatar_url: string;
+    desc: string;
+    id: number;
+    is_blocked: number;
+    is_blocking: number;
+    is_followed: number;
+    is_following: number;
+    is_friend: number;
+    live_info_type: number;
+    medals: unknown;
+    name: string;
+    remark_name: string;
+    schema: string;
+    screen_name: string;
+    theme_day: string;
+    user_auth_info: string;
+    user_decoration: string;
+    user_id: string;
+    user_verified: number;
+    verified_content: string;
+}
+
+interface UserInfoCell49 {
+    info: {
+        avatar_uri: string;
+        avatar_url: string;
+        ban_status: boolean;
+        desc: string;
+        media_id: string;
+        name: string;
+        origin_profile_url: boolean;
+        origin_user_id: string;
+        schema: string;
+        user_auth_info: string;
+        user_id: string;
+        user_verified: number;
+        verified_content: string;
+    };
+    relation: {
+        is_followed: number;
+        is_following: number;
+        is_friend: number;
+    };
+    relation_count: {
+        followers_count: number;
+        following_count: number;
+    };
+    user_id: string;
+}
+
+interface UserInfoCell60 {
+    avatar_url: string;
+    description: string;
+    follow: boolean;
+    name: string;
+    user_auth_info: string;
+    user_id: string;
+    user_verified: boolean;
+    verified_content: string;
+}
+
+interface ImageListItem {
+    height: number;
+    uri: string;
+    url: string;
+    url_list: UrlList[];
+    width: number;
+}
+
+interface UrlList {
+    url: string;
+}
+
+interface LargeImageListItem {
+    height: number;
+    uri: string;
+    url: string;
+    url_list: UrlList[];
+    width: number;
+}
+
+interface MiddleImage {
+    height: number;
+    uri: string;
+    url: string;
+    url_list: UrlList[];
+    width: number;
+}
+
+interface VideoDetailInfo {
+    detail_video_large_image: DetailVideoLargeImage;
+    direct_play: number;
+    group_flags: number;
+    show_pgc_subscribe: number;
+    video_id: string;
+}
+
+interface DetailVideoLargeImage {
+    height: number;
+    uri: string;
+    url: string;
+    url_list: UrlList[];
+    width: number;
+}

package/lib/routes/toutiao/user.ts

@@ -0,0 +1,127 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import randUserAgent from '@/utils/rand-user-agent';
+import { generate_a_bogus } from './a-bogus';
+import { Feed } from './types';
+import RejectError from '@/errors/types/reject';
+import { config } from '@/config';
+import path from 'node:path';
+import { getCurrentPath } from '@/utils/helpers';
+import { art } from '@/utils/render';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+export const route: Route = {
+    path: '/user/token/:token',
+    categories: ['new-media'],
+    example: '/toutiao/user/token/MS4wLjABAAAAEmbqJP2CmC8XXv1BpMvQ3sQHKAxFsq8wHxj8XVIQWja6tMcB-QEbFkzkRNgMl12M',
+    parameters: { token: '用户 token，可在用户主页 URL 找到' },
+    features: {
+        antiCrawler: true,
+    },
+    radar: [
+        {
+            source: ['www.toutiao.com/c/user/token/:token'],
+        },
+    ],
+    name: '头条主页',
+    maintainers: ['TonyRL'],
+    handler,
+};
+
+async function handler(ctx) {
+    const { token } = ctx.req.param();
+    const ua = randUserAgent({ browser: 'chrome', os: 'windows', device: 'desktop' });
+
+    const feed = (await cache.tryGet(
+        `toutiao:user:${token}`,
+        async () => {
+            const query = `category=profile_all&token=${token}&max_behot_time=0&entrance_gid&aid=24&app_name=toutiao_web`;
+
+            const data = await ofetch(`https://www.toutiao.com/api/pc/list/feed?${query}&a_bogus=${generate_a_bogus(query, ua)}`, {
+                headers: {
+                    'User-Agent': ua,
+                },
+            });
+
+            return data.data;
+        },
+        config.cache.routeExpire,
+        false
+    )) as Feed[];
+
+    if (!feed) {
+        throw new RejectError('无法获取用户信息');
+    }
+
+    const items = feed.map((item) => {
+        switch (item.cell_type) {
+            case 0:
+            case 49: {
+                const video = item.video.play_addr_list.sort((a, b) => b.bitrate - a.bitrate)[0];
+                return {
+                    title: item.title,
+                    description: art(path.join(__dirname, 'templates', 'video.art'), {
+                        poster: item.video.origin_cover.url_list[0],
+                        url: item.video.play_addr_list.sort((a, b) => b.bitrate - a.bitrate)[0].play_url_list[0],
+                    }),
+                    link: `https://www.toutiao.com/video/${item.id}/`,
+                    pubDate: parseDate(item.publish_time, 'X'),
+                    author: item.user?.info.name ?? item.source,
+                    enclosure_url: video?.play_url_list[0],
+                    enclosure_type: video?.play_url_list[0] ? 'video/mp4' : undefined,
+                    user: {
+                        name: item.user?.info.name,
+                        avatar: item.user?.info.avatar_url,
+                        description: item.user?.info.desc,
+                    },
+                };
+            }
+
+            // text w/o title
+            case 32: {
+                const enclosure = item.large_image_list?.pop();
+                return {
+                    title: item.content?.split('\n')[0],
+                    description: item.rich_content,
+                    link: `https://www.toutiao.com/w/${item.id}/`,
+                    pubDate: parseDate(item.publish_time, 'X'),
+                    author: item.user?.name,
+                    enclosure_url: enclosure?.url,
+                    enclosure_type: enclosure?.url ? `image/${new URL(enclosure.url).pathname.split('.').pop()}` : undefined,
+                    user: {
+                        name: item.user?.name,
+                        avatar: item.user?.avatar_url,
+                        description: item.user?.desc,
+                    },
+                };
+            }
+
+            // text w/ title
+            case 60:
+            default:
+                return {
+                    title: item.title,
+                    description: item.abstract,
+                    link: `https://www.toutiao.com/article/${item.id}/`,
+                    pubDate: parseDate(item.publish_time, 'X'),
+                    author: item.user_info?.name,
+                    user: {
+                        name: item.user_info?.name,
+                        avatar: item.user_info?.avatar_url,
+                        description: item.user_info?.description,
+                    },
+                };
+        }
+    });
+
+    return {
+        title: `${items[0].user.name}的头条主页 - 今日头条(www.toutiao.com)`,
+        description: items[0].user.description,
+        link: `https://www.toutiao.com/c/user/token/${token}/`,
+        image: items[0].user.avatar,
+        item: items,
+    };
+}

package/lib/routes/twitter/api/mobile-api/api.ts

@@ -160,6 +160,14 @@ function gatherLegacyFromData(entries, filterNested, userId) {
                         t.legacy.quoted_status = quote.legacy;
                         t.legacy.quoted_status.user = quote.core.user_result?.result?.legacy || quote.core.user_results?.result?.legacy;
                     }
+                    if (t.note_tweet) {
+                        const tmp = t.note_tweet.note_tweet_results.result;
+                        t.legacy.entities.hashtags = tmp.entity_set.hashtags;
+                        t.legacy.entities.symbols = tmp.entity_set.symbols;
+                        t.legacy.entities.urls = tmp.entity_set.urls;
+                        t.legacy.entities.user_mentions = tmp.entity_set.user_mentions;
+                        t.legacy.full_text = tmp.text;
+                    }
                 }
                 const legacy = tweet.legacy;
                 if (legacy) {

package/lib/routes/twitter/api/web-api/utils.ts

@@ -14,11 +14,12 @@ import login from './login';
 let authTokenIndex = 0;
 
 const token2Cookie = async (token) => {
-    if (cache.get(`twitter:cookie:${token}`)) {
-        return cache.get(`twitter:cookie:${token}`);
+    const c = await cache.get(`twitter:cookie:${token}`);
+    if (c) {
+        return c;
     }
     const jar = new CookieJar();
-    jar.setCookieSync(`auth_token=${token}`, 'https://x.com');
+    await jar.setCookie(`auth_token=${token}`, 'https://x.com');
     try {
         const agent = proxy.proxyUri
             ? new ProxyAgent({
@@ -56,6 +57,7 @@ const getAuth = async (retry: number) => {
         const token = config.twitter.authToken[index];
         const lock = await cache.get(`${lockPrefix}${token}`, false);
         if (lock) {
+            logger.debug(`twitter debug: twitter cookie for token ${token} is locked, retry: ${retry}`);
             await new Promise((resolve) => setTimeout(resolve, Math.random() * 500 + 500));
             return await getAuth(retry - 1);
         } else {
@@ -160,13 +162,15 @@ export const twitterGot = async (
             const remaining = response.headers.get('x-rate-limit-remaining');
             const remainingInt = Number.parseInt(remaining || '0');
             const reset = response.headers.get('x-rate-limit-reset');
-            logger.debug(`twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}`);
+            logger.debug(
+                `twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(response._data?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}`
+            );
             if (auth) {
                 if (remaining && remainingInt < 2 && reset) {
                     const resetTime = new Date(Number.parseInt(reset) * 1000);
                     const delay = (resetTime.getTime() - Date.now()) / 1000;
                     logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`);
-                    await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay));
+                    await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2);
                 } else if (response.status === 429 || JSON.stringify(response._data?.data) === '{"user":{}}') {
                     logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`);
                     await cache.set(`${lockPrefix}${auth.token}`, '1', 2000);
@@ -209,10 +213,10 @@ export const twitterGot = async (
         },
     });
 
-    // if (auth?.token) {
-    //     logger.debug(`twitter debug: update twitter cookie for token ${auth.token}`);
-    //     await cache.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers?.jar.serializeSync()), config.cache.contentExpire);
-    // }
+    if (auth?.token) {
+        logger.debug(`twitter debug: update twitter cookie for token ${auth.token}`);
+        await cache.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers?.jar.serializeSync()), config.cache.contentExpire);
+    }
 
     return response._data;
 };
@@ -282,6 +286,14 @@ export function gatherLegacyFromData(entries: any[], filterNested?: string[], us
                         t.legacy.quoted_status = quote.legacy;
                         t.legacy.quoted_status.user = quote.core.user_result?.result?.legacy || quote.core.user_results?.result?.legacy;
                     }
+                    if (t.note_tweet) {
+                        const tmp = t.note_tweet.note_tweet_results.result;
+                        t.legacy.entities.hashtags = tmp.entity_set.hashtags;
+                        t.legacy.entities.symbols = tmp.entity_set.symbols;
+                        t.legacy.entities.urls = tmp.entity_set.urls;
+                        t.legacy.entities.user_mentions = tmp.entity_set.user_mentions;
+                        t.legacy.full_text = tmp.text;
+                    }
                 }
                 const legacy = tweet.legacy;
                 if (legacy) {

package/lib/routes/twitter/namespace.ts

@@ -10,6 +10,7 @@ export const namespace: Namespace = {
 | \`readable\`                     | Enable readable layout                                                                                                               | \`0\`/\`1\`/\`true\`/\`false\` | \`false\`                                   |
 | \`authorNameBold\`               | Display author name in bold                                                                                                          | \`0\`/\`1\`/\`true\`/\`false\` | \`false\`                                   |
 | \`showAuthorInTitle\`            | Show author name in title                                                                                                            | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` (\`true\` in \`/twitter/followings\`) |
+| \`showAuthorAsTitleOnly\`        | Show only author name as title                                                                                                            | \`0\`/\`1\`/\`true\`/\`false\` | \`false\`                                   |
 | \`showAuthorInDesc\`             | Show author name in description (RSS body)                                                                                           | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` (\`true\` in \`/twitter/followings\`) |
 | \`showQuotedAuthorAvatarInDesc\` | Show avatar of quoted Tweet's author in description (RSS body) (Not recommended if your RSS reader extracts images from description) | \`0\`/\`1\`/\`true\`/\`false\` | \`false\`                                   |
 | \`showAuthorAvatarInDesc\`       | Show avatar of author in description (RSS body) (Not recommended if your RSS reader extracts images from description)                | \`0\`/\`1\`/\`true\`/\`false\` | \`false\`                                   |

package/lib/routes/twitter/utils.ts

@@ -35,7 +35,7 @@ const formatText = (item) => {
     const urls = item.entities.urls || [];
     for (const url of urls) {
         // trim link pointing to the tweet itself (usually appears when the tweet is truncated)
-        text = text.replaceAll(url.url, url.expanded_url.endsWith(id_str) ? '' : url.expanded_url);
+        text = text.replaceAll(url.url, url.expanded_url?.endsWith(id_str) ? '' : url.expanded_url);
     }
     const media = item.extended_entities?.media || [];
     for (const m of media) {
@@ -52,6 +52,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
         readable: fallback(params.readable, queryToBoolean(routeParams.get('readable')), false),
         authorNameBold: fallback(params.authorNameBold, queryToBoolean(routeParams.get('authorNameBold')), false),
         showAuthorInTitle: fallback(params.showAuthorInTitle, queryToBoolean(routeParams.get('showAuthorInTitle')), false),
+        showAuthorAsTitleOnly: fallback(params.showAuthorAsTitleOnly, queryToBoolean(routeParams.get('showAuthorAsTitleOnly')), false),
         showAuthorInDesc: fallback(params.showAuthorInDesc, queryToBoolean(routeParams.get('showAuthorInDesc')), false),
         showQuotedAuthorAvatarInDesc: fallback(params.showQuotedAuthorAvatarInDesc, queryToBoolean(routeParams.get('showQuotedAuthorAvatarInDesc')), false),
         showAuthorAvatarInDesc: fallback(params.showAuthorAvatarInDesc, queryToBoolean(routeParams.get('showAuthorAvatarInDesc')), false),
@@ -74,6 +75,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
         readable,
         authorNameBold,
         showAuthorInTitle,
+        showAuthorAsTitleOnly,
         showAuthorInDesc,
         showQuotedAuthorAvatarInDesc,
         showAuthorAvatarInDesc,
@@ -292,6 +294,10 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
             title += quoteInTitle;
         }
 
+        if (showAuthorAsTitleOnly) {
+            title = originalItem.user.name;
+        }
+
         // Make description
         let description = '';
         if (showAuthorInDesc && showAuthorAvatarInDesc) {