rsshub 1.0.0-master.f9003f9 → 1.0.0-master.f917bdc

package/lib/routes/wired/tag.ts

@@ -0,0 +1,97 @@
+import { Route } from '@/types';
+
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import * as cheerio from 'cheerio';
+import { Item } from './types';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/tag/:tag',
+    example: '/wired/tag/facebook',
+    parameters: { tag: 'Tag name' },
+    radar: [
+        {
+            source: ['www.wired.com/tag/:tag/'],
+        },
+    ],
+    name: 'Tags',
+    maintainers: ['Naiqus'],
+    handler,
+};
+
+async function handler(ctx) {
+    const baseUrl = 'https://www.wired.com';
+    const { tag } = ctx.req.param() as { tag: string };
+    const link = `${baseUrl}/tag/${tag}/`;
+
+    const response = await ofetch(link);
+    const $ = cheerio.load(response);
+    const preloadedState = JSON.parse(
+        $('script:contains("window.__PRELOADED_STATE__")')
+            .text()
+            .match(/window\.__PRELOADED_STATE__ = (.*);/)?.[1] ?? '{}'
+    );
+
+    const list = (preloadedState.transformed.tag.items as Item[]).map((item) => ({
+        title: item.dangerousHed,
+        description: item.dangerousDek,
+        link: `${baseUrl}${item.url}`,
+        pubDate: parseDate(item.date),
+        author: item.contributors.author.items.map((author) => author.name).join(', '),
+        category: [item.rubric.name],
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await ofetch(item.link);
+                const $ = cheerio.load(response);
+                const preloadedState = JSON.parse(
+                    $('script:contains("window.__PRELOADED_STATE__")')
+                        .text()
+                        .match(/window\.__PRELOADED_STATE__ = (.*);/)?.[1] ?? '{}'
+                );
+
+                const headerLeadAsset = $('div[data-testid*="ContentHeaderLeadAsset"]');
+                headerLeadAsset.find('button').remove();
+                // false postive: 'some' does not exist on type 'Cheerio<Element>'
+                // eslint-disable-next-line unicorn/prefer-array-some
+                if (headerLeadAsset.find('video')) {
+                    headerLeadAsset.find('video').attr('src', $('link[rel="preload"][as="video"]').attr('href'));
+                    headerLeadAsset.find('video').attr('controls', '');
+                    headerLeadAsset.find('video').attr('preload', 'metadata');
+                    headerLeadAsset.find('video').removeAttr('autoplay');
+                }
+
+                const content = $('.body__inner-container')
+                    .toArray()
+                    .map((el) => {
+                        const $el = $(el);
+                        $el.find('noscript').each((_, el) => {
+                            const $e = $(el);
+                            $e.replaceWith($e.html() || '');
+                        });
+
+                        return $el.html();
+                    })
+                    .join('');
+
+                item.description = ($('div[class^=ContentHeaderDek]').prop('outerHTML') || '') + headerLeadAsset.prop('outerHTML') + content;
+
+                item.category = [...new Set([...item.category, ...preloadedState.transformed.article.tagCloud.tags.map((t: { tag: string }) => t.tag)])];
+
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: preloadedState.transformed['head.title'],
+        description: preloadedState.transformed['head.description'],
+        link,
+        image: `${baseUrl}${preloadedState.transformed.logo.sources.sm.url}`,
+        language: 'en',
+        item: items,
+    };
+}

package/lib/routes/wired/types.ts

@@ -0,0 +1,81 @@
+interface ImageSource {
+    aspectRatio: string;
+    width: number;
+    url: string;
+    srcset: string;
+}
+
+interface Image {
+    contentType: string;
+    id: string;
+    altText: string;
+    credit: string;
+    caption: string;
+    metaData: string;
+    modelName: string;
+    sources: {
+        sm: ImageSource;
+        md: ImageSource;
+        lg: ImageSource;
+        xl: ImageSource;
+        xxl: ImageSource;
+    };
+    segmentedSources: {
+        sm: ImageSource[];
+        lg: ImageSource[];
+    };
+    showImageWithoutLink: boolean;
+    isLazy: boolean;
+    brandDetail: {
+        brandIcon: string;
+        brandName: string;
+    };
+}
+
+interface Contributor {
+    author: {
+        items: { name: string }[];
+    };
+    photographer: {
+        items: any[];
+    };
+}
+
+interface Rubric {
+    name: string;
+    url: string;
+}
+
+interface RecircMostPopular {
+    contentType: string;
+    dangerousHed: string;
+    dangerousDek: string;
+    url: string;
+    rubric: { name: string };
+    tout: Image;
+    image: Image;
+    contributors: {
+        author: {
+            brandName: string;
+            brandSlug: string;
+            preamble: string;
+            items: { name: string }[];
+        };
+    };
+}
+
+export interface Item {
+    contributors: Contributor;
+    contentType: string;
+    date: string;
+    dangerousDek: string;
+    dangerousHed: string;
+    id: string;
+    image: Image;
+    imageLabels: any[];
+    hasNoFollowOnSyndicated: boolean;
+    rating: string;
+    rubric: Rubric;
+    url: string;
+    recircMostPopular: RecircMostPopular[];
+}

package/lib/routes/wordpress/util.ts

@@ -239,7 +239,7 @@ const getFilterByKeyAndKeyword = async (key: string, keyword: string, rootUrl: s
 const getFilterKeyForSearchParams = (key: string, isApi: boolean = false): string | undefined => {
     const keys = isApi ? filterApiKeys : filterKeys;
 
-    return Object.hasOwn(keys, key) ? keys[key] ?? key : undefined;
+    return Object.hasOwn(keys, key) ? (keys[key] ?? key) : undefined;
 };
 
 /**

package/lib/routes/xiaoyuzhou/podcast.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
@@ -6,6 +6,7 @@ import { parseDate } from '@/utils/parse-date';
 export const route: Route = {
     path: '/podcast/:id',
     categories: ['multimedia', 'popular'],
+    view: ViewType.Audios,
     example: '/xiaoyuzhou/podcast/6021f949a789fca4eff4492c',
     parameters: { id: '播客id，可以在小宇宙播客的 URL 中找到' },
     features: {

package/lib/routes/ximalaya/album.ts

@@ -1,11 +1,12 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
-import { getUrl, getRandom16 } from './utils';
+import { getRandom16, decryptUrl } from './utils';
 const baseUrl = 'https://www.ximalaya.com';
 import { config } from '@/config';
 import { parseDate } from '@/utils/parse-date';
+import { RichIntro, TrackInfoResponse } from './types';
 
 // Find category from: https://help.apple.com/itc/podcasts_connect/?lang=en#/itc9267a2f12
 const categoryDict = {
@@ -50,9 +51,8 @@ async function parseAlbumData(category, album_id) {
     // 这里的 {category} 并不重要，系统会准确的返回该 id 对应专辑的信息
     // 现在 {category} 必须要精确到大的类别
     // https://www.ximalaya.com/{category}/{album_id}
-    const response = await got(`${baseUrl}/${category}/${album_id}`);
-    const data = response.body;
-    const $ = load(data);
+    const response = await ofetch(`${baseUrl}/${category}/${album_id}`);
+    const $ = load(response);
     const stateElement = $('script')
         .toArray()
         .filter((element) => getTextFromElement(element).startsWith('window.__INITIAL_STATE__'));
@@ -83,7 +83,7 @@ function judgeTrue(str, ...validStrings) {
 
 export const route: Route = {
     path: ['/:type/:id/:all/:shownote?'],
-    categories: ['multimedia', 'popular'],
+    categories: ['multimedia'],
     example: '/ximalaya/album/299146',
     parameters: { type: '专辑类型, 通常可以使用 `album`，可在对应专辑页面的 URL 中找到', id: '专辑 id, 可在对应专辑页面的 URL 中找到', all: '是否需要获取全部节目，填入 `1`、`true`、`all` 视为获取所有节目，填入其他则不获取。' },
     features: {
@@ -140,20 +140,26 @@ async function handler(ctx) {
     // const isAsc = albumData.store.AlbumDetailTrackList.sort === 0;
     // 喜马拉雅的 API 的 query 参数 isAsc=0 时才是升序，不写就是降序。
     const trackInfoApi = `http://mobile.ximalaya.com/mobile/v1/album/track/?albumId=${id}&pageSize=${pageSize}&pageId=`;
-    const trackInfoResponse = await got(trackInfoApi + '1');
-    const maxPageId = trackInfoResponse.data.data.maxPageId; // 最大页数
+    const trackInfoResponse = await ofetch<TrackInfoResponse>(trackInfoApi + '1', {
+        parseResponse: JSON.parse,
+    });
+    const maxPageId = trackInfoResponse.data.maxPageId; // 最大页数
 
-    let playList = trackInfoResponse.data.data.list;
+    let playList = trackInfoResponse.data.list;
 
     if (shouldAll) {
         const promises = [];
         for (let i = 2; i <= maxPageId; i++) {
             // string + number -> string
-            promises.push(got(trackInfoApi + i));
+            promises.push(
+                ofetch<TrackInfoResponse>(trackInfoApi + i, {
+                    parseResponse: JSON.parse,
+                })
+            );
         }
         const responses = await Promise.all(promises);
         for (const j of responses) {
-            playList = [...playList, ...j.data.data.list];
+            playList = [...playList, ...j.data.list];
         }
     }
 
@@ -164,8 +170,8 @@ async function handler(ctx) {
                 let _desc;
                 if (shouldShowNote) {
                     const trackRichInfoApi = `https://mobile.ximalaya.com/mobile-track/richIntro?trackId=${item.trackId}`;
-                    const trackRichInfoResponse = await got(trackRichInfoApi);
-                    _desc = trackRichInfoResponse.data.richIntro;
+                    const trackRichInfoResponse = await ofetch<RichIntro>(trackRichInfoApi);
+                    _desc = trackRichInfoResponse.richIntro;
                 }
                 if (!_desc) {
                     _desc = `<a href="${link}">在网页中查看</a>`;
@@ -180,22 +186,24 @@ async function handler(ctx) {
     if (isPaid && token) {
         await Promise.all(
             playList.map(async (item) => {
-                const trackPayInfoApi = `https://mpay.ximalaya.com/mobile/track/pay/${item.trackId}/?device=pc`;
+                const timestamp = Math.floor(Date.now());
+                const trackPayInfoApi = `https://www.ximalaya.com/mobile-playpage/track/v3/baseInfo/${timestamp}?device=www2&trackQualityLevel=2&trackId=${item.trackId}`;
                 const data = await cache.tryGet('ximalaya:trackPayInfo' + trackPayInfoApi, async () => {
-                    const trackPayInfoResponse = await got(trackPayInfoApi, {
+                    const trackPayInfoResponse = await ofetch(trackPayInfoApi, {
                         headers: {
                             'user-agent': 'ting_6.7.9(GM1900,Android29)',
                             cookie: `1&_device=android&${randomToken}&6.7.9;1&_token=${token}`,
                         },
                     });
+                    const trackInfo = trackPayInfoResponse.trackInfo;
                     const _item = {};
-                    if (trackPayInfoResponse.data.ep) {
-                        _item.playPathAacv224 = getUrl(trackPayInfoResponse.data);
-                    } else if (trackPayInfoResponse.data.msg) {
-                        _item.desc = item.desc + '<br/>' + trackPayInfoResponse.data.msg;
+                    if (!trackInfo.isAuthorized) {
+                        return _item;
                     }
+                    _item.playPathAacv224 = decryptUrl(trackInfo.playUrlList[0].url);
                     return _item;
                 });
+
                 if (data.playPathAacv224) {
                     item.playPathAacv224 = data.playPathAacv224;
                 }

package/lib/routes/ximalaya/types.ts

@@ -0,0 +1,68 @@
+interface Track {
+    trackId: number;
+    trackRecordId: number;
+    uid: number;
+    playUrl64: string;
+    playUrl32: string;
+    playPathAacv164: string;
+    playPathAacv224: string;
+    playPathHq: string;
+    title: string;
+    duration: number;
+    albumId: number;
+    albumTitle: string;
+    isPaid: boolean;
+    isFree: boolean;
+    isVideo: boolean;
+    isDraft: boolean;
+    isAuthorized: boolean;
+    priceTypeId: number;
+    sampleDuration: number;
+    priceTypeEnum: number;
+    type: number;
+    relatedId: number;
+    orderNo: number;
+    isHoldCopyright: boolean;
+    vipFirstStatus: number;
+    paidType: number;
+    isSample: boolean;
+    ximiFirstStatus: number;
+    coverLarge: string;
+    permissionSource: string;
+    playtimes: number;
+    labelType: number;
+    processState: number;
+    createdAt: number;
+    coverSmall: string;
+    coverMiddle: string;
+    nickname: string;
+    smallLogo: string;
+    userSource: number;
+    opType: number;
+    isPublic: boolean;
+    likes: number;
+    comments: number;
+    shares: number;
+    status: number;
+    videoCover: string;
+    intro: string;
+    labelList: string[];
+    isTrailer: number;
+}
+
+export interface TrackInfoResponse {
+    msg: string;
+    ret: number;
+    data: {
+        list: Track[];
+        pageId: number;
+        pageSize: number;
+        maxPageId: number;
+        totalCount: number;
+    };
+}
+
+export interface RichIntro {
+    ret: number;
+    richIntro: string;
+}

package/lib/routes/ximalaya/utils.ts

@@ -126,4 +126,41 @@ const getRandom16 = (len) =>
         .toString('hex')
         .slice(0, len);
 
-export { getUrl, getRandom16 };
+const decryptUrl = (encryptedUrl) => {
+    const o = [
+        183, 174, 108, 16, 131, 159, 250, 5, 239, 110, 193, 202, 153, 137, 251, 176, 119, 150, 47, 204, 97, 237, 1, 71, 177, 42, 88, 218, 166, 82, 87, 94, 14, 195, 69, 127, 215, 240, 225, 197, 238, 142, 123, 44, 219, 50, 190, 29,
+        181, 186, 169, 98, 139, 185, 152, 13, 141, 76, 6, 157, 200, 132, 182, 49, 20, 116, 136, 43, 155, 194, 101, 231, 162, 242, 151, 213, 53, 60, 26, 134, 211, 56, 28, 223, 107, 161, 199, 15, 229, 61, 96, 41, 66, 158, 254, 21, 165,
+        253, 103, 89, 3, 168, 40, 246, 81, 95, 58, 31, 172, 78, 99, 45, 148, 187, 222, 124, 55, 203, 235, 64, 68, 149, 180, 35, 113, 207, 118, 111, 91, 38, 247, 214, 7, 212, 209, 189, 241, 18, 115, 173, 25, 236, 121, 249, 75, 57,
+        216, 10, 175, 112, 234, 164, 70, 206, 198, 255, 140, 230, 12, 32, 83, 46, 245, 0, 62, 227, 72, 191, 156, 138, 248, 114, 220, 90, 84, 170, 128, 19, 24, 122, 146, 80, 39, 37, 8, 34, 22, 11, 93, 130, 63, 154, 244, 160, 144, 79,
+        23, 133, 92, 54, 102, 210, 65, 67, 27, 196, 201, 106, 143, 52, 74, 100, 217, 179, 48, 233, 126, 117, 184, 226, 85, 171, 167, 86, 2, 147, 17, 135, 228, 252, 105, 30, 192, 129, 178, 120, 36, 145, 51, 163, 77, 205, 73, 4, 188,
+        125, 232, 33, 243, 109, 224, 104, 208, 221, 59, 9,
+    ];
+
+    const a = [204, 53, 135, 197, 39, 73, 58, 160, 79, 24, 12, 83, 180, 250, 101, 60, 206, 30, 10, 227, 36, 95, 161, 16, 135, 150, 235, 116, 242, 116, 165, 171];
+
+    const padding = '='.repeat((4 - (encryptedUrl.length % 4)) % 4);
+    const encrypted_data = Buffer.from(encryptedUrl.replace('_', '/').replace('-', '+') + padding, 'base64');
+    if (encrypted_data.length < 16) {
+        return encryptedUrl;
+    }
+    const data = encrypted_data.slice(0, -16);
+    const iv = encrypted_data.slice(-16);
+    const decryptedData = new Uint8Array(data);
+    for (let i = 0; i < decryptedData.length; i++) {
+        decryptedData[i] = o[decryptedData[i]];
+    }
+    for (let i = 0; i < decryptedData.length; i += 16) {
+        const block = decryptedData.slice(i, i + 16);
+        for (const [j, element] of block.entries()) {
+            decryptedData[i + j] = element ^ iv[j];
+        }
+    }
+    for (let i = 0; i < decryptedData.length; i += 32) {
+        const block = decryptedData.slice(i, i + 32);
+        for (const [j, element] of block.entries()) {
+            decryptedData[i + j] = element ^ a[j];
+        }
+    }
+    return Buffer.from(decryptedData).toString('utf8');
+};
+export { getUrl, getRandom16, decryptUrl };

package/lib/routes/xinhuanet/app.ts

@@ -0,0 +1,109 @@
+import { Route } from '@/types';
+
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import timezone from '@/utils/timezone';
+import { parseDate } from '@/utils/parse-date';
+
+export const handler = async (ctx) => {
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 7;
+
+    const rootUrl = 'https://app.xinhuanet.com';
+    const currentUrl = new URL('news/index.html', rootUrl).href;
+
+    const { data: response } = await got(currentUrl);
+
+    const $ = load(response);
+
+    const language = $('html').prop('lang');
+
+    let items = $('a.article-item')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const title = item.find('div.article-title').text();
+            const guid = `xinhuanet-${item.prop('data-uuid')}`;
+
+            return {
+                title,
+                link: item.prop('href'),
+                guid,
+                id: guid,
+                language,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const { data: detailResponse } = await got(item.link);
+
+                const $$ = load(detailResponse);
+
+                const title = $$('div.article_title').text();
+                const description = $$('#detail-content').html();
+
+                item.title = title;
+                item.description = description;
+
+                const authorEl = $$('div.article_auth div').first();
+                item.author = authorEl.text();
+
+                authorEl.remove();
+
+                item.pubDate = timezone(parseDate($$('div.article_auth div').first().text()), +8);
+                item.content = {
+                    html: description,
+                    text: $$('#detail-content').text(),
+                };
+                item.language = language;
+
+                return item;
+            })
+        )
+    );
+
+    const image = $('meta[itemprop="image"]').prop('content');
+
+    return {
+        title: $('title').text(),
+        description: $('meta[name="description"]').prop('content'),
+        link: currentUrl,
+        item: items,
+        allowEmpty: true,
+        image,
+        author: $('meta[itemprop="name"]').prop('content'),
+        language,
+    };
+};
+
+export const route: Route = {
+    path: '/app',
+    name: '客户端',
+    url: 'app.xinhuanet.com',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/xinhuanet/app',
+    parameters: undefined,
+    description: '',
+    categories: ['traditional-media'],
+
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['app.xinhuanet.com'],
+            target: '/app',
+        },
+    ],
+};

package/lib/routes/xinhuanet/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '新华网',
+    url: 'xinhuanet.com',
+    categories: ['traditional-media'],
+    description: '',
+};

package/lib/routes/xsijishe/forum.ts

@@ -3,6 +3,7 @@ import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
+import { config } from '@/config';
 const baseUrl = 'https://xsijishe.com';
 
 export const route: Route = {
@@ -11,7 +12,16 @@ export const route: Route = {
     example: '/xsijishe/forum/51',
     parameters: { fid: '子论坛 id' },
     features: {
-        requireConfig: false,
+        requireConfig: [
+            {
+                name: 'XSIJISHE_COOKIE',
+                description: '',
+            },
+            {
+                name: 'XSIJISHE_USER_AGENT',
+                description: '',
+            },
+        ],
         requirePuppeteer: false,
         antiCrawler: false,
         supportBT: false,
@@ -29,7 +39,15 @@ export const route: Route = {
 async function handler(ctx) {
     const fid = ctx.req.param('fid');
     const url = `${baseUrl}/forum-${fid}-1.html`;
-    const resp = await got(url);
+    const headers = {
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+        Cookie: config.xsijishe.cookie,
+        'User-Agent': config.xsijishe.userAgent,
+    };
+    const resp = await got(url, {
+        headers,
+    });
     const $ = load(resp.data);
     const forumCategory = $('.nex_bkinterls_top .nex_bkinterls_ls a').text();
     let items = $('[id^="normalthread"]')
@@ -51,7 +69,9 @@ async function handler(ctx) {
     items = await Promise.all(
         items.map((item) =>
             cache.tryGet(item.link, async () => {
-                const resp = await got(item.link);
+                const resp = await got(item.link, {
+                    headers,
+                });
                 const $ = load(resp.data);
                 const firstViewBox = $('.t_f').first();
 

package/lib/routes/xsijishe/rank.ts

@@ -3,6 +3,7 @@ import { Route } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
+import { config } from '@/config';
 const baseUrl = 'https://xsijishe.com';
 
 export const route: Route = {
@@ -11,7 +12,16 @@ export const route: Route = {
     example: '/xsijishe/rank/weekly',
     parameters: { type: '排行榜类型: weekly | monthly' },
     features: {
-        requireConfig: false,
+        requireConfig: [
+            {
+                name: 'XSIJISHE_COOKIE',
+                description: '',
+            },
+            {
+                name: 'XSIJISHE_USER_AGENT',
+                description: '',
+            },
+        ],
         requirePuppeteer: false,
         antiCrawler: false,
         supportBT: false,
@@ -37,7 +47,15 @@ async function handler(ctx) {
         throw new InvalidParameterError('Invalid rank type');
     }
     const url = `${baseUrl}/portal.php`;
-    const resp = await got(url);
+    const headers = {
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+        Cookie: config.xsijishe.cookie,
+        'User-Agent': config.xsijishe.user_agent,
+    };
+    const resp = await got(url, {
+        headers,
+    });
     const $ = load(resp.data);
     let items = $(`#${rankId} dd`)
         .toArray()
@@ -53,7 +71,9 @@ async function handler(ctx) {
     items = await Promise.all(
         items.map((item) =>
             cache.tryGet(item.link, async () => {
-                const resp = await got(item.link);
+                const resp = await got(item.link, {
+                    headers,
+                });
                 const $ = load(resp.data);
                 const firstViewBox = $('.t_f').first();