rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa1dcc5

package/lib/routes/xiaohongshu/util.ts

@@ -2,6 +2,30 @@ import { config } from '@/config';
 import logger from '@/utils/logger';
 import { parseDate } from '@/utils/parse-date';
 import puppeteer from '@/utils/puppeteer';
+import { ofetch } from 'ofetch';
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+
+// Common headers for requests
+const getHeaders = (cookie?: string) => ({
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Cache-Control': 'no-cache',
+    Connection: 'keep-alive',
+    Host: 'www.xiaohongshu.com',
+    Pragma: 'no-cache',
+    'Sec-Ch-Ua': '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',
+    'Sec-Ch-Ua-Mobile': '?0',
+    'Sec-Ch-Ua-Platform': '"Windows"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Upgrade-Insecure-Requests': '1',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+    ...(cookie ? { Cookie: cookie } : {}),
+});
 
 const getUser = (url, cache) =>
     cache.tryGet(
@@ -23,7 +47,7 @@ const getUser = (url, cache) =>
                 });
                 await page.waitForSelector('div.reds-tab-item:nth-child(2)');
 
-                const initialState = await page.evaluate(() => window.__INITIAL_STATE__);
+                const initialState = await page.evaluate(() => (window as any).__INITIAL_STATE__);
 
                 if (!(await page.$('.lock-icon'))) {
                     await page.click('div.reds-tab-item:nth-child(2)');
@@ -68,7 +92,7 @@ const getBoard = (url, cache) =>
                 logger.http(`Requesting ${url}`);
                 await page.goto(url);
                 await page.waitForSelector('.pc-container');
-                const initialSsrState = await page.evaluate(() => window.__INITIAL_SSR_STATE__);
+                const initialSsrState = await page.evaluate(() => (window as any).__INITIAL_SSR_STATE__);
                 return initialSsrState.Main;
             } finally {
                 browser.close();
@@ -78,108 +102,6 @@ const getBoard = (url, cache) =>
         false
     );
 
-const setPageFilter = async (page) => {
-    await page.setRequestInterception(true);
-    page.on('request', (req) => {
-        req.resourceType() === 'document' || req.resourceType() === 'script' || req.resourceType() === 'xhr' || req.resourceType() === 'other' ? req.continue() : req.abort();
-    });
-};
-
-const getNotes = (url, cache) =>
-    cache.tryGet(
-        url + '/notes', // To avoid mixing with the cache for `user.js`
-        async () => {
-            let user = '';
-            let notes = [];
-
-            const browser = await puppeteer({ stealth: true });
-            try {
-                const page = await browser.newPage();
-                await setPageFilter(page);
-
-                logger.http(`Requesting ${url}`);
-                await page.goto(url);
-
-                let otherInfo = {};
-                let userPosted = {};
-                try {
-                    [otherInfo, userPosted] = await Promise.all(
-                        ['/api/sns/web/v1/user/otherinfo', '/api/sns/web/v1/user_posted'].map((p) =>
-                            page
-                                .waitForResponse((res) => {
-                                    const req = res.request();
-                                    return req.url().includes(p) && req.method() === 'GET';
-                                })
-                                .then((r) => r.json())
-                        )
-                    );
-                } catch (error) {
-                    throw new Error(`Could not get user information and note list\n${error}`);
-                }
-
-                await page.close();
-
-                // Get full text for each note
-                const notesPromise = userPosted.data.notes.map((n) => {
-                    const noteUrl = url + '/' + n.note_id;
-
-                    return cache.tryGet(noteUrl, async () => {
-                        const notePage = await browser.newPage();
-                        await setPageFilter(notePage);
-
-                        logger.http(`Requesting ${noteUrl}`);
-                        await notePage.goto(noteUrl);
-
-                        let feed = {};
-                        try {
-                            feed = await notePage.evaluate(() => window.__INITIAL_STATE__);
-
-                            // Sometimes the page is not server-side rendered
-                            if (feed?.note?.note === undefined || JSON.stringify(feed?.note?.note) === '{}') {
-                                const res = await notePage.waitForResponse((res) => {
-                                    const req = res.request();
-                                    return req.url().includes('/api/sns/web/v1/feed') && req.method() === 'POST';
-                                });
-
-                                const json = await res.json();
-                                const note_card = json.data.items[0].note_card;
-                                feed.note.note = {
-                                    title: note_card.title,
-                                    noteId: note_card.id,
-                                    desc: note_card.desc,
-                                    tagList: note_card.tag_list,
-                                    imageList: note_card.image_list,
-                                    user: note_card.user,
-                                    time: note_card.time,
-                                    lastUpdateTime: note_card.last_update_time,
-                                };
-                            }
-                        } catch (error) {
-                            throw new Error(`Could not get note ${n.note_id}\n${error}`);
-                        }
-
-                        await notePage.close();
-
-                        if (feed?.note?.note !== undefined && JSON.stringify(feed?.note?.note) !== '{}') {
-                            return feed.note.note;
-                        } else {
-                            throw new Error(`Could not get note ${n.note_id}`);
-                        }
-                    });
-                });
-
-                user = otherInfo.data.basic_info;
-                notes = await Promise.all(notesPromise);
-            } finally {
-                await browser.close();
-            }
-
-            return { user, notes };
-        },
-        config.cache.routeExpire,
-        false
-    );
-
 const formatText = (text) => text.replaceAll(/(\r\n|\r|\n)/g, '<br>').replaceAll('\t', '&emsp;');
 
 // tag_list.id has nothing to do with its url
@@ -196,4 +118,89 @@ const formatNote = (url, note) => ({
     updated: parseDate(note.lastUpdateTime, 'x'),
 });
 
-export { getUser, getBoard, getNotes, formatText, formatNote };
+async function renderNotesFulltext(notes, urlPrex) {
+    const data: Array<{
+        title: string;
+        link: string;
+        description: string;
+        author: string;
+        guid: string;
+        pubDate: Date;
+    }> = [];
+    const promises = notes.flatMap((note) =>
+        note.map(async ({ noteCard, id }) => {
+            const link = `${urlPrex}/${id}`;
+            const { title, description, pubDate } = await getFullNote(link);
+            return {
+                title,
+                link,
+                description,
+                author: noteCard.user.nickName,
+                guid: noteCard.noteId,
+                pubDate,
+            };
+        })
+    );
+    data.push(...(await Promise.all(promises)));
+    return data;
+}
+
+async function getFullNote(link) {
+    const data = (await cache.tryGet(link, async () => {
+        const res = await ofetch(link, {
+            headers: getHeaders(config.xiaohongshu.cookie),
+        });
+        const $ = load(res);
+        const script = extractInitialState($);
+        const state = JSON.parse(script);
+        const note = state.note.noteDetailMap[state.note.firstNoteId].note;
+        const images = note.imageList.map((image) => image.urlDefault);
+        const title = note.title;
+        let desc = note.desc;
+        desc = desc.replaceAll(/\[.*?\]/g, '');
+        desc = desc.replaceAll(/#(.*?)#/g, '#$1');
+        desc = desc.replaceAll('\n', '<br>');
+        const pubDate = new Date(note.time);
+        const description = `${images.map((image) => `<img src="${image}">`).join('')}<br>${title}<br>${desc}`;
+        return {
+            title,
+            description,
+            pubDate,
+        };
+    })) as Promise<{ title: string; description: string; pubDate: Date }>;
+    return data;
+}
+
+async function getUserWithCookie(url: string, cookie: string) {
+    const res = await ofetch(url, {
+        headers: getHeaders(cookie),
+    });
+    const $ = load(res);
+    const paths = $('#userPostedFeeds > section > div > a.cover.ld.mask').map((i, item) => item.attributes[3].value);
+    const script = extractInitialState($);
+    const state = JSON.parse(script);
+    let index = 0;
+    for (const item of state.user.notes.flat()) {
+        const path = paths[index];
+        if (path && path.includes('?')) {
+            item.id = item.id + path?.substring(path.indexOf('?'));
+        }
+        index = index + 1;
+    }
+    return state.user;
+}
+
+// Add helper function to extract initial state
+function extractInitialState($) {
+    let script = $('script')
+        .filter((i, script) => {
+            const text = script.children[0]?.data;
+            return text?.startsWith('window.__INITIAL_STATE__=');
+        })
+        .text();
+    script = script.slice('window.__INITIAL_STATE__='.length);
+    script = script.replaceAll('undefined', 'null');
+    return script;
+}
+
+export { getUser, getBoard, formatText, formatNote, renderNotesFulltext, getFullNote, getUserWithCookie };

package/lib/routes/xsijishe/rank.ts

@@ -8,9 +8,17 @@ const baseUrl = 'https://xsijishe.com';
 
 export const route: Route = {
     path: '/rank/:type',
-    categories: ['bbs'],
+    categories: ['bbs', 'popular'],
     example: '/xsijishe/rank/weekly',
-    parameters: { type: '排行榜类型: weekly | monthly' },
+    parameters: {
+        type: {
+            description: '排行榜类型',
+            options: [
+                { value: 'weekly', label: '周榜' },
+                { value: 'monthly', label: '月榜' },
+            ],
+        },
+    },
     features: {
         requireConfig: [
             {
@@ -29,23 +37,25 @@ export const route: Route = {
         supportScihub: false,
     },
     name: '排行榜',
-    maintainers: ['akynazh'],
+    maintainers: ['akynazh', 'AiraNadih'],
     handler,
 };
 
 async function handler(ctx) {
     const rankType = ctx.req.param('type');
     let title;
-    let rankId;
+    let index; // 用于选择第几个 li
+
     if (rankType === 'weekly') {
         title = '司机社综合周排行榜';
-        rankId = 'nex_recons_demens';
+        index = 0; // 第一个 li 是周榜
     } else if (rankType === 'monthly') {
         title = '司机社综合月排行榜';
-        rankId = 'nex_recons_demens1';
+        index = 1; // 第二个 li 是月榜
     } else {
         throw new InvalidParameterError('Invalid rank type');
     }
+
     const url = `${baseUrl}/portal.php`;
     const headers = {
         'Accept-Encoding': 'gzip, deflate, br',
@@ -53,11 +63,26 @@ async function handler(ctx) {
         Cookie: config.xsijishe.cookie,
         'User-Agent': config.xsijishe.user_agent,
     };
+
     const resp = await got(url, {
         headers,
     });
+
+    const redirectMatch = resp.data.match(/window\.location\.href\s*=\s*"([^"]+)"/);
+    if (redirectMatch) {
+        const redirectUrl = `${baseUrl}${redirectMatch[1]}`;
+        // 使用提取到的地址重新请求
+        const realResp = await got(redirectUrl, {
+            headers,
+        });
+        resp.data = realResp.data;
+    }
+
     const $ = load(resp.data);
-    let items = $(`#${rankId} dd`)
+    // 根据 index 选择对应的 li，然后获取其中的 dd 元素
+    let items = $('.nex_recon_lists ul li')
+        .eq(index)
+        .find('.nex_recons_demens dl dd')
         .toArray()
         .map((item) => {
             item = $(item);
@@ -68,6 +93,7 @@ async function handler(ctx) {
                 link: `${baseUrl}/${link}`,
             };
         });
+
     items = await Promise.all(
         items.map((item) =>
             cache.tryGet(item.link, async () => {

package/lib/routes/xueqiu/cookies.ts

@@ -1,29 +1,24 @@
-import ofetch from '@/utils/ofetch';
 import cache from '@/utils/cache';
 import { config } from '@/config';
-import { getAcwScV2ByArg1 } from '@/routes/5eplay/utils';
+import puppeteer from '@/utils/puppeteer';
+import { getCookies } from '@/utils/puppeteer-utils';
 
 export const parseToken = (link: string) =>
     cache.tryGet(
         'xueqiu:token',
         async () => {
-            const r = await ofetch(link);
-
-            let acw_sc__v2 = '';
-            const matches = r.match(/var arg1='(.*?)';/);
-            if (matches) {
-                acw_sc__v2 = getAcwScV2ByArg1(matches[1]);
-            }
-            const acw_sc__v2_cookie = `acw_sc__v2=${acw_sc__v2}`;
-            const res = await ofetch.raw(link, {
-                headers: {
-                    Cookie: acw_sc__v2_cookie,
-                },
+            const browser = await puppeteer({ stealth: true });
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' ? request.continue() : request.abort();
             });
-            const cookieArray = res.headers.getSetCookie();
-            const xq_a_token_cookie = cookieArray.find((c) => c.startsWith('xq_a_token='));
-
-            return `${acw_sc__v2_cookie}; ${xq_a_token_cookie}`;
+            await page.goto(link, {
+                waitUntil: 'domcontentloaded',
+            });
+            await page.evaluate(() => document.documentElement.innerHTML);
+            const cookies = await getCookies(page);
+            return cookies;
         },
         config.cache.routeExpire,
         false

package/lib/routes/yahoo/news/listid.ts

@@ -18,7 +18,10 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['hk.news.yahoo.com/', 'tw.news.yahoo.com/'],
+            source: ['hk.news.yahoo.com/'],
+        },
+        {
+            source: ['tw.news.yahoo.com/'],
         },
     ],
     name: '合作媒體',

package/lib/routes/yahoo/news/provider-helper.ts

@@ -18,7 +18,10 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['hk.news.yahoo.com/', 'tw.news.yahoo.com/'],
+            source: ['hk.news.yahoo.com/'],
+        },
+        {
+            source: ['tw.news.yahoo.com/'],
         },
     ],
     name: '新聞來源列表',

package/lib/routes/yahoo/news/provider.ts

@@ -18,7 +18,10 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['hk.news.yahoo.com/', 'tw.news.yahoo.com/'],
+            source: ['hk.news.yahoo.com/'],
+        },
+        {
+            source: ['tw.news.yahoo.com/'],
         },
     ],
     name: '新聞來源',

package/lib/routes/yande/post.ts

@@ -1,13 +1,23 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import queryString from 'query-string';
 
 export const route: Route = {
     path: '/post/popular_recent/:period?',
-    categories: ['picture'],
+    categories: ['picture', 'popular'],
+    view: ViewType.Pictures,
     example: '/yande/post/popular_recent/1d',
     parameters: {
-        period: '展示时间',
+        period: {
+            description: '展示时间',
+            options: [
+                { value: '1d', label: '最近 24 小时' },
+                { value: '1w', label: '最近一周' },
+                { value: '1m', label: '最近一月' },
+                { value: '1y', label: '最近一年' },
+            ],
+            default: '1d',
+        },
     },
     radar: [
         {

package/lib/routes/youtube/user.ts

@@ -3,9 +3,10 @@ import cache from '@/utils/cache';
 import utils from './utils';
 import { config } from '@/config';
 import { parseDate } from '@/utils/parse-date';
-import got from '@/utils/got';
-import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
+import * as cheerio from 'cheerio';
 import ConfigNotFoundError from '@/errors/types/config-not-found';
+import NotFoundError from '@/errors/types/not-found';
 
 export const route: Route = {
     path: '/user/:username/:embed?',
@@ -44,35 +45,46 @@ async function handler(ctx) {
     const username = ctx.req.param('username');
     const embed = !ctx.req.param('embed');
 
-    let playlistId;
-    let channelName;
-    let image;
-    let description;
+    let userHandleData;
     if (username.startsWith('@')) {
-        const link = `https://www.youtube.com/${username}`;
-        const response = await got(link);
-        const $ = load(response.data);
-        const ytInitialData = JSON.parse(
-            $('script')
-                .text()
-                .match(/ytInitialData = ({.*?});/)?.[1] || '{}'
-        );
-        const channelId = ytInitialData.metadata.channelMetadataRenderer.externalId;
-        channelName = ytInitialData.metadata.channelMetadataRenderer.title;
-        image = ytInitialData.metadata.channelMetadataRenderer.avatar?.thumbnails?.[0]?.url;
-        description = ytInitialData.metadata.channelMetadataRenderer.description;
-        playlistId = (await utils.getChannelWithId(channelId, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads;
+        userHandleData = await cache.tryGet(`youtube:handle:${username}`, async () => {
+            const link = `https://www.youtube.com/${username}`;
+            const response = await ofetch(link);
+            const $ = cheerio.load(response);
+            const ytInitialData = JSON.parse(
+                $('script')
+                    .text()
+                    .match(/ytInitialData = ({.*?});/)?.[1] || '{}'
+            );
+            const metadataRenderer = ytInitialData.metadata.channelMetadataRenderer;
+
+            const channelId = metadataRenderer.externalId;
+            const channelName = metadataRenderer.title;
+            const image = metadataRenderer.avatar?.thumbnails?.[0]?.url;
+            const description = metadataRenderer.description;
+            const playlistId = (await utils.getChannelWithId(channelId, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads;
+
+            return {
+                channelName,
+                image,
+                description,
+                playlistId,
+            };
+        });
     }
-    playlistId = playlistId || (await utils.getChannelWithUsername(username, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads;
+    const playlistId = userHandleData?.playlistId || (await utils.getChannelWithUsername(username, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads;
 
-    const data = (await utils.getPlaylistItems(playlistId, 'snippet', cache)).data.items;
+    const playlistItems = await utils.getPlaylistItems(playlistId, 'snippet', cache);
+    if (!playlistItems) {
+        throw new NotFoundError("This channel doesn't have any content.");
+    }
 
     return {
-        title: `${channelName || username} - YouTube`,
+        title: `${userHandleData?.channelName || username} - YouTube`,
         link: username.startsWith('@') ? `https://www.youtube.com/${username}` : `https://www.youtube.com/user/${username}`,
-        description: description || `YouTube user ${username}`,
-        image,
-        item: data
+        description: userHandleData?.description || `YouTube user ${username}`,
+        image: userHandleData?.image,
+        item: playlistItems.data.items
             .filter((d) => d.snippet.title !== 'Private video' && d.snippet.title !== 'Deleted video')
             .map((item) => {
                 const snippet = item.snippet;

package/lib/routes/zhonglun/index.ts

@@ -20,22 +20,21 @@ export const handler = async (ctx) => {
 
     const $ = load(response);
 
-    let items = $('div#dataList h3')
+    let items = $('div#dataList > dl > dd, div#dataList > ul > li')
         .slice(0, limit)
         .toArray()
         .map((item) => {
             item = $(item);
 
-            const title = item.text();
             const description = art(path.join(__dirname, 'templates/description.art'), {
-                intro: item.next().text(),
+                intro: item.find('p').text(),
             });
 
             return {
-                title,
+                title: item.find('h3 > a').text(),
                 description,
-                pubDate: parseDate(item.find('span').first().text()),
-                link: item.find('a').prop('href'),
+                pubDate: parseDate(item.find('span').text()),
+                link: item.find('h3 > a').prop('href'),
                 language,
             };
         });

package/lib/routes/zjut/cs/index.ts

@@ -27,7 +27,7 @@ export const route: Route = {
     handler,
     radar: [
         {
-            source: ['cs.zjut.edu.cn/jsp/newsclass.jsp?wcId=:type'],
+            source: ['cs.zjut.edu.cn/jsp/newsclass.jsp'],
             target: '/cs/:type',
         },
     ],

package/lib/utils/got.ts

@@ -1,5 +1,6 @@
 import { destr } from 'destr';
 import ofetch from '@/utils/ofetch';
+import { getSearchParamsString } from './helpers';
 
 const getFakeGot = (defaultOptions?: any) => {
     const fakeGot = (request, options?: any) => {
@@ -35,7 +36,7 @@ const getFakeGot = (defaultOptions?: any) => {
             delete options.form;
         }
         if (options?.searchParams) {
-            request += '?' + new URLSearchParams(options.searchParams).toString();
+            request += '?' + getSearchParamsString(options.searchParams);
             delete options.searchParams;
         }
 

package/lib/utils/helpers.test.ts

@@ -1,8 +1,20 @@
 import { describe, expect, it } from 'vitest';
-import { getRouteNameFromPath } from '@/utils/helpers';
+import { getRouteNameFromPath, getSearchParamsString } from '@/utils/helpers';
 
 describe('helpers', () => {
     it('getRouteNameFromPath', () => {
         expect(getRouteNameFromPath('/test/1')).toBe('test');
     });
+
+    it('getSearchParamsString', () => {
+        expect(getSearchParamsString({ a: 1, b: 2 })).toBe('a=1&b=2');
+        expect(getSearchParamsString({ a: 1, b: undefined })).toBe('a=1');
+        expect(getSearchParamsString({ a: undefined })).toBe('');
+        expect(getSearchParamsString({})).toBe('');
+
+        const searchParams = new URLSearchParams();
+        searchParams.append('ids[]', '1');
+        searchParams.append('ids[]', '2');
+        expect(getSearchParamsString(searchParams)).toBe('ids%5B%5D=1&ids%5B%5D=2');
+    });
 });

package/lib/utils/helpers.ts

@@ -1,5 +1,6 @@
 import { fileURLToPath } from 'url';
 import path from 'node:path';
+import { stringifyQuery } from 'ufo';
 
 export const getRouteNameFromPath = (path: string) => {
     const p = path.split('/').filter(Boolean);
@@ -30,3 +31,12 @@ export const getCurrentPath = (metaUrl: string) => {
     const __filename = path.join(fileURLToPath(metaUrl));
     return path.dirname(__filename);
 };
+
+function isPureObject(o: any) {
+    return Object.prototype.toString.call(o) === '[object Object]';
+}
+
+export function getSearchParamsString(searchParams: any) {
+    const searchParamsString = isPureObject(searchParams) ? stringifyQuery(searchParams) : null;
+    return searchParamsString ?? new URLSearchParams(searchParams).toString();
+}

package/lib/utils/ofetch.ts

@@ -1,6 +1,9 @@
 import { createFetch } from 'ofetch';
 import { config } from '@/config';
 import logger from '@/utils/logger';
+import { register } from 'node-network-devtools';
+
+config.enableRemoteDebugging && process.env.NODE_ENV === 'dev' && register();
 
 const rofetch = createFetch().create({
     retryStatusCodes: [400, 408, 409, 425, 429, 500, 502, 503, 504],