rsshub 1.0.0-master.f6c2ebb → 1.0.0-master.f70afef

package/lib/routes/xiaohongshu/util.ts

@@ -2,6 +2,30 @@ import { config } from '@/config';
 import logger from '@/utils/logger';
 import { parseDate } from '@/utils/parse-date';
 import puppeteer from '@/utils/puppeteer';
+import { ofetch } from 'ofetch';
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+
+// Common headers for requests
+const getHeaders = (cookie?: string) => ({
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Cache-Control': 'no-cache',
+    Connection: 'keep-alive',
+    Host: 'www.xiaohongshu.com',
+    Pragma: 'no-cache',
+    'Sec-Ch-Ua': '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',
+    'Sec-Ch-Ua-Mobile': '?0',
+    'Sec-Ch-Ua-Platform': '"Windows"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Upgrade-Insecure-Requests': '1',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+    ...(cookie ? { Cookie: cookie } : {}),
+});
 
 const getUser = (url, cache) =>
     cache.tryGet(
@@ -23,7 +47,7 @@ const getUser = (url, cache) =>
                 });
                 await page.waitForSelector('div.reds-tab-item:nth-child(2)');
 
-                const initialState = await page.evaluate(() => window.__INITIAL_STATE__);
+                const initialState = await page.evaluate(() => (window as any).__INITIAL_STATE__);
 
                 if (!(await page.$('.lock-icon'))) {
                     await page.click('div.reds-tab-item:nth-child(2)');
@@ -68,7 +92,7 @@ const getBoard = (url, cache) =>
                 logger.http(`Requesting ${url}`);
                 await page.goto(url);
                 await page.waitForSelector('.pc-container');
-                const initialSsrState = await page.evaluate(() => window.__INITIAL_SSR_STATE__);
+                const initialSsrState = await page.evaluate(() => (window as any).__INITIAL_SSR_STATE__);
                 return initialSsrState.Main;
             } finally {
                 browser.close();
@@ -78,108 +102,6 @@ const getBoard = (url, cache) =>
         false
     );
 
-const setPageFilter = async (page) => {
-    await page.setRequestInterception(true);
-    page.on('request', (req) => {
-        req.resourceType() === 'document' || req.resourceType() === 'script' || req.resourceType() === 'xhr' || req.resourceType() === 'other' ? req.continue() : req.abort();
-    });
-};
-
-const getNotes = (url, cache) =>
-    cache.tryGet(
-        url + '/notes', // To avoid mixing with the cache for `user.js`
-        async () => {
-            let user = '';
-            let notes = [];
-
-            const browser = await puppeteer({ stealth: true });
-            try {
-                const page = await browser.newPage();
-                await setPageFilter(page);
-
-                logger.http(`Requesting ${url}`);
-                await page.goto(url);
-
-                let otherInfo = {};
-                let userPosted = {};
-                try {
-                    [otherInfo, userPosted] = await Promise.all(
-                        ['/api/sns/web/v1/user/otherinfo', '/api/sns/web/v1/user_posted'].map((p) =>
-                            page
-                                .waitForResponse((res) => {
-                                    const req = res.request();
-                                    return req.url().includes(p) && req.method() === 'GET';
-                                })
-                                .then((r) => r.json())
-                        )
-                    );
-                } catch (error) {
-                    throw new Error(`Could not get user information and note list\n${error}`);
-                }
-
-                await page.close();
-
-                // Get full text for each note
-                const notesPromise = userPosted.data.notes.map((n) => {
-                    const noteUrl = url + '/' + n.note_id;
-
-                    return cache.tryGet(noteUrl, async () => {
-                        const notePage = await browser.newPage();
-                        await setPageFilter(notePage);
-
-                        logger.http(`Requesting ${noteUrl}`);
-                        await notePage.goto(noteUrl);
-
-                        let feed = {};
-                        try {
-                            feed = await notePage.evaluate(() => window.__INITIAL_STATE__);
-
-                            // Sometimes the page is not server-side rendered
-                            if (feed?.note?.note === undefined || JSON.stringify(feed?.note?.note) === '{}') {
-                                const res = await notePage.waitForResponse((res) => {
-                                    const req = res.request();
-                                    return req.url().includes('/api/sns/web/v1/feed') && req.method() === 'POST';
-                                });
-
-                                const json = await res.json();
-                                const note_card = json.data.items[0].note_card;
-                                feed.note.note = {
-                                    title: note_card.title,
-                                    noteId: note_card.id,
-                                    desc: note_card.desc,
-                                    tagList: note_card.tag_list,
-                                    imageList: note_card.image_list,
-                                    user: note_card.user,
-                                    time: note_card.time,
-                                    lastUpdateTime: note_card.last_update_time,
-                                };
-                            }
-                        } catch (error) {
-                            throw new Error(`Could not get note ${n.note_id}\n${error}`);
-                        }
-
-                        await notePage.close();
-
-                        if (feed?.note?.note !== undefined && JSON.stringify(feed?.note?.note) !== '{}') {
-                            return feed.note.note;
-                        } else {
-                            throw new Error(`Could not get note ${n.note_id}`);
-                        }
-                    });
-                });
-
-                user = otherInfo.data.basic_info;
-                notes = await Promise.all(notesPromise);
-            } finally {
-                await browser.close();
-            }
-
-            return { user, notes };
-        },
-        config.cache.routeExpire,
-        false
-    );
-
 const formatText = (text) => text.replaceAll(/(\r\n|\r|\n)/g, '<br>').replaceAll('\t', '&emsp;');
 
 // tag_list.id has nothing to do with its url
@@ -196,4 +118,149 @@ const formatNote = (url, note) => ({
     updated: parseDate(note.lastUpdateTime, 'x'),
 });
 
-export { getUser, getBoard, getNotes, formatText, formatNote };
+async function renderNotesFulltext(notes, urlPrex, displayLivePhoto) {
+    const data: Array<{
+        title: string;
+        link: string;
+        description: string;
+        author: string;
+        guid: string;
+        pubDate: Date;
+    }> = [];
+    const promises = notes.flatMap((note) =>
+        note.map(async ({ noteCard, id }) => {
+            const link = `${urlPrex}/${id}`;
+            const { title, description, pubDate } = await getFullNote(link, displayLivePhoto);
+            return {
+                title,
+                link,
+                description,
+                author: noteCard.user.nickName,
+                guid: noteCard.noteId,
+                pubDate,
+            };
+        })
+    );
+    data.push(...(await Promise.all(promises)));
+    return data;
+}
+
+async function getFullNote(link, displayLivePhoto) {
+    const data = (await cache.tryGet(link, async () => {
+        const res = await ofetch(link, {
+            headers: getHeaders(config.xiaohongshu.cookie),
+        });
+        const $ = load(res);
+        const script = extractInitialState($);
+        const state = JSON.parse(script);
+        const note = state.note.noteDetailMap[state.note.firstNoteId].note;
+        const title = note.title;
+        let desc = note.desc;
+        desc = desc.replaceAll(/\[.*?\]/g, '');
+        desc = desc.replaceAll(/#(.*?)#/g, '#$1');
+        desc = desc.replaceAll('\n', '<br>');
+        const pubDate = new Date(note.time);
+
+        let mediaContent = '';
+        if (note.type === 'video') {
+            const originVideoKey = note.video?.consumer?.originVideoKey;
+            const videoUrls: string[] = [];
+
+            if (originVideoKey) {
+                videoUrls.push(`http://sns-video-al.xhscdn.com/${originVideoKey}`);
+            }
+
+            const streamTypes = ['av1', 'h264', 'h265', 'h266'];
+            for (const type of streamTypes) {
+                const streams = note.video?.media?.stream?.[type];
+                if (streams?.length > 0) {
+                    const stream = streams[0];
+                    if (stream.masterUrl) {
+                        videoUrls.push(stream.masterUrl);
+                    }
+                    if (stream.backupUrls?.length) {
+                        videoUrls.push(...stream.backupUrls);
+                    }
+                }
+            }
+
+            const posterUrl = note.imageList?.[0]?.urlDefault;
+
+            if (videoUrls.length > 0) {
+                mediaContent = `<video controls ${posterUrl ? `poster="${posterUrl}"` : ''}>
+                    ${videoUrls.map((url) => `<source src="${url}" type="video/mp4">`).join('\n')}
+                </video><br>`;
+            }
+        } else {
+            mediaContent = note.imageList
+                .map((image) => {
+                    if (image.livePhoto && displayLivePhoto) {
+                        const videoUrls: string[] = [];
+
+                        const streamTypes = ['av1', 'h264', 'h265', 'h266'];
+                        for (const type of streamTypes) {
+                            const streams = image.stream?.[type];
+                            if (streams?.length > 0) {
+                                if (streams[0].masterUrl) {
+                                    videoUrls.push(streams[0].masterUrl);
+                                }
+                                if (streams[0].backupUrls?.length) {
+                                    videoUrls.push(...streams[0].backupUrls);
+                                }
+                            }
+                        }
+
+                        if (videoUrls.length > 0) {
+                            return `<video controls poster="${image.urlDefault}">
+                            ${videoUrls.map((url) => `<source src="${url}" type="video/mp4">`).join('\n')}
+                        </video>`;
+                        }
+                    }
+                    return `<img src="${image.urlDefault}">`;
+                })
+                .join('<br>');
+        }
+
+        const description = `${mediaContent}<br>${title}<br>${desc}`;
+        return {
+            title,
+            description,
+            pubDate,
+        };
+    })) as Promise<{ title: string; description: string; pubDate: Date }>;
+    return data;
+}
+
+async function getUserWithCookie(url: string, cookie: string) {
+    const res = await ofetch(url, {
+        headers: getHeaders(cookie),
+    });
+    const $ = load(res);
+    const paths = $('#userPostedFeeds > section > div > a.cover.ld.mask').map((i, item) => item.attributes[3].value);
+    const script = extractInitialState($);
+    const state = JSON.parse(script);
+    let index = 0;
+    for (const item of state.user.notes.flat()) {
+        const path = paths[index];
+        if (path && path.includes('?')) {
+            item.id = item.id + path?.substring(path.indexOf('?'));
+        }
+        index = index + 1;
+    }
+    return state.user;
+}
+
+// Add helper function to extract initial state
+function extractInitialState($) {
+    let script = $('script')
+        .filter((i, script) => {
+            const text = script.children[0]?.data;
+            return text?.startsWith('window.__INITIAL_STATE__=');
+        })
+        .text();
+    script = script.slice('window.__INITIAL_STATE__='.length);
+    script = script.replaceAll('undefined', 'null');
+    return script;
+}
+
+export { getUser, getBoard, formatText, formatNote, renderNotesFulltext, getFullNote, getUserWithCookie };

package/lib/routes/ximalaya/album.ts

@@ -104,11 +104,11 @@ export const route: Route = {
     handler,
     description: `目前喜马拉雅的 API 只能一集一集的获取各节目上的 ShowNote，会极大的占用系统资源，所以默认为不获取节目的 ShowNote。
 
-  :::warning
+::: warning
   专辑类型即 url 中的分类拼音，使用通用分类 \`album\` 通常是可行的，专辑 id 是跟在**分类拼音**后的那个 id, 不要输成某集的 id 了
 
   **付费内容需要配置好已购买账户的 token 才能收听，详情见部署页面的配置模块**
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/xinpianchang/index.ts

@@ -18,11 +18,11 @@ export const route: Route = {
     name: '发现',
     maintainers: ['nczitzk'],
     handler,
-    description: `:::tip
+    description: `::: tip
   跳转到欲订阅的分类页，将 URL 的 \`/discover\` 到末尾的部分填入 \`params\` 参数。
 
   如 [全部原创视频作品](https://www.xinpianchang.com/discover/article-0-0-all-all-0-0-score) 的 URL 为 \`https://www.xinpianchang.com/discover/article-0-0-all-all-0-0-score\`，其 \`/discover\` 到末尾的部分为 \`article-0-0-all-all-0-0-score\`，所以对应的路由为 [/xinpianchang/discover/article-0-0-all-all-0-0-score](https://rsshub.app/xinpianchang/discover/article-0-0-all-all-0-0-score)。
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/xmu/kydt.ts

@@ -0,0 +1,68 @@
+import { Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+
+export const route: Route = {
+    path: '/kydt',
+    categories: ['university'],
+    example: '/xmu/kydt',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['soe.xmu.edu.cn/kxyj/kydt.htm'],
+        },
+    ],
+    name: '科研动态',
+    maintainers: ['linsenwang'],
+    handler,
+};
+
+async function handler() {
+    const host = 'https://soe.xmu.edu.cn/kxyj/kydt.htm';
+    const response = await ofetch(host);
+    const $ = load(response);
+
+    const list = $('div.news li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const title = item.find('h4').first().text();
+            const time = item.find('h6').first().text();
+            const a = item.find('a').first().attr('href');
+            const fullUrl = new URL(a, host).href;
+
+            return {
+                title,
+                link: fullUrl,
+                pubDate: time,
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await ofetch(item.link);
+                const $ = load(response);
+
+                item.description = $('.v_news_content').first().html();
+
+                return item;
+            })
+        )
+    );
+
+    return {
+        allowEmpty: true,
+        title: '厦门大学经济学院科研动态',
+        link: host,
+        item: items,
+    };
+}

package/lib/routes/xmu/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Xiamen University',
+    url: 'soe.xmu.edu.cn',
+    zh: {
+        name: '厦门大学经济学院',
+    },
+};

package/lib/routes/xsijishe/forum.ts

@@ -31,9 +31,9 @@ export const route: Route = {
     name: '论坛',
     maintainers: ['akynazh'],
     handler,
-    description: `:::tip 关于子论坛 id 的获取方法
+    description: `::: tip 关于子论坛 id 的获取方法
   \`/xsijishe/forum/51\` 对应于论坛 \`https://xsijishe.com/forum-51-1.html\`，这个论坛的 fid 为 51，也就是 \`forum-{fid}-1\` 中的 fid。
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/xueqiu/column.ts

@@ -14,7 +14,7 @@ export const route: Route = {
     features: {
         requireConfig: false,
         requirePuppeteer: false,
-        antiCrawler: false,
+        antiCrawler: true,
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
@@ -25,7 +25,7 @@ export const route: Route = {
         },
     ],
     name: '用户专栏',
-    maintainers: ['TonyRL'],
+    maintainers: ['TonyRL', 'pseudoyu'],
     handler,
 };
 
@@ -33,6 +33,11 @@ async function handler(ctx) {
     const id = ctx.req.param('id');
     const pageUrl = `${baseUrl}/${id}/column`;
 
+    // Get cookie first
+    await got(baseUrl, {
+        cookieJar,
+    });
+
     const pageData = await got(pageUrl, {
         cookieJar,
     });
@@ -49,6 +54,10 @@ async function handler(ctx) {
         },
     });
 
+    if (!data.list) {
+        throw new Error('Error occurred, please refresh the page or try again after logging back into your account');
+    }
+
     const items = data.list.map((item) => ({
         title: item.title,
         description: item.description,

package/lib/routes/xueqiu/cookies.ts

@@ -1,29 +1,24 @@
-import ofetch from '@/utils/ofetch';
 import cache from '@/utils/cache';
 import { config } from '@/config';
-import { getAcwScV2ByArg1 } from '@/routes/5eplay/utils';
+import puppeteer from '@/utils/puppeteer';
+import { getCookies } from '@/utils/puppeteer-utils';
 
 export const parseToken = (link: string) =>
     cache.tryGet(
         'xueqiu:token',
         async () => {
-            const r = await ofetch(link);
-
-            let acw_sc__v2 = '';
-            const matches = r.match(/var arg1='(.*?)';/);
-            if (matches) {
-                acw_sc__v2 = getAcwScV2ByArg1(matches[1]);
-            }
-            const acw_sc__v2_cookie = `acw_sc__v2=${acw_sc__v2}`;
-            const res = await ofetch.raw(link, {
-                headers: {
-                    Cookie: acw_sc__v2_cookie,
-                },
+            const browser = await puppeteer({ stealth: true });
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' ? request.continue() : request.abort();
             });
-            const cookieArray = res.headers.getSetCookie();
-            const xq_a_token_cookie = cookieArray.find((c) => c.startsWith('xq_a_token='));
-
-            return `${acw_sc__v2_cookie}; ${xq_a_token_cookie}`;
+            await page.goto(link, {
+                waitUntil: 'domcontentloaded',
+            });
+            await page.evaluate(() => document.documentElement.innerHTML);
+            const cookies = await getCookies(page);
+            return cookies;
         },
         config.cache.routeExpire,
         false

package/lib/routes/xueqiu/timeline.ts

@@ -26,9 +26,9 @@ export const route: Route = {
     name: '用户关注时间线',
     maintainers: ['ErnestDong'],
     handler,
-    description: `:::warning
+    description: `::: warning
   用户关注动态需要登录后的 Cookie 值，所以只能自建，详情见部署页面的配置模块。
-  :::
+:::
 
   | -1   | -2       | 1             |
   | ---- | -------- | ------------- |

package/lib/routes/xueqiu/user-stock.ts

@@ -30,7 +30,7 @@ export const route: Route = {
     name: '用户自选动态',
     maintainers: ['hillerliao'],
     handler,
-    description: `:::warning
+    description: `::: warning
   用户自选动态需要登录后的 Cookie 值，所以只能自建，详情见部署页面的配置模块。
 :::`,
 };

package/lib/routes/xyzrank/namespace.ts

@@ -3,7 +3,7 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: '中文播客榜',
     url: 'xyzrank.com',
-    description: `:::tip
+    description: `::: tip
 可以通过指定 \`limit\` 参数确定榜单排名下限，默认为 250。
 
 若只查看榜单前 50，可在订阅 URL 后加入 \`?limit=50\`。

package/lib/routes/yahoo/news/index.ts

@@ -6,7 +6,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
     path: '/news/:region/:category?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/yahoo/news/hk/world',
     parameters: {
         region: 'Region, `hk/tw/au/ca/fr/malaysia/nz/sg/uk/en(us)`, the part represented by the asterisk (*) in *.news.yahoo.com',

package/lib/routes/yahoo/news/listid.ts

@@ -5,7 +5,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
     path: '/news/list/:region/:listId',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/yahoo/news/list/hk/09fcf7b0-0ab2-11e8-bf1f-4d52d4f79454',
     parameters: { region: '`hk`, `tw`', listId: '見下表' },
     features: {
@@ -18,7 +18,10 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['hk.news.yahoo.com/', 'tw.news.yahoo.com/'],
+            source: ['hk.news.yahoo.com/'],
+        },
+        {
+            source: ['tw.news.yahoo.com/'],
         },
     ],
     name: '合作媒體',

package/lib/routes/yahoo/news/provider-helper.ts

@@ -5,7 +5,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
     path: '/news/providers/:region/list',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/yahoo/news/providers/tw/list',
     parameters: { region: '地区, 同路由"新闻来源"中的支持地区, 即 hk 或 tw' },
     features: {
@@ -18,7 +18,10 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['hk.news.yahoo.com/', 'tw.news.yahoo.com/'],
+            source: ['hk.news.yahoo.com/'],
+        },
+        {
+            source: ['tw.news.yahoo.com/'],
         },
     ],
     name: '新聞來源列表',

package/lib/routes/yahoo/news/provider.ts

@@ -5,7 +5,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
     path: '/news/provider/:region/:providerId',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/yahoo/news/provider/tw/yahoo_tech_tw_942',
     parameters: { region: '地區, hk 或 tw, 分别表示香港雅虎和台湾雅虎', providerId: '新聞來源 ID, 可透過路由"新聞來源列表"獲得' },
     features: {
@@ -18,7 +18,10 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['hk.news.yahoo.com/', 'tw.news.yahoo.com/'],
+            source: ['hk.news.yahoo.com/'],
+        },
+        {
+            source: ['tw.news.yahoo.com/'],
         },
     ],
     name: '新聞來源',

package/lib/routes/yamibo/bbs/forum.ts

@@ -34,7 +34,7 @@ export const route: Route = {
             },
         ],
     },
-    description: `:::warning
+    description: `::: warning
 百合会BBS访问部分板块需要用户登录认证，请参考配置说明
 :::`,
 };

package/lib/routes/yamibo/bbs/thread.ts

@@ -30,7 +30,7 @@ export const route: Route = {
             },
         ],
     },
-    description: `:::warning
+    description: `::: warning
 百合会BBS访问部分讨论串需要用户登录认证，请参考配置说明
 :::`,
 };

package/lib/routes/yicai/feed.ts

@@ -27,9 +27,9 @@ export const route: Route = {
     name: '关注',
     maintainers: ['nczitzk'],
     handler,
-    description: `:::tip
+    description: `::: tip
   全部主题词见 [此处](https://www.yicai.com/feed/alltheme)
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {