rsshub 1.0.0-master.f71451d → 1.0.0-master.f75997f

package/lib/routes/u9a9/index.ts

@@ -9,17 +9,17 @@ const baseUrl = 'https://u9a9.com';
 
 export const route: Route = {
     path: ['/:preview?', '/search/:keyword/:preview?'],
+    example: '/u9a9/search/新片速递',
     radar: [
         {
             source: ['u9a9.com/'],
             target: '',
         },
     ],
-    name: 'Unknown',
+    name: 'Search',
     maintainers: ['TonyRL'],
     handler,
     url: 'u9a9.com/',
-    url: 'u9a9.com/',
 };
 
 async function handler(ctx) {

package/lib/routes/u9a9/namespace.ts

@@ -1,6 +1,6 @@
 import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
-    name: 'Unknown',
+    name: 'U9A9',
     url: 'u9a9.com',
 };

package/lib/routes/zsxq/group.ts

@@ -0,0 +1,63 @@
+import type { Data, Route } from '@/types';
+import type { Context } from 'hono';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import type { GroupInfoResponse, TopicsResponse } from './types';
+import { customFetch, generateTopicDataItem } from './utils';
+
+export const route: Route = {
+    name: '星球',
+    categories: ['social-media'],
+    path: '/group/:id/:scope?',
+    example: '/zsxq/group/88855458825252',
+    parameters: {
+        id: '星球id，从网页端url中获取',
+        scope: '栏目分类，默认为"all"，见下表',
+    },
+    maintainers: ['KarasuShin'],
+    radar: [
+        {
+            source: ['wx.zsxq.com/dweb2/index/group/:id'],
+        },
+    ],
+    features: {
+        requireConfig: [
+            {
+                name: 'ZSXQ_ACCESS_TOKEN',
+                description:
+                    '知识星球访问令牌,获取方式：\n1. 登录知识星球网页版\n2. 打开浏览器开发者工具，切换到 Application 面板\n3. 点击侧边栏中的Storage -> Cookies -> https://wx.zsxq.com\n4. 复制 Cookie 中的 zsxq_access_token 值',
+            },
+        ],
+    },
+    handler,
+    description: `| all  | digests | by_owner | questions | tasks |
+    | ---- | ------ | --------- | -------- | ------ |
+    | 最新 | 精华    | 只看星主    | 问答      | 作业   |`,
+};
+
+async function handler(ctx: Context): Promise<Data> {
+    const groupId = ctx.req.param('id');
+    const scope = ctx.req.param('scope') ?? 'all';
+    const accessToken = config.zsxq.accessToken;
+
+    if (!accessToken) {
+        throw new ConfigNotFoundError('该 RSS 源由于配置不正确而被禁用：令牌丢失。');
+    }
+
+    let count = Number(ctx.req.query('limit')) || 20;
+    if (count > 30) {
+        count = 30;
+    }
+
+    const { group } = await customFetch<GroupInfoResponse>(`/groups/${groupId}`);
+
+    const { topics } = await customFetch<TopicsResponse>(`/groups/${groupId}/topics?scope=${scope}&count=${count}`);
+
+    return {
+        title: `知识星球 - ${group.name}`,
+        description: group.description,
+        image: group.background_url,
+        link: `https://wx.zsxq.com/dweb2/index/group/${groupId}`,
+        item: generateTopicDataItem(topics),
+    };
+}

package/lib/routes/zsxq/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '知识星球',
+    url: 'zsxq.com',
+};

package/lib/routes/zsxq/types.ts

@@ -0,0 +1,149 @@
+export interface BasicResponse<T> {
+    succeeded: boolean;
+    resp_data: T;
+    code?: number;
+    info?: string;
+    error?: string;
+}
+
+interface BasicTopic {
+    comments_count: number;
+    create_time: string;
+    digested: boolean;
+    group: {
+        group_id: number;
+        name: string;
+        type: string;
+        background_url: string;
+    };
+    latest_likes: {
+        create_time: string;
+        owner: {
+            avatar_url: string;
+            name: string;
+            number: number;
+            user_id: number;
+        };
+    }[];
+    likes_count: number;
+    readers_count: number;
+    reading_count: number;
+    rewards_count: number;
+    topic_id: number;
+    type: string;
+    user_specific: {
+        liked: false;
+        subscribed: false;
+    };
+    title?: string;
+}
+
+interface TalkTopic extends BasicTopic {
+    type: 'talk';
+    talk: {
+        text?: string;
+        images?: TopicImage[];
+    };
+}
+
+export interface QATopic extends BasicTopic {
+    type: 'q&a';
+    answer?: {
+        owner: {
+            avatar_url: string;
+            description: string;
+            location: string;
+            name: string;
+            user_id: number;
+        };
+        text?: string;
+        images?: TopicImage[];
+    };
+    answered: boolean;
+    question: {
+        images?: TopicImage[];
+        text?: string;
+    };
+}
+
+interface TaskTopic extends BasicTopic {
+    type: 'task';
+    task: {
+        images?: TopicImage[];
+        text?: string;
+    };
+}
+
+interface SolutionTopic extends BasicTopic {
+    type: 'solution';
+    solution: {
+        task_id: number;
+        text?: string;
+        images?: TopicImage[];
+    };
+}
+
+interface UserInfo {
+    chat: {
+        identifier: string;
+    };
+    user: {
+        avatar_url: string;
+        introduction: string;
+        location: string;
+        name: string;
+        unique_id: string;
+        user_id: number;
+    };
+    user_specific: {
+        followed: boolean;
+    };
+}
+
+interface GroupInfo {
+    group: {
+        admin_ids: number[];
+        alive_time: string;
+        background_url: string;
+        category: {
+            category_id: number;
+            title: string;
+        };
+        create_time: string;
+        description: string;
+        group_id: number;
+        guest_ids: number[];
+        name: string;
+    };
+}
+
+export interface TopicImage {
+    image_id: number;
+    large: {
+        height: number;
+        width: number;
+        url: string;
+    };
+    original: {
+        height: number;
+        width: number;
+        url: string;
+        size: number;
+    };
+    thumbnail: {
+        height: number;
+        width: number;
+        url: string;
+    };
+    type: string;
+}
+
+export type Topic = TalkTopic | QATopic | TaskTopic | SolutionTopic;
+
+export type UserInfoResponse = BasicResponse<UserInfo>;
+
+export type GroupInfoResponse = BasicResponse<GroupInfo>;
+
+export type TopicsResponse = BasicResponse<{
+    topics: Topic[];
+}>;

package/lib/routes/zsxq/user.ts

@@ -0,0 +1,58 @@
+import type { Data, Route } from '@/types';
+import type { Context } from 'hono';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import type { TopicsResponse, UserInfoResponse } from './types';
+import { customFetch, generateTopicDataItem } from './utils';
+
+export const route: Route = {
+    name: '用户足迹',
+    categories: ['social-media'],
+    path: '/user/:id',
+    example: '/zsxq/user/2414218251',
+    parameters: {
+        id: '用户id，从网页端url中获取',
+    },
+    maintainers: ['KarasuShin'],
+    radar: [
+        {
+            source: ['wx.zsxq.com/dweb2/index/footprint/:id'],
+        },
+    ],
+    features: {
+        requireConfig: [
+            {
+                name: 'ZSXQ_ACCESS_TOKEN',
+                description:
+                    '知识星球访问令牌,获取方式：\n1. 登录知识星球网页版\n2. 打开浏览器开发者工具，切换到 Application 面板\n3. 点击侧边栏中的Storage -> Cookies -> https://wx.zsxq.com\n4. 复制 Cookie 中的 zsxq_access_token 值',
+            },
+        ],
+    },
+    handler,
+};
+
+async function handler(ctx: Context): Promise<Data> {
+    const uid = ctx.req.param('id');
+    const accessToken = config.zsxq.accessToken;
+
+    if (!accessToken) {
+        throw new ConfigNotFoundError('该 RSS 源由于配置不正确而被禁用：令牌丢失。');
+    }
+
+    let count = Number(ctx.req.query('limit')) || 20;
+    if (count > 30) {
+        count = 30;
+    }
+
+    const userInfo = await customFetch<UserInfoResponse>(`/users/${uid}`);
+
+    const { topics } = await customFetch<TopicsResponse>(`/users/${uid}/topics/footprint?count=${count}`);
+
+    return {
+        title: `知识星球 - ${userInfo.user.name}`,
+        description: userInfo.user.introduction,
+        image: userInfo.user.avatar_url,
+        link: `https://wx.zsxq.com/dweb2/index/footprint/${uid}`,
+        item: generateTopicDataItem(topics),
+    };
+}

package/lib/routes/zsxq/utils.ts

@@ -0,0 +1,70 @@
+import got from '@/utils/got';
+import type { TopicImage, Topic, BasicResponse } from './types';
+import { parseDate } from '@/utils/parse-date';
+import { config } from '@/config';
+import type { DataItem } from '@/types';
+
+export async function customFetch<T extends BasicResponse<any>>(path: string, retryCount = 0): Promise<T['resp_data']> {
+    const apiUrl = 'https://api.zsxq.com/v2';
+
+    const response = await got(apiUrl + path, {
+        headers: {
+            cookie: `zsxq_access_token=${config.zsxq.accessToken};`,
+        },
+    });
+    const { succeeded, code, resp_data } = response.data as T;
+    if (succeeded) {
+        return resp_data;
+    }
+    // sometimes the request will fail with code 1059, retry will solve the problem
+    if (code === 1059 && retryCount < 3) {
+        return customFetch(path, ++retryCount);
+    }
+    throw new Error('something wrong');
+}
+
+function parseTopicContent(text: string = '', images: TopicImage[] = []) {
+    let result = text.replaceAll('\n', '<br>');
+    result = result.replaceAll(/<e type="web" href="(.*?)" title="(.*?)" \/>/g, (_, p1, p2) => `<a href=${decodeURIComponent(p1)}>${decodeURIComponent(p2)}</a>`);
+    result = result.replaceAll(/<e type="hashtag".*?title="(.*?)" \/>/g, (_, p1) => {
+        const title = decodeURIComponent(p1);
+        return `<span>${title}</span>`;
+    });
+    result += images.map((image) => `<img src="${image.original?.url ?? image.large?.url ?? image.thumbnail?.url}">`).join('<br>');
+    return result;
+}
+
+export function generateTopicDataItem(topics: Topic[]): DataItem[] {
+    return topics.map((topic) => {
+        let description: string | undefined;
+        let title = '';
+        switch (topic.type) {
+            case 'talk':
+                title = topic.talk?.text?.split('\n')[0] ?? '文章';
+                description = parseTopicContent(topic.talk?.text, topic.talk?.images);
+                break;
+            case 'q&a':
+                title = topic.question?.text?.split('\n')[0] ?? '问答';
+                description = parseTopicContent(topic.question?.text, topic.question?.images);
+                if (topic.answered) {
+                    description += '<br><br>';
+                    description += parseTopicContent(topic.answer?.text, topic.answer?.images);
+                }
+                break;
+            case 'task':
+                title = topic.task?.text?.split('\n')[0] ?? '作业';
+                description = parseTopicContent(topic.task?.text, topic.task?.images);
+                break;
+            case 'solution':
+                title = topic.solution?.text?.split('\n')[0] ?? '写作业';
+                description = parseTopicContent(topic.solution?.text, topic.solution?.images);
+                break;
+            default:
+        }
+        return {
+            title: topic.title ?? title,
+            description,
+            pubDate: parseDate(topic.create_time),
+        };
+    });
+}

package/lib/setup.test.ts

@@ -2,6 +2,29 @@ import { afterAll, afterEach } from 'vitest';
 import { setupServer } from 'msw/node';
 import { http, HttpResponse } from 'msw';
 
+const genWeChatMpPage = (rich_media_content: string, scripts: string[] | string) => {
+    if (!Array.isArray(scripts)) {
+        scripts = [scripts];
+    }
+    let pageHtml = `
+<meta name="description" content="summary" />
+<meta name="author" content="author" />
+<meta property="og:title" content="title" />
+<meta property="og:image" content="https://mmbiz.qpic.cn/rsshub_test/og_img_1/0?wx_fmt=jpeg" />
+<meta property="twitter:card" content="summary" />
+<div class="rich_media_content" id="js_content" style="visibility: hidden;">
+${rich_media_content}
+</div>
+<div class="wx_follow_nickname">mpName</div>`;
+    for (const script of scripts) {
+        pageHtml += `
+<script type="text/javascript" nonce="000000000">
+${script}
+</script>`;
+    }
+    return pageHtml;
+};
+
 const server = setupServer(
     http.post(`https://api.openai.mock/v1/chat/completions`, () =>
         HttpResponse.json({
@@ -33,21 +56,169 @@ const server = setupServer(
             </ul>
         </div>`)
     ),
-    http.get(`https://mp.weixin.qq.com/rsshub_test/wechatMp_fetchArticle`, () =>
+    http.get(`https://mp.weixin.qq.com/rsshub_test/appMsg`, () =>
+        HttpResponse.text(
+            genWeChatMpPage(
+                `
+description
+<iframe class="video_iframe rich_pages" data-ratio="1.7777777777777777" data-w="864"data-src="https://v.qq.com/rsshub_test/?vid=fake"></iframe>
+<mpvoice name="title" voice_encode_fileid="rsshub_test"></mpvoice>
+`,
+                `
+var item_show_type = "0";
+var real_item_show_type = "0";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+var msg_source_url = "https://mp.weixin.qq.com/rsshub_test/fake";
+window.ip_wording = {
+  countryName: '中国',
+  countryId: '156',
+  provinceName: '福建',
+  provinceId: '',
+  cityName: '',
+  cityId: ''
+};`
+            )
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/rsshub_test/img`, () =>
+        HttpResponse.text(
+            genWeChatMpPage('fake_description', [
+                `
+var item_show_type = "8";
+var real_item_show_type = "8";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+`,
+                `
+window.picture_page_info_list = [
+{
+  cdn_url: 'https://mmbiz.qpic.cn/rsshub_test/fake_img_1/0?wx_fmt=jpeg',
+},
+{
+  cdn_url: 'https://mmbiz.qpic.cn/rsshub_test/fake_img_2/0?wx_fmt=jpeg',
+},
+].slice(0, 20);
+`,
+            ])
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/rsshub_test/audio`, () =>
+        HttpResponse.text(
+            genWeChatMpPage('fake_description', [
+                `
+var item_show_type = "7";
+var real_item_show_type = "7";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+`,
+                `
+reportOpt = {
+  voiceid: "",
+  uin: "",
+  biz: "",
+  mid: "",
+  idx: ""
+};
+window.cgiData = {
+  voiceid: "rsshub_test_voiceid_1",
+  duration: "6567" * 1,
+};
+`,
+            ])
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/rsshub_test/video`, () =>
+        HttpResponse.text(
+            genWeChatMpPage(
+                'fake_description',
+                `
+var item_show_type = "5";
+var real_item_show_type = "5";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+`
+            )
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/rsshub_test/fallback`, () =>
+        HttpResponse.text(
+            genWeChatMpPage(
+                'fake_description',
+                `
+var item_show_type = "99988877";
+var real_item_show_type = "99988877";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+`
+            )
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/s/rsshub_test`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/rsshub_test/fallback`)),
+    http.get(`https://mp.weixin.qq.com/s?__biz=rsshub_test&mid=1&idx=1&sn=1`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/rsshub_test/fallback`)),
+    http.get(`https://mp.weixin.qq.com/mp/rsshub_test/waf`, () =>
+        HttpResponse.text(
+            `<html>
+<head>
+<title>Title</title>
+<script>console.log</script>
+</head>
+<body class="zh_CN " ontouchstart="">
+<script>console.log</script>
+<style>.style{}</style>
+<div class="weui-msg">
+  <div id="tips" style="display:none;" class="top_tips warning"></div>
+        <div class="weui-msg__icon-area">
+      <i class="weui-icon-info-circle weui-icon_msg"></i>
+    </div>
+    <div class="weui-msg__text-area pc-area">
+        <h2 class="weui-msg__title">环境异常</h2>
+        <p class="weui-msg__desc">当前环境异常，完成验证后即可继续访问。</p>
+    </div>
+    <div class="weui-msg__opr-area">
+      <p class="weui-btn-area">
+        <a class="weui-btn weui-btn_primary" id="js_verify">去验证</a>
+      </p>
+    </div>
+</div>
+</body></html>`
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/s/rsshub_test_hit_waf`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/mp/rsshub_test/waf`)),
+    http.get(`https://mp.weixin.qq.com/s/unknown_page`, () =>
+        HttpResponse.text(
+            `<html>
+<head>
+<title>Title</title>
+<script>console.log</script>
+</head>
+<body class="zh_CN " ontouchstart="">
+<script>console.log</script>
+<style>.style{}</style>
+<p>
+Unknown paragraph
+</p>
+</body></html>`
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/s/deleted_page`, () =>
         HttpResponse.text(
-            '\n' +
-                '<meta name="description" content="summary" />\n' +
-                '<meta name="author" content="author" />\n' +
-                '<meta property="og:title" content="title" />\n' +
-                '<meta property="twitter:card" content="summary" />\n' +
-                '<div class="rich_media_content" id="js_content" style="visibility: hidden;">description</div>\n' +
-                '<div class="profile_inner"><strong class="profile_nickname">mpName</strong></div>\n' +
-                '<script type="text/javascript" nonce="000000000">\n' +
-                'var appmsg_type = "9";\n' +
-                `var ct = "${1_636_626_300}";\n` +
-                '</script>'
+            `<html>
+<head>
+<title>Title</title>
+<script>console.log</script>
+</head>
+<body class="zh_CN " ontouchstart="">
+<script>console.log</script>
+<style>.style{}</style>
+<p>
+该内容已被发布者删除
+</p>
+</body></html>`
         )
     ),
+    http.get(`https://mp.weixin.qq.com/s/rsshub_test_redirect_no_location`, () => HttpResponse.text('', { status: 302 })),
+    http.get(`https://mp.weixin.qq.com/s/rsshub_test_recursive_redirect`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/s/rsshub_test_recursive_redirect`)),
     http.get(`http://rsshub.test/headers`, ({ request }) =>
         HttpResponse.json({
             ...Object.fromEntries(request.headers.entries()),

package/lib/utils/render.ts

@@ -1,4 +1,4 @@
-export { default as rss3Ums } from '@/views/rss3-ums';
+export { default as rss3 } from '@/views/rss3';
 export { default as json } from '@/views/json';
 export { default as RSS } from '@/views/rss';
 export { default as Atom } from '@/views/atom';

package/lib/utils/request-rewriter/get.ts

@@ -59,7 +59,14 @@ const getWrappedGet: <T extends Get>(origin: T) => T = (origin) =>
         if (!options.agent && proxy.agent) {
             const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
 
-            if (proxyRegex.test(url.toString()) && url.protocol.startsWith('http') && url.host !== proxy.proxyUrlHandler?.host) {
+            if (
+                proxyRegex.test(url.toString()) &&
+                url.protocol.startsWith('http') &&
+                url.host !== proxy.proxyUrlHandler?.host &&
+                url.host !== 'localhost' &&
+                !url.host.startsWith('127.') &&
+                !(config.puppeteerWSEndpoint?.includes(url.host) ?? false)
+            ) {
                 options.agent = proxy.agent;
             }
         }