rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/utils/request-rewriter/fetch.ts

@@ -2,8 +2,19 @@ import logger from '@/utils/logger';
 import { config } from '@/config';
 import undici, { Request, RequestInfo, RequestInit } from 'undici';
 import proxy from '@/utils/proxy';
+import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
 
-const wrappedFetch: typeof undici.fetch = (input: RequestInfo, init?: RequestInit) => {
+const limiter = new RateLimiterMemory({
+    points: 10,
+    duration: 1,
+    execEvenly: true,
+});
+
+const limiterQueue = new RateLimiterQueue(limiter, {
+    maxQueueSize: 5000,
+});
+
+const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: RequestInit) => {
     const request = new Request(input, init);
     const options: RequestInit = {};
 
@@ -29,8 +40,14 @@ const wrappedFetch: typeof undici.fetch = (input: RequestInfo, init?: RequestIni
         }
     }
 
+    let isRetry = false;
+    if (request.headers.get('x-prefer-proxy')) {
+        isRetry = true;
+        request.headers.delete('x-prefer-proxy');
+    }
+
     // proxy
-    if (!options.dispatcher && proxy.dispatcher) {
+    if (!init?.dispatcher && proxy.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
         const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
         let urlHandler;
         try {
@@ -41,9 +58,11 @@ const wrappedFetch: typeof undici.fetch = (input: RequestInfo, init?: RequestIni
 
         if (proxyRegex.test(request.url) && request.url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
             options.dispatcher = proxy.dispatcher;
+            logger.debug(`Proxying request: ${request.url}`);
         }
     }
 
+    await limiterQueue.removeTokens(1);
     return undici.fetch(request, options);
 };
 

package/lib/utils/request-rewriter/get.ts

@@ -59,7 +59,14 @@ const getWrappedGet: <T extends Get>(origin: T) => T = (origin) =>
         if (!options.agent && proxy.agent) {
             const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
 
-            if (proxyRegex.test(url.toString()) && url.protocol.startsWith('http') && url.host !== proxy.proxyUrlHandler?.host) {
+            if (
+                proxyRegex.test(url.toString()) &&
+                url.protocol.startsWith('http') &&
+                url.host !== proxy.proxyUrlHandler?.host &&
+                url.host !== 'localhost' &&
+                !url.host.startsWith('127.') &&
+                !(config.puppeteerWSEndpoint?.includes(url.host) ?? false)
+            ) {
                 options.agent = proxy.agent;
             }
         }

package/lib/utils/request-rewriter.test.ts

@@ -134,4 +134,18 @@ describe('request-rewriter', () => {
             expect(options?.agent).toBeUndefined();
         }
     });
+
+    it('rate limiter', async () => {
+        const time = Date.now();
+        await Promise.all(
+            Array.from({ length: 20 }).map(async () => {
+                try {
+                    await fetch('http://rsshub.test/headers');
+                } catch {
+                    // ignore
+                }
+            })
+        );
+        expect(Date.now() - time).toBeGreaterThan(1500);
+    });
 });

package/lib/utils/wechat-mp.test.ts

@@ -1,6 +1,29 @@
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi, afterEach } from 'vitest';
 import { load } from 'cheerio';
-import { fixArticleContent, fetchArticle, finishArticleItem, normalizeUrl } from '@/utils/wechat-mp';
+import Parser from 'rss-parser';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+import { exportedForTestingOnly, WeChatMpError, fetchArticle, finishArticleItem, fixArticleContent, normalizeUrl } from '@/utils/wechat-mp';
+const { toggleWerror, ExtractMetadata, showTypeMapReverse } = exportedForTestingOnly;
+
+vi.mock('@/utils/request-rewriter', () => ({ default: null }));
+const { default: app } = await import('@/app');
+const parser = new Parser();
+
+afterEach(() => toggleWerror(false));
+
+const expectedItem: {
+    title: string;
+    summary: string;
+    author: string;
+    mpName: string;
+    link: string;
+} = {
+    title: 'title',
+    summary: 'summary',
+    author: 'author',
+    mpName: 'mpName',
+    link: '', // to be filled
+};
 
 // date from the cache will be an ISO8601 string, so we need to use this function
 const compareDate = (date1, date2) => {
@@ -8,8 +31,227 @@ const compareDate = (date1, date2) => {
     date2 = typeof date2 === 'string' ? new Date(date2) : date2;
     return date1.getTime() === date2.getTime();
 };
+const genScriptHtmlStr = (script: string) => `
+    <html lang="">
+        <script type="text/javascript" nonce="123456789">
+        ${script}
+        </script>
+    </html>
+`;
+const testFetchArticleFinishArticleItem = async (path: string, { setMpNameAsAuthor = false, skipLink = false } = {}) => {
+    const ct = 1_636_626_300;
+    const httpsUrl = `https://mp.weixin.qq.com/rsshub_test${path}`;
+    const httpUrl = 'http' + httpsUrl.slice(5);
+
+    const expectedDate = new Date(ct * 1000);
+
+    const expectedItem_ = {
+        ...expectedItem,
+        link: httpsUrl,
+    };
+
+    const fetchArticleItem = await fetchArticle(httpUrl);
+    expect(compareDate(fetchArticleItem.pubDate, expectedDate)).toBe(true);
+    expect(fetchArticleItem).toMatchObject(expectedItem_);
+
+    const ToBeFinishedArticleItem = { link: httpUrl };
+    const expectedFinishedArticleItem = { ...fetchArticleItem };
+    expectedFinishedArticleItem.author = setMpNameAsAuthor ? <string>expectedFinishedArticleItem.mpName : expectedFinishedArticleItem.author;
+    expectedFinishedArticleItem.link = skipLink ? ToBeFinishedArticleItem.link : expectedFinishedArticleItem.link;
+
+    const finishedArticleItem = await finishArticleItem(ToBeFinishedArticleItem, setMpNameAsAuthor, skipLink);
+    expect(compareDate(finishedArticleItem.pubDate, fetchArticleItem.pubDate)).toBe(true);
+    delete expectedFinishedArticleItem.pubDate;
+    expect(finishedArticleItem).toMatchObject(expectedFinishedArticleItem);
+
+    return fetchArticleItem;
+};
 
 describe('wechat-mp', () => {
+    it('ExtractMetadata.common', () => {
+        expect(ExtractMetadata.common(load(''))).toStrictEqual({});
+
+        expect(
+            ExtractMetadata.common(
+                load(
+                    genScriptHtmlStr(`
+            window.fake_item_show_type = '5' || '';
+            window.fake_real_item_show_type = '5' || '';
+            window.fake_ct = '1713009660' || '';
+        `)
+                )
+            )
+        ).toMatchObject({});
+
+        expect(
+            ExtractMetadata.common(
+                load(
+                    genScriptHtmlStr(`
+            window.item_show_type = '5' || '';
+            window.real_item_show_type = '5' || '';
+            window.ct = '1713009660' || '';
+        `)
+                )
+            )
+        ).toMatchObject({
+            showType: showTypeMapReverse['5'],
+            realShowType: showTypeMapReverse['5'],
+            createTime: '1713009660',
+        });
+
+        expect(
+            ExtractMetadata.common(
+                load(
+                    genScriptHtmlStr(`
+            var item_show_type = "5";
+            var real_item_show_type = "5";
+            var ct = "1713009660";
+            var msg_source_url = 'https://mp.weixin.qq.com/rsshub_test/fake';
+        `)
+                )
+            )
+        ).toMatchObject({
+            showType: showTypeMapReverse['5'],
+            realShowType: showTypeMapReverse['5'],
+            createTime: '1713009660',
+            sourceUrl: 'https://mp.weixin.qq.com/rsshub_test/fake',
+        });
+
+        expect(
+            ExtractMetadata.common(
+                load(
+                    genScriptHtmlStr(`
+            var item_show_type = "998877665544332211";
+            var real_item_show_type = "112233445566778899";
+            var ct = "1713009660";
+        `)
+                )
+            )
+        ).toMatchObject({
+            showType: '998877665544332211',
+            realShowType: '112233445566778899',
+            createTime: '1713009660',
+        });
+    });
+    it('ExtractMetadata.img', () => {
+        expect(ExtractMetadata.img(load(''))).toStrictEqual({});
+
+        expect(
+            ExtractMetadata.img(
+                load(
+                    genScriptHtmlStr(`
+            window.picture_page_info_list = [
+            {
+              cdn_url: 'https://mmbiz.qpic.cn/rsshub_test/fake_img_1/0?wx_fmt=jpeg',
+            },
+            {
+              cdn_url: 'https://mmbiz.qpic.cn/rsshub_test/fake_img_2/0?wx_fmt=jpeg',
+            },
+            ].slice(0, 20);
+        `)
+                )
+            )
+        ).toMatchObject({
+            imgUrls: ['https://mmbiz.qpic.cn/rsshub_test/fake_img_1/0?wx_fmt=jpeg', 'https://mmbiz.qpic.cn/rsshub_test/fake_img_2/0?wx_fmt=jpeg'],
+        });
+    });
+    it('ExtractMetadata.audio', () => {
+        expect(ExtractMetadata.audio(load(''))).toStrictEqual({});
+
+        expect(
+            ExtractMetadata.audio(
+                load(
+                    genScriptHtmlStr(`
+            reportOpt = {
+              voiceid: "",
+              uin: "",
+              biz: "",
+              mid: "",
+              idx: ""
+            };
+        `)
+                )
+            )
+        ).toMatchObject({});
+
+        expect(
+            ExtractMetadata.audio(
+                load(
+                    genScriptHtmlStr(`
+            window.cgiData = {
+              voiceid: "rsshub_test_voiceid_1",
+              duration: "6567" * 1,
+            };
+        `)
+                )
+            )
+        ).toMatchObject({
+            voiceId: 'rsshub_test_voiceid_1',
+            duration: '6567',
+        });
+
+        expect(
+            ExtractMetadata.audio(
+                load(
+                    genScriptHtmlStr(`
+            window.cgiData = {
+              voiceid: "rsshub_test_voiceid_1",
+            };
+        `)
+                )
+            )
+        ).toMatchObject({
+            voiceId: 'rsshub_test_voiceid_1',
+            duration: null,
+        });
+
+        expect(
+            ExtractMetadata.audio(
+                load(
+                    genScriptHtmlStr(`
+            reportOpt = {
+              voiceid: "",
+              uin: "",
+              biz: "",
+              mid: "",
+              idx: ""
+            };
+            window.cgiData = {
+              voiceid: "rsshub_test_voiceid_1",
+              duration: "6567" * 1,
+            };
+        `)
+                )
+            )
+        ).toMatchObject({
+            voiceId: 'rsshub_test_voiceid_1',
+            duration: '6567',
+        });
+    });
+    it('ExtractMetadata.location', () => {
+        expect(ExtractMetadata.location(load(''))).toStrictEqual({});
+
+        expect(
+            ExtractMetadata.location(
+                load(
+                    genScriptHtmlStr(`
+            window.ip_wording = {
+              countryName: '中国',
+              countryId: '156',
+              provinceName: '广东',
+              provinceId: '',
+              cityName: '',
+              cityId: ''
+            };
+        `)
+                )
+            )
+        ).toMatchObject({
+            countryName: '中国',
+            provinceName: '广东',
+            cityName: '',
+        });
+    });
     it('fixArticleContent', () => {
         const divHeader = '<div class="rich_media_content " id="js_content">';
         const divFooter = '</div>';
@@ -84,41 +326,178 @@ describe('wechat-mp', () => {
         expect(normalizeUrl(somethingElseWithHash.replace('https://', 'http://'))).toBe(somethingElse);
 
         const notWechatMp = 'https://im.not.wechat.mp/and/an/error/is/expected';
-        expect(() => normalizeUrl(notWechatMp)).toThrow();
+        expect(() => normalizeUrl(notWechatMp)).toThrow('URL host must be "mp.weixin.qq.com"');
         expect(normalizeUrl(notWechatMp, true)).toBe(notWechatMp);
+
+        const unknownSearchParam = mpArticleRoot + '?unknown=param';
+        toggleWerror(false);
+        expect(normalizeUrl(unknownSearchParam)).toBe(unknownSearchParam);
+        toggleWerror(true);
+        expect(() => normalizeUrl(unknownSearchParam)).toThrow('WarningAsError: unknown URL search parameters');
+
+        const unknownPath = mpRoot + '/unknown/path';
+        toggleWerror(false);
+        expect(normalizeUrl(unknownPath)).toBe(unknownPath);
+        toggleWerror(true);
+        expect(() => normalizeUrl(unknownPath, true)).toThrow('WarningAsError: unknown URL path');
+
+        const ampEscapedUrl = longUrl.replaceAll('&', '&amp;');
+        expect(normalizeUrl(ampEscapedUrl)).toBe(longUrlShortened);
     });
 
-    it('fetchArticle_&_finishArticleItem', async () => {
-        const ct = 1_636_626_300;
-        const httpsUrl = 'https://mp.weixin.qq.com/rsshub_test/wechatMp_fetchArticle';
-        const httpUrl = httpsUrl.replace(/^https:\/\//, 'http://');
+    it('fetchArticle_&_finishArticleItem_appMsg', async () => {
+        const fetchArticleItem = await testFetchArticleFinishArticleItem('/appMsg');
+        const $ = load(fetchArticleItem.description);
+        expect($('iframe').attr()).toMatchObject({
+            src:
+                'https://v.qq.com/txp/iframe/player.html?origin=https%3A%2F%2Fmp.weixin.qq.com' +
+                '&containerId=js_tx_video_container_0.3863487104715233&vid=fake&width=677&height=380.8125' +
+                '&autoplay=false&allowFullScreen=true&chid=17&full=true&show1080p=false&isDebugIframe=false',
+            width: '677',
+            height: '380.8125',
+        });
+        expect($('audio').attr()).toMatchObject({
+            src: 'https://res.wx.qq.com/voice/getvoice?mediaid=rsshub_test',
+            title: 'title',
+        });
+        expect($('a').attr()).toMatchObject({
+            href: 'https://mp.weixin.qq.com/rsshub_test/fake',
+        });
+        expect(fetchArticleItem.description).toContain('description');
+        expect(fetchArticleItem.description).toContain('📍发表于：中国 福建');
+        expect(fetchArticleItem.description).toContain('🔗️ 阅读原文');
+    });
 
-        const expectedItem: {
-            title: string;
-            summary: string;
-            author: string;
-            description: string;
-            mpName?: string;
-            link: string;
-        } = {
+    it('fetchArticle_&_finishArticleItem_img', async () => {
+        const fetchArticleItem = await testFetchArticleFinishArticleItem('/img');
+        const $ = load(fetchArticleItem.description);
+        expect($.text()).toBe('summary');
+        expect($('img:nth-of-type(1)').attr()).toMatchObject({
+            src: 'https://mmbiz.qpic.cn/rsshub_test/fake_img_1/0?wx_fmt=jpeg',
+        });
+        expect($('img:nth-of-type(2)').attr()).toMatchObject({
+            src: 'https://mmbiz.qpic.cn/rsshub_test/fake_img_2/0?wx_fmt=jpeg',
+        });
+    });
+
+    it('fetchArticle_&_finishArticleItem_audio', async () => {
+        const fetchArticleItem = await testFetchArticleFinishArticleItem('/audio');
+        const $ = load(fetchArticleItem.description);
+        expect($.text()).toBe('summary');
+        expect($('audio').attr()).toMatchObject({
+            controls: '',
+            src: 'https://res.wx.qq.com/voice/getvoice?mediaid=rsshub_test_voiceid_1',
+            style: 'width:100%',
             title: 'title',
-            summary: 'summary',
-            author: 'author',
-            description: 'description',
-            mpName: 'mpName',
-            link: httpsUrl,
+        });
+        expect(fetchArticleItem).toMatchObject({
+            enclosure_type: 'audio/mp3',
+            enclosure_url: 'https://res.wx.qq.com/voice/getvoice?mediaid=rsshub_test_voiceid_1',
+            itunes_duration: '6567',
+        });
+    });
+
+    it('fetchArticle_&_finishArticleItem_video', async () => {
+        const fetchArticleItem = await testFetchArticleFinishArticleItem('/video');
+        const $ = load(fetchArticleItem.description);
+        expect($.text()).toBe('summary');
+        expect($('img').attr()).toMatchObject({
+            src: 'https://mmbiz.qpic.cn/rsshub_test/og_img_1/0?wx_fmt=jpeg',
+        });
+    });
+
+    it('fetchArticle_&_finishArticleItem_fallback', async () => {
+        const fetchArticleItem = await testFetchArticleFinishArticleItem('/fallback');
+        const $ = load(fetchArticleItem.description);
+        expect($.text()).toBe('summary');
+        expect($('img').attr()).toMatchObject({
+            src: 'https://mmbiz.qpic.cn/rsshub_test/og_img_1/0?wx_fmt=jpeg',
+        });
+    });
+
+    it('finishArticleItem_param', async () => {
+        await testFetchArticleFinishArticleItem('/fallback', { setMpNameAsAuthor: false, skipLink: false });
+        await testFetchArticleFinishArticleItem('/fallback', { setMpNameAsAuthor: true, skipLink: false });
+        await testFetchArticleFinishArticleItem('/fallback', { setMpNameAsAuthor: false, skipLink: true });
+        await testFetchArticleFinishArticleItem('/fallback', { setMpNameAsAuthor: true, skipLink: true });
+    });
+
+    it('hit_waf', async () => {
+        try {
+            await fetchArticle('https://mp.weixin.qq.com/s/rsshub_test_hit_waf');
+            expect.unreachable('Should throw an error');
+        } catch (error) {
+            expect(error).toBeInstanceOf(WeChatMpError);
+            expect((<WeChatMpError>error).message).not.toContain('console.log');
+            expect((<WeChatMpError>error).message).not.toContain('.style');
+            expect((<WeChatMpError>error).message).not.toContain('Consider raise an issue');
+            expect((<WeChatMpError>error).message).toContain('request blocked by WAF:');
+            expect((<WeChatMpError>error).message).toContain('/mp/rsshub_test/waf');
+            expect((<WeChatMpError>error).message).toContain('Title');
+            expect((<WeChatMpError>error).message).toContain('环境异常');
+        }
+    });
+
+    it('unknown_page', async () => {
+        const unknownPageUrl = 'https://mp.weixin.qq.com/s/unknown_page';
+        try {
+            await fetchArticle(unknownPageUrl);
+            expect.unreachable('Should throw an error');
+        } catch (error) {
+            expect(error).toBeInstanceOf(WeChatMpError);
+            expect((<WeChatMpError>error).message).not.toContain('console.log');
+            expect((<WeChatMpError>error).message).not.toContain('.style');
+            expect((<WeChatMpError>error).message).toContain('Consider raise an issue');
+            expect((<WeChatMpError>error).message).toContain('unknown page,');
+            expect((<WeChatMpError>error).message).toContain('Title Unknown paragraph');
+            expect((<WeChatMpError>error).message).toContain(unknownPageUrl);
+        }
+    });
+
+    it('deleted_page', async () => {
+        const deletedPageUrl = 'https://mp.weixin.qq.com/s/deleted_page';
+
+        try {
+            await fetchArticle(deletedPageUrl);
+            expect.unreachable('Should throw an error');
+        } catch (error) {
+            expect(error).toBeInstanceOf(WeChatMpError);
+            expect((<WeChatMpError>error).message).not.toContain('console.log');
+            expect((<WeChatMpError>error).message).not.toContain('.style');
+            expect((<WeChatMpError>error).message).not.toContain('Consider raise an issue');
+            expect((<WeChatMpError>error).message).toContain('deleted by author:');
+            expect((<WeChatMpError>error).message).toContain('Title 该内容已被发布者删除');
+            expect((<WeChatMpError>error).message).toContain(deletedPageUrl);
+        }
+    });
+
+    it('redirect', () => {
+        expect(fetchArticle('https://mp.weixin.qq.com/s/rsshub_test_redirect_no_location')).rejects.toThrow('redirect without location');
+        expect(fetchArticle('https://mp.weixin.qq.com/s/rsshub_test_recursive_redirect')).rejects.toThrow('too many redirects');
+    });
+
+    it('route_test', async () => {
+        try {
+            await app.request('/test/wechat-mp');
+        } catch (error) {
+            expect(error).toBeInstanceOf(InvalidParameterError);
+        }
+
+        const responseShort = await app.request('/test/wechat-mp/rsshub_test');
+        const parsedShort = await parser.parseString(await responseShort.text());
+        const expectedItemShort = {
+            author: expectedItem.author,
+            title: expectedItem.title,
+            link: 'https://mp.weixin.qq.com/s/rsshub_test',
+        };
+        expect(parsedShort.items[0]).toMatchObject(expectedItemShort);
+
+        const responseLong = await app.request('/test/wechat-mp/__biz=rsshub_test&mid=1&idx=1&sn=1');
+        const parsedLong = await parser.parseString(await responseLong.text());
+        const expectedItemLong = {
+            ...expectedItemShort,
+            link: 'https://mp.weixin.qq.com/s?__biz=rsshub_test&mid=1&idx=1&sn=1',
         };
-        const expectedDate = new Date(ct * 1000);
-
-        const fetchArticleItem = await fetchArticle(httpUrl);
-        expect(compareDate(fetchArticleItem.pubDate, expectedDate)).toBe(true);
-        delete fetchArticleItem.pubDate;
-        expect(fetchArticleItem).toEqual(expectedItem);
-
-        delete expectedItem.mpName;
-        const finishedArticleItem = await finishArticleItem({ link: httpUrl });
-        expect(compareDate(finishedArticleItem.pubDate, expectedDate)).toBe(true);
-        delete finishedArticleItem.pubDate;
-        expect(finishedArticleItem).toEqual(expectedItem);
+        expect(parsedLong.items[0]).toMatchObject(expectedItemLong);
     });
 });