rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/setup.test.ts

@@ -2,13 +2,36 @@ import { afterAll, afterEach } from 'vitest';
 import { setupServer } from 'msw/node';
 import { http, HttpResponse } from 'msw';
 
+const genWeChatMpPage = (rich_media_content: string, scripts: string[] | string) => {
+    if (!Array.isArray(scripts)) {
+        scripts = [scripts];
+    }
+    let pageHtml = `
+<meta name="description" content="summary" />
+<meta name="author" content="author" />
+<meta property="og:title" content="title" />
+<meta property="og:image" content="https://mmbiz.qpic.cn/rsshub_test/og_img_1/0?wx_fmt=jpeg" />
+<meta property="twitter:card" content="summary" />
+<div class="rich_media_content" id="js_content" style="visibility: hidden;">
+${rich_media_content}
+</div>
+<div class="wx_follow_nickname">mpName</div>`;
+    for (const script of scripts) {
+        pageHtml += `
+<script type="text/javascript" nonce="000000000">
+${script}
+</script>`;
+    }
+    return pageHtml;
+};
+
 const server = setupServer(
     http.post(`https://api.openai.mock/v1/chat/completions`, () =>
         HttpResponse.json({
             choices: [
                 {
                     message: {
-                        content: 'Summary of the article.',
+                        content: 'AI processed content.',
                     },
                 },
             ],
@@ -33,21 +56,169 @@ const server = setupServer(
             </ul>
         </div>`)
     ),
-    http.get(`https://mp.weixin.qq.com/rsshub_test/wechatMp_fetchArticle`, () =>
+    http.get(`https://mp.weixin.qq.com/rsshub_test/appMsg`, () =>
+        HttpResponse.text(
+            genWeChatMpPage(
+                `
+description
+<iframe class="video_iframe rich_pages" data-ratio="1.7777777777777777" data-w="864"data-src="https://v.qq.com/rsshub_test/?vid=fake"></iframe>
+<mpvoice name="title" voice_encode_fileid="rsshub_test"></mpvoice>
+`,
+                `
+var item_show_type = "0";
+var real_item_show_type = "0";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+var msg_source_url = "https://mp.weixin.qq.com/rsshub_test/fake";
+window.ip_wording = {
+  countryName: '中国',
+  countryId: '156',
+  provinceName: '福建',
+  provinceId: '',
+  cityName: '',
+  cityId: ''
+};`
+            )
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/rsshub_test/img`, () =>
+        HttpResponse.text(
+            genWeChatMpPage('fake_description', [
+                `
+var item_show_type = "8";
+var real_item_show_type = "8";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+`,
+                `
+window.picture_page_info_list = [
+{
+  cdn_url: 'https://mmbiz.qpic.cn/rsshub_test/fake_img_1/0?wx_fmt=jpeg',
+},
+{
+  cdn_url: 'https://mmbiz.qpic.cn/rsshub_test/fake_img_2/0?wx_fmt=jpeg',
+},
+].slice(0, 20);
+`,
+            ])
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/rsshub_test/audio`, () =>
+        HttpResponse.text(
+            genWeChatMpPage('fake_description', [
+                `
+var item_show_type = "7";
+var real_item_show_type = "7";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+`,
+                `
+reportOpt = {
+  voiceid: "",
+  uin: "",
+  biz: "",
+  mid: "",
+  idx: ""
+};
+window.cgiData = {
+  voiceid: "rsshub_test_voiceid_1",
+  duration: "6567" * 1,
+};
+`,
+            ])
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/rsshub_test/video`, () =>
+        HttpResponse.text(
+            genWeChatMpPage(
+                'fake_description',
+                `
+var item_show_type = "5";
+var real_item_show_type = "5";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+`
+            )
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/rsshub_test/fallback`, () =>
+        HttpResponse.text(
+            genWeChatMpPage(
+                'fake_description',
+                `
+var item_show_type = "99988877";
+var real_item_show_type = "99988877";
+var appmsg_type = "9";
+var ct = "${1_636_626_300}";
+`
+            )
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/s/rsshub_test`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/rsshub_test/fallback`)),
+    http.get(`https://mp.weixin.qq.com/s?__biz=rsshub_test&mid=1&idx=1&sn=1`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/rsshub_test/fallback`)),
+    http.get(`https://mp.weixin.qq.com/mp/rsshub_test/waf`, () =>
+        HttpResponse.text(
+            `<html>
+<head>
+<title>Title</title>
+<script>console.log</script>
+</head>
+<body class="zh_CN " ontouchstart="">
+<script>console.log</script>
+<style>.style{}</style>
+<div class="weui-msg">
+  <div id="tips" style="display:none;" class="top_tips warning"></div>
+        <div class="weui-msg__icon-area">
+      <i class="weui-icon-info-circle weui-icon_msg"></i>
+    </div>
+    <div class="weui-msg__text-area pc-area">
+        <h2 class="weui-msg__title">环境异常</h2>
+        <p class="weui-msg__desc">当前环境异常，完成验证后即可继续访问。</p>
+    </div>
+    <div class="weui-msg__opr-area">
+      <p class="weui-btn-area">
+        <a class="weui-btn weui-btn_primary" id="js_verify">去验证</a>
+      </p>
+    </div>
+</div>
+</body></html>`
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/s/rsshub_test_hit_waf`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/mp/rsshub_test/waf`)),
+    http.get(`https://mp.weixin.qq.com/s/unknown_page`, () =>
+        HttpResponse.text(
+            `<html>
+<head>
+<title>Title</title>
+<script>console.log</script>
+</head>
+<body class="zh_CN " ontouchstart="">
+<script>console.log</script>
+<style>.style{}</style>
+<p>
+Unknown paragraph
+</p>
+</body></html>`
+        )
+    ),
+    http.get(`https://mp.weixin.qq.com/s/deleted_page`, () =>
         HttpResponse.text(
-            '\n' +
-                '<meta name="description" content="summary" />\n' +
-                '<meta name="author" content="author" />\n' +
-                '<meta property="og:title" content="title" />\n' +
-                '<meta property="twitter:card" content="summary" />\n' +
-                '<div class="rich_media_content" id="js_content" style="visibility: hidden;">description</div>\n' +
-                '<div class="profile_inner"><strong class="profile_nickname">mpName</strong></div>\n' +
-                '<script type="text/javascript" nonce="000000000">\n' +
-                'var appmsg_type = "9";\n' +
-                `var ct = "${1_636_626_300}";\n` +
-                '</script>'
+            `<html>
+<head>
+<title>Title</title>
+<script>console.log</script>
+</head>
+<body class="zh_CN " ontouchstart="">
+<script>console.log</script>
+<style>.style{}</style>
+<p>
+该内容已被发布者删除
+</p>
+</body></html>`
         )
     ),
+    http.get(`https://mp.weixin.qq.com/s/rsshub_test_redirect_no_location`, () => HttpResponse.text('', { status: 302 })),
+    http.get(`https://mp.weixin.qq.com/s/rsshub_test_recursive_redirect`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/s/rsshub_test_recursive_redirect`)),
     http.get(`http://rsshub.test/headers`, ({ request }) =>
         HttpResponse.json({
             ...Object.fromEntries(request.headers.entries()),
@@ -57,6 +228,7 @@ const server = setupServer(
         const formData = await request.formData();
         return HttpResponse.json({
             test: formData.get('test'),
+            req: { headers: Object.fromEntries(request.headers.entries()) },
         });
     }),
     http.post(`http://rsshub.test/json-post`, async ({ request }) => {

package/lib/types.ts

@@ -3,6 +3,7 @@ import type { Context } from 'hono';
 // Make sure it's synchronise with scripts/workflow/data.ts
 // and lib/routes/rsshub/routes.ts
 type Category =
+    | 'popular'
     | 'social-media'
     | 'new-media'
     | 'traditional-media'
@@ -34,7 +35,13 @@ export type DataItem = {
     pubDate?: number | string | Date;
     link?: string;
     category?: string[];
-    author?: string | { name: string }[];
+    author?:
+        | string
+        | {
+              name: string;
+              url?: string;
+              avatar?: string;
+          }[];
     doi?: string;
     guid?: string;
     id?: string;
@@ -118,6 +125,15 @@ interface Namespace extends NamespaceItem {
 
 export type { Namespace };
 
+export enum ViewType {
+    Articles = 0,
+    SocialMedia = 1,
+    Pictures = 2,
+    Videos = 3,
+    Audios = 4,
+    Notifications = 5,
+}
+
 // route
 interface RouteItem {
     /**
@@ -154,7 +170,18 @@ interface RouteItem {
     /**
      * The description of the route parameters
      */
-    parameters?: Record<string, string>;
+    parameters?: Record<
+        string,
+        | string
+        | {
+              description: string;
+              default?: string;
+              options?: {
+                  value: string;
+                  label: string;
+              }[];
+          }
+    >;
 
     /**
      * Hints and additional explanations for users using this route, it will be appended after the route component, supports markdown
@@ -205,6 +232,11 @@ interface RouteItem {
      * The [RSSHub-Radar](https://github.com/DIYgod/RSSHub-Radar) rule of the route
      */
     radar?: RadarItem[];
+
+    /**
+     * The [Follow](https://github.com/RSSNext/follow) default view of the route, default to `ViewType.Articles`
+     */
+    view?: ViewType;
 }
 
 interface Route extends RouteItem {
@@ -241,6 +273,7 @@ export type RadarItem = {
      *
      * Using `target` as a function is deprecated in RSSHub-Radar 2.0.19
      * @see https://github.com/DIYgod/RSSHub-Radar/commit/5a97647f900bb2bca792787a322b2b1ca512e40b#diff-f84e3c1e16af314bc4ed7c706d7189844663cde9b5142463dc5c0db34c2e8d54L10
+     * @see https://github.com/DIYgod/RSSHub-Radar/issues/692
      */
     target?:
         | string
@@ -253,3 +286,9 @@ export type RadarItem = {
               document: Document
           ) => string);
 };
+
+export type RadarDomain = {
+    _name: string;
+} & {
+    [subdomain: string]: RadarItem[];
+};

package/lib/utils/cache/index.ts

@@ -63,6 +63,14 @@ if (config.cache.type === 'redis') {
 // plz, write these tips in comments!
 export default {
     ...cacheModule,
+    /**
+     * Try to get the cache. If the cache does not exist, the `getValueFunc` function will be called to get the data, and the data will be cached.
+     * @param key The key used to store and retrieve the cache. You can use `:` as a separator to create a hierarchy.
+     * @param getValueFunc A function that returns data to be cached when a cache miss occurs.
+     * @param maxAge The maximum age of the cache in seconds. This should left to the default value in most cases which is `CACHE_CONTENT_EXPIRE`.
+     * @param refresh Whether to renew the cache expiration time when the cache is hit. `true` by default.
+     * @returns
+     */
     tryGet: async (key: string, getValueFunc: () => Promise<string | Record<string, any>>, maxAge = config.cache.contentExpire, refresh = true) => {
         if (typeof key !== 'string') {
             throw new TypeError('Cache key must be a string');

package/lib/utils/camelcase-keys.spec.ts

@@ -0,0 +1,127 @@
+import { describe, expect, it } from 'vitest';
+
+import { camelcase, camelcaseKeys } from './camelcase-keys';
+
+describe('test camelcase keys', () => {
+    it('case 1 normal', () => {
+        const obj = {
+            tool: 'too',
+            tool_name: 'too_name',
+
+            a_b: 1,
+            a: 1,
+            b: {
+                c_d: 1,
+            },
+        };
+
+        expect(camelcaseKeys(obj)).toStrictEqual({
+            tool: 'too',
+            toolName: 'too_name',
+
+            aB: 1,
+            a: 1,
+            b: {
+                cD: 1,
+            },
+        });
+    });
+
+    it('case 2: key has number', () => {
+        const obj = {
+            b147da0eaecbea00aeb62055: {
+                data: {},
+            },
+            a_c11ab_Ac: [
+                {
+                    a_b: 1,
+                },
+                1,
+            ],
+        };
+
+        expect(camelcaseKeys(obj)).toStrictEqual({
+            b147da0eaecbea00aeb62055: {
+                data: {},
+            },
+            aC11abAc: [
+                {
+                    aB: 1,
+                },
+                1,
+            ],
+        });
+    });
+
+    it('case 3: not a object', () => {
+        const value = 1;
+        expect(camelcaseKeys(value)).toBe(value);
+    });
+
+    it('case 4: nullable value', () => {
+        let value = null as any;
+        expect(camelcaseKeys(value)).toBe(value);
+
+        value = undefined;
+        expect(camelcaseKeys(value)).toBe(value);
+
+        value = Number.NaN;
+        expect(camelcaseKeys(value)).toBe(value);
+    });
+
+    it('case 5: array', () => {
+        const arr = [
+            {
+                a_b: 1,
+            },
+            null,
+            undefined,
+            +0,
+            -0,
+            Number.POSITIVE_INFINITY,
+            {
+                a_b: 1,
+            },
+        ];
+
+        expect(camelcaseKeys(arr)).toStrictEqual([
+            {
+                aB: 1,
+            },
+            null,
+            undefined,
+            +0,
+            -0,
+            Number.POSITIVE_INFINITY,
+            {
+                aB: 1,
+            },
+        ]);
+    });
+
+    it('case 6: filter out mongo id', () => {
+        const obj = {
+            _id: '123',
+            a_b: 1,
+            collections: {
+                posts: {
+                    '661bb93307d35005ba96731b': {},
+                },
+            },
+        };
+
+        expect(camelcaseKeys(obj)).toStrictEqual({
+            id: '123',
+            aB: 1,
+            collections: {
+                posts: {
+                    '661bb93307d35005ba96731b': {},
+                },
+            },
+        });
+    });
+
+    it('case 7: start with underscore should not camelcase', () => {
+        expect(camelcase('_id')).toBe('id');
+    });
+});

package/lib/utils/camelcase-keys.ts

@@ -0,0 +1,27 @@
+const isObject = (obj: any) => obj && typeof obj === 'object';
+const isPlainObject = (obj: any) => isObject(obj) && Object.prototype.toString.call(obj) === '[object Object]' && Object.getPrototypeOf(obj) === Object.prototype;
+
+/**
+ * A simple camelCase function that only handles strings, but not handling symbol, date, or other complex case.
+ * If you need to handle more complex cases, please use camelcase-keys package.
+ */
+export const camelcaseKeys = <T = any>(obj: any): T => {
+    if (Array.isArray(obj)) {
+        return obj.map((x) => camelcaseKeys(x)) as any;
+    }
+
+    if (isPlainObject(obj)) {
+        return Object.keys(obj).reduce((result: any, key) => {
+            const nextKey = isMongoId(key) ? key : camelcase(key);
+            result[nextKey] = camelcaseKeys(obj[key]);
+            return result;
+        }, {}) as any;
+    }
+
+    return obj;
+};
+
+export function camelcase(str: string) {
+    return str.replace(/^_+/, '').replaceAll(/([_-][a-z])/gi, ($1) => $1.toUpperCase().replace('-', '').replace('_', ''));
+}
+const isMongoId = (id: string) => id.length === 24 && /^[\dA-Fa-f]{24}$/.test(id);

package/lib/utils/common-utils.ts

@@ -39,6 +39,7 @@ const getLocalhostAddress = () => {
         .filter((iface) => iface?.family === 'IPv4' && !iface.internal)
         .map((iface) => iface?.address)
         .filter(Boolean);
+    address.push('[::]');
     return address;
 };
 

package/lib/utils/got.test.ts

@@ -36,8 +36,9 @@ describe('got', () => {
                 test: 'rsshub',
             },
         });
-        expect(response.body).toBe('{"test":"rsshub"}');
+        expect(response.body).toContain('"test":"rsshub"');
         expect(response.data.test).toBe('rsshub');
+        expect(response.data.req.headers['content-type']).toBe('application/x-www-form-urlencoded');
     });
 
     it('json-post', async () => {

package/lib/utils/got.ts

@@ -27,14 +27,11 @@ const getFakeGot = (defaultOptions?: any) => {
             delete options.json;
         }
         if (options?.form && !options.body) {
-            const body = new FormData();
-            for (const key in options.form) {
-                body.append(key, options.form[key]);
-            }
-            options.body = body;
+            options.body = new URLSearchParams(options.form as Record<string, string>).toString();
             if (!options.headers) {
                 options.headers = {};
             }
+            options.headers['content-type'] = 'application/x-www-form-urlencoded';
             delete options.form;
         }
         if (options?.searchParams) {

package/lib/utils/ofetch.ts

@@ -3,9 +3,23 @@ import { config } from '@/config';
 import logger from '@/utils/logger';
 
 const rofetch = createFetch().create({
+    retryStatusCodes: [400, 408, 409, 425, 429, 500, 502, 503, 504],
     retry: config.requestRetry,
     retryDelay: 1000,
-    timeout: config.requestTimeout,
+    // timeout: config.requestTimeout,
+    onResponseError({ request, response, options }) {
+        if (options.retry) {
+            logger.warn(`Request ${request} with error ${response.status} remaining retry attempts: ${options.retry}`);
+            if (!options.headers) {
+                options.headers = {};
+            }
+            if (options.headers instanceof Headers) {
+                options.headers.set('x-prefer-proxy', '1');
+            } else {
+                options.headers['x-prefer-proxy'] = '1';
+            }
+        }
+    },
     onRequestError({ request, error }) {
         logger.error(`Request ${request} fail: ${error}`);
     },

package/lib/utils/otel/index.ts

@@ -0,0 +1,2 @@
+export * from './metric';
+export * from './trace';

package/lib/utils/otel/metric.ts

@@ -0,0 +1,67 @@
+import { Resource } from '@opentelemetry/resources';
+import { PrometheusExporter, PrometheusSerializer } from '@opentelemetry/exporter-prometheus';
+import { SEMRESATTRS_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import { MeterProvider } from '@opentelemetry/sdk-metrics';
+import { Attributes } from '@opentelemetry/api';
+import { config } from '@/config';
+
+interface IMetricAttributes extends Attributes {
+    method: string;
+    path: string;
+    status: number;
+}
+
+interface IHistogramAttributes extends IMetricAttributes {
+    unit: string;
+}
+
+const METRIC_PREFIX = 'rsshub';
+
+const exporter = new PrometheusExporter({});
+
+const provider = new MeterProvider({
+    resource: new Resource({
+        [SEMRESATTRS_SERVICE_NAME]: 'rsshub',
+    }),
+    readers: [exporter],
+});
+
+const serializer = new PrometheusSerializer();
+
+const meter = provider.getMeter('rsshub');
+
+const requestTotal = meter.createCounter<IMetricAttributes>(`${METRIC_PREFIX}_request_total`);
+const requestErrorTotal = meter.createCounter<IMetricAttributes>(`${METRIC_PREFIX}_request_error_total`);
+const requestDurationSecondsBucket = meter.createHistogram<IHistogramAttributes>(`${METRIC_PREFIX}_request_duration_seconds_bucket`, {
+    advice: {
+        explicitBucketBoundaries: config.otel.seconds_bucket?.split(',').map(Number),
+    },
+});
+const request_duration_milliseconds_bucket = meter.createHistogram<IHistogramAttributes>(`${METRIC_PREFIX}_request_duration_milliseconds_bucket`, {
+    advice: {
+        explicitBucketBoundaries: config.otel.milliseconds_bucket?.split(',').map(Number),
+    },
+});
+
+export const requestMetric = {
+    success: (value: number, attributes: IMetricAttributes) => {
+        requestTotal.add(1, attributes);
+        request_duration_milliseconds_bucket.record(value, { unit: 'millisecond', ...attributes });
+        requestDurationSecondsBucket.record(value / 1000, { unit: 'second', ...attributes });
+    },
+    error: (attributes: IMetricAttributes) => {
+        requestErrorTotal.add(1, attributes);
+    },
+};
+
+export const getContext = () =>
+    new Promise<string>((resolve, reject) => {
+        exporter
+            .collect()
+            .then((value) => {
+                resolve(serializer.serialize(value.resourceMetrics));
+            })
+            .finally(() => {
+                reject('');
+            });
+    });

package/lib/utils/otel/trace.ts

@@ -0,0 +1,28 @@
+import { Resource } from '@opentelemetry/resources';
+import { BasicTracerProvider, BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { SEMRESATTRS_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+
+const provider = new BasicTracerProvider({
+    resource: new Resource({
+        [SEMRESATTRS_SERVICE_NAME]: 'rsshub',
+    }),
+});
+
+const exporter = new OTLPTraceExporter({
+    // optional OTEL_EXPORTER_OTLP_ENDPOINT=https://localhost:4318
+});
+
+provider.addSpanProcessor(
+    new BatchSpanProcessor(exporter, {
+        // The maximum queue size. After the size is reached spans are dropped.
+        maxQueueSize: 4096,
+        // The interval between two consecutive exports
+        scheduledDelayMillis: 30000,
+    })
+);
+
+provider.register();
+
+export const tracer = provider.getTracer('rsshub');
+export const mainSpan = tracer.startSpan('main');

package/lib/utils/render.ts

@@ -1,4 +1,4 @@
-export { default as rss3Ums } from '@/views/rss3-ums';
+export { default as rss3 } from '@/views/rss3';
 export { default as json } from '@/views/json';
 export { default as RSS } from '@/views/rss';
 export { default as Atom } from '@/views/atom';