rsshub 1.0.0-master.fc1b99f → 1.0.0-master.fc2cf18

package/lib/types.ts

@@ -1,5 +1,33 @@
 import type { Context } from 'hono';
 
+// Make sure it's synchronise with scripts/workflow/data.ts
+// and lib/routes/rsshub/routes.ts
+type Category =
+    | 'popular'
+    | 'social-media'
+    | 'new-media'
+    | 'traditional-media'
+    | 'bbs'
+    | 'blog'
+    | 'programming'
+    | 'design'
+    | 'live'
+    | 'multimedia'
+    | 'picture'
+    | 'anime'
+    | 'program-update'
+    | 'university'
+    | 'forecast'
+    | 'travel'
+    | 'shopping'
+    | 'game'
+    | 'reading'
+    | 'government'
+    | 'study'
+    | 'journal'
+    | 'finance'
+    | 'other';
+
 // rss
 export type DataItem = {
     title: string;
@@ -7,7 +35,13 @@ export type DataItem = {
     pubDate?: number | string | Date;
     link?: string;
     category?: string[];
-    author?: string | { name: string }[];
+    author?:
+        | string
+        | {
+              name: string;
+              url?: string;
+              avatar?: string;
+          }[];
     doi?: string;
     guid?: string;
     id?: string;
@@ -24,6 +58,8 @@ export type DataItem = {
     enclosure_title?: string;
     enclosure_length?: number;
     itunes_duration?: number | string;
+    itunes_item_image?: string;
+    media?: Record<string, Record<string, string>>;
 
     _extra?: Record<string, any> & {
         links?: {
@@ -45,6 +81,13 @@ export type Data = {
     language?: string;
     feedLink?: string;
     lastBuildDate?: string;
+    itunes_author?: string;
+    itunes_category?: string;
+    itunes_explicit?: string | boolean;
+    id?: string;
+
+    atomlink?: string;
+    ttl?: number;
 };
 
 // namespace
@@ -63,7 +106,7 @@ interface NamespaceItem {
     /**
      * The classification of the namespace, which will be written into the corresponding classification document
      */
-    categories?: string[];
+    categories?: Category[];
 
     /**
      * Hints and additional explanations for users using this namespace, it will be inserted into the documentation
@@ -82,6 +125,15 @@ interface Namespace extends NamespaceItem {
 
 export type { Namespace };
 
+export enum ViewType {
+    Articles = 0,
+    SocialMedia = 1,
+    Pictures = 2,
+    Videos = 3,
+    Audios = 4,
+    Notifications = 5,
+}
+
 // route
 interface RouteItem {
     /**
@@ -108,7 +160,7 @@ interface RouteItem {
     /**
      * The handler function of the route
      */
-    handler: (ctx?: Context) => Promise<Data> | Data;
+    handler: (ctx: Context) => Promise<Data> | Data;
 
     /**
      * An example URL of the route
@@ -118,7 +170,18 @@ interface RouteItem {
     /**
      * The description of the route parameters
      */
-    parameters?: Record<string, string>;
+    parameters?: Record<
+        string,
+        | string
+        | {
+              description: string;
+              default?: string;
+              options?: {
+                  value: string;
+                  label: string;
+              }[];
+          }
+    >;
 
     /**
      * Hints and additional explanations for users using this route, it will be appended after the route component, supports markdown
@@ -128,7 +191,7 @@ interface RouteItem {
     /**
      * The classification of the route, which will be written into the corresponding classification documentation
      */
-    categories?: string[];
+    categories?: Category[];
 
     /**
      * Special features of the route, such as what configuration items it depends on, whether it is strict anti-crawl, whether it supports a certain function and so on
@@ -169,6 +232,11 @@ interface RouteItem {
      * The [RSSHub-Radar](https://github.com/DIYgod/RSSHub-Radar) rule of the route
      */
     radar?: RadarItem[];
+
+    /**
+     * The [Follow](https://github.com/RSSNext/follow) default view of the route, default to `ViewType.Articles`
+     */
+    view?: ViewType;
 }
 
 interface Route extends RouteItem {
@@ -205,6 +273,7 @@ export type RadarItem = {
      *
      * Using `target` as a function is deprecated in RSSHub-Radar 2.0.19
      * @see https://github.com/DIYgod/RSSHub-Radar/commit/5a97647f900bb2bca792787a322b2b1ca512e40b#diff-f84e3c1e16af314bc4ed7c706d7189844663cde9b5142463dc5c0db34c2e8d54L10
+     * @see https://github.com/DIYgod/RSSHub-Radar/issues/692
      */
     target?:
         | string
@@ -212,8 +281,8 @@ export type RadarItem = {
               /** The parameters matched from the `source` field */
               params: any,
               /** The current webpage URL string */
-              url?: string,
+              url: string,
               /** @deprecated Temporary removed  @see https://github.com/DIYgod/RSSHub-Radar/commit/e6079ea1a8c96e89b1b2c2aa6d13c7967788ca3b */
-              document?: Document
+              document: Document
           ) => string);
 };

package/lib/utils/cache/index.ts

@@ -63,6 +63,14 @@ if (config.cache.type === 'redis') {
 // plz, write these tips in comments!
 export default {
     ...cacheModule,
+    /**
+     * Try to get the cache. If the cache does not exist, the `getValueFunc` function will be called to get the data, and the data will be cached.
+     * @param key The key used to store and retrieve the cache. You can use `:` as a separator to create a hierarchy.
+     * @param getValueFunc A function that returns data to be cached when a cache miss occurs.
+     * @param maxAge The maximum age of the cache in seconds. This should left to the default value in most cases which is `CACHE_CONTENT_EXPIRE`.
+     * @param refresh Whether to renew the cache expiration time when the cache is hit. `true` by default.
+     * @returns
+     */
     tryGet: async (key: string, getValueFunc: () => Promise<string | Record<string, any>>, maxAge = config.cache.contentExpire, refresh = true) => {
         if (typeof key !== 'string') {
             throw new TypeError('Cache key must be a string');

package/lib/utils/common-config.test.ts

@@ -1,6 +1,5 @@
 import { describe, expect, it } from 'vitest';
 import configUtils, { transElemText, replaceParams, getProp } from '@/utils/common-config';
-import nock from 'nock';
 
 describe('index', () => {
     it('transElemText', () => {
@@ -40,23 +39,6 @@ describe('index', () => {
     });
 
     it('buildData', async () => {
-        nock('http://rsshub.test')
-            .get('/buildData')
-            .reply(() => [
-                200,
-                `<div class="content">
-                <ul>
-                    <li>
-                        <a href="/1">1</a>
-                        <div class="description">RSSHub1</div>
-                    </li>
-                    <li>
-                        <a href="/2">2</a>
-                        <div class="description">RSSHub2</div>
-                    </li>
-                </ul>
-            </div>`,
-            ]);
         const data = await configUtils({
             link: 'http://rsshub.test/buildData',
             url: 'http://rsshub.test/buildData',

package/lib/utils/common-config.ts

@@ -1,5 +1,5 @@
-import cheerio from 'cheerio';
-import got from '@/utils/got';
+import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
 import iconv from 'iconv-lite';
 
 function transElemText($, prop) {
@@ -37,8 +37,8 @@ function getProp(data, prop, $) {
 }
 
 async function buildData(data) {
-    const response = await got.get(data.url);
-    const contentType = response.headers['content-type'] || '';
+    const response = await ofetch.raw(data.url);
+    const contentType = response.headers.get('content-type') || '';
     // 若没有指定编码，则默认utf-8
     let charset = 'utf-8';
     for (const attr of contentType.split(';')) {
@@ -47,8 +47,8 @@ async function buildData(data) {
         }
     }
     // @ts-expect-error custom property
-    const responseData = charset === 'utf-8' ? response.data : iconv.decode((await got.get({ url: data.url, responseType: 'buffer' })).data, charset);
-    const $ = cheerio.load(responseData);
+    const responseData = charset === 'utf-8' ? response._data : iconv.decode(await ofetch(data.url, { responseType: 'buffer' }), charset);
+    const $ = load(responseData);
     const $item = $(data.item.item);
     // 这里应该是可以通过参数注入一些代码的，不过应该无伤大雅
     return {

package/lib/utils/git-hash.ts

@@ -1,11 +1,11 @@
-import gitRevSync from 'git-rev-sync';
+import { execSync } from 'child_process';
 
 let gitHash = process.env.HEROKU_SLUG_COMMIT?.slice(0, 8) || process.env.VERCEL_GIT_COMMIT_SHA?.slice(0, 8);
 let gitDate: Date | undefined;
 if (!gitHash) {
     try {
-        gitHash = gitRevSync.short(undefined, 8);
-        gitDate = gitRevSync.date();
+        gitHash = execSync('git rev-parse HEAD').toString().trim().slice(0, 8);
+        gitDate = new Date(execSync('git log -1 --format=%cd').toString().trim());
     } catch {
         gitHash = 'unknown';
     }

package/lib/utils/got-deprecated.ts

@@ -0,0 +1,79 @@
+import logger from '@/utils/logger';
+import { config } from '@/config';
+import got, { CancelableRequest, Response as GotResponse, OptionsInit, Options, Got } from 'got';
+
+type Response<T> = GotResponse<string> & {
+    data: T;
+    status: number;
+};
+
+type GotRequestFunction = {
+    (url: string | URL, options?: Options): CancelableRequest<Response<Record<string, any>>>;
+    <T>(url: string | URL, options?: Options): CancelableRequest<Response<T>>;
+    (options: Options): CancelableRequest<Response<Record<string, any>>>;
+    <T>(options: Options): CancelableRequest<Response<T>>;
+};
+
+// @ts-expect-error got instance with custom response type
+const custom: {
+    all?: <T>(list: Array<Promise<T>>) => Promise<Array<T>>;
+    get: GotRequestFunction;
+    post: GotRequestFunction;
+    put: GotRequestFunction;
+    patch: GotRequestFunction;
+    head: GotRequestFunction;
+    delete: GotRequestFunction;
+} & GotRequestFunction &
+    Got = got.extend({
+    retry: {
+        limit: config.requestRetry,
+        statusCodes: [400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 421, 422, 423, 424, 426, 428, 429, 431, 451, 500, 501, 502, 503, 504, 505, 506, 507, 508, 510, 511, 521, 522, 524],
+    },
+    hooks: {
+        beforeRetry: [
+            (err, count) => {
+                logger.error(`Request ${err.options.url} fail, retry attempt #${count}: ${err}`);
+            },
+        ],
+        beforeRedirect: [
+            (options, response) => {
+                logger.http(`Redirecting to ${options.url} for ${response.requestUrl}`);
+            },
+        ],
+        afterResponse: [
+            // @ts-expect-error custom response type
+            (response: Response<Record<string, any>>) => {
+                try {
+                    response.data = typeof response.body === 'string' ? JSON.parse(response.body) : response.body;
+                } catch {
+                    // @ts-expect-error for compatibility
+                    response.data = response.body;
+                }
+                response.status = response.statusCode;
+                return response;
+            },
+        ],
+        init: [
+            (
+                options: OptionsInit & {
+                    data?: string;
+                }
+            ) => {
+                // compatible with axios api
+                if (options && options.data) {
+                    options.body = options.body || options.data;
+                }
+            },
+        ],
+    },
+    headers: {
+        'user-agent': config.ua,
+    },
+    timeout: {
+        request: config.requestTimeout,
+    },
+});
+custom.all = (list) => Promise.all(list);
+
+export default custom;
+export type { Response, Options } from 'got';

package/lib/utils/got.test.ts

@@ -1,88 +1,77 @@
-import { describe, expect, it, afterEach, vi } from 'vitest';
-import nock from 'nock';
-
-afterEach(() => {
-    vi.resetModules();
-});
+import { describe, expect, it, vi } from 'vitest';
+import { http, HttpResponse } from 'msw';
+import got from '@/utils/got';
+import { config } from '@/config';
+import { Cookie, CookieJar } from 'tough-cookie';
 
 describe('got', () => {
     it('headers', async () => {
-        const { default: got } = await import('@/utils/got');
-        const { config } = await import('@/config');
-        nock('http://rsshub.test')
-            .get('/test')
-            .reply(function () {
-                expect(this.req.headers['user-agent']).toBe(config.ua);
-                return [200, ''];
-            });
-
-        await got.get('http://rsshub.test/test');
+        const { data } = await got('http://rsshub.test/headers');
+        expect(data['user-agent']).toBe(config.ua);
     });
 
     it('retry', async () => {
-        const { default: got } = await import('@/utils/got');
-        const { config } = await import('@/config');
         const requestRun = vi.fn();
-        nock('http://rsshub.test')
-            .get('/testRerty')
-            .times(config.requestRetry + 1)
-            .reply(() => {
+        const { default: server } = await import('@/setup.test');
+        server.use(
+            http.get(`http://rsshub.test/retry-test`, () => {
                 requestRun();
-                return [503, '0'];
-            });
+                return HttpResponse.error();
+            })
+        );
 
         try {
-            await got.get('http://rsshub.test/testRerty');
+            await got.get('http://rsshub.test/retry-test');
         } catch (error: any) {
-            expect(error.name).toBe('HTTPError');
+            expect(error.name).toBe('FetchError');
         }
 
         // retries
         expect(requestRun).toHaveBeenCalledTimes(config.requestRetry + 1);
     });
 
-    it('axios', async () => {
-        const { default: got } = await import('@/utils/got');
-        nock('http://rsshub.test')
-            .post('/post')
-            .reply(() => [200, '{"code": 0}']);
-
-        const response1 = await got.post('http://rsshub.test/post', {
+    it('form-post', async () => {
+        const response = await got.post('http://rsshub.test/form-post', {
             form: {
-                test: 1,
+                test: 'rsshub',
             },
         });
-        expect(response1.statusCode).toBe(200);
-        // @ts-expect-error custom property
-        expect(response1.status).toBe(200);
-        expect(response1.body).toBe('{"code": 0}');
-        // @ts-expect-error custom property
-        expect(response1.data.code).toBe(0);
+        expect(response.body).toContain('"test":"rsshub"');
+        expect(response.data.test).toBe('rsshub');
+        expect(response.data.req.headers['content-type']).toBe('application/x-www-form-urlencoded');
     });
 
-    it('timeout', async () => {
-        process.env.REQUEST_TIMEOUT = '500';
-
-        const { default: got } = await import('@/utils/got');
-        nock('http://rsshub.test')
-            .get('/timeout')
-            .delay(600)
-            .reply(() => [200, '{"code": 0}']);
-
-        const logger = (await import('@/utils/logger')).default;
-        // @ts-expect-error unused
-        const loggerSpy = vi.spyOn(logger, 'error').mockReturnValue({});
+    it('json-post', async () => {
+        const response = await got.post('http://rsshub.test/json-post', {
+            json: {
+                test: 'rsshub',
+            },
+        });
+        expect(response.body).toBe('{"test":"rsshub"}');
+        expect(response.data.test).toBe('rsshub');
+    });
 
-        try {
-            await got.get('http://rsshub.test/timeout');
-            throw new Error('Timeout Invalid');
-        } catch (error: any) {
-            expect(error.name).toBe('RequestError');
-        }
-        expect(loggerSpy).toHaveBeenCalledWith(expect.stringContaining('http://rsshub.test/timeout'));
+    it('buffer-get', async () => {
+        const response = await got.get('http://example.com', {
+            responseType: 'buffer',
+        });
+        expect(response.body instanceof Buffer).toBe(true);
+        expect(response.data instanceof Buffer).toBe(true);
+    });
 
-        loggerSpy.mockRestore();
+    it('cookieJar', async () => {
+        const cookieJar = new CookieJar();
+        const cookie = Cookie.fromJSON({
+            key: 'cookie',
+            value: 'test',
+            domain: 'rsshub.test',
+            path: '/',
+        });
+        cookie && cookieJar.setCookie(cookie, 'http://rsshub.test');
+        const { data } = await got.get('http://rsshub.test/headers', {
+            cookieJar,
+        });
 
-        delete process.env.REQUEST_TIMEOUT;
+        expect(data.cookie).toBe('cookie=test; Domain=rsshub.test; Path=/');
     });
 });

package/lib/utils/got.ts

@@ -1,87 +1,86 @@
-import logger from '@/utils/logger';
-import { config } from '@/config';
-import got, { CancelableRequest, Response as GotResponse, NormalizedOptions, Options, Got } from 'got';
+import { destr } from 'destr';
+import ofetch from '@/utils/ofetch';
 
-type Response<T> = GotResponse<string> & {
-    data: T;
-    status: number;
-};
+const getFakeGot = (defaultOptions?: any) => {
+    const fakeGot = (request, options?: any) => {
+        if (!(typeof request === 'string' || request instanceof Request) && request.url) {
+            options = {
+                ...request,
+                ...options,
+            };
+            request = request.url;
+        }
+        if (options?.hooks?.beforeRequest) {
+            for (const hook of options.hooks.beforeRequest) {
+                hook(options);
+            }
+            delete options.hooks;
+        }
 
-type GotRequestFunction = {
-    (url: string | URL, options?: Options): CancelableRequest<Response<Record<string, any>>>;
-    <T>(url: string | URL, options?: Options): CancelableRequest<Response<T>>;
-    (options: Options): CancelableRequest<Response<Record<string, any>>>;
-    <T>(options: Options): CancelableRequest<Response<T>>;
-};
+        options = {
+            ...defaultOptions,
+            ...options,
+        };
 
-// @ts-expect-error got instance with custom response type
-const custom: {
-    all?: <T>(list: Array<Promise<T>>) => Promise<Array<T>>;
-    get: GotRequestFunction;
-    post: GotRequestFunction;
-    put: GotRequestFunction;
-    patch: GotRequestFunction;
-    head: GotRequestFunction;
-    delete: GotRequestFunction;
-} & GotRequestFunction &
-    Got = got.extend({
-    get: got.get,
-    retry: {
-        limit: config.requestRetry,
-        statusCodes: [400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 421, 422, 423, 424, 426, 428, 429, 431, 451, 500, 501, 502, 503, 504, 505, 506, 507, 508, 510, 511, 521, 522, 524],
-    },
-    hooks: {
-        beforeRetry: [
-            (
-                options: NormalizedOptions & {
-                    retryCount?: number;
-                },
-                err,
-                count
-            ) => {
-                logger.error(`Request ${options.url} fail, retry attempt #${count}: ${err}`);
-                options.retryCount = count;
-            },
-        ],
-        beforeRedirect: [
-            (options, response) => {
-                logger.http(`Redirecting to ${options.url} for ${response.requestUrl}`);
-            },
-        ],
-        afterResponse: [
-            // @ts-expect-error custom response type
-            (response: Response<Record<string, any>>) => {
-                try {
-                    response.data = typeof response.body === 'string' ? JSON.parse(response.body) : response.body;
-                } catch {
-                    // @ts-expect-error for compatibility
-                    response.data = response.body;
-                }
-                response.status = response.statusCode;
-                return response;
-            },
-        ],
-        init: [
-            (
-                options: Options & {
-                    data?: string;
-                }
-            ) => {
-                // compatible with axios api
-                if (options && options.data) {
-                    options.body = options.body || options.data;
+        if (options?.json && !options.body) {
+            options.body = options.json;
+            delete options.json;
+        }
+        if (options?.form && !options.body) {
+            options.body = new URLSearchParams(options.form as Record<string, string>).toString();
+            if (!options.headers) {
+                options.headers = {};
+            }
+            options.headers['content-type'] = 'application/x-www-form-urlencoded';
+            delete options.form;
+        }
+        if (options?.searchParams) {
+            request += '?' + new URLSearchParams(options.searchParams).toString();
+            delete options.searchParams;
+        }
+
+        // Add support for buffer responseType, to be compatible with got
+        options.parseResponse = (responseText) => ({
+            data: destr(responseText),
+            body: responseText,
+        });
+
+        if (options?.responseType === 'buffer' || options?.responseType === 'arrayBuffer') {
+            options.responseType = 'arrayBuffer';
+            delete options.parseResponse;
+        }
+
+        if (options.cookieJar) {
+            const cookies = options.cookieJar.getCookiesSync(request);
+            if (cookies.length) {
+                if (!options.headers) {
+                    options.headers = {};
                 }
-            },
-        ],
-    },
-    headers: {
-        'user-agent': config.ua,
-    },
-    timeout: {
-        request: config.requestTimeout,
-    },
-});
-custom.all = (list) => Promise.all(list);
+                options.headers.cookie = cookies.join('; ');
+            }
+            delete options.cookieJar;
+        }
+
+        const response = ofetch(request, options);
+
+        if (options?.responseType === 'arrayBuffer') {
+            return response.then((responseData) => ({
+                data: Buffer.from(responseData),
+                body: Buffer.from(responseData),
+            }));
+        }
+        return response;
+    };
+
+    fakeGot.get = (request, options?) => fakeGot(request, { ...options, method: 'GET' });
+    fakeGot.post = (request, options?) => fakeGot(request, { ...options, method: 'POST' });
+    fakeGot.put = (request, options?) => fakeGot(request, { ...options, method: 'PUT' });
+    fakeGot.patch = (request, options?) => fakeGot(request, { ...options, method: 'PATCH' });
+    fakeGot.head = (request, options?) => fakeGot(request, { ...options, method: 'HEAD' });
+    fakeGot.delete = (request, options?) => fakeGot(request, { ...options, method: 'DELETE' });
+    fakeGot.extend = (options) => getFakeGot(options);
+
+    return fakeGot;
+};
 
-export default custom;
-export type { Response, NormalizedOptions, Options } from 'got';
+export default getFakeGot();

package/lib/utils/helpers.ts

@@ -1,5 +1,5 @@
 import { fileURLToPath } from 'url';
-import * as path from 'node:path';
+import path from 'node:path';
 
 export const getRouteNameFromPath = (path: string) => {
     const p = path.split('/').filter(Boolean);

package/lib/utils/logger.ts

@@ -1,4 +1,4 @@
-import * as path from 'node:path';
+import path from 'node:path';
 import winston from 'winston';
 import { config } from '@/config';