rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa04a11

package/lib/routes/zaker/utils.ts

@@ -0,0 +1,186 @@
+import { DataItem } from '@/types';
+
+import CryptoJS from 'crypto-js';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { config } from '@/config';
+import logger from '@/utils/logger';
+import * as cheerio from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+const hints = ['globalThis', 'headless', 'languages', 'permHook', 'vendor', 'webDriverValue', 'webdriver'];
+export const baseUrl = 'https://www.myzaker.com';
+
+const generateSalt = (input: string, targetZeros = 20) => {
+    for (let nonce = 0; nonce < 100_000_000; nonce++) {
+        const hash = CryptoJS.SHA256(input + nonce).toString();
+        let leadingZeros = 0;
+
+        for (const char of hash) {
+            if (char !== '0') {
+                leadingZeros += 4 - Number.parseInt(char, 16).toString(2).length;
+                break;
+            }
+            leadingZeros += 4;
+        }
+
+        if (leadingZeros >= targetZeros) {
+            return nonce;
+        }
+    }
+    return 0;
+};
+
+const padSeed = (seed: string) => {
+    const padding = '0'.repeat(16);
+    return CryptoJS.enc.Utf8.parse((seed + padding).slice(0, 16));
+};
+
+const encrypt = (data, seed: string) => {
+    const iv = CryptoJS.enc.Utf8.parse('1234567890123456');
+    return CryptoJS.AES.encrypt(JSON.stringify(data), padSeed(seed), {
+        iv,
+        padding: CryptoJS.pad.Pkcs7,
+    });
+};
+
+const encryptPayload = (data, seed: string) => encrypt(data, seed).ciphertext.toString();
+
+export const getSafeLineCookieWithData = async (link): Promise<{ cookie: string; data: string }> => {
+    const cacheKey = 'zaker:cookie';
+    const cacheAge = 3600;
+    const cacheIn = await cache.get(cacheKey, false);
+    if (cacheIn) {
+        return JSON.parse(cacheIn);
+    }
+    const apiBaseUrl = 'https://challenge.rivers.chaitin.cn/captcha/api';
+
+    const headerResponse = await ofetch.raw(link);
+    const session = headerResponse.headers
+        .getSetCookie()
+        .find((e) => e.startsWith('sl-session'))
+        ?.split(';')[0]
+        .split('sl-session=')[1];
+    const onceId = headerResponse._data.match(/once_id:\s*"(.*?)",/)?.[1];
+    logger.debug(`getSafeLineCookie: sl-session=${session}, onceId=${onceId}`);
+    if (!/window\.captcha/.test(headerResponse._data)) {
+        logger.debug('getSafeLineCookie: Failed to get once_id');
+        return {
+            cookie: headerResponse.headers
+                .getSetCookie()
+                .map((c) => c.split(';')[0])
+                .join('; '),
+            data: headerResponse._data,
+        };
+    }
+
+    // await ofetch(`${apiBaseUrl}/index.html?${Math.random()}`);
+    // await ofetch('${apiBaseUrl}/sdk.js');
+    const seedResponse = await ofetch<{ req_id: string; seed: string }>(`${apiBaseUrl}/seed`, {
+        headers: {
+            Referer: `${baseUrl}/`,
+        },
+        query: {
+            once_id: onceId,
+            v: '1.0.0',
+            hints: hints.sort(() => Math.random() - 0.5).join(','),
+        },
+    });
+
+    const ua = config.ua;
+    const seed = seedResponse.seed;
+    const takeTime = Math.trunc(Math.random() * 2000 + 1000);
+    logger.debug(`getSafeLineCookie: ua=${ua}, seed=${seed}, takeTime=${takeTime}`);
+    const payload = encryptPayload(
+        {
+            resolution: '1920x1080',
+            languages: ['en-US'],
+            useragents: [ua, ua, ua],
+            hint: 0,
+            salt: String(generateSalt(seed, 16)),
+            taketime: takeTime,
+        },
+        seed
+    );
+
+    const inspectResponse = await ofetch<{ req_id: string; jwt: string; reason: string }>(`${apiBaseUrl}/inspect`, {
+        method: 'POST',
+        headers: {
+            Referer: `${baseUrl}/`,
+            'Content-Type': 'text/plain',
+        },
+        query: {
+            seed,
+        },
+        body: payload,
+    });
+    logger.debug(`getSafeLineCookie: inspectResponse=${JSON.stringify(inspectResponse)}`);
+    if (inspectResponse.reason) {
+        logger.error(`getSafeLineCookie: reason=${inspectResponse.reason}`);
+        return {
+            cookie: headerResponse.headers
+                .getSetCookie()
+                .map((c) => c.split(';')[0])
+                .join('; '),
+            data: headerResponse._data,
+        };
+    }
+
+    const response = await ofetch.raw(link, {
+        headers: {
+            Cookie: `sl-session=${session}; sl_waf_recap=${inspectResponse.jwt}`,
+        },
+    });
+
+    const cookie = response.headers
+        .getSetCookie()
+        .map((c) => c.split(';')[0])
+        .join('; ');
+    logger.debug(`getSafeLineCookie: ${cookie}`);
+
+    cache.set(cacheKey, JSON.stringify(cookie), cacheAge);
+    return {
+        cookie,
+        data: response._data,
+    };
+};
+
+export const parseList = ($: cheerio.CheerioAPI) => {
+    const winPageData = JSON.parse(
+        $('script:contains("window.WinPageData")')
+            .text()
+            .match(/window\.WinPageData\s*=\s*({.*})/)?.[1] ?? '{}'
+    );
+
+    return winPageData.data.article.map((item) => ({
+        title: item.title,
+        description: item.desc,
+        link: 'https:' + item.url,
+        author: item.author_name,
+        pubDate: timezone(parseDate(item.date, 'MM月DD日'), +8),
+        category: item.tag.map((t) => t.tag),
+        image: item.thumbnail_mpic,
+    })) as DataItem[];
+};
+
+export const fetchItem = async (item: DataItem, cookie: string) => {
+    const response = await ofetch(item.link!, {
+        headers: {
+            Cookie: cookie as string,
+        },
+    });
+
+    const $ = cheerio.load(response);
+
+    const content = $('div.article_content div');
+    content.find('img').each((_, img) => {
+        const $img = $(img);
+        $img.attr('src', $img.attr('data-original'));
+        $img.removeAttr('data-original');
+    });
+
+    item.description = content.html();
+
+    return item;
+};

package/lib/routes/zaobao/namespace.ts

@@ -3,7 +3,7 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: '联合早报',
     url: 'www.zaobao.com',
-    description: `:::warning
+    description: `::: warning
 由于 [RSSHub#10309](https://github.com/DIYgod/RSSHub/issues/10309) 中的问题，使用靠近香港的服务器部署将从 hk 版联合早报爬取内容，造成输出的新闻段落顺序错乱。如有订阅此源的需求，建议寻求部署在远离香港的服务器上的 RSSHub，或者在自建时选择远离香港的服务器。
 :::`,
     lang: 'zh-CN',

package/lib/routes/zaobao/util.ts

@@ -86,16 +86,22 @@ const parseList = async (
                 const article = await got_ins.get(link);
                 const $1 = load(article.data);
 
-                const time = (() =>
-                    $1("head script[type='application/json']").text() === ''
-                        ? new Date(JSON.parse($1("head script[type='application/ld+json']").eq(1).text())?.datePublished) // HK
-                        : new Date(Number(JSON.parse($1("head script[type='application/json']").text())?.articleDetails?.created) * 1000))(); // SG
+                let title, time;
+                if ($1('#seo-article-page').text() === '') {
+                    // HK
+                    title = $1('h1.article-title').text();
+                    time = new Date(JSON.parse($1("head script[type='application/ld+json']").eq(1).text())?.datePublished);
+                } else {
+                    // SG
+                    title = JSON.parse($1('#seo-article-page').text())['@graph'][0]?.headline;
+                    time = new Date(JSON.parse($1('#seo-article-page').text())['@graph'][0]?.datePublished);
+                }
 
                 $1('.overlay-microtransaction').remove();
                 $1('#video-freemium-player').remove();
                 $1('script').remove();
 
-                let articleBodyNode = $1('.article-content-rawhtml');
+                let articleBodyNode = $1('.articleBody');
                 if (articleBodyNode.length === 0) {
                     // for HK version
                     orderContent($1('.article-body'));
@@ -107,8 +113,8 @@ const parseList = async (
                 const imageDataArray = processImageData($1);
 
                 return {
-                    // <- for SG version  -> for HK version
-                    title: $1('h1', '.content').text().trim() || $1('h1.article-title').text(),
+                    // <- for HK version  -> for SG version
+                    title,
                     description: art(path.join(__dirname, 'templates/zaobao.art'), {
                         articleBody,
                         imageDataArray,

package/lib/routes/zcool/discover.ts

@@ -42,9 +42,9 @@ export const route: Route = {
 
   在 **精选** 分类下的 **运营设计** 子分类全部内容基础上，筛选出有 **视频**，且城市选择 **北京**，可直接使用路由 [\`/zcool/discover/0/617/1/北京\`](https://rsshub.app/zcool/discover/0/617/1/北京)
 
-  :::tip
+::: tip
   下方仅提供 **分类及其子分类** 参数的代码。**学校** 参数的代码可以在 [站酷发现页](https://www.zcool.com.cn/discover) 中选择跳转后，从浏览器地址栏中找到。
-  :::
+:::
 
   分类 cate
 

package/lib/routes/zhihu/namespace.ts

@@ -3,7 +3,7 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: '知乎',
     url: 'www.zhihu.com',
-    description: `:::tip
+    description: `::: tip
 自2024年7月，未登录状态下大部分路由[无法获取全文](https://github.com/DIYgod/RSSHub/issues/16260)。若有需要请在登陆知乎后寻找并添加包含\`z_c0\`的Cookies至环境变量\`ZHIHU_COOKIES\`。
 :::`,
     lang: 'zh-CN',

package/lib/routes/zhihu/timeline.ts

@@ -26,9 +26,9 @@ export const route: Route = {
     name: '用户关注时间线',
     maintainers: ['SeanChao'],
     handler,
-    description: `:::warning
+    description: `::: warning
   用户关注动态需要登录后的 Cookie 值，所以只能自建，详情见部署页面的配置模块。
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/zhihu/utils.ts

@@ -61,7 +61,7 @@ export const getCookieValueByKey = (key: string) =>
         ?.split(';')
         .map((e) => e.trim())
         .find((e) => e.startsWith(key + '='))
-        ?.slice(key.length + 1);
+        ?.slice(key.length + 1) || '';
 
 export const getSignedHeader = async (url: string, apiPath: string) => {
     // Because the API of zhihu.com has changed, we must use the value of `d_c0` (extracted from cookies) to calculate
@@ -76,7 +76,6 @@ export const getSignedHeader = async (url: string, apiPath: string) => {
         const response1 = await ofetch.raw('https://static.zhihu.com/zse-ck/v3.js');
         const script = await response1._data.text();
         const zseCk = script.match(/__g\.ck\|\|"([\w+/=\\]*?)",_=/)?.[1];
-
         const response2 = zseCk
             ? await ofetch.raw(url, {
                   headers: {
@@ -88,15 +87,13 @@ export const getSignedHeader = async (url: string, apiPath: string) => {
               })
             : null;
 
-        const dc0 = (response2 || response1).headers
-            .getSetCookie()
-            .find((s) => s.startsWith('d_c0='))
-            ?.split(';')[0]
-            .trim()
-            .slice('d_c0='.length);
-        if (!dc0) {
-            throw new Error('Failed to extract `d_c0` from cookies');
-        }
+        const dc0 =
+            (response2 || response1).headers
+                .getSetCookie()
+                .find((s) => s.startsWith('d_c0='))
+                ?.split(';')[0]
+                .trim()
+                .slice('d_c0='.length) || '';
 
         return dc0;
     });

package/lib/routes/zhihu/xhu/activities.ts

@@ -28,12 +28,12 @@ export const route: Route = {
     handler,
     description: `[xhu](https://github.com/REToys/xhu)
 
-  :::tip
+::: tip
   用户的 16 进制 id 获取方式：
 
   1.  可以通过 RSSHub Radar 扩展获取；
   2.  或者在用户主页打开 F12 控制台，执行以下代码：\`console.log(/"id":"([0-9a-f]*?)","urlToken"/.exec(document.getElementById('js-initialData').innerHTML)[1]);\` 即可获取用户的 16 进制 id。
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/zhonglun/index.ts

@@ -20,22 +20,21 @@ export const handler = async (ctx) => {
 
     const $ = load(response);
 
-    let items = $('div#dataList h3')
+    let items = $('div#dataList > dl > dd, div#dataList > ul > li')
         .slice(0, limit)
         .toArray()
         .map((item) => {
             item = $(item);
 
-            const title = item.text();
             const description = art(path.join(__dirname, 'templates/description.art'), {
-                intro: item.next().text(),
+                intro: item.find('p').text(),
             });
 
             return {
-                title,
+                title: item.find('h3 > a').text(),
                 description,
-                pubDate: parseDate(item.find('span').first().text()),
-                link: item.find('a').prop('href'),
+                pubDate: parseDate(item.find('span').text()),
+                link: item.find('h3 > a').prop('href'),
                 language,
             };
         });

package/lib/routes/zhubai/index.ts

@@ -20,9 +20,9 @@ export const route: Route = {
     name: '文章',
     maintainers: ['naixy28'],
     handler,
-    description: `:::tip
+    description: `::: tip
   在路由末尾处加上 \`?limit=限制获取数目\` 来限制获取条目数量，默认值为\`20\`
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/zimuxia/portfolio.ts

@@ -24,7 +24,7 @@ export const route: Route = {
     name: '剧集',
     maintainers: ['nczitzk'],
     handler,
-    description: `:::tip
+    description: `::: tip
 本路由以 \`magnet\` 为默认 linktype，可以通过在路由后方加上 \`?linktype=链接类型\` 指定导出的链接类型。比如路由为 [\`/zimuxia/portfolio/我们这一天?linktype=baidu\`](https://rsshub.app/zimuxia/portfolio/我们这一天?linktype=baidu) 来导出百度盘链接。目前，你可以选择的 \`链接类型\` 包括: \`magnet\`(默认), \`all\`(所有), \`ed2k\`(电驴), \`baidu\`(百度盘), \`quark\`(夸克盘), \`115\`(115 盘), \`subhd\`(字幕).
 :::`,
 };

package/lib/routes/zjut/cs/index.ts

@@ -27,7 +27,7 @@ export const route: Route = {
     handler,
     radar: [
         {
-            source: ['cs.zjut.edu.cn/jsp/newsclass.jsp?wcId=:type'],
+            source: ['cs.zjut.edu.cn/jsp/newsclass.jsp'],
             target: '/cs/:type',
         },
     ],

package/lib/routes/zyw/hot.ts

@@ -17,9 +17,9 @@ export const route: Route = {
     name: '今日热榜',
     maintainers: ['nczitzk'],
     handler,
-    description: `:::tip
+    description: `::: tip
   全部站点请见 [此处](https://hot.zyw.asia/#/list)
-  :::
+:::
 
   | 哔哩哔哩 | 微博 | 知乎 | 36 氪 | 百度 | 少数派 | IT 之家 | 澎湃新闻 | 今日头条 | 百度贴吧 | 稀土掘金 | 腾讯新闻 |
   | -------- | ---- | ---- | ----- | ---- | ------ | ------- | -------- | -------- | -------- | -------- | -------- |`,

package/lib/types.ts

@@ -52,7 +52,7 @@ export type DataItem = {
     image?: string;
     banner?: string;
     updated?: number | string | Date;
-    language?: string;
+    language?: Language;
     enclosure_url?: string;
     enclosure_type?: string;
     enclosure_title?: string;
@@ -78,14 +78,15 @@ export type Data = {
     allowEmpty?: boolean;
     image?: string;
     author?: string;
-    language?: string;
+    language?: Language;
     feedLink?: string;
     lastBuildDate?: string;
     itunes_author?: string;
     itunes_category?: string;
     itunes_explicit?: string | boolean;
     id?: string;
-
+    icon?: string;
+    logo?: string;
     atomlink?: string;
     ttl?: number;
 };
@@ -247,9 +248,9 @@ interface RouteItem {
 }
 
 interface Route extends RouteItem {
-    ja?: NamespaceItem;
-    zh?: NamespaceItem;
-    'zh-TW'?: NamespaceItem;
+    ja?: RouteItem;
+    zh?: RouteItem;
+    'zh-TW'?: RouteItem;
 }
 
 export type { Route };

package/lib/utils/got.ts

@@ -1,5 +1,6 @@
 import { destr } from 'destr';
 import ofetch from '@/utils/ofetch';
+import { getSearchParamsString } from './helpers';
 
 const getFakeGot = (defaultOptions?: any) => {
     const fakeGot = (request, options?: any) => {
@@ -35,7 +36,7 @@ const getFakeGot = (defaultOptions?: any) => {
             delete options.form;
         }
         if (options?.searchParams) {
-            request += '?' + new URLSearchParams(options.searchParams).toString();
+            request += '?' + getSearchParamsString(options.searchParams);
             delete options.searchParams;
         }
 

package/lib/utils/helpers.test.ts

@@ -1,8 +1,20 @@
 import { describe, expect, it } from 'vitest';
-import { getRouteNameFromPath } from '@/utils/helpers';
+import { getRouteNameFromPath, getSearchParamsString } from '@/utils/helpers';
 
 describe('helpers', () => {
     it('getRouteNameFromPath', () => {
         expect(getRouteNameFromPath('/test/1')).toBe('test');
     });
+
+    it('getSearchParamsString', () => {
+        expect(getSearchParamsString({ a: 1, b: 2 })).toBe('a=1&b=2');
+        expect(getSearchParamsString({ a: 1, b: undefined })).toBe('a=1');
+        expect(getSearchParamsString({ a: undefined })).toBe('');
+        expect(getSearchParamsString({})).toBe('');
+
+        const searchParams = new URLSearchParams();
+        searchParams.append('ids[]', '1');
+        searchParams.append('ids[]', '2');
+        expect(getSearchParamsString(searchParams)).toBe('ids%5B%5D=1&ids%5B%5D=2');
+    });
 });

package/lib/utils/helpers.ts

@@ -1,5 +1,6 @@
 import { fileURLToPath } from 'url';
 import path from 'node:path';
+import { stringifyQuery } from 'ufo';
 
 export const getRouteNameFromPath = (path: string) => {
     const p = path.split('/').filter(Boolean);
@@ -30,3 +31,12 @@ export const getCurrentPath = (metaUrl: string) => {
     const __filename = path.join(fileURLToPath(metaUrl));
     return path.dirname(__filename);
 };
+
+function isPureObject(o: any) {
+    return Object.prototype.toString.call(o) === '[object Object]';
+}
+
+export function getSearchParamsString(searchParams: any) {
+    const searchParamsString = isPureObject(searchParams) ? stringifyQuery(searchParams) : null;
+    return searchParamsString ?? new URLSearchParams(searchParams).toString();
+}

package/lib/utils/ofetch.ts

@@ -1,6 +1,9 @@
 import { createFetch } from 'ofetch';
 import { config } from '@/config';
 import logger from '@/utils/logger';
+import { register } from 'node-network-devtools';
+
+config.enableRemoteDebugging && process.env.NODE_ENV === 'dev' && register();
 
 const rofetch = createFetch().create({
     retryStatusCodes: [400, 408, 409, 425, 429, 500, 502, 503, 504],

package/lib/utils/request-rewriter/fetch.test.ts

@@ -0,0 +1,94 @@
+import { getCurrentCell, setCurrentCell } from 'node-network-devtools';
+import { useCustomHeader } from './fetch';
+import { describe, beforeEach, afterEach, test, expect } from 'vitest';
+
+const getInitRequest = () =>
+    ({
+        requestHeaders: {} as Record<string, string>,
+        id: '',
+        loadCallFrames: () => {},
+        cookies: '',
+        requestData: '',
+        responseData: '',
+        responseHeaders: {},
+        responseInfo: {},
+    }) satisfies NonNullable<ReturnType<typeof getCurrentCell>>['request'];
+
+enum Env {
+    dev = 'dev',
+    production = 'production',
+    test = 'test',
+}
+
+describe('useCustomHeader', () => {
+    let originalEnv: string;
+
+    beforeEach(() => {
+        originalEnv = process.env.NODE_ENV || Env.test;
+        process.env.ENABLE_REMOTE_DEBUGGING = 'true';
+    });
+
+    afterEach(() => {
+        process.env.NODE_ENV = originalEnv;
+    });
+
+    test('should register request with custom headers in dev environment', () => {
+        process.env.NODE_ENV = Env.dev;
+
+        const headers = new Headers();
+        const headerText = 'authorization';
+        const headerValue = 'Bearer token';
+        headers.set(headerText, headerValue);
+
+        const req = getInitRequest();
+        setCurrentCell({
+            request: req,
+            pipes: [],
+            isAborted: false,
+        });
+
+        useCustomHeader(headers);
+
+        const cell = getCurrentCell();
+        expect(cell).toBeDefined();
+
+        let request = req;
+        if (cell) {
+            for (const { pipe } of cell.pipes) {
+                request = pipe(request);
+            }
+        }
+
+        expect(request.requestHeaders[headerText]).toEqual(headerValue);
+    });
+
+    test('should not register request in non-dev environment', () => {
+        process.env.NODE_ENV = Env.production;
+
+        const headers = new Headers();
+        const headerText = 'content-type';
+        const headerValue = 'application/json';
+
+        headers.set(headerText, headerValue);
+        const req = getInitRequest();
+
+        setCurrentCell({
+            request: req,
+            pipes: [],
+            isAborted: false,
+        });
+        useCustomHeader(headers);
+
+        const cell = getCurrentCell();
+        expect(cell).toBeDefined();
+
+        let request = req;
+        if (cell) {
+            for (const { pipe } of cell.pipes) {
+                request = pipe(request);
+            }
+        }
+
+        expect(req.requestHeaders[headerText]).toBeUndefined();
+    });
+});

package/lib/utils/request-rewriter/fetch.ts

@@ -3,6 +3,7 @@ import { config } from '@/config';
 import undici, { Request, RequestInfo, RequestInit } from 'undici';
 import proxy from '@/utils/proxy';
 import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
+import { useRegisterRequest } from 'node-network-devtools';
 
 const limiter = new RateLimiterMemory({
     points: 10,
@@ -11,9 +12,19 @@ const limiter = new RateLimiterMemory({
 });
 
 const limiterQueue = new RateLimiterQueue(limiter, {
-    maxQueueSize: 5000,
+    maxQueueSize: 4800,
 });
 
+export const useCustomHeader = (headers: Headers) => {
+    process.env.NODE_ENV === 'dev' &&
+        useRegisterRequest((req) => {
+            for (const [key, value] of headers.entries()) {
+                req.requestHeaders[key] = value;
+            }
+            return req;
+        });
+};
+
 const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: RequestInit) => {
     const request = new Request(input, init);
     const options: RequestInit = {};
@@ -46,6 +57,8 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ
         request.headers.delete('x-prefer-proxy');
     }
 
+    config.enableRemoteDebugging && useCustomHeader(request.headers);
+
     // proxy
     if (!init?.dispatcher && proxy.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
         const proxyRegex = new RegExp(proxy.proxyObj.url_regex);