rsshub 1.0.0-master.fc1b99f → 1.0.0-master.fc2cf18

package/lib/routes/gov/{shanghai → sh}/yjj/index.ts

@@ -7,17 +7,17 @@ import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
-    path: '/shanghai/yjj/*',
+    path: ['/sh/yjj/*', '/shanghai/yjj/*'],
     name: 'Unknown',
     maintainers: [],
     handler,
 };
 
 async function handler(ctx) {
-    const params = getSubPath(ctx) === '/shanghai/yjj' ? '/shanghai/yjj/zx-ylqx' : getSubPath(ctx);
+    const params = getSubPath(ctx) === '/sh/yjj' ? '/sh/yjj/zx-ylqx' : getSubPath(ctx);
 
     const rootUrl = 'https://yjj.sh.gov.cn';
-    const currentUrl = `${rootUrl}${params.replace(/^\/shanghai\/yjj/, '')}/index.html`;
+    const currentUrl = `${rootUrl}${params.replace(/^\/sh\/yjj/, '')}/index.html`;
 
     const response = await got({
         method: 'get',

package/lib/routes/gov/shenzhen/xxgk/zfxxgj.ts

@@ -4,6 +4,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import timezone from '@/utils/timezone';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
 const rootUrl = 'http://www.sz.gov.cn/cn/xxgk/zfxxgj/';
 
 const config = {
@@ -49,7 +50,7 @@ export const route: Route = {
 async function handler(ctx) {
     const cfg = config[ctx.req.param('caty')];
     if (!cfg) {
-        throw new Error('Bad category. See <a href="https://docs.rsshub.app/routes/government#guang-dong-sheng-ren-min-zheng-fu-guang-dong-sheng-shen-zhen-shi-ren-min-zheng-fu">docs</a>');
+        throw new InvalidParameterError('Bad category. See <a href="https://docs.rsshub.app/routes/government#guang-dong-sheng-ren-min-zheng-fu-guang-dong-sheng-shen-zhen-shi-ren-min-zheng-fu">docs</a>');
     }
 
     const currentUrl = new URL(cfg.link, rootUrl).href;

package/lib/routes/gov/shenzhen/zjj/index.ts

@@ -3,6 +3,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import timezone from '@/utils/timezone';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 const config = {
     tzgg: {
@@ -41,7 +42,7 @@ async function handler(ctx) {
     const baseUrl = 'http://zjj.sz.gov.cn/xxgk/';
     const cfg = config[ctx.req.param('caty')];
     if (!cfg) {
-        throw new Error('Bad category. See <a href="https://docs.rsshub.app/routes/government#guang-dong-sheng-ren-min-zheng-fu-shen-zhen-shi-zhu-fang-he-jian-she-ju">docs</a>');
+        throw new InvalidParameterError('Bad category. See <a href="https://docs.rsshub.app/routes/government#guang-dong-sheng-ren-min-zheng-fu-shen-zhen-shi-zhu-fang-he-jian-she-ju">docs</a>');
     }
 
     const currentUrl = new URL(cfg.link, baseUrl).href;

package/lib/routes/gov/sichuan/deyang/govpublicinfo.ts

@@ -7,7 +7,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 import timezone from '@/utils/timezone';
 
 // 各地区url信息

package/lib/routes/gov/sichuan/deyang/mztoday.ts

@@ -7,7 +7,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 import timezone from '@/utils/timezone';
 
 const rootUrl = 'http://www.mztoday.gov.cn';

package/lib/routes/gov/stats/index.ts

@@ -4,12 +4,12 @@ const __dirname = getCurrentPath(import.meta.url);
 
 import { getSubPath } from '@/utils/common-utils';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 export const route: Route = {
     path: '/stats/*',
@@ -27,37 +27,39 @@ export const route: Route = {
         },
     ],
     description: `::: tip
-    路径处填写对应页面 URL 中 \`http://www.stats.gov.cn/\` 后的字段。下面是一个例子。
+路径处填写对应页面 URL 中 \`http://www.stats.gov.cn/\` 后的字段。下面是一个例子。
 
-    若订阅 [数据 > 数据解读](http://www.stats.gov.cn/sj/sjjd/) 则将对应页面 URL \`http://www.stats.gov.cn/sj/sjjd/\` 中 \`http://www.stats.gov.cn/\` 后的字段 \`sj/sjjd\` 作为路径填入。此时路由为 [\`/gov/stats/sj/sjjd\`](https://rsshub.app/gov/stats/sj/sjjd)
+若订阅 [数据 > 数据解读](http://www.stats.gov.cn/sj/sjjd/)
+则将对应页面 URL \`http://www.stats.gov.cn/sj/sjjd/\` 中 \`http://www.stats.gov.cn/\` 后的字段 \`sj/sjjd\` 作为路径填入。
+此时路由为 [\`/gov/stats/sj/sjjd\`](https://rsshub.app/gov/stats/sj/sjjd)
 
-    若订阅 [新闻 > 时政要闻 > 中央精神](http://www.stats.gov.cn/xw/szyw/zyjs/) 则将对应页面 URL \`http://www.stats.gov.cn/xw/szyw/zyjs/\` 中 \`http://www.stats.gov.cn/\` 后的字段 \`xw/szyw/zyjs\` 作为路径填入。此时路由为 [\`/gov/stats/xw/szyw/zyjs\`](https://rsshub.app/gov/stats/xw/szyw/zyjs)
-    :::`,
+若订阅 [新闻 > 时政要闻 > 中央精神](http://www.stats.gov.cn/xw/szyw/zyjs/)
+则将对应页面 URL \`http://www.stats.gov.cn/xw/szyw/zyjs/\` 中 \`http://www.stats.gov.cn/\`
+后的字段 \`xw/szyw/zyjs\` 作为路径填入。此时路由为 [\`/gov/stats/xw/szyw/zyjs\`](https://rsshub.app/gov/stats/xw/szyw/zyjs)
+:::`,
 };
 
 async function handler(ctx) {
     const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 15;
     const rootUrl = 'http://www.stats.gov.cn';
 
+    const { headers } = await ofetch.raw(rootUrl);
+    const sid = headers
+        ?.getSetCookie()
+        .find((s) => s.startsWith('wzws_sessionid='))
+        ?.split(';')[0] as string;
+
     const pathname = getSubPath(ctx) === '/stats' ? '/sj/zxfb/' : getSubPath(ctx).replace(/^\/stats(.*)/, '$1');
     const currentUrl = `${rootUrl}${pathname.endsWith('/') ? pathname : pathname + '/'}`;
 
-    let response = await got({
-        method: 'get',
-        url: rootUrl,
-    });
-
-    const headers = {
-        cookie: response.headers['set-cookie'].join(' ').match(/(wzws_sessionid=.*?);/)[1],
-    };
-
-    response = await got({
-        method: 'get',
-        url: currentUrl,
-        headers,
+    const response = await ofetch(currentUrl, {
+        headers: {
+            Cookie: sid,
+            Referer: currentUrl,
+        },
     });
 
-    const $ = load(response.data);
+    const $ = load(response);
 
     let items = $($('a.pchide').length === 0 ? 'a[title]' : '.list-content a.pchide')
         .slice(0, limit)
@@ -74,13 +76,14 @@ async function handler(ctx) {
     items = await Promise.all(
         items.map((item) =>
             cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                    headers,
+                const detailResponse = await ofetch(item.link, {
+                    headers: {
+                        Cookie: sid,
+                        Referer: rootUrl,
+                    },
                 });
 
-                const content = load(detailResponse.data);
+                const content = load(detailResponse);
 
                 // articles from www.news.cn or www.gov.cn
 

package/lib/routes/gov/suzhou/news.ts

@@ -4,6 +4,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import timezone from '@/utils/timezone';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
     path: '/suzhou/news/:uid',
@@ -119,7 +120,7 @@ async function handler(ctx) {
             title = '苏州市政府 - 民生资讯';
             break;
         default:
-            throw new Error('pattern not matched');
+            throw new InvalidParameterError('pattern not matched');
     }
     if (apiUrl) {
         const response = await got(apiUrl);

package/lib/routes/gov/zhengce/index.ts

@@ -70,19 +70,16 @@ async function handler(ctx) {
                     const agencyEl = content('table.bd1')
                         .find('td')
                         .toArray()
-                        .filter((a) => content(a).text().startsWith('发文机关'))
-                        .pop();
+                        .findLast((a) => content(a).text().startsWith('发文机关'));
 
                     const sourceEl = content('span.font-zyygwj')
                         .toArray()
-                        .filter((a) => content(a).text().startsWith('来源'))
-                        .pop();
+                        .findLast((a) => content(a).text().startsWith('来源'));
 
                     const subjectEl = content('table.bd1')
                         .find('td')
                         .toArray()
-                        .filter((a) => content(a).text().startsWith('主题分类'))
-                        .pop();
+                        .findLast((a) => content(a).text().startsWith('主题分类'));
 
                     const agency = agencyEl ? processElementText(agencyEl) : undefined;
                     const source = sourceEl ? processElementText(sourceEl) : undefined;

package/lib/routes/gov/zj/search.ts

@@ -0,0 +1,80 @@
+import { Route } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import dayjs from 'dayjs';
+
+export const route: Route = {
+    path: '/zj/search/:websiteid?/:word/:cateid?',
+    categories: ['government'],
+    example: '/gov/zj/search',
+    parameters: {
+        websiteid: '搜索范围-全省、各市各区、详细信息点击源网站https://www.zj.gov.cn/请求中寻找',
+        word: '搜索关键词-默认：人才',
+        cateid: '信息分类-默认：658（全部）',
+        sortType: '排序类型-默认：2（按时间）',
+    },
+    radar: [
+        {
+            source: ['search.zj.gov.cn/jsearchfront/search.do'],
+            target: '/zj/search/:websiteid?/:word/:cateid?',
+        },
+    ],
+    name: '浙江省人民政府-全省政府网站统一搜索',
+    url: 'search.zj.gov.cn/jsearchfront/search.do',
+    maintainers: ['HaoyuLee'],
+    description: `
+        | 行政区域         | websiteid |
+        | ------------ | -- |
+        | 宁波市本级     | 330201000000000  |
+
+        | 搜索关键词         | word    |
+
+        | 信息分类         | cateid    |
+
+        | 排序类型         | sortType    |
+        | ------------ | -- |
+        | 按相关度     | 1  |
+        | 按时间     | 2  |
+    `,
+    async handler(ctx) {
+        const { websiteid = '330201000000000', word = '人才', cateid = 658, sortType = 2 } = ctx.req.param();
+        const {
+            data: { result: list },
+        } = await got.post('https://search.zj.gov.cn/jsearchfront/interfaces/cateSearch.do', {
+            form: {
+                websiteid,
+                pg: '30',
+                p: '1',
+                cateid,
+                word,
+                checkError: 1,
+                isContains: 0,
+                q: word,
+                begin: dayjs().subtract(1, 'week').format('YYYYMMDD'),
+                end: dayjs().format('YYYYMMDD'),
+                timetype: 2,
+                pos: 'title,content,keyword',
+                sortType,
+            },
+        });
+        const items =
+            list?.map((item: string) => {
+                const $ = load(item);
+                const title = $('.titleWrapper>a');
+                const footer = $('.sourceTime>span');
+                return {
+                    title: title.text().trim() || '',
+                    link: title.attr('href') || '',
+                    pubDate: parseDate(footer.eq(1).text().trim().replace('时间:', '')) || '',
+                    author: footer.eq(0).text().trim().replace('来源:', '') || '',
+                    description: $('.newsDescribe>a').text() || '',
+                };
+            }) || [];
+        return {
+            title: '浙江省人民政府-全省政府网站统一搜索',
+            link: 'https://search.zj.gov.cn/jsearchfront/search.do',
+            item: items,
+        };
+    },
+};

package/lib/routes/gq/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'GQ',
+    url: 'gq.com',
+};

package/lib/routes/gq/news.ts

@@ -0,0 +1,59 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import parser from '@/utils/rss-parser';
+import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
+const host = 'https://www.gq.com';
+export const route: Route = {
+    path: '/news',
+    categories: ['traditional-media'],
+    example: '/gq/news',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['gq.com/'],
+        },
+    ],
+    name: 'News',
+    maintainers: ['EthanWng97'],
+    handler,
+};
+
+async function handler() {
+    const rssUrl = `${host}/feed/rss`;
+    const feed = await parser.parseURL(rssUrl);
+    const items = await Promise.all(
+        feed.items.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const data = await ofetch(item.link);
+                const $ = load(data);
+                const description = $('#main-content');
+                description.find('.article-body__footer').remove();
+                description.find('[class*="ContentHeaderContributorImage"]').remove();
+                description.find('h1').remove();
+                return {
+                    title: item.title,
+                    pubDate: item.pubDate,
+                    link: item.link,
+                    category: item.categories,
+                    description: description.html(),
+                };
+            })
+        )
+    );
+
+    return {
+        title: 'GQ',
+        link: host,
+        description: `GQ is the global flagship of men's fashion, the arbiter of cool for anyone who sees the world through the lens of taste and style.`,
+        item: items,
+    };
+}

package/lib/routes/grist/utils.ts

@@ -1,8 +1,8 @@
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { parseDate } from '@/utils/parse-date';
 import timezone from '@/utils/timezone';
 
-const getData = (url) => got.get(url).json();
+const getData = (url) => ofetch(url);
 
 const getList = (data) =>
     data.map((value) => {

package/lib/routes/guangdiu/search.ts

@@ -9,7 +9,7 @@ const host = 'https://guangdiu.com';
 export const route: Route = {
     path: '/search/:query?',
     categories: ['shopping'],
-    example: '/guangdiu/search/k=百度网盘',
+    example: '/guangdiu/search/q=百度网盘',
     parameters: { query: '链接参数，对应网址问号后的内容' },
     features: {
         requireConfig: false,

package/lib/routes/guduodata/daily.ts

@@ -6,7 +6,7 @@ import cache from '@/utils/cache';
 import got from '@/utils/got';
 import dayjs from 'dayjs';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 const host = 'http://data.guduodata.com';
 

package/lib/routes/gumroad/index.ts

@@ -5,8 +5,9 @@ const __dirname = getCurrentPath(import.meta.url);
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 import { isValidHost } from '@/utils/valid-host';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
     path: '/:username/:products',
@@ -31,7 +32,7 @@ async function handler(ctx) {
     const username = ctx.req.param('username');
     const products = ctx.req.param('products');
     if (!isValidHost(username)) {
-        throw new Error('Invalid username');
+        throw new InvalidParameterError('Invalid username');
     }
     const url = `https://${username}.gumroad.com/l/${products}`;
 

package/lib/routes/guokr/channel.ts

@@ -1,15 +1,7 @@
 import { Route } from '@/types';
-import cache from '@/utils/cache';
 import got from '@/utils/got';
-
-async function loadFullPage(id) {
-    const link = `https://apis.guokr.com/minisite/article/${id}.json`;
-    const content = await cache.tryGet(link, async () => {
-        const res = await got(link);
-        return res.data.result.content;
-    });
-    return content;
-}
+import { parseList, parseItem } from './utils';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 const channelMap = {
     calendar: 'pac',
@@ -19,21 +11,13 @@ const channelMap = {
 };
 
 export const route: Route = {
-    path: '/:channel',
+    path: '/column/:channel',
     categories: ['new-media'],
-    example: '/guokr/calendar',
+    example: '/guokr/column/calendar',
     parameters: { channel: '专栏类别' },
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
     radar: [
         {
-            source: ['guokr.com/'],
+            source: ['guokr.com/:channel'],
         },
     ],
     name: '果壳网专栏',
@@ -48,29 +32,28 @@ export const route: Route = {
 async function handler(ctx) {
     const channel = channelMap[ctx.req.param('channel')] ?? ctx.req.param('channel');
 
-    const response = await got(`https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_wx&channel_key=${channel}&offset=0&limit=10`);
-    const items = response.data.result;
+    const { data: response } = await got(`https://www.guokr.com/apis/minisite/article.json`, {
+        searchParams: {
+            retrieve_type: 'by_wx',
+            channel_key: channel,
+            offset: 0,
+            limit: 10,
+        },
+    });
+    const result = parseList(response.result);
 
-    if (items.length === 0) {
-        throw new Error('Unknown channel');
+    if (result.length === 0) {
+        throw new InvalidParameterError('Unknown channel');
     }
 
-    const channel_name = items[0].channels[0].name;
-    const channel_url = items[0].channels[0].url;
+    const channelName = result[0].channels[0].name;
+    const channelUrl = result[0].channels[0].url;
 
-    const result = await Promise.all(
-        items.map(async (item) => ({
-            title: item.title,
-            description: await loadFullPage(item.id), // Mercury 无法正确解析全文，故这里手动加载
-            pubDate: item.date_published,
-            link: item.url,
-            author: item.author.nickname,
-        }))
-    );
+    const items = await Promise.all(result.map((item) => parseItem(item)));
 
     return {
-        title: `果壳网 ${channel_name}`,
-        link: channel_url,
-        item: result,
+        title: `果壳网 ${channelName}`,
+        link: channelUrl,
+        item: items,
     };
 }

package/lib/routes/guokr/scientific.ts

@@ -1,21 +1,11 @@
 import { Route } from '@/types';
-import cache from '@/utils/cache';
 import got from '@/utils/got';
-import { load } from 'cheerio';
+import { parseList, parseItem } from './utils';
 
 export const route: Route = {
     path: '/scientific',
     categories: ['new-media'],
     example: '/guokr/scientific',
-    parameters: {},
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
     radar: [
         {
             source: ['guokr.com/scientific', 'guokr.com/'],
@@ -28,29 +18,21 @@ export const route: Route = {
 };
 
 async function handler() {
-    const response = await got('https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_subject&limit=20&offset=0');
+    const { data: response } = await got('https://www.guokr.com/beta/proxy/science_api/articles', {
+        searchParams: {
+            retrieve_type: 'by_category',
+            page: 1,
+        },
+    });
+
+    const result = parseList(response);
 
-    const result = response.data.result;
+    const items = await Promise.all(result.map((item) => parseItem(item)));
 
     return {
         title: '果壳网 科学人',
         link: 'https://www.guokr.com/scientific',
         description: '果壳网 科学人',
-        item: await Promise.all(
-            result.map((item) =>
-                cache.tryGet(item.url, async () => {
-                    const res = await got(item.url);
-                    const $ = load(res.data);
-                    item.description = $('.eflYNZ #js_content').css('visibility', 'visible').html() ?? $('.bxHoEL').html();
-                    return {
-                        title: item.title,
-                        description: item.description,
-                        pubDate: item.date_published,
-                        link: item.url,
-                        author: item.author.nickname,
-                    };
-                })
-            )
-        ),
+        item: items,
     };
 }

package/lib/routes/guokr/utils.ts

@@ -0,0 +1,37 @@
+import { parseDate } from '@/utils/parse-date';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import * as cheerio from 'cheerio';
+
+export const parseList = (result) =>
+    result.map((item) => ({
+        title: item.title,
+        description: item.summary,
+        pubDate: parseDate(item.date_published),
+        link: `https://www.guokr.com/article/${item.id}/`,
+        author: item.author.nickname,
+        category: item.subject?.name,
+        id: item.id,
+        channels: item.channels,
+    }));
+
+export const parseItem = (item) =>
+    cache.tryGet(item.link, async () => {
+        const { data: res } = await got(`https://apis.guokr.com/minisite/article/${item.id}.json`);
+        const $ = cheerio.load(res.result.content);
+
+        $('#meta_content').remove();
+        $('div').each((_, elem) => {
+            const $elem = $(elem);
+            $elem.attr('style', $elem.attr('style')?.replaceAll(/(?:display:\s*none|visibility:\s*hidden|opacity:\s*0);?/g, ''));
+        });
+        $('img').each((_, elem) => {
+            const $elem = $(elem);
+            if ($elem.attr('data-src')) {
+                $elem.attr('src', $elem.attr('data-src'));
+            }
+        });
+        item.description = $.html();
+
+        return item;
+    });

package/lib/routes/gxmzu/ai.ts

@@ -31,7 +31,7 @@ export const route: Route = {
 async function handler(ctx) {
     const out = await getNoticeList(ctx, url, host, 'a', '.timestyle55267', {
         title: '.titlestyle55269',
-        content: '#vsb_content',
+        content: '#vsb_newscontent',
         date: '.timestyle55269',
     });