rsshub 1.0.0-master.facc6d5 → 1.0.0-master.fad0dfa

package/lib/routes/gov/jinan/healthcommission/medical-exam-notice.ts

@@ -50,25 +50,23 @@ async function handler() {
     return {
         title: '济南卫建委-执业考试通知',
         link: `${baseUrl}/col/col14418/index.html`,
-        item: list
-            .map((_, item) => {
-                // 获取每个item对应的html字符串
-                item = $(item).text();
+        item: list.toArray().map((item) => {
+            // 获取每个item对应的html字符串
+            item = $(item).text();
 
-                // 解析上一步中的html
-                const html = load(item);
+            // 解析上一步中的html
+            const html = load(item);
 
-                const title = html('td[width="620"] a').attr('title');
-                const link = html('td[width="620"] a').attr('href');
-                const date = timezone(parseDate(html('td[width="100"]').text()), +8);
-                return {
-                    title,
-                    description: title,
-                    pubDate: date,
-                    link,
-                    author: '济南市卫生健康委员会',
-                };
-            })
-            .get(),
+            const title = html('td[width="620"] a').attr('title');
+            const link = html('td[width="620"] a').attr('href');
+            const date = timezone(parseDate(html('td[width="100"]').text()), +8);
+            return {
+                title,
+                description: title,
+                pubDate: date,
+                link,
+                author: '济南市卫生健康委员会',
+            };
+        }),
     };
 }

package/lib/routes/gov/mee/ywdt.ts

@@ -52,7 +52,8 @@ async function handler(ctx) {
     const list = all
         .find(`div:nth-child(${columns[cate].order})`)
         .find('.mobile_none li , .mobile_clear li')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             const title = $(item).find('a.cjcx_biaob').text().trim();
             const href = $(item).find('a').attr('href');
 
@@ -69,8 +70,7 @@ async function handler(ctx) {
                 title,
                 link,
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         list.map((item) =>

package/lib/routes/gov/miit/wjfb.ts

@@ -38,11 +38,11 @@ async function handler(ctx) {
     const indexContent = load(cookieResponse.data);
     const title = indexContent('div.dqwz > a:nth-child(4)').text();
     const dataRequestUrl = indexContent('div.lmy_main_rb > script:nth-child(2)')
-        .map((_, item) => ({
+        .toArray()
+        .map((item) => ({
             url: `${rootUrl}${indexContent(item).attr('url')}`,
             queryData: JSON.parse(indexContent(item).attr('querydata').replaceAll('"', '|').replaceAll("'", '"').replaceAll('|', '"')),
-        }))
-        .get()[0];
+        }))[0];
 
     const dataUrl = `${dataRequestUrl.url}?${Object.keys(dataRequestUrl.queryData)
         .map((key) => `${key}=${dataRequestUrl.queryData[key]}`)
@@ -56,12 +56,12 @@ async function handler(ctx) {
     });
     const $ = load(response.data.data.html);
     const list = $('ul > li')
-        .map((_, item) => ({
+        .toArray()
+        .map((item) => ({
             title: $(item).find('a').text(),
             link: new URL($(item).find('a').attr('href'), rootUrl).href,
             pubDate: parseDate($(item).find('span').text(), 'YYYY-MM-DD'),
-        }))
-        .get();
+        }));
 
     const items = await Promise.all(
         list.map((item) =>

package/lib/routes/gov/miit/yjzj.ts

@@ -37,11 +37,11 @@ async function handler() {
     const cookie = cookieResponse.headers['set-cookie'][0].split(';')[0];
     const indexContent = load(cookieResponse.data);
     const dataRequestUrl = indexContent('div.clist_con > script:nth-child(2)')
-        .map((_, item) => ({
+        .toArray()
+        .map((item) => ({
             url: `${rootUrl}${indexContent(item).attr('url')}`,
             queryData: JSON.parse(indexContent(item).attr('querydata').replaceAll('"', '|').replaceAll("'", '"').replaceAll('|', '"')),
-        }))
-        .get()[0];
+        }))[0];
 
     const dataUrl = `${dataRequestUrl.url}?${Object.keys(dataRequestUrl.queryData)
         .map((key) => `${key}=${dataRequestUrl.queryData[key]}`)
@@ -55,12 +55,12 @@ async function handler() {
     });
     const $ = load(response.data.data.html);
     const list = $('ul > li')
-        .map((_, item) => ({
+        .toArray()
+        .map((item) => ({
             title: $(item).find('a').text(),
             link: new URL($(item).find('a').attr('href'), rootUrl).href,
             pubDate: parseDate($(item).find('span').text(), 'YYYY-MM-DD'),
-        }))
-        .get();
+        }));
 
     const items = await Promise.all(
         list.map((item) =>

package/lib/routes/gov/miit/zcwj.ts

@@ -25,7 +25,7 @@ async function handler() {
     const base_url = 'http://www.miit.gov.cn/n1146295/n1652858/';
     const response = await got.get(base_url);
     const $ = load(response.data);
-    const list = $('.clist_con li').get();
+    const list = $('.clist_con li').toArray();
 
     const ProcessFeed = (data) => {
         const $ = load(data);

package/lib/routes/gov/moa/gjs.ts

@@ -5,7 +5,8 @@ import ofetch from '@/utils/ofetch';
 import { parseDate } from '@/utils/parse-date';
 import timezone from '@/utils/timezone';
 
-import { type CheerioAPI, type Cheerio, type Element, load } from 'cheerio';
+import { type CheerioAPI, type Cheerio, load } from 'cheerio';
+import type { Element } from 'domhandler';
 import { type Context } from 'hono';
 
 export const handler = async (ctx: Context): Promise<Data> => {

package/lib/routes/gov/moa/moa.ts

@@ -82,7 +82,8 @@ async function dealChannel(suburl, selectors) {
     const channelTitle = channelTitleText ?? $(channelTitleSelector).text();
 
     const pageInfos = $(listSelector)
-        .map((i, e) => {
+        .toArray()
+        .map((e) => {
             const element = $(e);
             const titleElement = element.find(titleSelector);
 
@@ -99,8 +100,7 @@ async function dealChannel(suburl, selectors) {
                 // 如果是公示文章或者站外文章的话只能用这个保底了
                 pubDate: parseRelativeDate(dateraw),
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         pageInfos.map(async (item) => {

package/lib/routes/gov/moa/zdscxx.ts

@@ -17,7 +17,7 @@ export const handler = async (ctx) => {
     const currentUrl = new URL('nyb/pc/messageList.jsp', rootUrl).href;
     const frameUrl = new URL('iframe/top_sj/', rootFrameUrl).href;
 
-    let filterForm = {};
+    const filterForm = {};
 
     if (category) {
         const apiFilterUrl = new URL('nyb/getMessageFilters', rootUrl).href;
@@ -29,17 +29,19 @@ export const handler = async (ctx) => {
             },
         });
 
-        const filters = filterResponse.result.reduce((filters, f) => {
+        const filters: Record<string, string[]> = {};
+        for (const f of filterResponse.result) {
             filters[f.name.trim()] = f.data.map((d) => d.name.trim());
-            return filters;
-        }, {});
-
-        filterForm = category.split(/\//).reduce((form, c) => {
-            for (const key of Object.keys(filters).filter((key) => filters[key].includes(c))) {
-                form[key] = c;
+        }
+
+        const categories = category.split(/\//);
+        for (const c of categories) {
+            for (const key of Object.keys(filters)) {
+                if (filters[key].includes(c)) {
+                    filterForm[key] = c;
+                }
             }
-            return form;
-        }, {});
+        }
     }
 
     const { data: response } = await got.post(apiUrl, {
@@ -108,7 +110,7 @@ export const route: Route = {
     parameters: { category: '分类，默认为全部，见下表' },
     description: `::: tip
   若订阅 [中华人民共和国农业农村部数据](http://zdscxx.moa.gov.cn:8080/nyb/pc/messageList.jsp) 的 \`价格指数\` 报告主题。此时路由为 [\`/gov/moa/zdscxx/价格指数\`](https://rsshub.app/gov/moa/zdscxx/价格指数)。
-  
+
   若订阅 \`央视网\` 报告来源 的 \`蔬菜生产\` 报告主题。此时路由为 [\`/gov/moa/zdscxx/央视网/蔬菜生产\`](https://rsshub.app/gov/moa/zdscxx/央视网/蔬菜生产)。
 :::
 

package/lib/routes/gov/moe/moe.ts

@@ -61,48 +61,46 @@ async function handler(ctx) {
         title: name,
         link: moeUrl,
         item: await Promise.all(
-            newsLis
-                .map(async (_, item) => {
-                    item = $(item);
+            newsLis.toArray().map(async (item) => {
+                item = $(item);
 
-                    const firstA = item.find('a');
-                    const itemUrl = new URL(firstA.attr('href'), moeUrl).href;
+                const firstA = item.find('a');
+                const itemUrl = new URL(firstA.attr('href'), moeUrl).href;
 
-                    // some live pages have no content, just return the liva page url
-                    const infos = itemUrl.includes('/live/')
-                        ? {
-                              description: firstA.html(),
-                          }
-                        : await cache.tryGet(itemUrl, async () => {
-                              const res = {};
-                              const response = await got({
-                                  method: 'get',
-                                  url: itemUrl,
-                                  headers: {
-                                      Referer: moeUrl,
-                                  },
-                              });
-                              const data = load(response.data);
+                // some live pages have no content, just return the liva page url
+                const infos = itemUrl.includes('/live/')
+                    ? {
+                          description: firstA.html(),
+                      }
+                    : await cache.tryGet(itemUrl, async () => {
+                          const res = {};
+                          const response = await got({
+                              method: 'get',
+                              url: itemUrl,
+                              headers: {
+                                  Referer: moeUrl,
+                              },
+                          });
+                          const data = load(response.data);
 
-                              if (itemUrl.includes('www.gov.cn')) {
-                                  res.description = data('#UCAP-CONTENT').html();
-                              } else if (itemUrl.includes('srcsite')) {
-                                  res.description = data('div#content_body_xxgk').html();
-                              } else if (itemUrl.includes('jyb_')) {
-                                  res.description = data('div.moe-detail-box').html() || data('div#moe-detail-box').html();
-                              }
+                          if (itemUrl.includes('www.gov.cn')) {
+                              res.description = data('#UCAP-CONTENT').html();
+                          } else if (itemUrl.includes('srcsite')) {
+                              res.description = data('div#content_body_xxgk').html();
+                          } else if (itemUrl.includes('jyb_')) {
+                              res.description = data('div.moe-detail-box').html() || data('div#moe-detail-box').html();
+                          }
 
-                              return res;
-                          });
+                          return res;
+                      });
 
-                    return {
-                        title: firstA.text(),
-                        description: infos.description,
-                        link: itemUrl,
-                        pubDate: parseDate(item.find('span').text(), 'MM-DD'),
-                    };
-                })
-                .get()
+                return {
+                    title: firstA.text(),
+                    description: infos.description,
+                    link: itemUrl,
+                    pubDate: parseDate(item.find('span').text(), 'MM-DD'),
+                };
+            })
         ),
     };
 }

package/lib/routes/gov/mof/gss.ts

@@ -0,0 +1,82 @@
+import { Data, Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import timezone from '@/utils/timezone';
+import { parseDate } from '@/utils/parse-date';
+import { Context } from 'hono';
+
+const DOMAIN = 'gss.mof.gov.cn';
+
+const handler = async (ctx: Context): Promise<Data | null> => {
+    const { category = 'zhengcefabu' } = ctx.req.param();
+    const currentUrl = `https://${DOMAIN}/gzdt/${category}/`;
+    const { data: response } = await got(currentUrl);
+    const $ = load(response);
+    const title = $('title').text();
+    const author = $('div.zzName').text();
+    const siteName = $('meta[name="SiteName"]').prop('content');
+    const description = $('meta[name="ColumnDescription"]').prop('content');
+    const indexes = $('ul.liBox li')
+        .toArray()
+        .map((li) => {
+            const a = $(li).find('a');
+            const pubDate = $(li).find('span').text();
+            const href = a.prop('href') as string;
+            const link = href.startsWith('http') ? href : new URL(href, currentUrl).href;
+            return {
+                title: a.prop('title'),
+                link,
+                pubDate: timezone(parseDate(pubDate), +8),
+            };
+        });
+
+    const items = await Promise.all(
+        indexes.map((item: Data) =>
+            cache.tryGet(item.link!, async () => {
+                const { data: detailResponse } = await got(item.link);
+                const content = load(detailResponse);
+                item.description = content('div.my_doccontent').html() ?? '';
+                item.author = author;
+                return item;
+            })
+        )
+    );
+
+    return {
+        item: items,
+        title,
+        link: currentUrl,
+        description: `${description} - ${siteName}`,
+        author,
+    } as Data;
+};
+
+export const route: Route = {
+    path: '/mof/gss/:category?',
+    categories: ['government'],
+    example: '/gov/mof/gss',
+    parameters: { category: '列表标签，默认为政策发布' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '关税政策文件',
+    maintainers: ['la3rence'],
+    handler,
+    description: `#### 关税文件发布
+
+| 政策发布 | 政策解读 |
+| ------------- | -------------- |
+| zhengcefabu   | zhengcejiedu   |`,
+    radar: [
+        {
+            source: ['gss.mof.gov.cn/gzdt/:category/'],
+            target: '/mof/gss/:category',
+        },
+    ],
+};

package/lib/routes/gov/nea/ghs.ts

@@ -1,9 +1,10 @@
-import { Route } from '@/types';
+import type { Route, DataItem } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
-import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
+import { URL } from 'node:url';
+import timezone from '@/utils/timezone';
 
 export const route: Route = {
     path: '/nea/sjzz/ghs',
@@ -21,64 +22,96 @@ export const route: Route = {
     radar: [
         {
             source: ['nea.gov.cn/sjzz/ghs/'],
+            target: '/nea/sjzz/ghs',
         },
     ],
     name: '发展规划司',
-    maintainers: ['nczitzk'],
+    maintainers: ['nczitzk', 'pseudoyu'],
     handler,
-    url: 'nea.gov.cn/sjzz/ghs/',
+    url: 'www.nea.gov.cn/sjzz/ghs/',
 };
 
 async function handler(ctx) {
     const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 35;
 
     const rootUrl = 'https://www.nea.gov.cn';
-    const currentUrl = new URL('sjzz/ghs/', rootUrl).href;
-
-    const { data: response } = await got(currentUrl);
+    const targetUrl: string = new URL('sjzz/ghs/', rootUrl).href;
 
+    const response = await ofetch(targetUrl);
     const $ = load(response);
 
-    let items = $('div.right_box ul li')
-        .slice(0, limit)
-        .toArray()
-        .map((item) => {
-            item = $(item);
+    const dataSourceId: string | undefined = $('ul#showData0').attr('data')?.split(/:/).pop();
 
-            const a = item.find('a');
+    if (!dataSourceId) {
+        throw new Error('Data source ID not found');
+    }
 
-            return {
-                title: a.text(),
-                link: a.prop('href'),
-                pubDate: parseDate(item.find('span.date-tex').text()),
-            };
-        });
+    const jsonUrl = new URL(`ds_${dataSourceId}.json`, targetUrl).href;
 
-    items = await Promise.all(
-        items.map((item) =>
-            cache.tryGet(item.link, async () => {
-                const { data: detailResponse } = await got(item.link);
+    const jsonData: NeaGhsResponse = await ofetch(jsonUrl);
+
+    const list: DataItem[] = jsonData.datasource.slice(0, limit).map((item) => {
+        const itemLink = new URL(item.publishUrl, rootUrl).href;
+
+        const $title = load(item.showTitle);
+        const titleText = $title.text();
 
-                const content = load(detailResponse);
+        return {
+            title: titleText,
+            link: itemLink,
+            pubDate: item.publishTime ? timezone(parseDate(item.publishTime), +8) : undefined,
+            description: item.summary?.trim() || titleText,
+            author: [...new Set([item.sourceText, item.author, item.editor, item.responsibleEditor].filter(Boolean))].map((author) => ({
+                name: author,
+            })),
+            category: item.keywords.split(/,/),
+        };
+    });
 
-                item.title = content('meta[name="ArticleTitle"]').prop('content');
-                item.description = content('td.detail').html() || content('div.article-content td').html();
-                item.author = content('meta[name="ContentSource"]').prop('content');
-                item.category = content('meta[name="keywords"]').prop('content').split(/,/);
-                item.pubDate = timezone(parseDate(content('meta[name="PubDate"]').prop('content')), +8);
+    const items = await Promise.all(
+        list.map((item: DataItem) =>
+            cache.tryGet(item.link, async () => {
+                try {
+                    const detailResponse = await ofetch(item.link);
+                    const content = load(detailResponse);
 
+                    item.title = content('meta[name="ArticleTitle"]').prop('content') || item.title;
+                    item.description = content('td.detail').html() || content('div.article-content').html() || item.description;
+                    item.category = [...new Set([...(item.category ?? []), ...(content('meta[name="keywords"]').attr('conetnt')?.split(/,/) ?? [])])];
+                    const detailPubDate = content('meta[name="PubDate"]').prop('content');
+                    item.pubDate = detailPubDate ? timezone(parseDate(detailPubDate), +8) : item.pubDate;
+                } catch {
+                    // logger.error(`Failed to fetch detail for ${item.link}`);
+                }
                 return item;
             })
         )
     );
 
+    const filteredItems: DataItem[] = items.filter(Boolean) as DataItem[];
+
     return {
-        item: items,
-        title: $('title').text(),
-        link: currentUrl,
-        description: $('meta[name="ColumnDescription"]').prop('content'),
-        language: 'zh',
-        subtitle: $('meta[name="ColumnType"]').prop('content'),
-        author: $('meta[name="ColumnKeywords"]').prop('content'),
+        item: filteredItems,
+        title: '国家能源局 - 发展规划司工作进展',
+        link: targetUrl,
+        description: '国家能源局 - 发展规划司工作进展',
     };
 }
+
+interface NeaGhsItem {
+    showTitle: string;
+    publishUrl: string;
+    publishTime: string;
+    summary?: string;
+    sourceText?: string;
+    author?: string;
+    editor?: string;
+    responsibleEditor?: string;
+    keywords: string;
+}
+
+interface NeaGhsResponse {
+    categoryName?: string;
+    categoryDesc?: string;
+    datasource: NeaGhsItem[];
+}

package/lib/routes/gov/nrta/dsj.ts

@@ -3,7 +3,7 @@ import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
-import asyncPool from 'tiny-async-pool';
+import pMap from 'p-map';
 
 export const route: Route = {
     path: '/nrta/dsj/:category?',
@@ -52,23 +52,22 @@ async function handler(ctx) {
             };
         });
 
-    const results = [];
+    const results = await pMap(
+        items,
+        (item) =>
+            cache.tryGet(item.link, async () => {
+                const { data: detailResponse } = await got(item.link);
 
-    for await (const item of asyncPool(5, items, (item) =>
-        cache.tryGet(item.link, async () => {
-            const { data: detailResponse } = await got(item.link);
+                const content = load(detailResponse);
 
-            const content = load(detailResponse);
+                content('table').last().remove();
 
-            content('table').last().remove();
+                item.description = content('td.newstext').html() || content('table').last().parent().parent().html();
 
-            item.description = content('td.newstext').html() || content('table').last().parent().parent().html();
-
-            return item;
-        })
-    )) {
-        results.push(item);
-    }
+                return item;
+            }),
+        { concurrency: 5 }
+    );
 
     return {
         item: results,

package/lib/routes/gov/nrta/news.ts

@@ -52,13 +52,13 @@ async function handler(ctx) {
     });
 
     const list = $('a', 'record')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             return {
                 link: item.attr('href'),
             };
-        })
-        .get();
+        });
     const items = await Promise.all(
         list.map((item) =>
             cache.tryGet(item.link, async () => {

package/lib/routes/gov/pbc/goutongjiaoliu.ts

@@ -45,15 +45,15 @@ async function handler() {
 
     const $ = load(html);
     const list = $('font.newslist_style')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             const a = item.find('a[title]');
             return {
                 title: a.attr('title'),
                 link: new URL(a.attr('href'), 'http://www.pbc.gov.cn').href,
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         list.map((item) =>

package/lib/routes/gov/pbc/gzlw.ts

@@ -35,12 +35,12 @@ async function handler() {
     const response = await got.post(url);
     const $ = load(response.data);
     const list = $('li.clearfix')
-        .map((_index, item) => ({
+        .toArray()
+        .map((item) => ({
             title: $(item).find('a').text(),
             link: new URL($(item).find('a').attr('href'), host).href,
             author: $(item).find('span.fr').text().replaceAll('…', ''),
-        }))
-        .get();
+        }));
 
     const items = await processItems(list);
 

package/lib/routes/gov/pbc/trade-announcement.ts

@@ -38,15 +38,15 @@ async function handler() {
     const html = await page.evaluate(() => document.documentElement.innerHTML);
     const $ = load(html);
     const list = $('font.newslist_style')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             const a = item.find('a[title]');
             return {
                 title: a.attr('title'),
                 link: new URL(a.attr('href'), 'http://www.pbc.gov.cn').href,
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         list.map((item) =>