rsshub 1.0.0-master.fc1b99f → 1.0.0-master.fc2cf18

package/lib/routes/bjnews/column.ts

@@ -0,0 +1,43 @@
+import { Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+import { fetchArticle } from './utils';
+
+export const route: Route = {
+    path: '/column/:column',
+    categories: ['traditional-media'],
+    example: '/bjnews/column/204',
+    parameters: { column: '栏目ID, 可从手机版网页URL中找到' },
+    features: {},
+    radar: [
+        {
+            source: ['m.bjnews.com.cn/column/:column.htm'],
+        },
+    ],
+    name: '分类',
+    maintainers: ['dzx-dzx'],
+    handler,
+    url: 'www.bjnews.com.cn',
+};
+
+async function handler(ctx) {
+    const columnID = ctx.req.param('column');
+    const url = `https://api.bjnews.com.cn/api/v101/news/column_news.php?column_id=${columnID}`;
+    const res = await ofetch(url);
+    const list = res.data.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 15).map((e) => ({
+        title: e.row.title,
+        guid: e.uuid,
+        pubDate: timezone(parseDate(e.row.publish_time), +8),
+        updated: timezone(parseDate(e.row.update_time), +8),
+        link: `https://www.bjnews.com.cn/detail/${e.uuid}.html`,
+    }));
+
+    const out = await Promise.all(list.map((item) => fetchArticle(item)));
+    return {
+        title: `新京报 - 栏目 - ${res.data[0].row.column_info[0].column_name}`,
+        link: `https://m.bjnews.com.cn/column/${columnID}.html`,
+        item: out,
+    };
+}

package/lib/routes/bjnews/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '新京报',
+    url: 'www.bjnews.com.cn',
+};

package/lib/routes/bjnews/utils.ts

@@ -0,0 +1,19 @@
+import cache from '@/utils/cache';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+
+export function fetchArticle(item) {
+    return cache.tryGet(item.link, async () => {
+        const responses = await ofetch(item.link);
+        const $d = load(responses);
+        // $d('img').each((i, e) => $(e).attr('referrerpolicy', 'no-referrer'));
+
+        item.pubDate = timezone(parseDate($d('.left-info .timer').text()), +8);
+        item.author = $d('.left-info .reporter').text();
+        item.description = $d('#contentStr').html();
+
+        return item;
+    });
+}

package/lib/routes/bjtu/gs.ts

@@ -0,0 +1,224 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+const rootURL = 'https://gs.bjtu.edu.cn';
+const urlCms = `${rootURL}/cms/item/?tag=`;
+const urlZszt = `${rootURL}/cms/zszt/item/?cat=`;
+const title = ' - 北京交通大学研究生院';
+const zsztRegex = /_zszt/;
+const struct = {
+    noti_zs: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 1,
+        name: '通知公告_招生',
+    },
+    noti: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 2,
+        name: '通知公告',
+    },
+    news: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 3,
+        name: '新闻动态',
+    },
+    zsxc: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 4,
+        name: '招生宣传',
+    },
+    py: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 5,
+        name: '培养',
+    },
+    zs: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 6,
+        name: '招生',
+    },
+    xw: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 7,
+        name: '学位',
+    },
+    ygb: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 8,
+        name: '研工部',
+    },
+    ygbtzgg: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 9,
+        name: '通知公告 - 研工部',
+    },
+    ygbnews: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 10,
+        name: '新闻动态 - 研工部',
+    },
+    ygbnewscover: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 11,
+        name: '新闻封面 - 研工部',
+    },
+    all: {
+        selector: {
+            list: '.tab-content li',
+        },
+        tag: 12,
+        name: '文章列表',
+    },
+    bszs_zszt: {
+        selector: {
+            list: '.mainleft_box li',
+        },
+        tag: 2,
+        name: '博士招生 - 招生专题',
+    },
+    sszs_zszt: {
+        selector: {
+            list: '.mainleft_box li',
+        },
+        tag: 3,
+        name: '硕士招生 - 招生专题',
+    },
+    zsjz_zszt: {
+        selector: {
+            list: '.mainleft_box li',
+        },
+        tag: 4,
+        name: '招生简章 - 招生专题',
+    },
+    zcfg_zszt: {
+        selector: {
+            list: '.mainleft_box li',
+        },
+        tag: 5,
+        name: '政策法规 - 招生专题',
+    },
+};
+
+const getItem = (item, selector) => {
+    const newsInfo = item.find('a');
+    const newsDate = item
+        .find('span')
+        .text()
+        .match(/\d{4}(-|\/|.)\d{1,2}\1\d{1,2}/)[0];
+
+    const infoTitle = newsInfo.text();
+    const link = rootURL + newsInfo.attr('href');
+    return cache.tryGet(link, async () => {
+        const resp = await ofetch(link);
+        const $$ = load(resp);
+        const infoText = $$(selector).html();
+
+        return {
+            title: infoTitle,
+            pubDate: parseDate(newsDate),
+            link,
+            description: infoText,
+        };
+    }) as any;
+};
+
+export const route: Route = {
+    path: '/gs/:type?',
+    categories: ['university'],
+    example: '/bjtu/gs/noti',
+    parameters: { type: 'Article type' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['gs.bjtu.edu.cn'],
+        },
+    ],
+    name: '研究生院',
+    maintainers: ['E1nzbern'],
+    handler,
+    description: `
+  | 文章来源           | 参数         |
+  | ----------------- | ------------ |
+  | 通知公告_招生      | noti_zs      |
+  | 通知公告           | noti         |
+  | 新闻动态           | news         |
+  | 招生宣传           | zsxc         |
+  | 培养               | py           |
+  | 招生               | zs           |
+  | 学位               | xw           |
+  | 研工部             | ygb          |
+  | 通知公告 - 研工部   | ygbtzgg      |
+  | 新闻动态 - 研工部   | ygbnews      |
+  | 新闻封面 - 研工部   | ygbnewscover |
+  | 文章列表           | all          |
+  | 博士招生 - 招生专题 | bszs_zszt    |
+  | 硕士招生 - 招生专题 | sszs_zszt    |
+  | 招生简章 - 招生专题 | zsjz_zszt    |
+  | 政策法规 - 招生专题 | zcfg_zszt    |
+
+  :::tip 文章来源
+  文章来源的命名均来自研究生院网站标题。
+  最常用的几项有“通知公告_招生”、“通知公告”、“博士招生 - 招生专题”、“硕士招生 - 招生专题”。
+  :::`,
+};
+
+async function handler(ctx) {
+    const { type = 'noti' } = ctx.req.param();
+    let url = urlCms;
+    let selectorArticle = 'div.main_left.main_left_list';
+    if (zsztRegex.test(type)) {
+        url = urlZszt;
+        selectorArticle = 'div.mainleft_box';
+    }
+    const urlAddr = `${url}${struct[type].tag}`;
+    const resp = await ofetch(urlAddr);
+    const $ = load(resp);
+
+    const list = $(struct[type].selector.list);
+
+    const items = await Promise.all(
+        list.toArray().map((i) => {
+            const item = $(i);
+            return getItem(item, selectorArticle);
+        })
+    );
+
+    return {
+        title: `${struct[type].name}${title}`,
+        link: urlAddr,
+        item: items,
+        allowEmpty: true,
+    };
+}

package/lib/routes/bjtu/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+export const namespace: Namespace = {
+    name: 'Beijing Jiaotong University',
+    url: 'bjtu.edu.cn',
+    zh: {
+        name: '北京交通大学',
+    },
+};

package/lib/routes/bjx/fd.ts

@@ -0,0 +1,63 @@
+import { DataItem, Route } from '@/types';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
+
+export const route: Route = {
+    path: '/fd/:type',
+    categories: ['traditional-media'],
+    example: '/bjx/fd/yw',
+    parameters: { type: '文章分类，详见下表' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '风电',
+    maintainers: ['hualiong'],
+    description: `\`:type\` 类型可选如下
+
+  | 要闻 | 政策 | 数据 | 市场 | 企业 | 招标 | 技术 | 报道 |
+| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| yw   | zc   | sj   | sc   | mq   | zb   | js   | bd   |`,
+    handler: async (ctx) => {
+        const type = ctx.req.param('type');
+        const response = await ofetch(`https://fd.bjx.com.cn/${type}/`);
+
+        const $ = load(response);
+        const typeName = $('div.box2 em:last-child').text();
+        const list = $('div.cc-list-content ul li:nth-child(-n+20)')
+            .toArray()
+            .map((item): DataItem => {
+                const each = $(item);
+                return {
+                    title: each.find('a').attr('title')!,
+                    link: each.find('a').attr('href'),
+                    pubDate: parseDate(each.find('span').text()),
+                };
+            });
+
+        const items = await Promise.all(
+            list.map((item) =>
+                cache.tryGet(item.link!, async () => {
+                    const response = await ofetch(item.link!);
+                    const $ = load(response);
+
+                    item.description = $('#article_cont').html()!;
+                    return item;
+                })
+            )
+        );
+
+        return {
+            title: `北极星风力发电网${typeName}`,
+            description: $('meta[name="Description"]').attr('content'),
+            link: `https://fd.bjx.com.cn/${type}/`,
+            item: items as DataItem[],
+        };
+    },
+};

package/lib/routes/bjx/namespace.ts

@@ -2,5 +2,5 @@ import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
     name: '北极星电力网',
-    url: 'guangfu.bjx.com.cn',
+    url: 'www.bjx.com.cn',
 };

package/lib/{routes-deprecated/blogread/newest.js → routes/blogread/index.ts}

@@ -1,8 +1,23 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
+import { Route } from '@/types';
+import got from '@/utils/got';
+import * as cheerio from 'cheerio';
 
-module.exports = async (ctx) => {
-    const url = 'http://blogread.cn/news/newest.php';
+export const route: Route = {
+    path: '/newest',
+    categories: ['programming'],
+    example: '/blogread/newest',
+    radar: [
+        {
+            source: ['blogread.cn/news/newest.php'],
+        },
+    ],
+    name: '最新文章',
+    maintainers: ['fashioncj'],
+    handler,
+};
+
+async function handler() {
+    const url = 'https://blogread.cn/news/newest.php';
     const response = await got({
         method: 'get',
         url,
@@ -12,19 +27,19 @@ module.exports = async (ctx) => {
         .map((index, elem) => {
             elem = $(elem);
             const $link = elem.find('dt a');
-
             return {
                 title: $link.text(),
                 description: elem.find('dd').eq(0).text(),
                 link: $link.attr('href'),
                 author: elem.find('.small a').eq(0).text(),
+                pubDate: elem.find('dd').eq(1).text().split('\n')[2],
             };
         })
         .get();
 
-    ctx.state.data = {
+    return {
         title: '技术头条',
         link: url,
         item: resultItem,
     };
-};
+}

package/lib/routes/blogread/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '技术头条',
+    url: 'blogread.cn',
+};

package/lib/routes/bloomberg/index.ts

@@ -16,9 +16,38 @@ const site_title_mapping = {
 };
 
 export const route: Route = {
-    path: ['/:site', '/'],
-    name: 'Unknown',
-    maintainers: [],
+    path: '/:site?',
+    categories: ['finance'],
+    example: '/bloomberg/bbiz',
+    parameters: {
+        site: 'Site ID, can be found below',
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Bloomberg Site',
+    maintainers: ['bigfei'],
+    description: `
+    | Site ID      | Title        |
+    | ------------ | ------------ |
+    | /            | News         |
+    | bpol         | Politics     |
+    | bbiz         | Business     |
+    | markets      | Markets      |
+    | technology   | Technology   |
+    | green        | Green        |
+    | wealth       | Wealth       |
+    | pursuits     | Pursuits     |
+    | bview        | Opinion      |
+    | equality     | Equality     |
+    | businessweek | Businessweek |
+    | citylab      | CityLab      |
+    `,
     handler,
 };
 

package/lib/routes/bloomberg/utils.ts

@@ -3,11 +3,13 @@ const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
 import { load } from 'cheerio';
-import * as path from 'node:path';
+import path from 'node:path';
 import asyncPool from 'tiny-async-pool';
+import { destr } from 'destr';
 
 import { parseDate } from '@/utils/parse-date';
 import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { art } from '@/utils/render';
 
 const rootUrl = 'https://www.bloomberg.com/feeds';
@@ -60,6 +62,15 @@ const regex = [pageTypeRegex1, pageTypeRegex2];
 const capRegex = /<p>|<\/p>/g;
 const emptyRegex = /<p\b[^>]*>(&nbsp;|\s)<\/p>/g;
 
+const redirectGot = (url) =>
+    ofetch.raw(url, {
+        headers,
+        parseResponse: (responseText) => ({
+            data: destr(responseText),
+            body: responseText,
+        }),
+    });
+
 const parseNewsList = async (url, ctx) => {
     const resp = await got(url);
     const $ = load(resp.data, {
@@ -74,9 +85,9 @@ const parseNewsList = async (url, ctx) => {
         .map((u) => {
             u = $(u);
             const item = {
-                title: u.find('news\\:title').text(),
+                title: u.find(String.raw`news\:title`).text(),
                 link: u.find('loc').text(),
-                pubDate: parseDate(u.find('news\\:publication_date').text()),
+                pubDate: parseDate(u.find(String.raw`news\:publication_date`).text()),
             };
             return item;
         });
@@ -96,12 +107,12 @@ const parseArticle = (item) =>
 
                 try {
                     const apiUrl = `${api.url}${link}`;
-                    res = await got(apiUrl, { headers });
+                    res = await redirectGot(apiUrl);
                 } catch (error) {
                     // fallback
-                    if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) {
+                    if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError' || error.name === 'FetchError')) {
                         try {
-                            res = await got(item.link, { headers });
+                            res = await redirectGot(item.link);
                         } catch {
                             // return the default one
                             return {
@@ -114,8 +125,7 @@ const parseArticle = (item) =>
                 }
 
                 // Blocked by PX3, or 404 by both api and direct link, return the default
-                const redirectUrls = res.redirectUrls.map(String);
-                if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html') || res.statusCode === 404) {
+                if ((res.redirected && new URL(res.url).pathname === '/tosv2.html') || res.status === 404) {
                     return {
                         title: item.title,
                         link: item.link,
@@ -125,15 +135,15 @@ const parseArticle = (item) =>
 
                 switch (page) {
                     case 'audio':
-                        return parseAudioPage(res, api, item);
+                        return parseAudioPage(res._data, api, item);
                     case 'videos':
-                        return parseVideoPage(res, api, item);
+                        return parseVideoPage(res._data, api, item);
                     case 'photo-essays':
-                        return parsePhotoEssaysPage(res, api, item);
+                        return parsePhotoEssaysPage(res._data, api, item);
                     case 'features/': // single features page
-                        return parseReactRendererPage(res, api, item);
+                        return parseReactRendererPage(res._data, api, item);
                     default: // use story api to get json
-                        return parseStoryJson(res.data, item);
+                        return parseStoryJson(res._data.data, item);
                 }
             }
         }
@@ -210,11 +220,11 @@ const parseReactRendererPage = async (res, api, item) => {
     const json = load(res.data)(api.sel).text().trim();
     const story_id = JSON.parse(json)[api.prop];
     try {
-        const res = await got(`${idUrl}${story_id}`, { headers });
-        return await parseStoryJson(res.data, item);
+        const res = await redirectGot(`${idUrl}${story_id}`);
+        return await parseStoryJson(res._data, item);
     } catch (error) {
         // fallback
-        if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) {
+        if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError' || error.name === 'FetchError')) {
             return {
                 title: item.title,
                 link: item.link,
@@ -364,11 +374,10 @@ const processBody = async (body_html, story_json) => {
 
 const processVideo = async (bmmrId, summary) => {
     const api = `https://www.bloomberg.com/multimedia/api/embed?id=${bmmrId}`;
-    const res = await got(api, { headers });
+    const res = await redirectGot(api);
 
     // Blocked by PX3, return the default
-    const redirectUrls = res.redirectUrls.map(String);
-    if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) {
+    if ((res.redirected && new URL(res.url).pathname === '/tosv2.html') || res.status === 404) {
         return {
             stream: '',
             mp4: '',
@@ -377,8 +386,8 @@ const processVideo = async (bmmrId, summary) => {
         };
     }
 
-    if (res.data) {
-        const video_json = res.data;
+    if (res._data.data) {
+        const video_json = res._data.data;
         return {
             stream: video_json.streams ? video_json.streams[0]?.url : '',
             mp4: video_json.downloadURLs ? video_json.downloadURLs['600'] : '',
@@ -386,7 +395,12 @@ const processVideo = async (bmmrId, summary) => {
             caption: video_json.description || video_json.title || summary,
         };
     }
-    return {};
+    return {
+        stream: '',
+        mp4: '',
+        coverUrl: '',
+        caption: summary,
+    };
 };
 
 const nodeRenderers = {

package/lib/routes/bluearchive/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Blue Archive',
+    url: 'bluearchive.jp',
+    categories: ['game'],
+};