rsshub 1.0.0-master.f7add90 → 1.0.0-master.f7bfacc

package/lib/v2/openai/common.js

@@ -0,0 +1,72 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const getApiUrl = async () => {
+    const blogRootUrl = 'https://openai.com/blog';
+
+    // Find API base URL
+    const initResponse = await got({
+        method: 'get',
+        url: blogRootUrl,
+    });
+
+    const apiBaseUrl = initResponse.data
+        .toString()
+        .match(/(?<=TWILL_API_BASE:").+?(?=")/)[0]
+        .replaceAll('\\u002F', '/');
+
+    return new URL(apiBaseUrl);
+};
+
+const parseArticle = (ctx, rootUrl, attributes) =>
+    ctx.cache.tryGet(attributes.slug, async () => {
+        const textUrl = `${rootUrl}/${attributes.slug}`;
+        const detailResponse = await got({
+            method: 'get',
+            url: textUrl,
+        });
+        let content = cheerio.load(detailResponse.data);
+
+        const authors = content('[aria-labelledby="metaAuthorsHeading"] > li > a > span > span')
+            .toArray()
+            .map((entry) => content(entry).text())
+            .join(', ');
+
+        // Leave out comments
+        const comments = content('*')
+            .contents()
+            .filter(function () {
+                return this.nodeType === 8;
+            });
+        comments.remove();
+
+        content = content('#content');
+
+        const imageSrc = attributes.seo.ogImageSrc;
+        const imageAlt = attributes.seo.ogImageAlt;
+
+        const article = art(path.join(__dirname, 'templates/article.art'), {
+            content,
+            imageSrc,
+            imageAlt,
+        });
+
+        // Not all article has tags
+        attributes.tags = attributes.tags || [];
+
+        return {
+            title: attributes.title,
+            author: authors,
+            description: article,
+            pubDate: attributes.createdAt,
+            category: attributes.tags.map((tag) => tag.title),
+            link: textUrl,
+        };
+    });
+
+module.exports = {
+    getApiUrl,
+    parseArticle,
+};

package/lib/v2/openai/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
     '/blog/:tag?': ['StevenRCE0', 'nczitzk'],
+    '/research': ['yuguorui'],
 };

package/lib/v2/openai/radar.js

@@ -23,5 +23,13 @@ module.exports = {
                 target: () => '/openai/chatgpt/release-notes',
             },
         ],
+        research: [
+            {
+                title: 'Research',
+                docs: 'https://docs.rsshub.app/en/new-media.html#openai',
+                source: '/research',
+                target: () => '/openai/research',
+            },
+        ],
     },
 };

package/lib/v2/openai/research.js

@@ -0,0 +1,32 @@
+const got = require('@/utils/got');
+const { getApiUrl, parseArticle } = require('./common');
+
+module.exports = async (ctx) => {
+    const apiUrl = new URL('/api/v1/research-publications', await getApiUrl());
+    const researchRootUrl = 'https://openai.com/research';
+
+    // Construct API query
+    apiUrl.searchParams.append('sort', '-publicationDate,-createdAt');
+    apiUrl.searchParams.append('include', 'media');
+
+    const resp = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+    const obj = resp.data;
+
+    const items = await Promise.all(
+        obj.data.map((item) => {
+            const attributes = item.attributes;
+            return parseArticle(ctx, researchRootUrl, attributes);
+        })
+    );
+
+    const title = 'OpenAI Research';
+
+    ctx.state.data = {
+        title,
+        link: researchRootUrl,
+        item: items,
+    };
+};

package/lib/v2/openai/router.js

@@ -1,4 +1,5 @@
 module.exports = function (router) {
     router.get('/blog/:tag?', require('./blog'));
     router.get('/chatgpt/release-notes', require('./chatgpt'));
+    router.get('/research', require('./research'));
 };

package/lib/v2/pumc/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/mdadmission': ['nczitzk'],
+};

package/lib/v2/pumc/mdadmission.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 100;
+
+    const rootUrl = 'https://mdadmission.pumc.edu.cn';
+    const currentUrl = `${rootUrl}/mdweb/site!noticeList?param.infoTypeId=&rows=${limit}&pages=1`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+        https: {
+            rejectUnauthorized: false,
+        },
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('div.media')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const a = item.find('h4.media-heading a');
+
+            return {
+                title: a.text(),
+                link: new URL(a.attr('href'), currentUrl).href,
+                pubDate: parseDate(item.find('span').first().text(), 'DDYYYY-MM'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('h3').remove();
+
+                item.description = content('div.media-body div').last().html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '北京协和医学院招生网 - 通知公告',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/pumc/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'pumc.edu.cn': {
+        _name: '北京协和医学院',
+        mdadmission: [
+            {
+                title: '“4+4”试点班招生网通知公告',
+                docs: 'https://docs.rsshub.app/university.html#bei-jing-xie-he-yi-xue-yuan',
+                source: ['/mdweb/site', '/'],
+                target: '/pumc/mdadmission',
+            },
+        ],
+    },
+};

package/lib/v2/pumc/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/mdadmission', require('./mdadmission'));
+};

package/lib/v2/rattibha/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/user/:user': ['yshalsager'],
+};

package/lib/v2/rattibha/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'rattibha.com': {
+        _name: 'رتبها - Rattibha',
+        '.': [
+            {
+                title: 'User Threads',
+                docs: 'https://docs.rsshub.app/en/social-media.html#rattibha',
+                source: ['/:user'],
+                target: '/rattibha/user/:user',
+            },
+        ],
+    },
+};

package/lib/v2/rattibha/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/user/:user', require('./user'));
+};

package/lib/v2/rattibha/user.js

@@ -0,0 +1,45 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://rattibha.com';
+    const { user: twitterUser } = ctx.params;
+
+    const {
+        data: { user: userData },
+    } = await got(`${baseUrl}/user?id=${twitterUser}`, {
+        headers: {
+            accept: 'application/json',
+        },
+    });
+    const { data: userThreads } = await got(`${baseUrl}/u_threads?id=${userData.account_user_id}`, { headers: { accept: 'application/json' } });
+
+    // extract the relevant data from the API response
+    const list = userThreads.map((item) => ({
+        title: item.thread.t.info.text,
+        link: `${baseUrl}/thread/${item.thread_id}`,
+        pubDate: parseDate(item.thread.created_at),
+        updated: parseDate(item.thread.updated_at),
+        author: userData.name,
+        threadData1URL: `${baseUrl}/thread?id=${item.thread_id}`,
+        threadData2URL: `${baseUrl}/threads?id=${item.thread_id}`,
+    }));
+
+    // Get tweet full text
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: data1 } = await got(item.threadData1URL, { headers: { accept: 'application/json' } });
+                const { data: data2 } = await got(item.threadData2URL, { headers: { accept: 'application/json' } });
+                item.description = [...data1.tweets, ...data2].reduce((accumulator, tweet) => `${accumulator}${tweet.tweet_detail.info.text}<br>`, '');
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `سلاسل تغريدات ${twitterUser}`,
+        link: `${baseUrl}/${twitterUser}`,
+        item: items,
+    };
+};

package/lib/v2/readhub/index.js

@@ -15,7 +15,7 @@ module.exports = async (ctx) => {
     let overview = false;
     if (titles[ctx.params.category]) {
         category = ctx.params.category;
-        overview = ctx.params.overview ? true : false;
+        overview = !!ctx.params.overview;
         category = category === 'tech' ? 'technews' : category;
     } else if (ctx.params.category) {
         overview = true;

package/lib/{routes → v2}/right/forum.js

@@ -1,58 +1,56 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const iconv = require('iconv-lite');
 const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const id = ctx.params.id || '31';
+    const id = ctx.params.id ?? '31';
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 20;
 
     const rootUrl = 'https://www.right.com.cn';
     const currentUrl = `${rootUrl}/forum/forum-${id}-1.html`;
+
     const response = await got({
         method: 'get',
         url: currentUrl,
-        responseType: 'buffer',
     });
 
-    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+    const $ = cheerio.load(response.data);
 
     $('a[title="隐藏置顶帖"]').each(function () {
         $(this).parents('tbody').remove();
     });
 
-    const list = $('.s')
-        .map((_, item) => {
+    let items = $('.s')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
             item = $(item);
+
             return {
                 title: item.text(),
                 link: `${rootUrl}/forum/${item.attr('href')}`,
             };
-        })
-        .get();
+        });
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',
                     url: item.link,
-                    responseType: 'buffer',
                 });
 
-                detailResponse.data = iconv.decode(detailResponse.data, 'gbk');
-                const postId = detailResponse.data.match(/<div id="post_(\d+)" >/)[1];
                 const content = cheerio.load(detailResponse.data);
 
-                item.author = content('.authi').eq(0).text();
-                item.description = content('#postmessage_' + postId).html();
-                item.pubDate = timezone(
-                    new Date(
-                        content('#authorposton' + postId)
-                            .text()
-                            .replace('发表于 ', '')
-                    ),
-                    +8
-                );
+                content('.pstatus').remove();
+
+                item.author = content('.authi').first().text();
+                item.description = content('.t_f').first().html();
+                item.pubDate = timezone(parseDate(content('.authi em').first().text().replace('发表于 ', '')), +8);
+                item.category = content('.ptg a')
+                    .toArray()
+                    .map((a) => content(a).text());
 
                 return item;
             })

package/lib/v2/right/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/forum/:id?': ['nczitzk'],
+};

package/lib/v2/right/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'right.com.cn': {
+        _name: '恩山无线论坛',
+        '.': [
+            {
+                title: '板块',
+                docs: 'https://docs.rsshub.app/bbs.html#en-shan-wu-xian-lun-tan',
+                source: ['/forum', '/'],
+                target: (params, url) => `/right/forum/${new URL(url).href.match(/\/forum-(\d+)-\d+.html/)[1]}`,
+            },
+        ],
+    },
+};

package/lib/v2/right/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/forum/:id?', require('./forum'));
+};

package/lib/v2/rsshub/radar.js

@@ -5,13 +5,13 @@ module.exports = {
             {
                 title: '有新路由啦',
                 docs: 'https://docs.rsshub.app/program-update.html#rsshub',
-                source: ['', '/*tpath'],
+                source: ['/*'],
                 target: '/rsshub/routes',
             },
             {
                 title: '有新赞助商啦',
                 docs: 'https://docs.rsshub.app/program-update.html#rsshub',
-                source: ['', '/*tpath'],
+                source: ['/*'],
                 target: '/rsshub/sponsors',
             },
         ],

package/lib/v2/sdust/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/yjsy/zhaosheng/:id?': ['nczitzk'],
+};

package/lib/v2/sdust/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'sdust.edu.cn': {
+        _name: '山东科技大学',
+        '.': [
+            {
+                title: '研究生招生网',
+                docs: 'https://docs.rsshub.app/university.html#shan-dong-ke-ji-da-xue-yan-jiu-sheng-zhao-sheng-wang',
+                source: ['/zhaosheng', '/'],
+                target: (params, url) => `/sdust/yjsy/zhaosheng/${new URL(url).href.match(/zhaosheng\/(.*)\.htm/)[1]}`,
+            },
+        ],
+    },
+};

package/lib/v2/sdust/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/yjsy/zhaosheng/:id?', require('./yjsy/zhaosheng'));
+};

package/lib/v2/sdust/yjsy/zhaosheng.js

@@ -0,0 +1,52 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? 'tzgg';
+
+    const rootUrl = 'https://yjsy.sdust.edu.cn';
+    const currentUrl = `${rootUrl}/zhaosheng/${id}.htm`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.pageUl ul li a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.attr('title'),
+                link: new URL(item.attr('href'), currentUrl).href,
+                pubDate: parseDate(item.find('span').text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('.txt, .v_news_content').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/sehuatang/maintainer.js

@@ -1,6 +1,7 @@
 module.exports = {
     '/bt/:subforumid?': ['qiwihui', 'junfengP', 'nczitzk'],
     '/picture/:subforumid': ['qiwihui', 'junfengP', 'nczitzk'],
+    '/user/:uid': ['JamYiz'],
     '/:subforumid?/:type?': ['qiwihui', 'junfengP', 'nczitzk'],
     '/:subforumid?': ['qiwihui', 'junfengP', 'nczitzk'],
     '/': ['qiwihui', 'junfengP', 'nczitzk'],

package/lib/v2/sehuatang/router.js

@@ -1,6 +1,7 @@
 module.exports = function (router) {
     router.get('/bt/:subforumid?', require('./index'));
     router.get('/picture/:subforumid', require('./index'));
+    router.get('/user/:uid', require('./user'));
     router.get('/:subforumid?/:type?', require('./index'));
     router.get('/:subforumid?', require('./index'));
     router.get('', require('./index'));

package/lib/v2/sehuatang/user.js

@@ -0,0 +1,113 @@
+// 导入必要的模组
+const got = require('@/utils/got'); // 自订的 got
+const cheerio = require('cheerio'); // 可以使用类似 jQuery 的 API HTML 解析器
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+
+const baseUrl = 'https://sehuatang.org/';
+
+module.exports = async (ctx) => {
+    if (!config.sehuatang.cookie) {
+        throw 'Sehuatang RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>';
+    }
+    // 从Url参数中获取uid
+    const uid = ctx.params.uid;
+    const link = `${baseUrl}home.php?mod=space&uid=${uid}&do=thread&view=me&from=space`;
+
+    const response = await got(link, {
+        headers: {
+            Cookie: config.sehuatang.cookie,
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+        },
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('#delform tr:not(.th)')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 25)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('th>a').first();
+            const category = item.find('.xg1').first().text();
+            return {
+                title: `[${category}] ${a.text()}`,
+                // `link` 需要一个绝对 URL，但 `a.attr('href')` 返回一个相对 URL。
+                link: baseUrl + a.attr('href'),
+                // pubDate: '',
+                author: $('.mt').first().text(),
+            };
+        });
+
+    const out = await Promise.all(
+        list.map((info) =>
+            ctx.cache.tryGet(info.link, async () => {
+                const response = await got(info.link, {
+                    headers: {
+                        Cookie: config.sehuatang.cookie,
+                        'Accept-Encoding': 'gzip, deflate, br',
+                        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+                    },
+                });
+
+                const $ = cheerio.load(response.data);
+                const postMessage = $("[id^='postmessage']").slice(0, 1);
+                const images = $(postMessage).find('img');
+                for (const image of images) {
+                    const file = $(image).attr('file');
+                    if (!file || file === 'undefined') {
+                        $(image).replaceWith('');
+                    } else {
+                        $(image).replaceWith($(`<img src="${file}">`));
+                    }
+                }
+                // if postMessage does not have any images, try to parse image url from `.pattl`
+                if (images.length === 0) {
+                    const pattl = $('.pattl');
+                    const pattlImages = $(pattl).find('img');
+                    for (const pattlImage of pattlImages) {
+                        const file = $(pattlImage).attr('file');
+                        if (!file || file === 'undefined') {
+                            $(pattlImage).replaceWith('');
+                        } else {
+                            $(pattlImage).replaceWith($(`<img src="${file}" />`));
+                        }
+                    }
+                    postMessage.append($(pattl));
+                }
+                $('em[onclick]').remove();
+
+                info.description = (postMessage.html() || '抓取原帖失败').replace(/ignore_js_op/g, 'div');
+
+                const dateString = $('.authi em').first().text();
+                const datestampString = dateString.split(' ')[1];
+                const timestampString = dateString.split(' ')[2];
+                const datetimeString = `${datestampString} ${timestampString}`;
+                const timestamp = new Date(datetimeString).getTime();
+
+                info.pubDate = $('.authi em span').length > 0 ? parseDate($('.authi em span').attr('title')) : parseDate(timestamp);
+
+                const magnet = postMessage.find('div.blockcode li').first().text();
+                const isMag = magnet.startsWith('magnet');
+                const torrent = postMessage.find('p.attnm a').attr('href');
+
+                const hasEnclosureUrl = isMag || torrent !== undefined;
+                if (hasEnclosureUrl) {
+                    const enclosureUrl = isMag ? magnet : new URL(torrent, baseUrl).href;
+                    info.enclosure_url = enclosureUrl;
+                    info.enclosure_type = isMag ? 'application/x-bittorrent' : 'application/octet-stream';
+                }
+
+                return info;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        // 在此处输出您的 RSS
+        title: `${$('.mt').text()}的帖子-色花堂`,
+        link,
+        item: out,
+    };
+};