rsshub 1.0.0-master.f961961 → 1.0.0-master.f992838

package/lib/v2/mixcloud/radar.js

@@ -0,0 +1,37 @@
+module.exports = {
+    'mixcloud.com': {
+        _name: 'Mixcloud',
+        www: [
+            {
+                title: 'User',
+                docs: 'https://docs.rsshub.app/multimedia.html#mixcloud',
+                source: ['/:username/:type?'],
+                target: (params) => {
+                    if (params.username !== undefined) {
+                        if (['stream', 'uploads', 'favorites', 'listens'].includes(params.type)) {
+                            return `/mixcloud/${params.username}/${params.type}`;
+                        } else if (params.type === undefined) {
+                            return `/mixcloud/${params.username}/uploads`;
+                        }
+                    }
+                },
+            },
+        ],
+        '.': [
+            {
+                title: 'User',
+                docs: 'https://docs.rsshub.app/multimedia.html#mixcloud',
+                source: ['/:username/:type?'],
+                target: (params) => {
+                    if (params.username !== undefined) {
+                        if (['stream', 'uploads', 'favorites', 'listens'].includes(params.type)) {
+                            return `/mixcloud/${params.username}/${params.type}`;
+                        } else if (params.type === undefined) {
+                            return `/mixcloud/${params.username}/uploads`;
+                        }
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/mixcloud/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:username/:type?', require('./index'));
+};

package/lib/v2/mrdx/daily.js

@@ -0,0 +1,79 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { getElementChildrenInnerText } = require('./utils');
+const dayjs = require('dayjs');
+const utc = require('dayjs/plugin/utc');
+dayjs.extend(utc);
+
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    const dateFormatted = dayjs().utcOffset(8).format('YYYYMMDD');
+
+    const baseUrl = `http://mrdx.cn/content/${dateFormatted}`;
+    const link = `${baseUrl}/Page01DK.htm`;
+
+    const indexPageResponse = await got({
+        url: link,
+        method: 'get',
+    });
+    const $ = cheerio.load(indexPageResponse.data);
+
+    // 当日所有文章Url在<a>的`daoxiang` attribute中
+    // 其站点使用**非标准**的HTML语法
+    // eg: `<a class="foo" daoxiang="foo-bar.htm">Lorem ipsum</a>`
+    const passagesList = $('.listdaohang > ul > li a')
+        .toArray()
+        .map((item) => ({
+            link: new URL(`${baseUrl}/${item.attribs.daoxiang}`).href,
+        }));
+
+    const passages = await Promise.all(
+        passagesList.map((passage) =>
+            ctx.cache.tryGet(passage.link, async () => {
+                const response = await got({
+                    url: passage.link,
+                    method: 'get',
+                });
+
+                const $ = cheerio.load(response.data);
+
+                // 标题在`h2`,`h3`,`h4`,`h5`中，有多余空格和换行符
+                // 但是Middleware会处理
+                const title =
+                    getElementChildrenInnerText($('.bggray > .margin15 > h2')[0]) +
+                    getElementChildrenInnerText($('.bggray > .margin15 > h3')[0]) +
+                    getElementChildrenInnerText($('.bggray > .margin15 > h4')[0]) +
+                    getElementChildrenInnerText($('.bggray > .margin15 > h5')[0]);
+
+                // 作者和发布时间
+                // 中间用超多空格分隔
+                // 头尾均有多余空格
+                // eg: 新华每日电讯     2022年12月21日
+                const [author, pubDate] = getElementChildrenInnerText($('.shijian')[0])
+                    .split(' ')
+                    .filter((item) => item !== '')
+                    .map((item) => item.trim());
+
+                const content = $('.contenttext').html();
+
+                return {
+                    title,
+                    author,
+                    pubDate: parseDate(pubDate, 'YYYY年MM月DD日'),
+                    guid: passage.link,
+                    link: passage.link,
+                    description: content,
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '新华每日电讯',
+        link,
+        item: passages,
+        description:
+            '中国报纸发行前三强。位列《人民日报》，《参考消息》之后。《新华每日电讯》是新华社出版的一份新闻电讯报，1993年创办，具有很高的权威性和准确性，有“一报在手，便知天下”之美誉。《新华每日电讯》为对开八版日报，集中刊登[新华社]每天向国内播发的电讯稿及图片稿。打开《新华每日电讯》，中国和全世界每天发生的重大事件便将一目了然。在人类生活节奏日益加快的当今世界，《新华每日电讯》以最便捷、最醒目的方式为公众提供最重要的新闻报道，受到公众的喜爱和拥护。',
+    };
+};

package/lib/v2/mrdx/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/today': ['Dustin-Jiang'],
+};

package/lib/v2/mrdx/radar.js

@@ -0,0 +1,13 @@
+module.exportss = {
+    'mrdx.cn': {
+        _name: '新华每日电讯',
+        '.': [
+            {
+                title: '今日',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xin-hua-mei-ri-dian-xun',
+                source: ['*'],
+                target: '/mrdx/today',
+            },
+        ],
+    },
+};

package/lib/v2/mrdx/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/today', require('./daily'));
+};

package/lib/v2/mrdx/utils.js

@@ -0,0 +1,17 @@
+function getElementChildrenInnerText(element) {
+    let text = '';
+    for (const child of element.children) {
+        if (child.type === 'text') {
+            text += child.data.trim();
+        }
+        if (child.children !== undefined) {
+            text += getElementChildrenInnerText(child);
+        }
+    }
+
+    return text;
+}
+
+module.exports = {
+    getElementChildrenInnerText,
+};

package/lib/v2/mtime/news.js

@@ -5,7 +5,7 @@ const { parseDate } = require('@/utils/parse-date');
 const asyncPool = require('tiny-async-pool');
 
 module.exports = async (ctx) => {
-    const link = 'https://news.mtime.com';
+    const link = 'http://news.mtime.com';
     const response = await got(link, {
         https: {
             rejectUnauthorized: false,

package/lib/v2/myfigurecollection/activity.js

@@ -4,6 +4,7 @@ const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const category = ctx.params.category ?? '-1';
@@ -13,6 +14,10 @@ module.exports = async (ctx) => {
     const latestAlerts = ctx.params.latestAlerts ?? '1';
     const latestPictures = ctx.params.latestPictures ?? '1';
 
+    if (language && !isValidHost(language)) {
+        throw Error('Invalid language');
+    }
+
     const rootUrl = `https://${language === 'en' || language === '' ? '' : `${language}.`}myfigurecollection.net`;
     const currentUrl = `${rootUrl}/browse.v4.php?mode=activity&latestAdditions=${latestAdditions}&latestEdits=${latestEdits}&latestAlerts=${latestAlerts}&latestPictures=${latestPictures}&rootId=${category}`;
 

package/lib/v2/myfigurecollection/index.js

@@ -2,6 +2,7 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { isValidHost } = require('@/utils/valid-host');
 
 const shortcuts = {
     potd: 'picture/browse/potd/',
@@ -12,6 +13,9 @@ const shortcuts = {
 module.exports = async (ctx) => {
     const language = ctx.params.language ?? '';
     const category = ctx.params.category ?? 'figure';
+    if (language && !isValidHost(language)) {
+        throw Error('Invalid language');
+    }
 
     const rootUrl = `https://${language === 'en' || language === '' ? '' : `${language}.`}myfigurecollection.net`;
     const currentUrl = `${rootUrl}/${shortcuts.hasOwnProperty(category) ? shortcuts[category] : category}`;

package/lib/v2/ndss-symposium/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/ndss': ['ZeddYu'],
+};

package/lib/v2/ndss-symposium/ndss.js

@@ -0,0 +1,80 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const url = 'https://www.ndss-symposium.org';
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    // use wordpress api to get data
+    const { data } = await got(`${url}/wp-json/wp/v2/pages?slug=accepted-papers`);
+
+    const items = await Promise.all(
+        data.map(async (elem) => {
+            const $ = cheerio.load(elem.content.rendered);
+            const link = elem.link;
+            const pubDate = parseDate(elem.date);
+
+            const divMatch = $('div h3')
+                .toArray()
+                .map((item) => {
+                    item = $(item);
+                    const title = item.text().trim();
+                    return {
+                        title,
+                        author: item.siblings().text().trim().replaceAll('\n', '').replace(/\s+/g, ' '),
+                        link: item.siblings('a').attr('href'),
+                        pubDate,
+                    };
+                });
+            if (divMatch.length > 0) {
+                const items = await Promise.all(
+                    divMatch.map((item) => {
+                        if (item.link) {
+                            return ctx.cache.tryGet(item.link, async () => {
+                                // some titles and authos are not complete
+                                const response = await got(item.link);
+                                const $ = cheerio.load(response.body);
+                                item.description = $('.paper-data').text().trim().replaceAll('\n', '');
+                                item.title = $('h1.entry-title').text().trim().replaceAll('\n', '').replace(/\s+/g, ' ');
+                                item.author = $('h1.entry-title').siblings().text().trim().replaceAll('\n', '').replace(/\s+/g, ' ');
+                                return item;
+                            });
+                        } else {
+                            item.link = `${link}#${item.title}`;
+                            return item;
+                        }
+                    })
+                );
+                return items;
+            } else {
+                const pMatch = $('p strong')
+                    .toArray()
+                    .map((item) => {
+                        item = $(item);
+                        const title = item.text().trim();
+                        return {
+                            title,
+                            author: item
+                                .parent()
+                                .contents()
+                                .filter((_, e) => e.nodeType === 3)
+                                .text()
+                                .trim()
+                                .replaceAll('\n', '')
+                                .replace(/\s+/g, ' '),
+                            link: `${link}#${title}`,
+                            pubDate,
+                        };
+                    });
+                return pMatch;
+            }
+        })
+    );
+
+    ctx.state.data = {
+        title: 'NDSS',
+        link: url,
+        description: 'The Network and Distributed System Security (NDSS) Symposium Accpeted Papers',
+        allowEmpty: true,
+        item: items.flat(),
+    };
+};

package/lib/v2/ndss-symposium/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'ndss-symposium.org': {
+        _name: 'Network and Distributed System Security (NDSS) Symposium',
+        '.': [
+            {
+                title: 'Accepted papers',
+                docs: 'https://docs.rsshub.app/journal.html#network-and-distributed-system-security-ndss-symposium',
+                source: ['/'],
+                target: '/ndss-symposium/ndss',
+            },
+        ],
+    },
+};

package/lib/v2/ndss-symposium/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/ndss', require('./ndss'));
+};

package/lib/v2/nikkei-cn/index.js

@@ -2,6 +2,7 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
+const { isValidHost } = require('@/utils/valid-host');
 
 const cleanContent = (language, content) => {
     switch (language) {
@@ -23,6 +24,9 @@ module.exports = async (ctx) => {
     const language = ctx.params.language ?? 'cn';
     const category = ctx.params.category ?? '';
     const type = ctx.params.type ?? '';
+    if (!isValidHost(language)) {
+        throw Error('Invalid language');
+    }
 
     const rootUrl = `https://${language === 'zh' ? `zh.cn` : language}.nikkei.com`;
     const currentUrl = `${rootUrl}/${category ? (category === 'rss' ? 'rss.html' : `${category}${type ? `/${type}` : ''}.html`) : ''}`;

package/lib/v2/nuaa/utils/pypasswaf.js

@@ -1,3 +1,5 @@
+const puppeteerUtils = require('@/utils/puppeteer-utils');
+
 /**
  * async function 获取cookie
  * @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.headers.cookie`即可
@@ -14,8 +16,7 @@ module.exports = async function getCookie(host) {
         waitUntil: 'networkidle0',
     });
 
-    let cookie = await page.cookies();
-    cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; ');
+    const cookie = await puppeteerUtils.getCookies(page);
     await browser.close();
     return cookie;
 };

package/lib/v2/paradigm/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/writing': ['Fatpandac'],
+};

package/lib/v2/paradigm/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'paradigm.xyz': {
+        _name: 'Paradigm',
+        '.': [
+            {
+                title: 'Writing',
+                docs: 'https://docs.rsshub.app/finance.html#paradigm',
+                source: ['/writing'],
+                target: '/paradigm/writing',
+            },
+        ],
+    },
+};

package/lib/v2/paradigm/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/writing', require('./writing'));
+};

package/lib/v2/paradigm/writing.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.paradigm.xyz';
+
+module.exports = async (ctx) => {
+    const url = `${baseUrl}/writing`;
+
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+
+    const nextData = JSON.parse($('#__NEXT_DATA__').text());
+    const buildId = nextData.buildId;
+    const list = nextData.props.pageProps.posts.map((item) => ({
+        title: item.title,
+        link: `${baseUrl}${item.slug}`,
+        api: `${baseUrl}/_next/data/${buildId}${item.slug}.json`,
+        author: item.authors.map((author) => author.name).join(', '),
+        pubDate: parseDate(item.originalDate),
+        category: item.tags,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.api);
+                const $ = cheerio.load(response.data.pageProps.content, null, false);
+
+                // Remove the TOC
+                $('.toc').remove();
+                item.description = $.html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'Paradigm - Writing',
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/people/index.js

@@ -3,12 +3,16 @@ const cheerio = require('cheerio');
 const iconv = require('iconv-lite');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const site = ctx.params[0] ?? 'www';
     let category = ctx.params[1] ?? (site === 'www' ? '59476' : '');
     category = site === 'cpc' && category === '24h' ? '87228' : category;
 
+    if (!isValidHost(site)) {
+        throw Error('Invalid site');
+    }
     const rootUrl = `http://${site}.people.com.cn`;
     const currentUrl = `${rootUrl}/GB/${category}/index.html`;
 

package/lib/v2/picuki/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/profile/:id/:functionalFlag?': ['hoilc', 'Rongronggg9'],
+    '/profile/:id/:functionalFlag?': ['hoilc', 'Rongronggg9', 'devinmugen'],
 };

package/lib/v2/picuki/profile.js

@@ -4,8 +4,12 @@ const chrono = require('chrono-node');
 const { art } = require('@/utils/render');
 const path = require('path');
 const config = require('@/config').value;
+const { puppeteerGet } = require('./utils');
 
 module.exports = async (ctx) => {
+    // use Puppeteer due to the obstacle by cloudflare challenge
+    const browser = await require('@/utils/puppeteer')();
+
     const id = ctx.params.id;
     const displayVideo = ctx.params.functionalFlag !== '0';
     const includeStories = ctx.params.functionalFlag === '10';
@@ -15,8 +19,8 @@ module.exports = async (ctx) => {
     const data = await ctx.cache.tryGet(
         profileUrl,
         async () => {
-            const _r = await got(profileUrl);
-            return _r.data;
+            const _r = await puppeteerGet(profileUrl, browser);
+            return _r;
         },
         config.cache.routeExpire,
         false
@@ -31,12 +35,8 @@ module.exports = async (ctx) => {
 
     async function getMedia(url) {
         const getPost = async () => {
-            const response = await got(url, {
-                headers: {
-                    Referer: profileUrl,
-                },
-            });
-            return response.data;
+            const response = await puppeteerGet(url, browser);
+            return response;
         };
         let data = await ctx.cache.tryGet(url, getPost);
         if (Object.prototype.toString.call(data) === '[object Object]') {
@@ -178,6 +178,8 @@ module.exports = async (ctx) => {
         }
     }
 
+    await browser.close();
+
     ctx.state.data = {
         title: `${profileName} (@${id}) - Picuki`,
         link: profileUrl,

package/lib/v2/picuki/utils.js

@@ -0,0 +1,21 @@
+const puppeteerGet = async (url, browser) => {
+    const page = await browser.newPage();
+    // await page.setExtraHTTPHeaders({ referer: host });
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    await page.waitForSelector('.wrapper');
+
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    return html;
+};
+
+module.exports = {
+    puppeteerGet,
+};

package/lib/v2/pixiv/novels.js

@@ -4,32 +4,41 @@ const baseUrl = 'https://www.pixiv.net';
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/all`);
+    const url = `${baseUrl}/users/${id}/novels`;
+    const { data: allData } = await got(`${baseUrl}/ajax/user/${id}/profile/all`, {
+        headers: {
+            referer: url,
+        },
+    });
 
-    const items = [
-        ...data.body.novelSeries.map((item) => ({
-            title: item.title,
-            description: item.caption,
-            link: `https://www.pixiv.net/novel/series/${item.id}`,
-            author: item.userName,
-            pubDate: parseDate(item.createDate),
-            updated: parseDate(item.updateDate),
-            category: item.tags,
-        })),
-        ...data.body.pickup.map((item) => ({
-            title: item.title,
-            description: item.description,
-            link: item.contentUrl,
-            author: item.userName,
-            pubDate: parseDate(item.createDate),
-            updated: parseDate(item.updateDate),
-            category: item.tags,
-        })),
-    ];
+    const novels = Object.keys(allData.body.novels);
+    const searchParams = new URLSearchParams();
+    novels.forEach((novel) => {
+        searchParams.append('ids[]', novel);
+    });
+
+    const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/novels`, {
+        headers: {
+            referer: url,
+        },
+        searchParams,
+    });
+
+    const items = Object.values(data.body.works).map((item) => ({
+        title: item.seriesTitle || item.title,
+        description: item.description || item.title,
+        link: `${baseUrl}/novel/series/${item.id}`,
+        author: item.userName,
+        pubDate: parseDate(item.createDate),
+        updated: parseDate(item.updateDate),
+        category: item.tags,
+    }));
 
     ctx.state.data = {
-        title: `${data.body.novelSeries[0].userName || data.body.pickup[0].userName} 的小说`,
-        link: `${baseUrl}/users/${id}/novels`,
+        title: data.body.extraData.meta.title,
+        description: data.body.extraData.meta.ogp.description,
+        image: Object.values(data.body.works)[0].profileImageUrl,
+        link: url,
         item: items,
     };
 };

package/lib/v2/qlu/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/notice': ['SunBK201'],
+};

package/lib/v2/qlu/notice.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+const host = 'https://www.qlu.edu.cn';
+
+module.exports = async (ctx) => {
+    const response = await got({
+        method: 'get',
+        url: `${host}/tzggsh/list1.htm`,
+    });
+
+    const $ = cheerio.load(response.data);
+    const list = $('ul.news_list.list2').children();
+
+    const items = await Promise.all(
+        list.map((i, item) => {
+            item = $(item);
+            const itemTitle = item.find('.news_title').children().text();
+            const itemDate = item.find('.news_year').text() + item.find('.news_days').text();
+            const path = item.find('.news_title').children().attr('href');
+            let itemUrl = '';
+            if (!path.startsWith('https://') && !path.startsWith('http://')) {
+                itemUrl = host + path;
+            } else {
+                itemUrl = path;
+            }
+            return ctx.cache.tryGet(itemUrl, async () => {
+                let description = '';
+                if (!path.startsWith('https://') && !path.startsWith('http://')) {
+                    const result = await got(itemUrl);
+                    const $ = cheerio.load(result.data);
+                    description = $('.read').html().trim();
+                } else {
+                    description = itemTitle;
+                }
+                return {
+                    title: itemTitle,
+                    link: itemUrl,
+                    pubDate: timezone(parseDate(itemDate), 8),
+                    description,
+                };
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: `齐鲁工业大学 - 通知公告`,
+        link: `${host}/tzggsh/list1.htm`,
+        description: '齐鲁工业大学 - 通知公告',
+        item: items,
+    };
+};

package/lib/v2/qlu/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'qlu.edu.cn': {
+        _name: '齐鲁工业大学',
+        '.': [
+            {
+                title: '通知公告',
+                docs: 'https://docs.rsshub.app/university.html#qi-lu-gong-ye-da-xue',
+                source: ['/tzggsh/list1.htm'],
+                target: '/qlu/notice',
+            },
+        ],
+    },
+};