rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rsshub",
-  "version": "1.0.0-master.f7c0247",
+  "version": "1.0.0-master.f7c4a93",
   "description": "Make RSS Great Again!",
   "main": "lib/pkg.js",
   "files": [
@@ -43,22 +43,22 @@
   "devDependencies": {
     "@napi-rs/pinyin": "1.7.0",
     "@types/koa": "2.13.4",
-    "@vercel/nft": "0.18.2",
+    "@vercel/nft": "0.20.0",
     "@vuepress/plugin-back-to-top": "1.9.7",
     "@vuepress/plugin-google-analytics": "1.9.7",
     "@vuepress/plugin-pwa": "1.9.7",
     "@vuepress/shared-utils": "1.9.7",
     "cross-env": "7.0.3",
-    "eslint": "8.15.0",
+    "eslint": "8.18.0",
     "eslint-config-prettier": "8.5.0",
     "eslint-plugin-prettier": "4.0.0",
-    "eslint-plugin-yml": "0.15.0",
+    "eslint-plugin-yml": "1.0.0",
     "fs-extra": "10.1.0",
-    "jest": "28.1.0",
+    "jest": "28.1.1",
     "mockdate": "3.0.5",
-    "nock": "13.2.4",
+    "nock": "13.2.7",
     "nodemon": "2.0.16",
-    "prettier": "2.6.2",
+    "prettier": "2.7.1",
     "prettier-check": "2.0.0",
     "pretty-quick": "3.1.3",
     "remark": "13.0.0",
@@ -79,31 +79,31 @@
   "dependencies": {
     "@koa/router": "10.1.1",
     "@postlight/mercury-parser": "2.2.1",
-    "@sentry/node": "6.19.7",
+    "@sentry/node": "7.2.0",
     "aes-js": "3.1.2",
     "art-template": "4.13.2",
     "bbcodejs": "0.0.4",
-    "cheerio": "1.0.0-rc.10",
+    "cheerio": "1.0.0-rc.11",
     "chrono-node": "2.3.8",
     "city-timezones": "1.2.0",
     "crypto-js": "4.1.1",
-    "currency-symbol-map": "5.0.1",
-    "dayjs": "1.11.2",
+    "currency-symbol-map": "5.1.0",
+    "dayjs": "1.11.3",
     "dotenv": "16.0.1",
     "emailjs-imap-client": "3.1.0",
     "entities": "3.0.1",
     "etag": "1.8.1",
     "fanfou-sdk": "4.2.0",
     "git-rev-sync": "3.0.2",
-    "googleapis": "100.0.0",
-    "got": "11.8.3",
+    "googleapis": "104.0.0",
+    "got": "11.8.5",
     "https-proxy-agent": "5.0.1",
     "iconv-lite": "0.6.3",
     "instagram-private-api": "1.45.3",
-    "ioredis": "5.0.4",
+    "ioredis": "5.0.6",
     "ip-regex": "4.3.0",
     "is-localhost-ip": "1.4.0",
-    "jsdom": "19.0.0",
+    "jsdom": "20.0.0",
     "json-bigint": "1.0.0",
     "json5": "2.2.1",
     "koa": "2.13.4",
@@ -119,21 +119,22 @@
     "pidusage": "3.0.0",
     "plist": "3.0.5",
     "puppeteer": "13.7.0",
-    "puppeteer-extra": "3.2.3",
-    "puppeteer-extra-plugin-stealth": "2.9.0",
+    "puppeteer-extra": "3.3.0",
+    "puppeteer-extra-plugin-stealth": "2.10.1",
     "query-string": "7.1.1",
-    "rand-user-agent": "1.0.63",
+    "rand-user-agent": "1.0.68",
+    "re2": "1.17.7",
     "require-all": "3.0.0",
     "rss-parser": "3.12.0",
     "showdown": "2.1.0",
     "simplecc-wasm": "0.1.4",
-    "socks-proxy-agent": "6.2.0",
+    "socks-proxy-agent": "7.0.0",
     "string-similarity": "4.0.4",
     "tiny-async-pool": "2.1.0",
     "torrent-search-api": "2.1.4",
     "tough-cookie": "4.0.0",
     "tunnel": "0.0.6",
-    "twitter-api-v2": "1.12.0",
+    "twitter-api-v2": "1.12.2",
     "winston": "3.7.2",
     "xml2js": "0.4.23"
   },

package/lib/routes/apnews/topics.js

@@ -1,43 +0,0 @@
-const cheerio = require('cheerio');
-
-const HOME_PAGE = 'https://apnews.com';
-
-module.exports = async (ctx) => {
-    const topic = ctx.params.topic;
-
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    const url_link = `${HOME_PAGE}/hub/${topic}`;
-    await page.goto(url_link);
-    const html = await page.evaluate(() => document.documentElement.innerHTML);
-    browser.close();
-
-    const $ = cheerio.load(html);
-
-    const list = $('div.FeedCard');
-
-    ctx.state.data = {
-        title: $('.Body div').find('h1[class^=hubTitle]').text(),
-        link: HOME_PAGE,
-        item: list
-            .slice(0, 5)
-            .map((index, item) => {
-                item = $(item);
-                const title = item.find('h3[class^=Component-heading-]').text();
-                const link = item.find('a[class^=Component-headline-]').attr('href');
-                const pubDate = item.find('span[class^=Timestamp]').attr('title');
-                const text = item.find('div[class^=content]').text();
-                const imageUrl = item.find('img[class^=image-]').attr('src');
-                const author = item.find('span[class^=Component-bylines-]').text().slice(3);
-
-                return {
-                    title,
-                    description: `<img src="${imageUrl}">` + text,
-                    link: `${HOME_PAGE}${link}`,
-                    pubDate,
-                    author,
-                };
-            })
-            .get(),
-    };
-};

package/lib/routes/behance/index.js

@@ -1,58 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const dayjs = require('dayjs');
-
-module.exports = async (ctx) => {
-    const user = ctx.params.user || '';
-    const type = ctx.params.type || 'projects';
-
-    const response = await got({
-        method: 'get',
-        url: `https://www.behance.net/${user}/${type}`, // 接口只获取12个项目
-        headers: {
-            'X-Requested-With': 'XMLHttpRequest',
-        },
-    });
-    const data = response.data;
-    const list = data.profile.activeSection.work.projects;
-    const articledata = await Promise.all(
-        list.map(async (item) => {
-            if (type !== 'projects') {
-                item = item.project;
-            }
-            const url = `${item.url}?ilo0=1`;
-            const cache = await ctx.cache.get(url);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-            const response2 = await got({
-                method: 'get',
-                url,
-            });
-            const articleHtml = response2.data;
-            const $2 = cheerio.load(articleHtml);
-
-            const content = $2('div.project-styles').html();
-            const single = {
-                content,
-            };
-            ctx.cache.set(url, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
-    );
-    ctx.state.data = {
-        title: `${data.profile.owner.first_name} ${data.profile.owner.last_name}'s Work`,
-        link: data.profile.owner.url,
-        item: list.map((item, index) => {
-            if (type !== 'projects') {
-                item = item.project;
-            }
-            return {
-                title: item.name,
-                description: articledata[index].content,
-                link: item.url,
-                pubDate: dayjs.unix(item.published_on).format(),
-            };
-        }),
-    };
-};

package/lib/routes/gov/customs/list.js

@@ -1,77 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-const host = 'http://www.customs.gov.cn/';
-module.exports = async (ctx) => {
-    const gchannel = ctx.params.gchannel;
-    let channelName = ``;
-    let link = `http://www.customs.gov.cn/customs/302249/2476857/302309/index.html`;
-
-    switch (gchannel) {
-        case 'paimai':
-            channelName = '拍卖信息';
-            link = `http://www.customs.gov.cn/customs/302249/302309/index.html`;
-
-            break;
-        case 'fagui':
-            channelName = '海关法规';
-            link = `http://www.customs.gov.cn/customs/302249/302266/index.html`;
-            break;
-        default:
-            break;
-    }
-    const response = await got({
-        method: 'get',
-        url: link,
-        headers: {
-            Referer: host,
-        },
-    });
-    const $ = cheerio.load(response.data);
-
-    const list = $('[class^="conList_ul"] li')
-        .map(function () {
-            const info = {
-                title: $(this).find('a').text(),
-                link: $(this).find('a').attr('href'),
-                date: $(this).find('span').text(),
-            };
-            return info;
-        })
-        .get();
-    const out = await Promise.all(
-        list.map(async (info) => {
-            const title = info.title;
-            const date = info.date;
-            const itemUrl = url.resolve(host, info.link);
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-            const response = await got({
-                method: 'get',
-                url: itemUrl,
-                headers: {
-                    Referer: host,
-                },
-            });
-            const $ = cheerio.load(response.data);
-            $('.easysite-news-operation').remove();
-            const description = $('.easysite-news-peruse').html();
-
-            const single = {
-                title,
-                link: itemUrl,
-                description,
-                pubDate: new Date(date).toDateString(),
-            };
-            ctx.cache.set(itemUrl, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
-    );
-    ctx.state.data = {
-        title: `中国海关-${channelName}`,
-        link,
-        item: out,
-    };
-};

package/lib/routes/hk01/channel.js

@@ -1,22 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id;
-
-    const response = await got(`https://web-data.api.hk01.com/v2/page/category/${id}`);
-    const data = response.data;
-    const list = data.sections[0].items;
-
-    ctx.state.data = {
-        title: `香港01 - ${data.category.publishName}`,
-        description: data.meta.metaDesc,
-        link: data.category.publishUrl,
-        item: list.map((item) => ({
-            title: item.data.title,
-            author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
-            description: `<p>${item.data.description}…</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`,
-            pubDate: new Date(item.data.lastModifyTime * 1000),
-            link: item.data.canonicalUrl,
-        })),
-    };
-};

package/lib/routes/hk01/issue.js

@@ -1,22 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id;
-
-    const response = await got(`https://web-data.api.hk01.com/v2/page/issue/${id}`);
-    const data = response.data;
-    const list = data.issue.blocks[0].articles;
-
-    ctx.state.data = {
-        title: `香港01 - ${data.issue.title}`,
-        description: data.meta.metaDesc,
-        link: data.issue.publishUrl,
-        item: list.map((item) => ({
-            title: item.data.title,
-            author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
-            description: `<p>${item.data.description}…</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`,
-            pubDate: new Date(item.data.lastModifyTime * 1000),
-            link: item.data.canonicalUrl,
-        })),
-    };
-};

package/lib/routes/mirror/entries.js

@@ -1,80 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id;
-    let link;
-    let subDomain = 0;
-    if (id.endsWith('.eth') || id.length === 42 || id.length === 40) {
-        link = 'https://mirror.xyz/' + id;
-    } else {
-        link = 'https://' + id + '.mirror.xyz';
-        subDomain = 1;
-    }
-
-    const response = await got({
-        method: 'get',
-        url: link,
-    });
-
-    const data = response.body;
-    const $ = cheerio.load(data);
-
-    const author = $('div#__next > div.GlobalNavigation > div.GlobalNavigationContainer > div > div > div > div > div > a').text();
-
-    // 标题
-    const article = $('h1')
-        .map(function () {
-            return $(this).text();
-        })
-        .get();
-
-    // 地址
-    const addr = $('h1')
-        .parent()
-        .parent()
-        .parent()
-        .map(function () {
-            return $(this).attr('href');
-        })
-        .get();
-
-    // 发布日期
-    const date = $('div#__next > div > div > div > div > div > div > div > div > div')
-        .map(function () {
-            return $(this).text();
-        })
-        .get();
-
-    // 摘要
-    const des = $('h1')
-        .parent()
-        .parent()
-        .parent()
-        .parent()
-        .next()
-        .next()
-        .map(function () {
-            return $(this).text();
-        })
-        .get();
-
-    const items = [];
-    article.forEach((item, index) => {
-        items.push({
-            title: item,
-            author,
-            description: des[index],
-            pubDate: date[index],
-            link: subDomain ? link + addr[index] : 'https://mirror.xyz' + addr[index],
-        });
-    });
-
-    const title = author + ' - Mirror';
-    ctx.state.data = {
-        title,
-        link,
-        allowEmpty: true,
-        item: items,
-    };
-};

package/lib/routes/mobilism/release.js

@@ -1,42 +0,0 @@
-const url = require('url');
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const currentUrl = `https://forum.mobilism.org/viewforum.php?f=19`;
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-    const $ = cheerio.load(response.data);
-    const list = $('tbody')
-        .eq(1)
-        .find('tr')
-        .map((_, item) => {
-            item = $(item);
-            const a = item.find('a');
-            return {
-                title: a.text(),
-                link: url.resolve(`https://forum.mobilism.org`, a.attr('href')),
-            };
-        })
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const res = await got({ method: 'get', url: item.link });
-                const content = cheerio.load(res.data);
-
-                item.description = content('div.content').html();
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: 'Mobilism - eBook Releases',
-        link: currentUrl,
-        item: items,
-    };
-};

package/lib/routes/pincong/hot.js

@@ -1,33 +0,0 @@
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    let url = 'https://pincong.rocks/hot/list/';
-
-    url += ctx.params.category ? 'category-' + ctx.params.category : 'category-0';
-
-    // use Puppeteer due to the obstacle by cloudflare challenge
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    await page.goto(url);
-    const html = await page.evaluate(
-        // eslint-disable-next-line no-undef
-        () => document.documentElement.innerHTML
-    );
-
-    browser.close();
-
-    const $ = cheerio.load(html);
-    const list = $('div.aw-item');
-
-    ctx.state.data = {
-        title: '品葱 - 精选',
-        link: 'https://pincong.rocks/hot/',
-        item: list
-            .map((_, item) => ({
-                title: $(item).find('h2 a').text().trim(),
-                description: $(item).find('div.markitup-box').html(),
-                link: 'https://pincong.rocks' + $(item).find('div.mod-head h2 a').attr('href'),
-            }))
-            .get(),
-    };
-};

package/lib/routes/pincong/topic.js

@@ -1,32 +0,0 @@
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const url = 'https://pincong.rocks/topic/' + ctx.params.topic;
-
-    // use Puppeteer due to the obstacle by cloudflare challenge
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    await page.goto(url);
-    const html = await page.evaluate(
-        () =>
-            // eslint-disable-next-line no-undef
-            (document.querySelector('div.aw-common-list') && document.querySelector('div.aw-common-list').innerHTML) || ''
-    );
-
-    browser.close();
-
-    const $ = cheerio.load(html);
-    const list = $('div.aw-item');
-
-    ctx.state.data = {
-        title: `品葱 - ${ctx.params.topic}`,
-        link: url,
-        item: list
-            .map((_, item) => ({
-                title: $(item).find('h4 a').text().trim(),
-                link: 'https://pincong.rocks' + $(item).find('h4 a').attr('href'),
-                pubDate: new Date($(item).attr('data-timestamp') * 1000).toISOString(),
-            }))
-            .get(),
-    };
-};

package/lib/routes/reuters/channel.js

@@ -1,155 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const utils = require('./utils');
-
-module.exports = async (ctx) => {
-    const { site } = ctx.params || 'cn';
-    const homePage = `https://${site === 'us' ? 'www' : site}.reuters.com`;
-
-    let title = 'Reuters ',
-        link = `https://${site === 'us' ? 'www' : site}.reuters.com/news/`,
-        linkSelector;
-
-    const { channel } = ctx.params;
-
-    if (channel) {
-        if (site === 'cn') {
-            title = '路透社 ';
-            linkSelector = '.inlineLinks a';
-            switch (channel) {
-                case 'china':
-                    title += '中国财经';
-                    break;
-                case 'internationalbusiness':
-                    title += '国际财经';
-                    break;
-                case 'newsmaker':
-                    title += '新闻人物';
-                    break;
-                case 'opinions':
-                    title += '财经视点';
-                    break;
-                case 'analyses':
-                    title += '深度分析';
-                    break;
-                case 'generalnews':
-                    title += '时事要闻';
-                    break;
-                case 'CnColumn':
-                    title += '中国财经专栏';
-                    break;
-                case 'ComColumn':
-                    title += '大宗商品专栏';
-                    break;
-                case 'IntColumn':
-                    title += '国际财经专栏';
-                    break;
-                case 'investing':
-                    title += '投资';
-                    link = homePage + '/';
-                    break;
-                case 'life':
-                    title += '生活';
-                    link = homePage + '/';
-                    break;
-                default:
-                    break;
-            }
-        } else if (site === 'uk') {
-            title += 'UK ';
-            linkSelector = '.story-content a';
-            switch (channel) {
-                case 'business':
-                    title += 'Business';
-                    link = homePage;
-                    break;
-                case 'markets':
-                    linkSelector = '.moduleBody a, .story-content a';
-                    title += 'Markets';
-                    link = 'https://uk.reuters.com/business/';
-                    break;
-                case 'world':
-                    title += 'World';
-                    break;
-                case 'uk':
-                    title += 'UK';
-                    break;
-                case 'technology':
-                    title += 'Tech';
-                    break;
-                case 'personalFinance':
-                    title += 'Money';
-                    link = 'https://uk.reuters.com/business';
-                    break;
-                case 'breakingviews':
-                    title += 'Breakingviews';
-                    link = homePage;
-                    break;
-                case 'sports':
-                    linkSelector = '.story-content a, .story-title a';
-                    title += 'Sport';
-                    break;
-                case 'lifestyle':
-                    title += 'Life';
-                    break;
-                default:
-                    break;
-            }
-        } else if (site === 'us') {
-            linkSelector = '.story-content a';
-            switch (channel) {
-                case 'business':
-                    title += 'Business';
-                    link = homePage;
-                    break;
-                case 'markets':
-                    linkSelector = '.moduleBody a, .story-content a';
-                    title += 'Markets';
-                    link = 'https://www.reuters.com/finance/';
-                    break;
-                case 'world':
-                    title += 'World';
-                    break;
-                case 'politics':
-                    title += 'Politics';
-                    link = 'https://www.reuters.com/';
-                    break;
-                case 'technology':
-                    title += 'Tech';
-                    break;
-                case 'breakingviews':
-                    title += 'Breakingviews';
-                    link = homePage;
-                    break;
-                case 'wealth':
-                    linkSelector = '.story-content a, .story-title a';
-                    title += 'Wealth';
-                    link = 'https://www.reuters.com/finance/';
-                    break;
-                case 'lifestyle':
-                    title += 'Life';
-                    break;
-                default:
-                    break;
-            }
-        }
-        link += channel;
-    }
-
-    const response = await got.get(link);
-
-    const $ = cheerio.load(response.data);
-
-    const links = $(linkSelector)
-        .map((i, e) => homePage + e.attribs.href)
-        .splice(0, 10);
-
-    const items = await Promise.all(links.map((link) => ctx.cache.tryGet(link, () => utils.ProcessFeed(link))));
-
-    ctx.state.data = {
-        title,
-        link,
-        description: title,
-        item: items,
-    };
-};