rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rsshub",
-  "version": "1.0.0-master.fbcba24",
+  "version": "1.0.0-master.fc648b5",
   "description": "Make RSS Great Again!",
   "main": "lib/pkg.js",
   "files": [
@@ -19,9 +19,10 @@
     "format": "eslint \"**/*.{js,yml}\" --fix && node docs/.format/format.js && prettier \"**/*.{js,json}\" --write",
     "format:staged": "eslint \"**/*.{js,yml}\" --fix && node docs/.format/format.js --staged && pretty-quick --staged --verbose --pattern \"**/*.{js,json}\"",
     "format:check": "eslint \"**/*.{js,yml}\" && prettier-check \"**/*.{js,json}\"",
-    "test": "npm run format:check && cross-env NODE_ENV=test jest --coverage --runInBand --forceExit",
-    "jest": "cross-env NODE_ENV=test jest --runInBand --forceExit",
-    "jest:watch": "cross-env NODE_ENV=test jest --watch"
+    "jest": "cross-env NODE_ENV=test jest --runInBand --forceExit --detectOpenHandles",
+    "jest:coverage": "cross-env NODE_ENV=test jest --coverage --runInBand --forceExit --detectOpenHandles",
+    "jest:watch": "cross-env NODE_ENV=test jest --watch",
+    "test": "npm run format:check && npm run jest:coverage"
   },
   "repository": {
     "type": "git",
@@ -41,97 +42,100 @@
   "homepage": "https://github.com/DIYgod/RSSHub#readme",
   "devDependencies": {
     "@napi-rs/pinyin": "1.7.0",
-    "@types/cheerio": "0.22.31",
-    "@types/got": "9.6.12",
     "@types/koa": "2.13.4",
-    "@vercel/nft": "0.17.5",
+    "@vercel/nft": "0.20.1",
     "@vuepress/plugin-back-to-top": "1.9.7",
     "@vuepress/plugin-google-analytics": "1.9.7",
     "@vuepress/plugin-pwa": "1.9.7",
-    "ci-info": "3.3.0",
+    "@vuepress/shared-utils": "1.9.7",
     "cross-env": "7.0.3",
-    "eslint": "8.10.0",
+    "eslint": "8.18.0",
     "eslint-config-prettier": "8.5.0",
-    "eslint-plugin-prettier": "4.0.0",
-    "eslint-plugin-yml": "0.14.0",
-    "fs-extra": "10.0.1",
-    "jest": "27.5.1",
+    "eslint-plugin-prettier": "4.2.1",
+    "eslint-plugin-yml": "1.0.0",
+    "fs-extra": "10.1.0",
+    "jest": "28.1.2",
     "mockdate": "3.0.5",
-    "nock": "13.2.4",
-    "nodemon": "2.0.15",
-    "prettier": "2.5.1",
+    "nock": "13.2.8",
+    "nodemon": "2.0.18",
+    "prettier": "2.7.1",
     "prettier-check": "2.0.0",
     "pretty-quick": "3.1.3",
     "remark": "13.0.0",
     "remark-frontmatter": "3.0.0",
     "remark-gfm": "1.0.0",
     "remark-pangu": "2.2.0",
+    "remark-parse": "9.0.0",
     "remark-preset-prettier": "0.5.1",
     "remark-stringify": "9.0.1",
     "staged-git-files": "1.3.0",
     "string-width": "4.2.3",
-    "supertest": "6.2.2",
+    "supertest": "6.2.3",
     "tosource": "2.0.0-alpha.3",
+    "unified": "9.2.2",
     "vuepress": "1.9.7",
     "yorkie": "2.0.0"
   },
   "dependencies": {
     "@koa/router": "10.1.1",
     "@postlight/mercury-parser": "2.2.1",
-    "@sentry/node": "6.18.1",
+    "@sentry/node": "7.4.1",
     "aes-js": "3.1.2",
     "art-template": "4.13.2",
     "bbcodejs": "0.0.4",
-    "cheerio": "1.0.0-rc.10",
-    "chrono-node": "2.3.7",
+    "cheerio": "1.0.0-rc.12",
+    "chrono-node": "2.3.8",
     "city-timezones": "1.2.0",
     "crypto-js": "4.1.1",
-    "currency-symbol-map": "5.0.1",
-    "dayjs": "1.10.8",
-    "dotenv": "16.0.0",
+    "currency-symbol-map": "5.1.0",
+    "dayjs": "1.11.3",
+    "dotenv": "16.0.1",
     "emailjs-imap-client": "3.1.0",
     "entities": "3.0.1",
     "etag": "1.8.1",
     "fanfou-sdk": "4.2.0",
     "git-rev-sync": "3.0.2",
-    "googleapis": "95.0.0",
-    "got": "11.8.3",
-    "https-proxy-agent": "5.0.0",
+    "googleapis": "104.0.0",
+    "got": "11.8.5",
+    "https-proxy-agent": "5.0.1",
     "iconv-lite": "0.6.3",
     "instagram-private-api": "1.45.3",
-    "ioredis": "4.28.5",
-    "ip": "*",
+    "ioredis": "5.1.0",
     "ip-regex": "4.3.0",
     "is-localhost-ip": "1.4.0",
-    "jsdom": "19.0.0",
+    "jsdom": "20.0.0",
     "json-bigint": "1.0.0",
-    "json5": "2.2.0",
+    "json5": "2.2.1",
     "koa": "2.13.4",
     "koa-basic-auth": "4.0.0",
     "koa-favicon": "2.1.0",
     "koa-mount": "4.0.0",
-    "lru-cache": "6.0.0",
+    "lru-cache": "7.12.0",
     "lz-string": "1.4.4",
-    "mailparser": "3.4.0",
-    "markdown-it": "12.2.0",
+    "mailparser": "3.5.0",
+    "markdown-it": "13.0.1",
     "module-alias": "2.2.2",
-    "parse-torrent": "9.1.4",
+    "parse-torrent": "9.1.5",
     "pidusage": "3.0.0",
-    "plist": "3.0.4",
-    "puppeteer": "13.4.1",
+    "plist": "3.0.5",
+    "puppeteer": "13.7.0",
+    "puppeteer-extra": "3.3.0",
+    "puppeteer-extra-plugin-stealth": "2.10.1",
     "query-string": "7.1.1",
+    "rand-user-agent": "1.0.69",
+    "re2": "1.17.7",
     "require-all": "3.0.0",
     "rss-parser": "3.12.0",
-    "showdown": "2.0.2",
+    "showdown": "2.1.0",
     "simplecc-wasm": "0.1.4",
-    "socks-proxy-agent": "6.1.1",
+    "socks-proxy-agent": "7.0.0",
     "string-similarity": "4.0.4",
-    "tiny-async-pool": "1.2.0",
+    "tiny-async-pool": "2.1.0",
     "torrent-search-api": "2.1.4",
     "tough-cookie": "4.0.0",
     "tunnel": "0.0.6",
-    "twit": "2.2.11",
-    "winston": "3.6.0",
+    "twitter-api-v2": "1.12.3",
+    "winston": "3.8.1",
     "xml2js": "0.4.23"
   },
   "jest": {
@@ -148,7 +152,10 @@
     ],
     "collectCoverageFrom": [
       "lib/**/*.js",
-      "!lib/routes/**/*.js"
+      "!lib/routes/**/*.js",
+      "!lib/v2/**/*.js",
+      "!lib/radar-rules.js",
+      "!lib/router.js"
     ],
     "bail": true,
     "testEnvironment": "node",
@@ -158,5 +165,8 @@
   },
   "nodemonConfig": {
     "ext": "art,js,mjs,json"
+  },
+  "engines": {
+    "node": ">=14"
   }
 }

package/lib/routes/9to5/subsite.js

@@ -1,67 +0,0 @@
-const got = require('@/utils/got');
-const parser = require('@/utils/rss-parser');
-const utils = require('./utils');
-
-module.exports = async (ctx) => {
-    let title = '9To5',
-        link,
-        description;
-
-    switch (ctx.params.subsite) {
-        case 'mac':
-            link = 'https://9to5mac.com';
-            title += 'Mac';
-            description = 'Apple News & Mac Rumors Breaking All Day';
-            break;
-
-        case 'google':
-            link = 'https://9to5google.com';
-            title += 'Google';
-            description = 'Google, Pixel news, Android, Home, Chrome OS, apps, more';
-            break;
-
-        case 'toys':
-            link = 'https://9to5toys.com';
-            title += 'Toys';
-            description = 'New Gear, reviews and deals';
-            break;
-
-        default:
-            break;
-    }
-
-    if (ctx.params.tag) {
-        link = `${link}/guides/${ctx.params.tag}/feed`;
-        title = `${ctx.params.tag} | ${title}`;
-    } else {
-        link = `${link}/feed`;
-    }
-
-    const feed = await parser.parseURL(link);
-
-    const items = await Promise.all(
-        feed.items.splice(0, 10).map(async (item) => {
-            const response = await got({
-                method: 'get',
-                url: item.link,
-            });
-            const description = utils.ProcessFeed(response.data);
-
-            const single = {
-                title: item.title,
-                description,
-                pubDate: item.pubDate,
-                link: item.link,
-                author: item['dc:creator'],
-            };
-            return Promise.resolve(single);
-        })
-    );
-
-    ctx.state.data = {
-        title,
-        link,
-        description,
-        item: items,
-    };
-};

package/lib/routes/9to5/utils.js

@@ -1,33 +0,0 @@
-const cheerio = require('cheerio');
-
-const ProcessFeed = (data) => {
-    const $ = cheerio.load(data);
-    const content = $('div[itemprop="articleBody"]');
-
-    const cover = $('meta[property="og:image"]');
-    if (cover.length > 0) {
-        $(`<img src=${cover[0].attribs.content}>`).insertBefore(content[0].firstChild);
-    }
-
-    // remove useless DOMs
-    content.find('hr').nextAll().remove();
-
-    content.find('hr, ins.adsbygoogle, script').each((i, e) => {
-        $(e).remove();
-    });
-
-    content.find('div').each((i, e) => {
-        if ($(e)[0].attribs.class) {
-            const classes = $(e)[0].attribs.class;
-            if (classes.match(/\w{10}\s\w{10}/g)) {
-                $(e).remove();
-            }
-        }
-    });
-
-    return content.html();
-};
-
-module.exports = {
-    ProcessFeed,
-};

package/lib/routes/acfun/bangumi.js

@@ -1,28 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const { id } = ctx.params;
-    const url = `http://www.acfun.cn/bangumi/aa${id}`;
-
-    const bangumiPage = await got({
-        method: 'get',
-        url,
-        headers: {
-            Referer: 'http://www.acfun.cn',
-        },
-    });
-    const bangumiData = JSON.parse(bangumiPage.data.match(/window.bangumiData = (.*?);\n/)[1]);
-    const bangumiList = JSON.parse(bangumiPage.data.match(/window.bangumiList = (.*?);\n/)[1]);
-
-    ctx.state.data = {
-        title: bangumiData.bangumiTitle,
-        link: url,
-        description: bangumiData.bangumiIntro,
-        item: bangumiList.items.map((item) => ({
-            title: `${item.episodeName} - ${item.title}`,
-            description: `<img src="${item.image.split('?')[0]}">`,
-            link: `http://www.acfun.cn/bangumi/aa${id}_36188_${item.itemId}`,
-            pubDate: new Date(item.updateTime).toUTCString(),
-        })),
-    };
-};

package/lib/routes/acfun/video.js

@@ -1,44 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const uid = ctx.params.uid;
-    const url = `http://www.acfun.cn/u/${uid}.aspx`;
-    const host = 'http://www.acfun.cn';
-    const response = await got({
-        method: 'get',
-        url,
-        headers: {
-            Referer: host,
-        },
-    });
-    const data = response.data;
-
-    const $ = cheerio.load(data);
-    const title = $('title').text();
-    const description = $('.preview').text();
-    const list = $('#ac-space-video-list a').get();
-
-    ctx.state.data = {
-        title,
-        link: url,
-        description,
-        item: list
-            .map((item) => {
-                const $ = cheerio.load(item);
-
-                const itemTitle = $('p.title').text();
-                const itemImg = $('figure img').attr('src');
-                const itemUrl = $('a').attr('href');
-                const itemDate = $('.date').text();
-
-                return {
-                    title: itemTitle,
-                    description: `<img src="${itemImg}">`,
-                    link: host + itemUrl,
-                    pubDate: new Date(itemDate).toUTCString(),
-                };
-            })
-            .reverse(),
-    };
-};

package/lib/routes/aliyun/database_month.js

@@ -1,45 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const url = 'http://mysql.taobao.org/monthly/';
-    const response = await got({ method: 'get', url });
-    const $ = cheerio.load(response.data);
-
-    const list = $("ul[class='posts'] > li")
-        .map((i, e) => {
-            const element = $(e);
-            const title = element.find('a').text().trim();
-            const link = 'http://mysql.taobao.org' + element.find('a').attr('href').trim();
-            return {
-                title,
-                description: '',
-                link,
-            };
-        })
-        .get();
-
-    const result = await Promise.all(
-        list.map(async (item) => {
-            const link = item.link;
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const itemReponse = await got.get(link);
-            const itemElement = cheerio.load(itemReponse.data);
-            item.description = itemElement('.content').html();
-
-            ctx.cache.set(link, JSON.stringify(item));
-            return item;
-        })
-    );
-
-    ctx.state.data = {
-        title: $('title').text(),
-        link: url,
-        item: result.reverse(),
-    };
-};

package/lib/routes/aliyun/developer/group.js

@@ -1,44 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const type = ctx.params.type;
-    const link = `https://developer.aliyun.com/group/${type}`;
-
-    // 发起 HTTP GET 请求
-    const response = await got({
-        method: 'get',
-        url: link,
-    });
-
-    const data = response.data;
-
-    // 使用 cheerio 加载返回的 HTML
-    const $ = cheerio.load(data);
-    const title = $('div[class="header-information-title"]')
-        .contents()
-        .filter(function () {
-            return this.nodeType === 3;
-        })
-        .text()
-        .trim();
-    const desc = $('div[class="header-information"]').find('span').last().text().trim();
-    const list = $('ul[class^="content-tab-list"] > li');
-
-    ctx.state.data = {
-        title: `阿里云开发者社区-${title}`,
-        link,
-        description: desc,
-        item:
-            list &&
-            list
-                .map((index, item) => {
-                    item = $(item);
-                    return {
-                        title: item.find('p').first().text().trim(),
-                        link: item.find('a').attr('href'),
-                    };
-                })
-                .get(),
-    };
-};

package/lib/routes/aliyun/notice.js

@@ -1,64 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const dayjs = require('dayjs');
-const customParseFormat = require('dayjs/plugin/customParseFormat');
-
-dayjs.extend(customParseFormat);
-
-const typeMap = {
-    0: '9004748',
-    1: '9004749',
-    2: '9213612',
-    3: '8314815',
-    4: '9222707',
-};
-
-/**
- *
- * @param ctx {import('koa').Context}
- */
-module.exports = async (ctx) => {
-    const type = ctx.params.type;
-    const url = `https://help.aliyun.com/noticelist/${typeMap[type] || typeMap[0]}.html`;
-    const response = await got({ method: 'get', url });
-    const $ = cheerio.load(response.data);
-    const list = $('ul > li.y-clear')
-        .map((i, e) => {
-            const element = $(e);
-            const title = element.find('a').text().trim();
-            const link = 'https://help.aliyun.com' + element.find('a').attr('href').trim();
-            const date = element.find('.y-right').text();
-            const pubDate = dayjs(`${date} +0800`, 'YYYY-MM-DDHH:mm:ss ZZ');
-            return {
-                title,
-                description: '',
-                link,
-                pubDate: pubDate.toString(),
-            };
-        })
-        .get();
-
-    const result = await Promise.all(
-        list.map(async (item) => {
-            const link = item.link;
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const itemReponse = await got.get(link);
-            const itemElement = cheerio.load(itemReponse.data);
-            item.description = itemElement('#se-knowledge').html();
-
-            ctx.cache.set(link, JSON.stringify(item));
-            return item;
-        })
-    );
-
-    ctx.state.data = {
-        title: $('title').text().trim(),
-        link: url,
-        item: result,
-    };
-};

package/lib/routes/apnews/topics.js

@@ -1,43 +0,0 @@
-const cheerio = require('cheerio');
-
-const HOME_PAGE = 'https://apnews.com';
-
-module.exports = async (ctx) => {
-    const topic = ctx.params.topic;
-
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    const url_link = `${HOME_PAGE}/hub/${topic}`;
-    await page.goto(url_link);
-    const html = await page.evaluate(() => document.documentElement.innerHTML);
-    browser.close();
-
-    const $ = cheerio.load(html);
-
-    const list = $('div.FeedCard');
-
-    ctx.state.data = {
-        title: $('.Body div').find('h1[class^=hubTitle]').text(),
-        link: HOME_PAGE,
-        item: list
-            .slice(0, 5)
-            .map((index, item) => {
-                item = $(item);
-                const title = item.find('h3[class^=Component-heading-]').text();
-                const link = item.find('a[class^=Component-headline-]').attr('href');
-                const pubDate = item.find('span[class^=Timestamp]').attr('title');
-                const text = item.find('div[class^=content]').text();
-                const imageUrl = item.find('img[class^=image-]').attr('src');
-                const author = item.find('span[class^=Component-bylines-]').text().slice(3);
-
-                return {
-                    title,
-                    description: `<img src="${imageUrl}">` + text,
-                    link: `${HOME_PAGE}${link}`,
-                    pubDate,
-                    author,
-                };
-            })
-            .get(),
-    };
-};

package/lib/routes/arknights/japan.js

@@ -1,24 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: 'https://www.arknights.jp:10014/news?lang=ja&limit=9&page=1',
-    });
-
-    const items = response.data.data.items;
-    const newsList = items.map((item) => ({
-        title: item.title,
-        description: item.content[0].value,
-        pubDate: item.publishedAt,
-        link: `https://www.arknights.jp/news/${item.id}`,
-    }));
-
-    ctx.state.data = {
-        title: 'アークナイツ',
-        link: 'https://www.arknights.jp/news',
-        description: 'アークナイツ ニュース',
-        language: 'ja',
-        item: newsList,
-    };
-};

package/lib/routes/arknights/news.js

@@ -1,41 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: 'https://ak.hypergryph.com/news.html',
-    });
-
-    let newslist = response.data;
-
-    const $ = cheerio.load(newslist);
-    newslist = $('.articleItem > .articleItemLink');
-
-    newslist = await Promise.all(
-        newslist
-            .slice(0, 9) // limit article count to a single page
-            .map(async (index, item) => {
-                const sth = $(item);
-                const link = `https://ak.hypergryph.com${sth.attr('href')}`;
-                const description = await ctx.cache.tryGet(link, async () => {
-                    const result = await got.get(link);
-                    const $ = cheerio.load(result.data);
-                    return $('.article-content').html();
-                });
-                return {
-                    title: `[${sth.find('.articleItemCate').first().text()}] ${sth.find('.articleItemTitle').first().text()}`,
-                    description,
-                    link,
-                    pubDate: new Date(sth.find('.articleItemDate').first().text()),
-                };
-            })
-            .get()
-    );
-
-    ctx.state.data = {
-        title: '《明日方舟》游戏公告与新闻',
-        link: 'https://ak.hypergryph.com/news.html',
-        item: newslist,
-    };
-};