rsshub 1.0.0-master.fe69819 → 1.0.0-master.febd623

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rsshub",
-  "version": "1.0.0-master.fe69819",
+  "version": "1.0.0-master.febd623",
   "description": "Make RSS Great Again!",
   "main": "lib/pkg.js",
   "files": [
@@ -19,9 +19,10 @@
     "format": "eslint \"**/*.{js,yml}\" --fix && node docs/.format/format.js && prettier \"**/*.{js,json}\" --write",
     "format:staged": "eslint \"**/*.{js,yml}\" --fix && node docs/.format/format.js --staged && pretty-quick --staged --verbose --pattern \"**/*.{js,json}\"",
     "format:check": "eslint \"**/*.{js,yml}\" && prettier-check \"**/*.{js,json}\"",
-    "test": "npm run format:check && cross-env NODE_ENV=test jest --coverage --runInBand --forceExit",
-    "jest": "cross-env NODE_ENV=test jest --runInBand --forceExit",
-    "jest:watch": "cross-env NODE_ENV=test jest --watch"
+    "jest": "cross-env NODE_ENV=test jest --runInBand --forceExit --detectOpenHandles",
+    "jest:coverage": "cross-env NODE_ENV=test jest --coverage --runInBand --forceExit --detectOpenHandles",
+    "jest:watch": "cross-env NODE_ENV=test jest --watch",
+    "test": "npm run format:check && npm run jest:coverage"
   },
   "repository": {
     "type": "git",
@@ -41,97 +42,100 @@
   "homepage": "https://github.com/DIYgod/RSSHub#readme",
   "devDependencies": {
     "@napi-rs/pinyin": "1.7.0",
-    "@types/cheerio": "0.22.31",
-    "@types/got": "9.6.12",
     "@types/koa": "2.13.4",
-    "@vercel/nft": "0.17.5",
+    "@vercel/nft": "0.19.1",
     "@vuepress/plugin-back-to-top": "1.9.7",
     "@vuepress/plugin-google-analytics": "1.9.7",
     "@vuepress/plugin-pwa": "1.9.7",
-    "ci-info": "3.3.0",
+    "@vuepress/shared-utils": "1.9.7",
     "cross-env": "7.0.3",
-    "eslint": "8.9.0",
-    "eslint-config-prettier": "8.3.0",
+    "eslint": "8.16.0",
+    "eslint-config-prettier": "8.5.0",
     "eslint-plugin-prettier": "4.0.0",
-    "eslint-plugin-yml": "0.14.0",
-    "fs-extra": "10.0.0",
-    "jest": "27.5.1",
+    "eslint-plugin-yml": "1.0.0",
+    "fs-extra": "10.1.0",
+    "jest": "28.1.0",
     "mockdate": "3.0.5",
-    "nock": "13.2.4",
-    "nodemon": "2.0.15",
-    "prettier": "2.5.1",
+    "nock": "13.2.6",
+    "nodemon": "2.0.16",
+    "prettier": "2.6.2",
     "prettier-check": "2.0.0",
     "pretty-quick": "3.1.3",
     "remark": "13.0.0",
     "remark-frontmatter": "3.0.0",
     "remark-gfm": "1.0.0",
     "remark-pangu": "2.2.0",
+    "remark-parse": "9.0.0",
     "remark-preset-prettier": "0.5.1",
     "remark-stringify": "9.0.1",
     "staged-git-files": "1.3.0",
     "string-width": "4.2.3",
-    "supertest": "6.2.2",
+    "supertest": "6.2.3",
     "tosource": "2.0.0-alpha.3",
+    "unified": "9.2.2",
     "vuepress": "1.9.7",
     "yorkie": "2.0.0"
   },
   "dependencies": {
     "@koa/router": "10.1.1",
     "@postlight/mercury-parser": "2.2.1",
-    "@sentry/node": "6.17.9",
+    "@sentry/node": "7.0.0",
     "aes-js": "3.1.2",
     "art-template": "4.13.2",
     "bbcodejs": "0.0.4",
-    "cheerio": "1.0.0-rc.10",
-    "chrono-node": "2.3.7",
+    "cheerio": "1.0.0-rc.11",
+    "chrono-node": "2.3.8",
     "city-timezones": "1.2.0",
     "crypto-js": "4.1.1",
-    "currency-symbol-map": "5.0.1",
-    "dayjs": "1.10.7",
-    "dotenv": "16.0.0",
+    "currency-symbol-map": "5.1.0",
+    "dayjs": "1.11.2",
+    "dotenv": "16.0.1",
     "emailjs-imap-client": "3.1.0",
     "entities": "3.0.1",
     "etag": "1.8.1",
     "fanfou-sdk": "4.2.0",
     "git-rev-sync": "3.0.2",
-    "googleapis": "95.0.0",
-    "got": "11.8.3",
-    "https-proxy-agent": "5.0.0",
+    "googleapis": "100.0.0",
+    "got": "11.8.5",
+    "https-proxy-agent": "5.0.1",
     "iconv-lite": "0.6.3",
     "instagram-private-api": "1.45.3",
-    "ioredis": "4.28.5",
-    "ip": "*",
+    "ioredis": "5.0.6",
     "ip-regex": "4.3.0",
     "is-localhost-ip": "1.4.0",
     "jsdom": "19.0.0",
     "json-bigint": "1.0.0",
-    "json5": "2.2.0",
+    "json5": "2.2.1",
     "koa": "2.13.4",
     "koa-basic-auth": "4.0.0",
     "koa-favicon": "2.1.0",
     "koa-mount": "4.0.0",
-    "lru-cache": "6.0.0",
+    "lru-cache": "7.10.1",
     "lz-string": "1.4.4",
-    "mailparser": "3.4.0",
-    "markdown-it": "12.2.0",
+    "mailparser": "3.5.0",
+    "markdown-it": "13.0.1",
     "module-alias": "2.2.2",
-    "parse-torrent": "9.1.4",
+    "parse-torrent": "9.1.5",
     "pidusage": "3.0.0",
-    "plist": "3.0.4",
-    "puppeteer": "11.0.0",
+    "plist": "3.0.5",
+    "puppeteer": "13.7.0",
+    "puppeteer-extra": "3.2.3",
+    "puppeteer-extra-plugin-stealth": "2.9.0",
     "query-string": "7.1.1",
+    "rand-user-agent": "1.0.65",
     "require-all": "3.0.0",
     "rss-parser": "3.12.0",
-    "showdown": "2.0.0",
+    "showdown": "2.1.0",
     "simplecc-wasm": "0.1.4",
-    "socks-proxy-agent": "6.1.1",
+    "socks-proxy-agent": "7.0.0",
     "string-similarity": "4.0.4",
-    "tiny-async-pool": "1.2.0",
+    "tiny-async-pool": "2.1.0",
     "torrent-search-api": "2.1.4",
     "tough-cookie": "4.0.0",
     "tunnel": "0.0.6",
-    "twit": "2.2.11",
-    "winston": "3.5.1"
+    "twitter-api-v2": "1.12.2",
+    "winston": "3.7.2",
+    "xml2js": "0.4.23"
   },
   "jest": {
     "testMatch": [
@@ -147,7 +151,10 @@
     ],
     "collectCoverageFrom": [
       "lib/**/*.js",
-      "!lib/routes/**/*.js"
+      "!lib/routes/**/*.js",
+      "!lib/v2/**/*.js",
+      "!lib/radar-rules.js",
+      "!lib/router.js"
     ],
     "bail": true,
     "testEnvironment": "node",
@@ -157,5 +164,8 @@
   },
   "nodemonConfig": {
     "ext": "art,js,mjs,json"
+  },
+  "engines": {
+    "node": ">=14"
   }
 }

package/lib/routes/36kr/motif.js

@@ -1,35 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const motifUrl = `https://36kr.com/motif/${ctx.params.mid}`;
-    const response = await got({
-        method: 'get',
-        url: motifUrl,
-    });
-    const data = JSON.parse(response.data.match(/"motifDetailData":{"code":0,"data":(.*?)},"channel":/)[1]);
-
-    const motifInfo = data.motifInfo.data;
-    const motifArticleList = data.motifArticleList.data.itemList;
-    const articleList = motifArticleList.map((item) => ({
-        title: item.templateMaterial.widgetTitle,
-        link: `https://36kr.com/${item.route.split('?')[0] === 'detail_video' ? 'video' : 'p'}/${item.itemId}`,
-        pubDate: new Date(item.templateMaterial.publishTime).toUTCString(),
-    }));
-
-    ctx.state.data = {
-        title: `36氪专题 - ${motifInfo.categoryTitle}`,
-        link: motifUrl,
-        item: await Promise.all(
-            articleList.map((item) =>
-                ctx.cache.tryGet(item.link, async () => {
-                    const contentResponse = await got({ method: 'get', url: item.link });
-                    const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
-                    const contentData = JSON.parse(result[1]);
-                    item.description = contentData.data.widgetContent;
-                    return item;
-                })
-            )
-        ),
-        description: motifInfo.categoryTitle,
-    };
-};

package/lib/routes/36kr/news.js

@@ -1,94 +0,0 @@
-const url = require('url');
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-const config = {
-    latest: {
-        link: '/information/web_news/latest',
-        title: '最新',
-    },
-    recommend: {
-        link: '/information/web_recommend',
-        title: '推荐',
-    },
-    contact: {
-        link: '/information/contact',
-        title: '创投',
-    },
-    ccs: {
-        link: '/information/ccs',
-        title: '中概股',
-    },
-    travel: {
-        link: '/information/travel',
-        title: '汽车',
-    },
-    technology: {
-        link: '/technology',
-        title: '科技',
-    },
-    enterpriseservice: {
-        link: '/information/enterpriseservice',
-        title: '企服',
-    },
-    banking: {
-        link: '/information/banking',
-        title: '金融',
-    },
-    life: {
-        link: '/information/happy_life',
-        title: '生活',
-    },
-    innovate: {
-        link: '/information/innovate',
-        title: '创新',
-    },
-    estate: {
-        link: '/information/real_estate',
-        title: '房产',
-    },
-    workplace: {
-        link: '/information/web_zhichang',
-        title: '职场',
-    },
-    other: {
-        link: '/information/other',
-        title: '其他',
-    },
-};
-
-module.exports = async (ctx) => {
-    const cfg = config[ctx.params.caty];
-    if (!cfg) {
-        throw Error('Bad category. See <a href="https://docs.rsshub.app/government.html#bei-jing-shi-wei-sheng-jian-kang-wei-yuan-hui">docs</a>');
-    }
-
-    const newsUrl = url.resolve(`https://36kr.com/`, cfg.link);
-    const response = await got({
-        method: 'get',
-        url: newsUrl,
-    });
-
-    const data = JSON.parse(response.data.match(/<script>window\.initialState=(.*?)<\/script>/)[1]);
-
-    const informationList = data.information.informationList.itemList.map((item) => ({
-        title: item.templateMaterial.widgetTitle,
-        link: `https://36kr.com/p/${item.itemId}`,
-        pubDate: new Date(item.templateMaterial.publishTime).toUTCString(),
-    }));
-
-    ctx.state.data = {
-        title: `36氪资讯 - ${cfg.title}`,
-        link: newsUrl,
-        item: await Promise.all(
-            informationList.map((item) =>
-                ctx.cache.tryGet(item.link, async () => {
-                    const contentResponse = await got({ method: 'get', url: item.link });
-                    const content = cheerio.load(contentResponse.data);
-                    item.description = content('div.common-width.content.articleDetailContent.kr-rich-text-wrapper').html();
-                    return item;
-                })
-            )
-        ),
-    };
-};

package/lib/routes/36kr/newsflashes.js

@@ -1,36 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: `https://36kr.com/newsflashes`,
-        headers: {
-            Host: '36kr.com',
-            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25',
-        },
-    });
-    const regexp = /<script>window\.initialState=(.*?)<\/script>/;
-    const newsflashes = JSON.parse(response.data.match(regexp)[1]);
-
-    const out = newsflashes.newsflashList.flow.itemList.map((item) => {
-        const link = item.templateMaterial.sourceUrlRoute ? new URL(`http://www.example.com/${item.templateMaterial.sourceUrlRoute}`).searchParams.get('url') : `https://36kr.com/newsflashes/${item.itemId}`;
-        const date = item.templateMaterial.publishTime;
-        const title = item.templateMaterial.widgetTitle;
-        const description = item.templateMaterial.widgetContent;
-
-        const single = {
-            title,
-            link,
-            pubDate: new Date(date).toUTCString(),
-            description,
-        };
-
-        return single;
-    });
-
-    ctx.state.data = {
-        title: `快讯 - 36氪`,
-        link: `https://36kr.com/newsflashes`,
-        item: out,
-    };
-};

package/lib/routes/36kr/search/article.js

@@ -1,69 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const { keyword = '' } = ctx.params;
-    const csrfresponse = await got.get('https://36kr.com/pp/api/csrf');
-    const cookies = csrfresponse.headers['set-cookie'].toString();
-    const token = cookies.match(/M-XSRF-TOKEN=.{64}/)[0];
-    const acw_tc = cookies.match(/acw_tc=.{61}/)[0];
-    const krnewsfrontss = cookies.match(/krnewsfrontss=.{32}/)[0];
-
-    const timestamp = new Date().getTime();
-
-    const response = await got({
-        method: 'post',
-        url: `https://gateway.36kr.com/api/mis/nav/search/resultbytype`,
-        headers: {
-            'Content-Type': 'application/json',
-            cookie: `${token};${acw_tc};${krnewsfrontss};`,
-            'M-X-XSRF-TOKEN': token.replace('M-XSRF-TOKEN=', ''),
-        },
-        data: JSON.stringify({
-            param: {
-                pageCallback: Buffer.from(
-                    JSON.stringify({
-                        firstId: 1,
-                        lastId: 1,
-                        firstCreateTime: timestamp,
-                        lastCreateTime: timestamp,
-                    })
-                ).toString('base64'),
-                pageEvent: 1,
-                pageSize: 20,
-                platformId: 2,
-                searchType: 'article',
-                searchWord: keyword,
-                siteId: 1,
-                sort: 'date',
-            },
-            partner_id: 'web',
-            timestamp,
-        }),
-    });
-
-    const load = async (link) => {
-        const response = await got.get(link);
-        const description = response.data.match(/"widgetContent":"(.*?)","sourceType":/)[1];
-        return { description };
-    };
-
-    const items = await Promise.all(
-        (response.data.data.itemList || []).map(async (item) => {
-            const link = `https://www.36kr.com/p/${item.itemId}`;
-            const single = {
-                title: item.widgetTitle,
-                link,
-                pubDate: new Date(item.publishTime).toUTCString(),
-            };
-
-            const other = await ctx.cache.tryGet(link, () => load(link));
-            return Promise.resolve(Object.assign({}, single, other));
-        })
-    );
-
-    ctx.state.data = {
-        title: `36kr - ${keyword}`,
-        link: `https://www.36kr.com/search/articles/${encodeURIComponent(keyword)}`,
-        item: items,
-    };
-};

package/lib/routes/36kr/user.js

@@ -1,35 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const userUrl = `https://36kr.com/user/${ctx.params.uid}`;
-    const response = await got({
-        method: 'get',
-        url: userUrl,
-    });
-    const data = JSON.parse(response.data.match(/"authorDetailData":(.*?),"channel":/)[1]);
-
-    const authorInfo = data.authorInfo.data;
-    const authorFlowList = data.authorFlowList.data.itemList;
-    const articleList = authorFlowList.map((item) => ({
-        title: item.templateMaterial.widgetTitle,
-        link: `https://36kr.com/${item.route.split('?')[0] === 'detail_video' ? 'video' : 'p'}/${item.itemId}`,
-        pubDate: new Date(item.templateMaterial.publishTime).toUTCString(),
-    }));
-
-    ctx.state.data = {
-        title: `36氪用户 - ${authorInfo.userNick}`,
-        link: userUrl,
-        item: await Promise.all(
-            articleList.map((item) =>
-                ctx.cache.tryGet(item.link, async () => {
-                    const contentResponse = await got({ method: 'get', url: item.link });
-                    const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
-                    const contentData = JSON.parse(result[1]);
-                    item.description = contentData.data.widgetContent;
-                    return item;
-                })
-            )
-        ),
-        description: `${authorInfo.label} | ${authorInfo.summary}`,
-    };
-};

package/lib/routes/8kcos/article.js

@@ -1,17 +0,0 @@
-const Cheerio = require('cheerio');
-const got = require('got');
-async function loadArticle(link) {
-    const resp = await got(link);
-    const article = Cheerio.load(resp.body);
-    const title = article('.entry-title').text();
-    const entry_children = article('div.entry-content').children().toArray();
-    const imgs = Cheerio.load(entry_children.shift()).text(); // 去除懒加载代码减少返回体积
-    const txt = Cheerio.load(entry_children).html();
-    return {
-        title,
-        description: imgs.concat(txt),
-        pubDate: article('time')[0].attribs.datetime.replace('>', ''),
-        link,
-    };
-}
-module.exports = loadArticle;

package/lib/routes/8kcos/cat.js

@@ -1,28 +0,0 @@
-const got = require('got');
-const Cheerio = require('cheerio');
-const { SUB_NAME_PREFIX, SUB_URL } = require('./const');
-const loadArticle = require('./article');
-module.exports = async (ctx) => {
-    const { cat = '8kasianidol' } = ctx.params;
-    const url = `${SUB_URL}category/${cat}/`;
-    const resp = await got(url);
-    const $ = Cheerio.load(resp.body);
-    ctx.state.data = resp.body
-        ? {
-              title: `${SUB_NAME_PREFIX}-${$('#wrap > div > div > div > div.sec-panel-head > h1 > span').text()}`,
-              link: url,
-              item: await Promise.all(
-                  Cheerio.load(resp.body)('#wrap > div > div > div > ul > li.item')
-                      .map((_, e) => {
-                          const { href } = Cheerio.load(e)('h2 > a')[0].attribs;
-                          return ctx.cache.tryGet(href, () => loadArticle(href));
-                      })
-                      .toArray()
-              ),
-          }
-        : {
-              title: `${SUB_NAME_PREFIX}-${cat}`,
-              link: url,
-              item: [{ title: '获取失败' }],
-          };
-};

package/lib/routes/8kcos/latest.js

@@ -1,27 +0,0 @@
-const got = require('got');
-const Cheerio = require('cheerio');
-const { SUB_NAME_PREFIX, SUB_URL } = require('./const');
-const loadArticle = require('./article');
-const url = SUB_URL;
-
-module.exports = async (ctx) => {
-    const response = await got(url);
-    ctx.state.data = {
-        title: `${SUB_NAME_PREFIX}-最新`,
-        link: url,
-        item: response.body
-            ? await Promise.all(
-                  Cheerio.load(response.body)('#wrap > div > div > div > ul > li.item')
-                      .map((_, e) => {
-                          const { href } = Cheerio.load(e)('div > h2 > a')[0].attribs;
-                          return ctx.cache.tryGet(href, () => loadArticle(href));
-                      })
-                      .toArray()
-              )
-            : [
-                  {
-                      title: '获取失败！',
-                  },
-              ],
-    };
-};

package/lib/routes/95mm/category.js

@@ -1,18 +0,0 @@
-const utils = require('./utils');
-
-module.exports = async (ctx) => {
-    const categories = {
-        1: '清纯唯美',
-        2: '摄影私房',
-        4: '明星写真',
-        5: '三次元',
-        6: '异域美景',
-        7: '性感妖姬',
-        9: '游戏主题',
-        11: '美女壁纸',
-    };
-
-    const rootUrl = `https://www.95mm.net/category-${ctx.params.category}/list-1/index.html?page=1`;
-
-    ctx.state.data = await utils(ctx, categories[ctx.params.category], rootUrl);
-};

package/lib/routes/95mm/tab.js

@@ -1,9 +0,0 @@
-const utils = require('./utils');
-
-module.exports = async (ctx) => {
-    ctx.params.tab = ctx.params.tab || '最新';
-
-    const rootUrl = `https://www.95mm.net/home-ajax/index.html?tabcid=${ctx.params.tab}&page=1`;
-
-    ctx.state.data = await utils(ctx, ctx.params.tab, rootUrl);
-};

package/lib/routes/95mm/tag.js

@@ -1,7 +0,0 @@
-const utils = require('./utils');
-
-module.exports = async (ctx) => {
-    const rootUrl = `https://www.95mm.net/tag-${ctx.params.tag}/page-1/index.html`;
-
-    ctx.state.data = await utils(ctx, ctx.params.tag, rootUrl);
-};

package/lib/routes/95mm/utils.js

@@ -1,51 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx, title, rootUrl) => {
-    const response = await got({
-        method: 'get',
-        url: rootUrl,
-        header: {
-            Referer: 'https://www.95mm.net',
-        },
-    });
-
-    const $ = cheerio.load(response.data);
-
-    const list = $('div.list-body')
-        .map((_, item) => {
-            item = $(item);
-            const a = item.find('a');
-            return {
-                title: a.text(),
-                link: a.attr('href'),
-            };
-        })
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-                const images = detailResponse.data.match(/src": '(.*?)',"width/g);
-
-                item.description = '';
-
-                for (const i of images) {
-                    item.description += `<img src="${i.split("'")[1]}">`;
-                }
-
-                return item;
-            })
-        )
-    );
-
-    return {
-        title: `${title} - MM范`,
-        link: rootUrl,
-        item: items,
-    };
-};