rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rsshub",
-  "version": "1.0.0-master.fb68956",
+  "version": "1.0.0-master.fbabfc2",
   "description": "Make RSS Great Again!",
   "main": "lib/pkg.js",
   "files": [
@@ -51,23 +51,23 @@
     "@napi-rs/pinyin": "1.7.1",
     "@types/koa": "2.13.5",
     "@vercel/nft": "0.22.6",
-    "@vuepress/plugin-back-to-top": "1.9.7",
-    "@vuepress/plugin-google-analytics": "1.9.7",
-    "@vuepress/plugin-pwa": "1.9.7",
-    "@vuepress/shared-utils": "1.9.7",
+    "@vuepress/plugin-back-to-top": "1.9.8",
+    "@vuepress/plugin-google-analytics": "1.9.8",
+    "@vuepress/plugin-pwa": "1.9.8",
+    "@vuepress/shared-utils": "1.9.8",
     "cross-env": "7.0.3",
-    "eslint": "8.30.0",
-    "eslint-config-prettier": "8.5.0",
+    "eslint": "8.33.0",
+    "eslint-config-prettier": "8.6.0",
     "eslint-plugin-prettier": "4.2.1",
     "eslint-plugin-yml": "1.4.0",
     "fs-extra": "11.1.0",
-    "jest": "29.3.1",
+    "jest": "29.4.1",
     "jest-junit": "15.0.0",
-    "meilisearch": "0.30.0",
+    "meilisearch": "0.31.1",
     "mockdate": "3.0.5",
-    "nock": "13.2.9",
+    "nock": "13.3.0",
     "nodemon": "2.0.20",
-    "prettier": "2.8.1",
+    "prettier": "2.8.3",
     "prettier-check": "2.0.0",
     "pretty-quick": "3.1.3",
     "remark": "13.0.0",
@@ -83,7 +83,7 @@
     "supertest": "6.3.3",
     "tosource": "2.0.0-alpha.3",
     "unified": "9.2.2",
-    "vuepress": "1.9.7",
+    "vuepress": "1.9.8",
     "vuepress-plugin-meilisearch": "0.13.0",
     "vuepress-plugin-sitemap": "2.3.1",
     "yorkie": "2.0.0"
@@ -91,7 +91,7 @@
   "dependencies": {
     "@koa/router": "12.0.0",
     "@postlight/parser": "2.2.3",
-    "@sentry/node": "7.28.1",
+    "@sentry/node": "7.36.0",
     "aes-js": "3.1.2",
     "art-template": "4.13.2",
     "bbcodejs": "0.0.4",
@@ -112,12 +112,12 @@
     "https-proxy-agent": "5.0.1",
     "iconv-lite": "0.6.3",
     "instagram-private-api": "1.45.3",
-    "ioredis": "5.2.4",
+    "ioredis": "5.3.0",
     "ip-regex": "4.3.0",
     "is-localhost-ip": "2.0.0",
-    "jsdom": "20.0.3",
+    "jsdom": "21.1.0",
     "json-bigint": "1.0.0",
-    "json5": "2.2.2",
+    "json5": "2.2.3",
     "koa": "2.14.1",
     "koa-basic-auth": "4.0.0",
     "koa-favicon": "2.1.0",
@@ -131,22 +131,22 @@
     "pidusage": "3.0.2",
     "plist": "3.0.6",
     "proxy-chain": "2.2.0",
-    "puppeteer": "19.4.1",
+    "puppeteer": "19.6.3",
     "puppeteer-extra": "3.3.4",
     "puppeteer-extra-plugin-stealth": "2.11.1",
     "query-string": "7.1.3",
-    "rand-user-agent": "1.0.96",
+    "rand-user-agent": "1.0.104",
     "re2": "1.18.0",
     "require-all": "3.0.0",
     "rss-parser": "3.12.0",
     "showdown": "2.1.0",
-    "simplecc-wasm": "0.1.4",
+    "simplecc-wasm": "0.1.5",
     "socks-proxy-agent": "7.0.0",
     "string-similarity": "4.0.4",
     "tiny-async-pool": "2.1.0",
     "torrent-search-api": "2.1.4",
     "tough-cookie": "4.1.2",
-    "twitter-api-v2": "1.13.0",
+    "twitter-api-v2": "1.14.1",
     "winston": "3.8.2",
     "xml2js": "0.4.23"
   },

package/lib/routes/baidu/topwords.js

@@ -1,38 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const { boardId = 1 } = ctx.params;
-    const response = await got({
-        method: 'get',
-        url: `http://top.baidu.com/mobile_v2/buzz?b=${boardId}`,
-    });
-
-    const { board, topwords, descs } = response.data.result;
-    const items = topwords.map((item, index) => {
-        const title = item.keyword;
-        const content = descs[index].content;
-        const desc = content
-            ? content.data[0]
-            : {
-                  originlink: `https://www.baidu.com/s?ie=utf-8&wd=${encodeURIComponent(title)}`,
-                  title,
-                  description: title,
-                  pubDate: Date.now(),
-              };
-
-        return {
-            title,
-            description: `
-        <a href="${desc.originlink}">${desc.title}</a><br>
-        ${desc.description || ''}
-      `,
-            link: desc.originlink,
-            pubDate: new Date(desc.pubDate).toUTCString(),
-        };
-    });
-
-    ctx.state.data = {
-        title: `百度搜索风云榜-${board.boardname}`,
-        item: items,
-    };
-};

package/lib/routes/bjp/apod.js

@@ -1,48 +0,0 @@
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const baseUrl = 'http://www.bjp.org.cn';
-    const indexUrl = baseUrl + '/mryt/index.shtml';
-
-    const res = await got.get(indexUrl);
-
-    const $ = cheerio.load(res.data);
-
-    const list = $('b > a')
-        .slice(0, 10)
-        .get()
-        .map((e) => $(e));
-
-    const items = await Promise.all(
-        list.map(async (e) => {
-            const link = `${baseUrl}${e.attr('href')}`;
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const data = await got.get(link);
-            const $ = cheerio.load(data.data);
-
-            const content = $($('body > table')[1]);
-
-            const item = {
-                title: $(e).attr('title'),
-                pubDate: new Date(data.data.match(/\d{4}-\d{2}-\d{2}/)[0]).toUTCString(),
-                description: content.find('table').last().html(),
-                link,
-                guid: link,
-            };
-            ctx.cache.set(link, JSON.stringify(item));
-
-            return item;
-        })
-    );
-    ctx.state.data = {
-        title: '北京天文馆每日一图',
-        link: indexUrl,
-        item: items,
-    };
-};

package/lib/routes/caixin/article.js

@@ -1,46 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: 'http://mapiv5.caixin.com//m/api/getWapIndexListByPage?page=1&callback=&_=1560140003179',
-        headers: {
-            Referer: `http://mapiv5.caixin.com/`,
-            Host: 'mapiv5.caixin.com',
-        },
-    });
-
-    const data = response.data.data.list;
-
-    const items = await Promise.all(
-        data.map(async (item) => {
-            const link = item.web_url;
-            const summary = `<blockquote><p>${item.summary}</p></blockquote><img src="${item.pics}">`;
-            const key = 'caixin_article_' + link;
-
-            const fullText = await ctx.cache.tryGet(key, async () => {
-                const result = await got.get(link);
-
-                const $ = cheerio.load(result.data);
-
-                return $('div#Main_Content_Val.text').html();
-            });
-
-            return {
-                title: item.title,
-                description: fullText ? summary + fullText : summary,
-                link,
-                pubDate: new Date(item.time * 1000),
-                author: item.author_name,
-            };
-        })
-    );
-
-    ctx.state.data = {
-        title: `财新网 - 首页`,
-        link: `http://www.caixin.com/`,
-        description: '财新网 - 首页',
-        item: items,
-    };
-};

package/lib/routes/caixin/blog.js

@@ -1,58 +0,0 @@
-const got = require('@/utils/got');
-const parser = require('@/utils/rss-parser');
-const cheerio = require('cheerio');
-
-async function load(link, need_feed_description) {
-    const response = await got.get(link);
-    const $ = cheerio.load(response.data);
-    const article = $('.blog_content').removeAttr('style');
-    article.find('img').removeAttr('style');
-    article
-        .find('div')
-        // Non-breaking space U+00A0, ` ` in html
-        // element.children[0].data === $(element, article).text()
-        .filter((_, element) => element.children[0].data === String.fromCharCode(160))
-        .remove();
-    const description = article.html();
-
-    const item = { description };
-
-    if (need_feed_description) {
-        const author = $('div.widget.author_detail p:nth-child(2)');
-        // workaround to split author name and author article
-        const author_name = author.find('strong');
-        author_name.text(author_name.text() + '，');
-        item.feed_description = author.text();
-    }
-    return item;
-}
-
-module.exports = async (ctx) => {
-    const { column } = ctx.params;
-    const link = `http://${column}.blog.caixin.com`;
-    const feed_url = `${link}/feed`;
-    const feed = await parser.parseURL(feed_url);
-
-    const title = `财新博客 - ${/[^»]*$/.exec(feed.title)[0]}`;
-
-    const items = await Promise.all(
-        feed.items.slice(0, 10).map(async (item, index) => {
-            const link = item.link;
-            const single = {
-                title: item.title,
-                pubDate: item.pubDate,
-                link,
-                author: item['dc:creator'],
-            };
-            const other = await ctx.cache.tryGet(link, () => load(link, index === 0));
-            return Promise.resolve(Object.assign({}, single, other));
-        })
-    );
-
-    ctx.state.data = {
-        title,
-        link,
-        description: items[0].feed_description,
-        item: items,
-    };
-};

package/lib/routes/caixin/category.js

@@ -1,44 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const category = ctx.params.category;
-    const column = ctx.params.column;
-    const url = `http://${column}.caixin.com/${category}`;
-
-    const response = await got({
-        method: 'get',
-        url,
-        headers: {
-            Referer: url,
-        },
-    });
-
-    const $ = cheerio.load(response.data);
-    const title = $('title').text();
-    const list = $('.stitXtuwen_list dl dd');
-    const items = [];
-
-    for (let i = 0; i < list.length; i++) {
-        const item = {
-            title: $(list[i]).children('h4').children().text(),
-            desc: $(list[i]).children('p').text(),
-            pubDate: new Date($(list[i]).children('span').text().replace('年', '-').replace('月', '-').replace('日', '')).toUTCString(),
-            url: $(list[i]).children('h4').children().attr('href'),
-        };
-        items.push(item);
-    }
-
-    ctx.state.data = {
-        title,
-        link: url,
-        description: '财新网 - 提供财经新闻及资讯服务',
-        item: items.map((item) => ({
-            title: item.title,
-            description: item.desc,
-            guid: item.title,
-            link: item.url,
-            pubDate: item.pubDate,
-        })),
-    };
-};

package/lib/routes/caixin/latest.js

@@ -1,75 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
-
-module.exports = async (ctx) => {
-    const li_r = await got({
-        method: 'get',
-        url: 'https://www.caixin.com/2021-08-22/101758426.html',
-    });
-
-    const $ = cheerio.load(li_r.data);
-    const list = $('div.columnBox a[name="new_artical"]~ul.list li');
-
-    const tasks = [];
-
-    list.map((_index, li) => $(li).find('a').first().attr('href'))
-        .filter((_index, link) => !link.includes('fm.caixin.com') && !link.includes('video.caixin.com') && !link.includes('datanews.caixin.com')) // content filter
-        .each((_index, link) => {
-            const entry = ctx.cache.tryGet(link, async () => {
-                const entry_r = await got.get(link);
-                const $ = cheerio.load(entry_r.data);
-                // title
-                const h1 = $('div#conTit h1').text();
-                // desc items
-                const subhead = $('div#subhead.subhead').text();
-                let pic_url = $('img.cx-img-loader').attr('src');
-                if (pic_url === undefined) {
-                    pic_url = $('img.cx-img-loader').attr('data-src');
-                }
-                const pic_alt = $('dl.media_pic dd').text();
-                const content = $('div#Main_Content_Val.text').html();
-
-                /*
-                const [, count] =
-                    $('a.box_titlecontent.cost-uibtn')
-                        .text()
-                        .match(/本文共计(\d+)字/) || [];
-                if (count !== 0 && count !== undefined) {
-                    content = `${content}<blockquote><p>此乃财新通收费文章，全文共计${count}字。</p></blockquote>`;
-                }
-                */
-                // desc
-                let desc;
-                if (pic_url === undefined) {
-                    desc = `<blockquote><p>${subhead}</p></blockquote>${content}`;
-                } else {
-                    desc = `<blockquote><p>${subhead}</p></blockquote><img src="${pic_url}" alt="${pic_alt}">${content}`;
-                }
-                // time
-                /*
-                const [, year, month, day, hour, minute] = $('div#artInfo.artInfo')
-                    .text()
-                    .match(/(\d+)年(\d+)月(\d+)日 *(\d+):(\d+)/);
-                */
-
-                return {
-                    title: h1,
-                    description: desc,
-                    pubDate: timezone(parseDate($('div#artInfo.artInfo').text(), 'YYYY年MM月DD日 HH:mm'), +8),
-                    link,
-                };
-            });
-            tasks.push(entry);
-        });
-
-    const rss = await Promise.all(tasks);
-
-    ctx.state.data = {
-        title: '财新网 - 最新文章',
-        link: 'http://www.caixin.com/',
-        item: rss,
-    };
-};

package/lib/routes/caixin/yxnews.js

@@ -1,30 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: 'http://k.caixin.com/app/v1/list?productIdList=8,28&uid=&unit=1&name=&code=&deviceType=&device=&userTag=&p=1&c=20',
-        headers: {
-            Referer: `http://k.caixin.com/web/`,
-            Host: 'k.caixin.com',
-        },
-    });
-
-    const data = response.data.data.list;
-    const items = await Promise.all(
-        data.map((item) => ({
-            title: item.title,
-            description: item.text,
-            link: `http://k.caixin.com/web/detail_${item.oneline_news_code}`,
-            pubDate: new Date(item.ts),
-            author: '财新一线',
-        }))
-    );
-
-    ctx.state.data = {
-        title: `财新网 - 财新一线新闻`,
-        link: `http://k.caixin.com/`,
-        description: `财新网 - 财新一线新闻`,
-        item: items,
-    };
-};

package/lib/routes/clickme/index.js

@@ -1,76 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const site = ctx.params.site === 'r18' ? 'r18' : '';
-    const grouping = ctx.params.grouping === 'tag' ? 'tag' : 'category';
-    const name = ctx.params.name;
-
-    const url = `https://${site ? 'r18.' : ''}clickme.net/${grouping.slice(0, 1)}/${encodeURIComponent(name)}`;
-
-    const response = await got({
-        method: 'post',
-        url: 'https://api.clickme.net/article/list?key=clickme',
-        headers: {
-            Referer: url,
-        },
-        form: {
-            articleType: site ? 'r18' : 'article',
-            subtype: grouping,
-            subtypeSlug: name,
-            device: '',
-            limit: 10,
-            page: 1,
-        },
-    });
-
-    const category_name = name === 'new' ? '最新' : response.data.data.items[0].categoryName[0].name;
-    const displayed_name = grouping === 'tag' ? name : category_name;
-
-    const list = response.data.data.items;
-
-    const parseContent = (htmlString) => {
-        const $ = cheerio.load(htmlString);
-        const content = $('.article-detail-content');
-        return {
-            description: content.html(),
-        };
-    };
-
-    const out = await Promise.all(
-        list.map(async (item) => {
-            const link = item.url;
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const rssitem = {
-                title: item.title,
-                link,
-                author: item.userNick,
-                pubDate: new Date(item.date * 1000),
-            };
-
-            try {
-                const response = await got.get(link);
-                const result = parseContent(response.data);
-                if (!result.description) {
-                    return Promise.resolve('');
-                }
-                rssitem.description = result.description;
-            } catch (err) {
-                return Promise.resolve('');
-            }
-            ctx.cache.set(link, JSON.stringify(rssitem));
-            return Promise.resolve(rssitem);
-        })
-    );
-
-    ctx.state.data = {
-        title: `ClickMe ${site ? 'R18 ' : ''}- ${displayed_name}`,
-        link: url,
-        item: out.filter((item) => item !== ''),
-    };
-};

package/lib/routes/copymanga/comic.js

@@ -1,74 +0,0 @@
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-
-// 直接调用拷贝漫画的接口
-module.exports = async (ctx) => {
-    const { id } = ctx.params;
-    // 用于控制返回的章节数量
-    const chapterCnt = Number(ctx.params.chapterCnt || 0);
-    // 获取漫画列表
-    let bHasNextPage = false;
-    const iReqLimit = (ctx.queries && ctx.queries.limit) || 100;
-    let iReqOffSet = 0;
-    const strBaseUrl = `https://api.copymanga.com/api/v3/comic/${id}/group/default/chapters?`;
-    let chapterArray = [];
-
-    do {
-        bHasNextPage = false;
-        // eslint-disable-next-line no-await-in-loop
-        const { data } = await got.get(strBaseUrl + 'limit=' + iReqLimit + '&offset=' + iReqOffSet);
-        const { code, results } = data;
-
-        if (code !== 200) {
-            break;
-        }
-
-        if (results.limit + results.offset < results.total) {
-            bHasNextPage = true;
-        }
-        iReqOffSet += iReqLimit;
-
-        chapterArray = chapterArray.concat(results.list);
-    } while (bHasNextPage);
-
-    chapterArray = chapterArray
-        .map(({ comic_path_word, uuid, name, size, datetime_created, index }) => ({
-            link: 'https://copymanga.com/comic/' + comic_path_word + '/chapter/' + uuid,
-            title: name,
-            num: size,
-            updateTime: datetime_created,
-            index,
-        }))
-        .reverse();
-
-    // 获取漫画标题、介绍
-    const { bookTitle, bookIntro } = await ctx.cache.tryGet(`https://copymanga.com/comic/${id}`, async () => {
-        const { data } = await got.get(`https://copymanga.com/comic/${id}`);
-        const $ = cheerio.load(data);
-        return {
-            bookTitle: $('.comicParticulars-title-right > ul > li > h6').text(),
-            bookIntro: $('.intro').text(),
-        };
-    });
-
-    const genResult = (chapter) => ({
-        link: chapter.link,
-        title: chapter.title,
-        description: `
-            <h1>${chapter.num}p</h1>
-        `.trim(),
-    });
-
-    let itemsLen = chapterArray.length;
-
-    if (chapterCnt > 0) {
-        itemsLen = chapterCnt;
-    }
-
-    ctx.state.data = {
-        title: `拷贝漫画 - ${bookTitle}`,
-        link: `https://copymanga.com/comic/${id}`,
-        description: bookIntro,
-        item: chapterArray.map(genResult).slice(0, itemsLen),
-    };
-};

package/lib/routes/eleduck/jobs.js

@@ -1,38 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: `https://svc.eleduck.com/api/v1/posts?category=5&page=1`,
-    });
-
-    const data = response.data.posts;
-
-    const out = await Promise.all(
-        data.map(async (job) => {
-            const url = `https://svc.eleduck.com/api/v1/posts/${job.id}`;
-            const cache = await ctx.cache.get(url);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-            const response = await got.get(url);
-            const jobDetail = response.data.post;
-
-            const single = {
-                title: jobDetail.title,
-                link: `https://eleduck.com/posts/${job.id}`,
-                author: jobDetail.user.nickname,
-                description: jobDetail.content.main,
-                pubDate: new Date(jobDetail.published_at).toUTCString(),
-            };
-            ctx.cache.set(url, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
-    );
-
-    ctx.state.data = {
-        title: '电鸭社区-工作机会',
-        link: 'https://eleduck.com/?category=5',
-        item: out,
-    };
-};