rsshub 1.0.0-master.ff97759 → 1.0.0-master.ffea261

package/lib/v2/tju/radar.js

@@ -0,0 +1,97 @@
+module.exports = {
+    'tju.edu.cn': {
+        _name: '天津大学',
+        cic: [
+            {
+                title: '智能与计算学部 - 学部新闻',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-zhi-neng-yu-ji-suan-xue-bu',
+                source: ['/xwzx/xyxw.htm', '/'],
+                target: '/tju/cic/news',
+            },
+            {
+                title: '智能与计算学部 - 通知公告',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-zhi-neng-yu-ji-suan-xue-bu',
+                source: ['/xwzx/tzgg.htm', '/'],
+                target: '/tju/cic/notification',
+            },
+            {
+                title: '智能与计算学部 - 北洋智算论坛',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-zhi-neng-yu-ji-suan-xue-bu',
+                source: ['/byzslt.htm', '/'],
+                target: '/tju/cic/forum',
+            },
+        ],
+        news: [
+            {
+                title: '新闻网 - 聚焦天大',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-xin-wen-wang',
+                source: ['/jjtd.htm', '/'],
+                target: '/tju/news/focus',
+            },
+            {
+                title: '新闻网 - 综合新闻',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-xin-wen-wang',
+                source: ['/zhxw.htm', '/'],
+                target: '/tju/news/general',
+            },
+            {
+                title: '新闻网 - 校内新闻',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-xin-wen-wang',
+                source: ['/xnxw1/qb.htm', '/'],
+                target: '/tju/news/internal',
+            },
+            {
+                title: '新闻网 - 媒体报道',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-xin-wen-wang',
+                source: ['/mtbd.htm', '/'],
+                target: '/tju/news/media',
+            },
+            {
+                title: '新闻网 - 图说天大',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-xin-wen-wang',
+                source: ['/tstd.htm', '/'],
+                target: '/tju/news/picture',
+            },
+        ],
+        oaa: [
+            {
+                title: '教务处 - 新闻动态',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-jiao-wu-chu',
+                source: ['/xwdt.htm', '/'],
+                target: '/tju/oaa/news',
+            },
+            {
+                title: '教务处 - 通知公告',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-jiao-wu-chu',
+                source: ['/tzgg.htm', '/'],
+                target: '/tju/oaa/notification',
+            },
+        ],
+        yzb: [
+            {
+                title: '研究生招生网 - 校级公告',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-yan-jiu-sheng-zhao-sheng-wang',
+                source: ['/xwzx/zxxx/', '/'],
+                target: '/tju/yzb/notice',
+            },
+            {
+                title: '研究生招生网 - 统考硕士',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-yan-jiu-sheng-zhao-sheng-wang',
+                source: ['/xwzx/tkss_xw/', '/'],
+                target: '/tju/yzb/master',
+            },
+            {
+                title: '研究生招生网 - 统考博士',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-yan-jiu-sheng-zhao-sheng-wang',
+                source: ['/xwzx/tkbs_xw/', '/'],
+                target: '/tju/yzb/doctor',
+            },
+            {
+                title: '研究生招生网 - 在职学位',
+                docs: 'https://docs.rsshub.app/university.html#tian-jin-da-xue-yan-jiu-sheng-zhao-sheng-wang',
+                source: ['/xwzx/zzxw/', '/'],
+                target: '/tju/yzb/job',
+            },
+        ],
+    },
+};

package/lib/v2/tju/router.js

@@ -0,0 +1,6 @@
+module.exports = function (router) {
+    router.get('/cic/:type?', require('./cic'));
+    router.get('/news/:type?', require('./news'));
+    router.get('/oaa/:type?', require('./oaa'));
+    router.get('/yzb/:type?', require('./yzb'));
+};

package/lib/v2/tju/yzb/index.js

@@ -0,0 +1,121 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const iconv = require('iconv-lite');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const yzb_base_url = 'http://yzb.tju.edu.cn/';
+const repo_url = 'https://github.com/DIYgod/RSSHub/issues';
+
+const pageType = (href) => {
+    if (href === undefined) {
+        return 'unknown';
+    } else if (!href.startsWith('http')) {
+        return 'in-site';
+    }
+    const url = new URL(href);
+    if (url.hostname === 'yzb.tju.edu.cn') {
+        return 'tju-yzb';
+    } else {
+        return 'unknown';
+    }
+};
+
+module.exports = async (ctx) => {
+    const type = ctx.params && ctx.params.type;
+    let path, subtitle;
+
+    switch (type) {
+        case 'notice':
+            subtitle = '校级公告';
+            path = 'xwzx/zxxx/';
+            break;
+        case 'master':
+            subtitle = '统考硕士';
+            path = 'xwzx/tkss_xw/';
+            break;
+        case 'doctor':
+            subtitle = '统考博士';
+            path = 'xwzx/tkbs_xw/';
+            break;
+        case 'job':
+            subtitle = '在职学位';
+            path = 'xwzx/zzxw/';
+            break;
+        default:
+            subtitle = '校级公告';
+            path = 'xwzx/zxxx/';
+    }
+    let response = null;
+    try {
+        response = await got(yzb_base_url + path, {
+            headers: {
+                Referer: yzb_base_url,
+            },
+            responseType: 'buffer',
+        });
+    } catch (e) {
+        // ignore error handler
+        // console.log(e);
+    }
+
+    if (response === null) {
+        ctx.state.data = {
+            title: '天津大学研究生招生网 - ' + subtitle,
+            link: yzb_base_url + path,
+            description: '链接失效' + yzb_base_url + path,
+            item: [
+                {
+                    title: '提示信息',
+                    link: repo_url,
+                    description: `<h2>请到<a href=${repo_url}>此处</a>提交Issue</h2>`,
+                },
+            ],
+        };
+    } else {
+        const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+        const list = $('body > table:nth-child(3) > tbody > tr > td.table_left_right > table > tbody > tr:nth-child(3) > td > table > tbody > tr:nth-child(1) > td > table > tbody > tr')
+            .slice(1, -1)
+            .map((_index, item) => {
+                const href = $('td > a', item).attr('href');
+                const type = pageType(href);
+                return {
+                    title: $('td > a', item).text(),
+                    link: type === 'in-site' ? yzb_base_url + path + href : href,
+                    pubDate: timezone(parseDate($('.font_10_time', item).text().slice(2, -2), 'YYYY-MM-DD'), +8),
+                    type,
+                };
+            })
+            .get();
+
+        const items = await Promise.all(
+            list.map((item) => {
+                switch (item.type) {
+                    case 'tju-yzb':
+                    case 'in-site':
+                        return ctx.cache.tryGet(item.link, async () => {
+                            let detailResponse = null;
+                            try {
+                                detailResponse = await got(item.link, { responseType: 'buffer' });
+                                const content = cheerio.load(iconv.decode(detailResponse.data, 'gbk'));
+                                content('.font_18_b').remove();
+                                content('.font_grey_en').remove();
+                                item.description = content('.nav_a10 > table > tbody').html();
+                            } catch (e) {
+                                // ignore error handler
+                            }
+                            return item;
+                        });
+                    default:
+                        return item;
+                }
+            })
+        );
+
+        ctx.state.data = {
+            title: '天津大学研究生招生网 - ' + subtitle,
+            link: yzb_base_url + path,
+            item: items,
+        };
+    }
+};

package/lib/v2/tradingview/blog.js

@@ -0,0 +1,60 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const asyncPool = require('tiny-async-pool');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const language = ctx.params.language ?? 'en';
+
+    const rootUrl = 'https://www.tradingview.com';
+    const currentUrl = `${rootUrl}/blog/${language}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('.articles-grid-item a[rel="bookmark"]')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.find('.title').text(),
+                link: item.attr('href'),
+                pubDate: parseDate(item.find('.date').text(), 'MMMM D, YYYY'),
+            };
+        });
+
+    const items = [];
+    for await (const item of asyncPool(3, list, (item) =>
+        ctx.cache.tryGet(item.link, async () => {
+            const detailResponse = await got({
+                method: 'get',
+                url: item.link,
+            });
+
+            const content = cheerio.load(detailResponse.data);
+
+            item.description = art(path.join(__dirname, 'templates/description.art'), {
+                image: content('.single-img img').attr('src'),
+                description: content('.entry-content').html(),
+            });
+
+            return item;
+        })
+    )) {
+        items.push(item);
+    }
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/tradingview/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/blog/:language?': ['nczitzk'],
+};

package/lib/v2/tradingview/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'tradingview.com': {
+        _name: 'TradingView',
+        '.': [
+            {
+                title: 'Blog',
+                docs: 'https://docs.rsshub.app/program-update.html#tradingview-blog',
+                source: ['/blog/:language', '/'],
+                target: '/tradingview/blog',
+            },
+        ],
+    },
+};

package/lib/v2/tradingview/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/blog/:language?', require('./blog'));
+};

package/lib/v2/tradingview/templates/description.art

@@ -0,0 +1,4 @@
+{{ if image }}
+<img src="{{ image }}">
+{{ /if }}
+{{@ description }}

package/lib/v2/trending/allTrending.js

@@ -0,0 +1,196 @@
+const dayjs = require('dayjs');
+dayjs.extend(require('dayjs/plugin/utc'));
+dayjs.extend(require('dayjs/plugin/timezone'));
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const config = require('@/config').value;
+const md5 = require('@/utils/md5');
+
+// Constants
+const CACHE_KEY = 'trending-all-in-one';
+const DATA_REPO_BASE_URL = 'https://raw.githubusercontent.com/huqi-pr/trending-in-one/master/raw';
+const DATE_FORMAT = 'YYYY-MM-DD';
+// TODO: support custom data repo urls
+const CHANNELS = {
+    'toutiao-search': {
+        baseUrl: 'https://so.toutiao.com/search?keyword=',
+        name: '今日头条热搜',
+    },
+    'weibo-search': {
+        baseUrl: 'https://s.weibo.com/weibo?q=',
+        name: '微博热搜',
+    },
+    'zhihu-search': {
+        baseUrl: 'https://www.zhihu.com/search?q=',
+        name: '知乎热搜',
+    },
+    'zhihu-questions': {
+        baseUrl: 'https://www.zhihu.com/search?type=question&q=',
+        name: '知乎热门话题',
+    },
+    'zhihu-video': {
+        baseUrl: 'https://www.zhihu.com/search?type=video&q=',
+        name: '知乎热门视频',
+    },
+};
+
+// Helper Functions
+const processRawDataByChannel = {
+    'toutiao-search': ({ word: title }) => ({
+        // 源 url 存在 encoding 问题，暂时不使用
+        url: CHANNELS['toutiao-search'].baseUrl + encodeURIComponent(title),
+        title,
+    }),
+    'weibo-search': ({ title }) => ({
+        // 源 url 存在 encoding 问题，暂时不使用
+        url: CHANNELS['weibo-search'].baseUrl + encodeURIComponent(title),
+        title,
+    }),
+    'zhihu-questions': (item) => item,
+    'zhihu-search': ({ query }) => {
+        const title = query.trim();
+        return {
+            // 源 url 存在 encoding 问题，暂时不使用
+            url: CHANNELS['zhihu-search'].baseUrl + encodeURIComponent(title),
+            title,
+        };
+    },
+    'zhihu-video': (item) => item,
+};
+
+const hasKeyword = (str, keywordList) => keywordList.some((keyword) => str.includes(keyword));
+const toShanghaiTimezone = (date) => dayjs.tz(date, 'Asia/Shanghai');
+const processRawData = (channel) => (res) => res.map((item) => processRawDataByChannel[channel](item));
+const filterKeyword = (keywordList) => (res) => res.filter(({ title }) => hasKeyword(title, keywordList));
+
+// Data Fetcher
+// TODO: support channel selection
+const fetchAllData = async (keywordList = [], dateList = [], cache) => {
+    const cachedGetData = (url) => cache.tryGet(url, () => got(url).json(), config.cache.contentExpire, false);
+
+    let data = await Promise.all(
+        dateList.map(async (dateTime) => ({
+            dateTime,
+            data: await Promise.all(
+                Object.keys(CHANNELS).map(async (channel) => ({
+                    name: CHANNELS[channel].name,
+                    data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`)
+                        .then(processRawData(channel))
+                        .then(filterKeyword(keywordList)),
+                }))
+            ),
+        }))
+    );
+
+    for (const i of data) {
+        i.count = i.data.reduce((acc, { data }) => acc + data.length, 0);
+    }
+
+    data = data.filter(({ count }) => count > 0);
+
+    if (data.length === 0) {
+        return data;
+    }
+
+    const prev = cache.get(CACHE_KEY + ':latest-items');
+
+    const latest = data[0];
+    latest.newItemCount = 0;
+    if (latest.count > 0 && prev) {
+        // Mark new items in latest
+        for (const channel in latest.data) {
+            for (const i of latest.data[channel].data) {
+                i.new = !(i.url in prev);
+                latest.newItemCount += i.new ? 1 : 0;
+            }
+        }
+        // Save latest data to cache
+        const cachedItems = latest.data.reduce((acc, { data: channel }) => {
+            for (const item of channel) {
+                acc[item.url] = true;
+            }
+            return acc;
+        }, {});
+        cache.set(CACHE_KEY + ':latest-items', cachedItems, config.cache.contentExpire);
+
+        latest.count = Object.keys(cachedItems).length;
+    }
+
+    return data;
+};
+
+// Generate Feed Items
+const searchLinkUrls = (keyword) => [
+    `https://tophub.today/search?e=tophub&q=${keyword}`,
+    `https://www.baidu.com/s?wd=${keyword}`,
+    `https://www.google.com/search?q=${keyword}`,
+    `https://www.zhihu.com/search?type=content&q=${keyword}`,
+    `https://s.weibo.com/weibo/${keyword}`,
+    `https://www.douyin.com/search/${keyword}`,
+    `https://so.toutiao.com/search?keyword=${keyword}`,
+];
+
+const searchLinkNames = ['热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条'];
+
+const createItem = ({ dateTime, data, count, newItemCount }, keywords, isToday) => {
+    const EOD = dateTime.endOf('day');
+    const pubDate = isToday ? new Date() : EOD.toDate();
+    const countStr = isToday && newItemCount ? newItemCount + '🆕' : count;
+
+    return {
+        title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${countStr})`,
+        author: 'Trending All In One',
+        pubDate,
+        description: art(path.join(__dirname, 'templates/content.art'), {
+            data,
+            queries: keywords.map((query) => ({
+                links: searchLinkUrls(encodeURIComponent(query)).map((url, index) => `<a href="${url}" rel="noopener noreferrer" target="_blank">${searchLinkNames[index]}</a>`),
+                key: query,
+            })),
+        }),
+        guid: `trending-all-in-one-${EOD.toISOString()}-${md5(JSON.stringify(data))}-${keywords.join('-')}`,
+    };
+};
+
+// Main
+module.exports = async (ctx) => {
+    // Prevent making over 100 requests per invocation
+    if (ctx.params.numberOfDays > 14) {
+        throw new Error('days must be less than 14');
+    }
+    const numberOfDays = ctx.params.numberOfDays || 3;
+    const currentShanghaiDateTime = dayjs(toShanghaiTimezone(new Date()));
+    const currentShanghaiDateStr = currentShanghaiDateTime.format(DATE_FORMAT);
+    const dateList = [];
+    for (let i = 0; i < numberOfDays; i++) {
+        const d = currentShanghaiDateTime.subtract(i, 'day');
+        dateList.push(d);
+    }
+
+    const keywordList = ctx.params.keywords
+        .replace('，', ',')
+        .split(',')
+        .map((keyword) => keyword.trim());
+    const keywordStr = keywordList.join(', ');
+
+    const data = await fetchAllData(keywordList, dateList, ctx.cache).catch(() => []);
+    const item =
+        data.length > 0
+            ? data.map((i, index) => createItem(i, keywordList, index === 0))
+            : [
+                  {
+                      title: `${keywordStr} | ${currentShanghaiDateStr} 热点追踪 (0)`,
+                      author: 'Trending All In One',
+                      description: `近${numberOfDays}日的热搜都不包含下列关键词：${keywordStr}<br>请耐心等待，或添加更多关键词试试。`,
+                      guid: `trending-all-in-one-${md5(JSON.stringify(data))}-${keywordList.join('-')}`,
+                  },
+              ];
+
+    ctx.state.data = {
+        title: `${keywordStr} | 热点聚合`,
+        description: `${keywordStr} | 今日头条热搜，知乎热门视频，知乎热搜榜，知乎热门话题，微博热搜榜聚合追踪`,
+        language: 'zh-cn',
+        item,
+    };
+};

package/lib/v2/trending/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:keywords/:numberOfDays?': ['Jkker'],
+};

package/lib/v2/trending/radar.js

@@ -0,0 +1,35 @@
+module.exports = {
+    'toutiao.com': {
+        _name: '今日头条',
+        so: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: ['/search'],
+                target: (params, url) => `/trending/${new URL(url).searchParams.get('keyword')}`,
+            },
+        ],
+    },
+    'weibo.com': {
+        _name: '微博',
+        s: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: '/weibo/:keyword',
+                target: (params) => `/trending/${params.keyword}}`,
+            },
+        ],
+    },
+    'zhihu.com': {
+        _name: '知乎',
+        www: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: ['/search'],
+                target: (params, url) => `/trending/${new URL(url).searchParams.get('q')}`,
+            },
+        ],
+    },
+};

package/lib/v2/trending/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:keywords/:numberOfDays?', require('./allTrending'));
+};

package/lib/v2/trending/templates/content.art

@@ -0,0 +1,25 @@
+<article>
+  <h2>热榜内容</h2>
+  {{each data channel}}
+  {{if channel.data.length !== 0}}
+  <h3>{{channel.name}}</h3>
+    <ul>
+      {{each channel.data item}}
+      <li>
+        <a href="{{item.url}}" rel="noopener noreferrer" target="_blank">{{item.title}}</a>
+        {{if item.new}}🆕{{/if}}
+      </li>
+      {{/each}}
+    </ul>
+  {{/if}}
+  {{/each}}
+  <h2>搜索关键词</h2>
+  {{each queries q}}
+  <h3>{{q.key}}</h3>
+  <p>
+    {{each q.links l}}
+      {{@l}}&nbsp&nbsp
+    {{/each}}
+  </p>
+  {{/each}}
+</article>

package/lib/v2/tribalfootball/latest.js

@@ -0,0 +1,68 @@
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const rssUrl = 'https://www.tribalfootball.com/rss/mediafed/general/rss.xml';
+
+module.exports = async (ctx) => {
+    const rss = await got(rssUrl);
+    const $ = cheerio.load(rss.data, { xmlMode: true });
+    const items = $('rss > channel > item')
+        .map((_, item) => {
+            const $item = $(item);
+            let link = $item.find('link').text();
+            link = new URL(link);
+            link.search = '';
+            link = link.href;
+            return {
+                title: $item.find('title').text(),
+                description: $item.find('description').text(),
+                link,
+                guid: $item.find('guid').text(),
+                pubDate: parseDate($item.find('pubDate').text()),
+                author: $item.find('dc\\:creator').text(),
+                _header_image: $item.find('enclosure').attr('url'),
+            };
+        })
+        .get();
+
+    await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+
+                const title = $('head > title').text().replace(' - Tribal Football', '');
+
+                let desc = $('.articleBody');
+                desc.find('.ad').remove();
+                // <p><br><i>AD</i><span></span></p>
+                const ad = desc.find('p > br:first-child').next('i');
+                const adNextSpan = ad.next('span');
+                if (adNextSpan.length && !adNextSpan.text() && !adNextSpan.next().length) {
+                    ad.parent().remove();
+                }
+                desc = desc.html();
+                desc = art(path.join(__dirname, 'templates/plus_header.art'), {
+                    desc,
+                    header_image: item._header_image,
+                });
+
+                item.title = title || item.title;
+                item.description = desc || item.description;
+                delete item._header_image;
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'Tribal Football - Latest',
+        description: 'Tribal Football - Football News, Soccer News, Transfers & Rumours',
+        link: 'https://www.tribalfootball.com/articles',
+        image: 'https://www.tribalfootball.com/images/tribal-logo-rss.png',
+        item: items,
+    };
+};

package/lib/v2/tribalfootball/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['Rongronggg9'],
+};

package/lib/v2/tribalfootball/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'tribalfootball.com': {
+        _name: 'Tribal Football',
+        '.': [
+            {
+                title: 'Latest News',
+                docs: 'https://docs.rsshub.app/new-media.html#tribal-football',
+                source: ['/'],
+                target: '/tribalfootball',
+            },
+        ],
+    },
+};

package/lib/v2/tribalfootball/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/', require('./latest'));
+};