rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/v2/tingshuitz/yangjiang.js

@@ -0,0 +1,29 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const url = 'https://www.yjsswjt.com/zxdt_list.jsp?flbz=7';
+    const response = await got(url);
+
+    const $ = cheerio.load(response.data);
+    const list = $('div.list_ul_div > ul > li');
+
+    ctx.state.data = {
+        title: '停水通知 - 阳江市水务集团有限公司',
+        link: 'https://www.yjsswjt.com/zxdt_list.jsp?flbz=7',
+        item: list
+            .map((_, el) => {
+                const item = $(el);
+
+                const id = item.find('a').attr('href').slice(17, -1);
+                return {
+                    title: item.find('span').text().trim(),
+                    description: item.find('span').text().trim(),
+                    link: 'http://www.yjsswjt.com/list.jsp?id=' + id,
+                    pubDate: parseDate(item.find('.datetime').text(), 'YYYY.MM.DD'),
+                };
+            })
+            .get(),
+    };
+};

package/lib/v2/trending/allTrending.js

@@ -0,0 +1,169 @@
+const dayjs = require('dayjs');
+dayjs.extend(require('dayjs/plugin/utc'));
+dayjs.extend(require('dayjs/plugin/timezone'));
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const config = require('@/config').value;
+const md5 = require('@/utils/md5');
+
+// Constants
+const DATA_REPO_BASE_URL = 'https://raw.githubusercontent.com/huqi-pr/trending-in-one/master/raw';
+const DATE_FORMAT = 'YYYY-MM-DD';
+// TODO: support custom data repo urls
+const CHANNELS = {
+    'toutiao-search': {
+        baseUrl: 'https://so.toutiao.com/search?keyword=',
+        name: '今日头条热搜',
+    },
+    'weibo-search': {
+        baseUrl: 'https://s.weibo.com/weibo?q=',
+        name: '微博热搜',
+    },
+    'zhihu-search': {
+        baseUrl: 'https://www.zhihu.com/search?q=',
+        name: '知乎热搜',
+    },
+    'zhihu-questions': {
+        baseUrl: 'https://www.zhihu.com/search?type=question&q=',
+        name: '知乎热门话题',
+    },
+    'zhihu-video': {
+        baseUrl: 'https://www.zhihu.com/search?type=video&q=',
+        name: '知乎热门视频',
+    },
+};
+
+// Helper Functions
+// TODO: integrate into CHANNELS
+const processRawDataByChannel = {
+    'toutiao-search': ({ word: title }) => ({
+        // 源 url 存在 encoding 问题，暂时不使用
+        url: CHANNELS['toutiao-search'].baseUrl + encodeURIComponent(title),
+        title,
+    }),
+    'weibo-search': ({ title }) => ({
+        // 源 url 存在 encoding 问题，暂时不使用
+        url: CHANNELS['weibo-search'].baseUrl + encodeURIComponent(title),
+        title,
+    }),
+    'zhihu-questions': (item) => item,
+    'zhihu-search': ({ query }) => {
+        const title = query.trim();
+        return {
+            // 源 url 存在 encoding 问题，暂时不使用
+            url: CHANNELS['zhihu-search'].baseUrl + encodeURIComponent(title),
+            title,
+        };
+    },
+    'zhihu-video': (item) => item,
+};
+
+const hasKeyword = (str, keywordList) => keywordList.some((keyword) => str.includes(keyword));
+const toShanghaiTimezone = (date) => dayjs.tz(date, 'Asia/Shanghai');
+
+// Data Fetcher
+// TODO: support channel selection
+const fetchAllData = async (keywordList = [], dateList = [], cache) => {
+    const cachedGetData = (url, dataMappingFn) =>
+        cache.tryGet(
+            url,
+            () =>
+                got(url)
+                    .json()
+                    // Normalize data to { title, url } format
+                    .then((res) => res.map((item) => dataMappingFn(item))),
+            config.cache.contentExpire,
+            false
+        );
+
+    const data = await Promise.all(
+        dateList.map(async (dateTime) => ({
+            dateTime,
+            data: await Promise.all(
+                Object.keys(CHANNELS).map(async (channel) => ({
+                    name: CHANNELS[channel].name,
+                    data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`, processRawDataByChannel[channel]).then((res) => res.filter(({ title }) => hasKeyword(title, keywordList))),
+                }))
+            ),
+        }))
+    );
+    for (const i of data) {
+        i.count = i.data.reduce((acc, { data }) => acc + data.length, 0);
+    }
+
+    return data.filter(({ count }) => count > 0);
+};
+
+// Generate Feed Items
+const searchLinkUrls = (keyword) => [
+    `https://tophub.today/search?e=tophub&q=${keyword}`,
+    `https://www.baidu.com/s?wd=${keyword}`,
+    `https://www.google.com/search?q=${keyword}`,
+    `https://www.zhihu.com/search?type=content&q=${keyword}`,
+    `https://s.weibo.com/weibo/${keyword}`,
+    `https://www.douyin.com/search/${keyword}`,
+    `https://so.toutiao.com/search?keyword=${keyword}`,
+];
+
+const searchLinkNames = ['热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条'];
+
+const createItem = ({ dateTime, data, count }, keywords, isToday) => {
+    const EOD = dateTime.endOf('day');
+    const pubDate = isToday ? new Date() : EOD.toDate();
+    return {
+        title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${count})`,
+        author: 'Trending All In One',
+        pubDate,
+        description: art(path.join(__dirname, 'templates/content.art'), {
+            data,
+            queries: keywords.map((query) => ({
+                links: searchLinkUrls(encodeURIComponent(query)).map((url, index) => `<a href="${url}" rel="noopener noreferrer" target="_blank">${searchLinkNames[index]}</a>`),
+                key: query,
+            })),
+        }),
+        guid: `trending-all-in-one-${EOD.toISOString()}-${md5(JSON.stringify(data))}-${keywords.join('-')}`,
+    };
+};
+
+// Main
+module.exports = async (ctx) => {
+    // Prevent making over 100 requests per invocation
+    if (ctx.params.numberOfDays > 14) {
+        throw new Error('days must be less than 14');
+    }
+    const numberOfDays = ctx.params.numberOfDays || 3;
+    const currentShanghaiDateTime = dayjs(toShanghaiTimezone(new Date()));
+    const currentShanghaiDateStr = currentShanghaiDateTime.format(DATE_FORMAT);
+    const dateList = [];
+    for (let i = 0; i < numberOfDays; i++) {
+        const d = currentShanghaiDateTime.subtract(i, 'day');
+        dateList.push(d);
+    }
+
+    const keywordList = ctx.params.keywords
+        .replace('，', ',')
+        .split(',')
+        .map((keyword) => keyword.trim());
+    const keywordStr = keywordList.join(', ');
+
+    const data = await fetchAllData(keywordList, dateList, ctx.cache);
+    const item =
+        data.length > 0
+            ? data.map((i, index) => createItem(i, keywordList, index === 0))
+            : [
+                  {
+                      title: `${keywordStr} | ${currentShanghaiDateStr} 热点追踪 (0)`,
+                      author: 'Trending All In One',
+                      description: `近${numberOfDays}日的热搜都不包含下列关键词：${keywordStr}<br>请耐心等待，或添加更多关键词试试。`,
+                      guid: `trending-all-in-one-${md5(JSON.stringify(data))}-${keywordList.join('-')}`,
+                  },
+              ];
+
+    ctx.state.data = {
+        title: `${keywordStr} | 热点聚合`,
+        description: `${keywordStr} | 今日头条热搜，知乎热门视频，知乎热搜榜，知乎热门话题，微博热搜榜聚合追踪`,
+        language: 'zh-cn',
+        item,
+    };
+};

package/lib/v2/trending/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:keywords/:numberOfDays?': ['Jkker'],
+};

package/lib/v2/trending/radar.js

@@ -0,0 +1,35 @@
+module.exports = {
+    'toutiao.com': {
+        _name: '今日头条',
+        so: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: ['/search'],
+                target: (params, url) => `/trending/${new URL(url).searchParams.get('keyword')}`,
+            },
+        ],
+    },
+    'weibo.com': {
+        _name: '微博',
+        s: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: '/weibo/:keyword',
+                target: (params) => `/trending/${params.keyword}}`,
+            },
+        ],
+    },
+    'zhihu.com': {
+        _name: '知乎',
+        www: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: ['/search'],
+                target: (params, url) => `/trending/${new URL(url).searchParams.get('q')}`,
+            },
+        ],
+    },
+};

package/lib/v2/trending/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:keywords/:numberOfDays?', require('./allTrending'));
+};

package/lib/v2/trending/templates/content.art

@@ -0,0 +1,24 @@
+<article>
+  <h2>热榜内容</h2>
+  {{each data channel}}
+  {{if channel.data.length !== 0}}
+  <h3>{{channel.name}}</h3>
+    <ul>
+      {{each channel.data item}}
+      <li>
+        <a href="{{item.url}}" rel="noopener noreferrer" target="_blank">{{item.title}}</a>
+      </li>
+      {{/each}}
+    </ul>
+  {{/if}}
+  {{/each}}
+  <h2>搜索关键词</h2>
+  {{each queries q}}
+  <h3>{{q.key}}</h3>
+  <p>
+    {{each q.links l}}
+      {{@l}}&nbsp&nbsp
+    {{/each}}
+  </p>
+  {{/each}}
+</article>

package/lib/v2/twreporter/category.js

@@ -0,0 +1,31 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+const fetch = require('./fetch_article');
+
+module.exports = async (ctx) => {
+    const baseURL = 'https://www.twreporter.org';
+    const url = baseURL + `/categories/${ctx.params.cid}`;
+    const res = await got(url);
+    const $ = cheerio.load(res.data);
+    const list = $('.lnKPLr').get();
+    const category = $('.kCfkTU').text();
+
+    const out = await Promise.all(
+        list.map((item) => {
+            const $ = cheerio.load(item);
+            const address = baseURL + $('a').attr('href');
+            const title = $('.list-item__Title-sc-1dx5lew-5').text();
+            return ctx.cache.tryGet(address, async () => {
+                const single = await fetch(address);
+                single.title = title;
+                return single;
+            });
+        })
+    );
+    ctx.state.data = {
+        title: `報導者 | ${category}`,
+        link: url,
+        item: out,
+    };
+};

package/lib/v2/twreporter/fetch_article.js

@@ -0,0 +1,67 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async function fetch(address) {
+    const res = await got(address);
+    const capture = cheerio.load(res.data);
+    capture('.gIMvvS').remove();
+
+    let metaInfoBox = capture('.ffAPnj')
+        .filter((index) => index === 0)
+        .get();
+
+    // For photography
+    if (metaInfoBox.length === 0) {
+        metaInfoBox = capture('.deNvJY')
+            .filter((index) => index === 0)
+            .get();
+    }
+
+    const acquire = cheerio.load(metaInfoBox);
+    let time = acquire('.gimsRe').text();
+
+    // For `photography`
+    if (!time) {
+        time = acquire('.kHluJP').text();
+    }
+    // # Author(s) of the article
+    //
+    // There exists two formats for this section.
+    // 1. 文字 /...` with `摄影 / ..` in separate line, or
+    // 2. just a simple line starts with `文/ ...`
+    // For the first condition, we use a array
+    // to record the list of author(s), and use
+    // `；` to connect two lines.
+    const authors = [];
+
+    if (acquire('.loxoWO').text() === '') {
+        for (const item of acquire('.flciyI').get()) {
+            const $ = cheerio.load(item);
+            const job = $('.hGsNtm').text();
+            // An article may have multiple authors
+            const name = [];
+            for (const item of $('.cidPTd > a').get()) {
+                const $ = cheerio.load(item);
+                name.push($('.fJSaZP').text());
+            }
+
+            const author = job + '／' + name.join('，');
+            authors.push(author);
+        }
+    } else {
+        authors.push(acquire('.loxoWO').text());
+    }
+    const author = authors.join('；');
+
+    // contents = cover photo + italic intro + text
+    const contents = '<em>' + capture('.hxFBKc').html() + '</em><br>' + capture('.irqyDp').html();
+
+    return {
+        author,
+        description: contents,
+        link: address,
+        guid: address,
+        pubDate: parseDate(time, 'M/D/YYYY'),
+    };
+};

package/lib/v2/twreporter/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/category/:cid': ['emdoe'],
+    '/newest': ['emdoe'],
+    '/photography': ['emdoe'],
+};

package/lib/v2/twreporter/newest.js

@@ -0,0 +1,29 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+const fetch = require('./fetch_article');
+
+module.exports = async (ctx) => {
+    const url = 'https://www.twreporter.org';
+    const res = await got(url);
+    const $ = cheerio.load(res.data);
+    const list = $('.gKMjSz').get();
+
+    const out = await Promise.all(
+        list.map((item) => {
+            const $ = cheerio.load(item);
+            const address = url + $('a').attr('href');
+            const title = $('.latest-section__Title-hzxpx3-6').text();
+            return ctx.cache.tryGet(address, async () => {
+                const single = await fetch(address);
+                single.title = title;
+                return single;
+            });
+        })
+    );
+    ctx.state.data = {
+        title: `報導者 | 最新`,
+        link: url,
+        item: out,
+    };
+};

package/lib/v2/twreporter/photography.js

@@ -0,0 +1,46 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+const fetch = require('./fetch_article');
+
+module.exports = async (ctx) => {
+    const baseURL = 'https://www.twreporter.org';
+    const url = baseURL + `/photography`;
+    const res = await got(url);
+    const $ = cheerio.load(res.data);
+    const coverList = $('.WPJvn').get();
+    const commonList = $('.eVNsZf').get();
+
+    const coverView = await Promise.all(
+        coverList.map((item) => {
+            const $ = cheerio.load(item);
+            const address = baseURL + $('li > a').attr('href');
+            const title = $('.sc-1aojo4z-4').text();
+            return ctx.cache.tryGet(address, async () => {
+                const single = await fetch(address);
+                single.title = title;
+                return single;
+            });
+        })
+    );
+
+    const listView = await Promise.all(
+        commonList.map((item) => {
+            const $ = cheerio.load(item);
+            const address = baseURL + $('li > a').attr('href');
+            const title = $('.ii0887-4').text();
+
+            return ctx.cache.tryGet(address, async () => {
+                const single = await fetch(address);
+                single.title = title;
+                return single;
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: `報導者 | 影像`,
+        link: url,
+        item: coverView.concat(listView),
+    };
+};

package/lib/v2/twreporter/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'twreporter.org': {
+        _name: '報導者',
+        '.': [
+            {
+                title: '最新',
+                docs: 'https://docs.rsshub.app/new-media.html#bao-dao-zhe',
+                source: ['/'],
+                target: '/twreporter/newest',
+            },
+            {
+                title: '摄影',
+                docs: 'https://docs.rsshub.app/new-media.html#bao-dao-zhe',
+                source: ['/photography'],
+                target: '/twreporter',
+            },
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/new-media.html#bao-dao-zhe',
+                source: ['/categories/:tid'],
+                target: '/twreporter/category/:tid',
+            },
+        ],
+    },
+};

package/lib/v2/twreporter/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/category/:cid', require('./category'));
+    router.get('/newest', require('./newest'));
+    router.get('/photography', require('./photography'));
+};

package/lib/v2/u3c3/index.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type;
+    // should be:
+    // undefined
+    // U3C3
+    // Videos
+    // Photo
+    // Book
+    // Game
+    // Software
+    // Other
+    const rootURL = 'https://www.u3c3.com';
+    let currentURL;
+    let title;
+    if (typeof type === 'undefined') {
+        currentURL = rootURL;
+        title = 'home - u3c3';
+    } else if (type === 'search') {
+        const keyword = typeof ctx.params.keyword === 'undefined' ? '' : ctx.params.keyword;
+        currentURL = `${rootURL}/?search=${keyword}`;
+        title = `search ${keyword} - u3c3`;
+    } else {
+        currentURL = `${rootURL}/?type=${type}&p=1`;
+        title = `${type} - u3c3`;
+    }
+
+    const response = await got(currentURL);
+    const $ = cheerio.load(response.data);
+
+    const items = $('body > div.container > div.table-responsive > table > tbody > tr')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const title = item.find('td:nth-of-type(2) > a ').attr('title');
+            const guid = rootURL + item.find('td:nth-of-type(2) > a').attr('href');
+            const link = guid;
+            const pubDate = item.find('td:nth-of-type(5)').text();
+            const enclosure_url = item.find('td:nth-of-type(3) > a:nth-of-type(2)').attr('href');
+            return {
+                title,
+                guid,
+                link,
+                pubDate,
+
+                enclosure_url,
+                enclosure_type: 'application/x-bittorrent',
+            };
+        });
+
+    ctx.state.data = {
+        title,
+        description: title,
+        link: currentURL,
+        item: items,
+    };
+};

package/lib/v2/u3c3/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:type?/:keyword?': ['noname1897'],
+};

package/lib/v2/u3c3/radar.js

@@ -0,0 +1,29 @@
+module.exports = {
+    'u3c3.com': {
+        _name: 'u3c3',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/multimedia.html#u3c3-fen-lei',
+                source: '/',
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    const type = searchParams.has('type') ? searchParams.get('type') : '';
+                    return `/u3c3/${type}`;
+                },
+            },
+            {
+                title: '关键词搜索',
+                docs: 'https://docs.rsshub.app/multimedia.html#u3c3-guan-jian-ci-sou-suo',
+                source: '/',
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    if (searchParams.has('search')) {
+                        const keyword = searchParams.get('search');
+                        return `/u3c3/search/${keyword}`;
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/u3c3/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:type?/:keyword?', require('./index'));
+};

package/lib/v2/uraaka-joshi/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/': ['SettingDust', 'Halcao'],
+    '/:id': ['SettingDust', 'Halcao'],
+};

package/lib/v2/uraaka-joshi/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'uraaka-joshi.com': {
+        _name: '裏垢女子まとめ',
+        '.': [
+            {
+                title: '主页',
+                docs: 'https://docs-rsshub.pages.dev/other.html#li-gou-nu-zi-まとめ',
+                source: ['/'],
+                target: '/uraaka-joshi',
+            },
+            {
+                title: '用户',
+                docs: 'https://docs-rsshub.pages.dev/other.html#li-gou-nu-zi-まとめ',
+                source: ['/:id'],
+                target: '/uraaka-joshi/:id',
+            },
+        ],
+    },
+};

package/lib/v2/uraaka-joshi/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/', require('./uraaka-joshi'));
+    router.get('/:id', require('./uraaka-joshi-user'));
+};