rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/lib/{routes → v2}/tingshuitz/xian.js

@@ -4,7 +4,7 @@ const iconv = require('iconv-lite');
 
 module.exports = async (ctx) => {
     const url = 'http://www.xazls.com/tsgg/index.htm';
-    const response = await got.get(url, {
+    const response = await got(url, {
         responseType: 'buffer',
     });
 

package/lib/{routes → v2}/tingshuitz/yangjiang.js

@@ -1,20 +1,17 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const iconv = require('iconv-lite');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const url = 'http://www.yjsswjt.com/zxdt_list.jsp?flbz=7';
-    const response = await got.get(url, {
-        responseType: 'buffer',
-    });
+    const url = 'https://www.yjsswjt.com/zxdt_list.jsp?flbz=7';
+    const response = await got(url);
 
-    const data = response.data;
-    const $ = cheerio.load(iconv.decode(data, 'gb2312'));
+    const $ = cheerio.load(response.data);
     const list = $('div.list_ul_div > ul > li');
 
     ctx.state.data = {
         title: '停水通知 - 阳江市水务集团有限公司',
-        link: 'http://www.yjsswjt.com/zxdt_list.jsp?flbz=7',
+        link: 'https://www.yjsswjt.com/zxdt_list.jsp?flbz=7',
         item: list
             .map((_, el) => {
                 const item = $(el);
@@ -24,6 +21,7 @@ module.exports = async (ctx) => {
                     title: item.find('span').text().trim(),
                     description: item.find('span').text().trim(),
                     link: 'http://www.yjsswjt.com/list.jsp?id=' + id,
+                    pubDate: parseDate(item.find('.datetime').text(), 'YYYY.MM.DD'),
                 };
             })
             .get(),

package/lib/v2/trending/allTrending.js

@@ -0,0 +1,169 @@
+const dayjs = require('dayjs');
+dayjs.extend(require('dayjs/plugin/utc'));
+dayjs.extend(require('dayjs/plugin/timezone'));
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const config = require('@/config').value;
+const md5 = require('@/utils/md5');
+
+// Constants
+const DATA_REPO_BASE_URL = 'https://raw.githubusercontent.com/huqi-pr/trending-in-one/master/raw';
+const DATE_FORMAT = 'YYYY-MM-DD';
+// TODO: support custom data repo urls
+const CHANNELS = {
+    'toutiao-search': {
+        baseUrl: 'https://so.toutiao.com/search?keyword=',
+        name: '今日头条热搜',
+    },
+    'weibo-search': {
+        baseUrl: 'https://s.weibo.com/weibo?q=',
+        name: '微博热搜',
+    },
+    'zhihu-search': {
+        baseUrl: 'https://www.zhihu.com/search?q=',
+        name: '知乎热搜',
+    },
+    'zhihu-questions': {
+        baseUrl: 'https://www.zhihu.com/search?type=question&q=',
+        name: '知乎热门话题',
+    },
+    'zhihu-video': {
+        baseUrl: 'https://www.zhihu.com/search?type=video&q=',
+        name: '知乎热门视频',
+    },
+};
+
+// Helper Functions
+// TODO: integrate into CHANNELS
+const processRawDataByChannel = {
+    'toutiao-search': ({ word: title }) => ({
+        // 源 url 存在 encoding 问题，暂时不使用
+        url: CHANNELS['toutiao-search'].baseUrl + encodeURIComponent(title),
+        title,
+    }),
+    'weibo-search': ({ title }) => ({
+        // 源 url 存在 encoding 问题，暂时不使用
+        url: CHANNELS['weibo-search'].baseUrl + encodeURIComponent(title),
+        title,
+    }),
+    'zhihu-questions': (item) => item,
+    'zhihu-search': ({ query }) => {
+        const title = query.trim();
+        return {
+            // 源 url 存在 encoding 问题，暂时不使用
+            url: CHANNELS['zhihu-search'].baseUrl + encodeURIComponent(title),
+            title,
+        };
+    },
+    'zhihu-video': (item) => item,
+};
+
+const hasKeyword = (str, keywordList) => keywordList.some((keyword) => str.includes(keyword));
+const toShanghaiTimezone = (date) => dayjs.tz(date, 'Asia/Shanghai');
+
+// Data Fetcher
+// TODO: support channel selection
+const fetchAllData = async (keywordList = [], dateList = [], cache) => {
+    const cachedGetData = (url, dataMappingFn) =>
+        cache.tryGet(
+            url,
+            () =>
+                got(url)
+                    .json()
+                    // Normalize data to { title, url } format
+                    .then((res) => res.map((item) => dataMappingFn(item))),
+            config.cache.contentExpire,
+            false
+        );
+
+    const data = await Promise.all(
+        dateList.map(async (dateTime) => ({
+            dateTime,
+            data: await Promise.all(
+                Object.keys(CHANNELS).map(async (channel) => ({
+                    name: CHANNELS[channel].name,
+                    data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`, processRawDataByChannel[channel]).then((res) => res.filter(({ title }) => hasKeyword(title, keywordList))),
+                }))
+            ),
+        }))
+    );
+    for (const i of data) {
+        i.count = i.data.reduce((acc, { data }) => acc + data.length, 0);
+    }
+
+    return data.filter(({ count }) => count > 0);
+};
+
+// Generate Feed Items
+const searchLinkUrls = (keyword) => [
+    `https://tophub.today/search?e=tophub&q=${keyword}`,
+    `https://www.baidu.com/s?wd=${keyword}`,
+    `https://www.google.com/search?q=${keyword}`,
+    `https://www.zhihu.com/search?type=content&q=${keyword}`,
+    `https://s.weibo.com/weibo/${keyword}`,
+    `https://www.douyin.com/search/${keyword}`,
+    `https://so.toutiao.com/search?keyword=${keyword}`,
+];
+
+const searchLinkNames = ['热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条'];
+
+const createItem = ({ dateTime, data, count }, keywords, isToday) => {
+    const EOD = dateTime.endOf('day');
+    const pubDate = isToday ? new Date() : EOD.toDate();
+    return {
+        title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${count})`,
+        author: 'Trending All In One',
+        pubDate,
+        description: art(path.join(__dirname, 'templates/content.art'), {
+            data,
+            queries: keywords.map((query) => ({
+                links: searchLinkUrls(encodeURIComponent(query)).map((url, index) => `<a href="${url}" rel="noopener noreferrer" target="_blank">${searchLinkNames[index]}</a>`),
+                key: query,
+            })),
+        }),
+        guid: `trending-all-in-one-${EOD.toISOString()}-${md5(JSON.stringify(data))}-${keywords.join('-')}`,
+    };
+};
+
+// Main
+module.exports = async (ctx) => {
+    // Prevent making over 100 requests per invocation
+    if (ctx.params.numberOfDays > 14) {
+        throw new Error('days must be less than 14');
+    }
+    const numberOfDays = ctx.params.numberOfDays || 3;
+    const currentShanghaiDateTime = dayjs(toShanghaiTimezone(new Date()));
+    const currentShanghaiDateStr = currentShanghaiDateTime.format(DATE_FORMAT);
+    const dateList = [];
+    for (let i = 0; i < numberOfDays; i++) {
+        const d = currentShanghaiDateTime.subtract(i, 'day');
+        dateList.push(d);
+    }
+
+    const keywordList = ctx.params.keywords
+        .replace('，', ',')
+        .split(',')
+        .map((keyword) => keyword.trim());
+    const keywordStr = keywordList.join(', ');
+
+    const data = await fetchAllData(keywordList, dateList, ctx.cache);
+    const item =
+        data.length > 0
+            ? data.map((i, index) => createItem(i, keywordList, index === 0))
+            : [
+                  {
+                      title: `${keywordStr} | ${currentShanghaiDateStr} 热点追踪 (0)`,
+                      author: 'Trending All In One',
+                      description: `近${numberOfDays}日的热搜都不包含下列关键词：${keywordStr}<br>请耐心等待，或添加更多关键词试试。`,
+                      guid: `trending-all-in-one-${md5(JSON.stringify(data))}-${keywordList.join('-')}`,
+                  },
+              ];
+
+    ctx.state.data = {
+        title: `${keywordStr} | 热点聚合`,
+        description: `${keywordStr} | 今日头条热搜，知乎热门视频，知乎热搜榜，知乎热门话题，微博热搜榜聚合追踪`,
+        language: 'zh-cn',
+        item,
+    };
+};

package/lib/v2/trending/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:keywords/:numberOfDays?': ['Jkker'],
+};

package/lib/v2/trending/radar.js

@@ -0,0 +1,35 @@
+module.exports = {
+    'toutiao.com': {
+        _name: '今日头条',
+        so: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: ['/search'],
+                target: (params, url) => `/trending/${new URL(url).searchParams.get('keyword')}`,
+            },
+        ],
+    },
+    'weibo.com': {
+        _name: '微博',
+        s: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: '/weibo/:keyword',
+                target: (params) => `/trending/${params.keyword}}`,
+            },
+        ],
+    },
+    'zhihu.com': {
+        _name: '知乎',
+        www: [
+            {
+                title: '热搜关键词聚合追踪',
+                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                source: ['/search'],
+                target: (params, url) => `/trending/${new URL(url).searchParams.get('q')}`,
+            },
+        ],
+    },
+};

package/lib/v2/trending/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:keywords/:numberOfDays?', require('./allTrending'));
+};

package/lib/v2/trending/templates/content.art

@@ -0,0 +1,24 @@
+<article>
+  <h2>热榜内容</h2>
+  {{each data channel}}
+  {{if channel.data.length !== 0}}
+  <h3>{{channel.name}}</h3>
+    <ul>
+      {{each channel.data item}}
+      <li>
+        <a href="{{item.url}}" rel="noopener noreferrer" target="_blank">{{item.title}}</a>
+      </li>
+      {{/each}}
+    </ul>
+  {{/if}}
+  {{/each}}
+  <h2>搜索关键词</h2>
+  {{each queries q}}
+  <h3>{{q.key}}</h3>
+  <p>
+    {{each q.links l}}
+      {{@l}}&nbsp&nbsp
+    {{/each}}
+  </p>
+  {{/each}}
+</article>

package/lib/v2/uraaka-joshi/uraaka-joshi-user.js

@@ -11,8 +11,14 @@ module.exports = async (ctx) => {
         async () => {
             const browser = await require('@/utils/puppeteer')();
             const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' ? request.continue() : request.abort();
+            });
 
-            await page.goto(link);
+            await page.goto(link, {
+                waitUntil: 'domcontentloaded',
+            });
             await page.waitForSelector('#pickup03 .grid-cell');
             await page.waitForSelector('#pickup04 .grid-cell');
             await page.waitForSelector('#main-block .grid-cell');

package/lib/v2/uraaka-joshi/uraaka-joshi.js

@@ -8,7 +8,14 @@ module.exports = async (ctx) => {
     const browser = await require('@/utils/puppeteer')();
 
     const page = await browser.newPage();
-    await page.goto(link);
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' ? request.continue() : request.abort();
+    });
+
+    await page.goto(link, {
+        waitUntil: 'domcontentloaded',
+    });
 
     await page.waitForSelector('#pickup03 .grid-cell');
     await page.waitForSelector('#pickup04 .grid-cell');

package/lib/v2/usts/jwch.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const rootURL = 'http://jwch.usts.edu.cn/index';
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type ?? 'jwdt';
+    const url = `${rootURL}/${type}.htm`;
+    const response = await got(url);
+
+    const $ = cheerio.load(response.data);
+    const title = $('div.mainWrap.cleafix > div > div.right.fr > div.local.fl > h3').text();
+    const list = $('div.list > ul > li')
+        .map((_index, item) => ({
+            title: $(item).find('a').text(),
+            link: new URL($(item).find('a').attr('href'), rootURL).href,
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                let author = null;
+                let pubDate = null;
+                content('div.content-title.fl > i')
+                    .text()
+                    .split('  ')
+                    .forEach((item) => {
+                        if (item.includes('作者：')) {
+                            author = item.split('：')[1];
+                        }
+                        if (item.includes('时间：')) {
+                            pubDate = item.split('：')[1];
+                        }
+                    });
+
+                item.description = content('div#vsb_content').html();
+                item.author = author;
+                item.pubDate = timezone(parseDate(pubDate), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `苏州科技大学 教务处 - ${title}`,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/usts/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/juch/:type?': ['Fatpandac'],
+};

package/lib/v2/usts/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'usts.edu.cn': {
+        _name: '苏州科技大学',
+        jwch: [
+            {
+                title: '教务动态',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/jwdt.htm'],
+                target: '/wsts/jwch/jwdt',
+            },
+            {
+                title: '公告在线',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/ggzx.htm'],
+                target: '/wsts/jwch/ggzx',
+            },
+            {
+                title: '选课通知',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/xktz.htm'],
+                target: '/wsts/jwch/xktz',
+            },
+        ],
+    },
+};

package/lib/v2/usts/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/jwch/:type?', require('./jwch'));
+};

package/lib/v2/wallpaperhub/index.js

@@ -0,0 +1,28 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const link = 'https://wallpaperhub.app/api/v1/wallpapers/?limit=20&page=&query=&width=&height=&tags=';
+    const response = await got({
+        method: 'get',
+        url: link,
+    });
+
+    const list = response.data.entities.map((item) => ({
+        title: item.entity.title,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            description: item.entity.description,
+            img: item.entity.variations[0].resolutions[0].url || item.entity.thumbnail,
+        }),
+        pubDate: parseDate(item.entity.created),
+        link: `https://wallpaperhub.app/wallpapers/${item.entity.id}`,
+    }));
+
+    ctx.state.data = {
+        title: 'WallpaperHub',
+        link: 'https://wallpaperhub.app/wallpapers',
+        item: list,
+    };
+};

package/lib/v2/wallpaperhub/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['nczitzk'],
+};

package/lib/v2/wallpaperhub/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'wallpaperhub.app': {
+        _name: 'WallpaperHub',
+        '.': [
+            {
+                title: '壁纸',
+                docs: 'https://docs.rsshub.app/picture.html#wallpaperhub',
+                source: ['/wallpaperhub', '/'],
+                target: '/wallpaperhub',
+            },
+        ],
+    },
+};

package/lib/v2/wallpaperhub/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/', require('./index'));
+};

package/lib/v2/wallpaperhub/templates/description.art

@@ -0,0 +1 @@
+<p>{{ description }}</p><img src="{{ img }}">

package/lib/{routes → v2}/xiaohongshu/board.js

@@ -1,11 +1,10 @@
-const { getContent } = require('@/routes/xiaohongshu/util');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { getContent } = require('./util');
 
 module.exports = async (ctx) => {
     const url = `https://www.xiaohongshu.com/board/${ctx.params.board_id}`;
-    const content = await getContent(url);
-    const regex = /_SSR_STATE__\S*=([\s\S]+)<\/script>/gm;
-    const match = regex.exec(content);
-    const main = JSON.parse(match[1]).Main;
+    const main = await getContent(url, ctx.cache);
 
     const albumInfo = main.albumInfo;
     const title = albumInfo.name;
@@ -15,8 +14,11 @@ module.exports = async (ctx) => {
     const resultItem = list.map((item) => ({
         title: item.title,
         link: `https://www.xiaohongshu.com/discovery/item/${item.id}`,
-        description: `<img src ="${item.cover.url}"><br>${item.title}`,
+        description: `<img src ="${item.cover.url.split('?imageView2')[0]}"><br>${item.title}`,
+        author: item.user.nickname,
+        pubDate: timezone(parseDate(item.time), 8),
     }));
+
     ctx.state.data = {
         title,
         link: url,

package/lib/v2/xiaohongshu/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/board/:board_id': ['lotosbin'],
+    '/user/:user_id/:category': ['lotosbin'],
+};

package/lib/v2/xiaohongshu/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'xiaohongshu.com': {
+        _name: '小红书',
+        '.': [
+            {
+                title: '用户笔记',
+                docs: 'https://docs.rsshub.app/social-media.html#xiao-hong-shu',
+                source: '/user/profile/:user_id',
+                target: '/xiaohongshu/user/:user_id/notes',
+            },
+            {
+                title: '用户专辑',
+                docs: 'https://docs.rsshub.app/social-media.html#xiao-hong-shu',
+                source: '/user/profile/:user_id',
+                target: '/xiaohongshu/user/:user_id/album',
+            },
+            {
+                title: '专辑',
+                docs: 'https://docs.rsshub.app/social-media.html#xiao-hong-shu',
+                source: '/board/:board_id',
+                target: '/xiaohongshu/board/:board_id',
+            },
+        ],
+    },
+};

package/lib/v2/xiaohongshu/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/board/:board_id', require('./board'));
+    router.get('/user/:user_id/:category', require('./user'));
+};

package/lib/{routes → v2}/xiaohongshu/user.js

@@ -1,25 +1,23 @@
-const { getContent } = require('@/routes/xiaohongshu/util');
-
-const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { getContent } = require('./util');
 
 module.exports = async (ctx) => {
     const userId = ctx.params.user_id;
     const category = ctx.params.category;
     const url = `https://www.xiaohongshu.com/user/profile/${userId}`;
 
-    const content = await getContent(url);
-    const $ = cheerio.load(content);
-    const description = $('head title').html();
-    const regex = /_SSR_STATE__\S*=([\s\S]+)<\/script>/gm;
-    const match = regex.exec(content);
-    const main = JSON.parse(match[1]).Main;
+    const main = await getContent(url, ctx.cache);
     const userDetail = main.userDetail;
+    const description = `${userDetail.nickname} • 小红书 / RED`;
     if (category === 'notes') {
         const list = main.notesDetail;
         const resultItem = list.map((item) => ({
             title: item.title,
             link: `https://www.xiaohongshu.com/discovery/item/${item.id}`,
-            description: `<img src ="${item.cover.url}"><br>${item.title}`,
+            description: `<img src ="${item.cover.url.split('?imageView2')[0]}"><br>${item.title}`,
+            author: item.user.nickname,
+            pubDate: timezone(parseDate(item.time), 8),
         }));
 
         const title = `小红书-${userDetail.nickname}-笔记`;
@@ -34,7 +32,7 @@ module.exports = async (ctx) => {
         const resultItem = list.map((item) => ({
             title: item.title,
             link: `https://www.xiaohongshu.com/board/${item.id}`,
-            description: item.images.map((it) => `<img src ="${it}">`).join(`<br />`),
+            description: item.images.map((it) => `<img src ="${it.split('?imageView2')[0]}">`).join(`<br>`),
         }));
 
         const title = `小红书-${userDetail.nickname}-专辑`;

package/lib/v2/xiaohongshu/util.js

@@ -0,0 +1,32 @@
+const config = require('@/config').value;
+
+const getContent = (url, cache) =>
+    cache.tryGet(
+        url,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            try {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+                });
+                await page.goto(url);
+                await page.waitForSelector('.pc-container');
+                let content = await page.content();
+
+                content = content.replace('"RedAppLayout":undefined,', ''); // fix json parse
+                const regex = /_SSR_STATE__\S*=([\s\S]+)<\/script>/gm;
+                const match = regex.exec(content);
+                return JSON.parse(match[1]).Main;
+            } finally {
+                browser.close();
+            }
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+module.exports = {
+    getContent,
+};