rsshub 1.0.0-master.f7add90 → 1.0.0-master.f7bfacc

package/lib/v2/twitter/web-api/tweet.js

@@ -0,0 +1,25 @@
+// const config = require('@/config').value;
+const { getUser, getUserTweet } = require('./twitter-api');
+const utils = require('../utils');
+const { fallback, queryToBoolean } = require('@/utils/readable-social');
+const config = require('@/config').value;
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+    const status = ctx.params.status;
+    const routeParams = new URLSearchParams(ctx.params.original);
+    const original = fallback(undefined, queryToBoolean(routeParams.get('original')), false);
+    const params = { focalTweetId: status };
+    const userInfo = await getUser(ctx.cache, id);
+    const data = await getUserTweet(ctx.cache, id, params);
+    const profileImageUrl = userInfo.profile_image_url || userInfo.profile_image_url_https;
+    const item = original && config.isPackage ? data : utils.ProcessFeed(ctx, { data });
+
+    ctx.state.data = {
+        title: `Twitter @${userInfo.name}`,
+        link: `https://twitter.com/${id}/status/${status}`,
+        image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
+        description: userInfo.description,
+        item,
+    };
+};

package/lib/v2/twitter/web-api/twitter-api.js

@@ -111,11 +111,47 @@ const timelineKeywords = (keywords, params = {}) =>
         pc: 1,
     });
 
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L795-L805
+const tweetDetail = (userId, params) =>
+    paginationTweets(
+        '/graphql/ItejhtHVxU7ksltgMmyaLA/TweetDetail',
+        userId,
+        {
+            ...params,
+            with_rux_injections: false,
+            withCommunity: true,
+            withQuickPromoteEligibilityTweetFields: false,
+            withBirdwatchNotes: false,
+        },
+        ['threaded_conversation_with_injections']
+    );
+
 function gatherLegacyFromData(entries, filter = 'tweet-') {
     const tweets = [];
+    const filte_entries = [];
     entries.forEach((entry) => {
-        if (entry.entryId && entry.entryId.includes(filter)) {
-            const tweet = entry.content.itemContent.tweet_results?.result;
+        const entryId = entry.entryId;
+        if (entryId) {
+            if (filter === 'none') {
+                if (entryId.startsWith('tweet-')) {
+                    filte_entries.push(entry);
+                } else if (entryId.startsWith('homeConversation-') || entryId.startsWith('conversationthread-')) {
+                    filte_entries.push(...entry.content.items);
+                }
+            } else {
+                if (entryId.startsWith(filter)) {
+                    filte_entries.push(entry);
+                }
+            }
+        }
+    });
+    filte_entries.forEach((entry) => {
+        if (entry.entryId) {
+            const content = entry.content || entry.item;
+            let tweet = content?.itemContent?.tweet_results?.result;
+            if (tweet && tweet.tweet) {
+                tweet = tweet.tweet;
+            }
             if (tweet) {
                 const retweet = tweet.legacy?.retweeted_status_result?.result;
                 for (const t of [tweet, retweet]) {
@@ -175,6 +211,7 @@ const getUserTweetsByID = async (id, params = {}) => gatherLegacyFromData(await
 const getUserTweetsAndRepliesByID = async (id, params = {}) => gatherLegacyFromData(await timelineTweetsAndReplies(id, params));
 const getUserMediaByID = async (id, params = {}) => gatherLegacyFromData(await timelineMedia(id, params));
 const getUserLikesByID = async (id, params = {}) => gatherLegacyFromData(await timelineLikes(id, params));
+const getUserTweetByStatus = async (id, params = {}) => gatherLegacyFromData(await tweetDetail(id, params), 'none');
 
 const excludeRetweet = function (tweets) {
     const excluded = [];
@@ -206,6 +243,7 @@ const getUserTweets = (cache, screenName, params = {}) => cacheTryGet(cache, scr
 const getUserTweetsAndReplies = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserTweetsAndRepliesByID);
 const getUserMedia = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserMediaByID);
 const getUserLikes = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserLikesByID);
+const getUserTweet = (cache, screenName, params) => cacheTryGet(cache, screenName, params, getUserTweetByStatus);
 
 const getSearch = async (keywords, params = {}) => gatherLegacyFromLegacyApiData(await timelineKeywords(keywords, params), 'sq-I-t-');
 
@@ -217,4 +255,5 @@ module.exports = {
     getUserLikes,
     excludeRetweet,
     getSearch,
+    getUserTweet,
 };

package/lib/v2/ustb/maintainer.js

@@ -1,25 +1,5 @@
 module.exports = {
-    '/yjsy/news/adma': ['DA1Y1'],
-    '/yjsy/news/all': ['DA1Y1'],
-    '/yjsy/news/cicst': ['DA1Y1'],
-    '/yjsy/news/cres': ['DA1Y1'],
-    '/yjsy/news/huasheng': ['DA1Y1'],
-    '/yjsy/news/iet': ['DA1Y1'],
-    '/yjsy/news/ihmm': ['DA1Y1'],
-    '/yjsy/news/marx': ['DA1Y1'],
-    '/yjsy/news/me': ['DA1Y1'],
-    '/yjsy/news/metall': ['DA1Y1'],
-    '/yjsy/news/mse': ['DA1Y1'],
-    '/yjsy/news/ncms': ['DA1Y1'],
-    '/yjsy/news/saee': ['DA1Y1'],
-    '/yjsy/news/scce': ['DA1Y1'],
-    '/yjsy/news/sd': ['DA1Y1'],
-    '/yjsy/news/seee': ['DA1Y1'],
-    '/yjsy/news/sem': ['DA1Y1'],
-    '/yjsy/news/sfs': ['DA1Y1'],
-    '/yjsy/news/shuli': ['DA1Y1'],
-    '/yjsy/news/skl': ['DA1Y1'],
-    '/yjsy/news/slam': ['DA1Y1'],
-    '/yjsy/news/wenfa': ['DA1Y1'],
+    '/tj/news/:type?': ['henbf'],
+    '/yjsy/news/:type': ['DA1Y1'],
     '/yzxc/tzgg': ['yanbot-team'],
 };

package/lib/v2/ustb/radar.js

@@ -9,6 +9,14 @@ module.exports = {
                 target: '/ustb/yjsy/news/:type',
             },
         ],
+        tj: [
+            {
+                title: '天津学院',
+                docs: 'https://docs.rsshub.app/university.html#bei-jing-ke-ji-da-xue',
+                source: ['/*'],
+                target: '/ustb/tj/news/all',
+            },
+        ],
         yzxc: [
             {
                 title: '研究生招生信息网',

package/lib/v2/ustb/router.js

@@ -1,4 +1,5 @@
 module.exports = function (router) {
+    router.get('/tj/news/:type?', require('./tj/news'));
     router.get('/yjsy/news/:type', require('./yjsy/news'));
     router.get('/yzxc/tzgg', require('./yzxc/tzgg'));
 };

package/lib/{routes/universities → v2}/ustb/tj/news.js

@@ -1,7 +1,7 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const dayjs = require('dayjs');
-const _ = require('lodash');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 const baseUrl = 'http://tj.ustb.edu.cn';
 const maps = {
@@ -22,18 +22,17 @@ const maps = {
 function getNews(data) {
     const $ = cheerio.load(data);
     return $('div[class="classnews"] ul li a')
-        .slice(0, 5)
-        .map((_, elem) => ({
+        .toArray()
+        .map((elem) => ({
             link: baseUrl + elem.attribs.href,
             title: elem.children[0].data,
-            pubDate: dayjs(elem.attribs.href.split('/')[3].split('.')[0]).toString(),
-        }))
-        .get();
+            pubDate: timezone(parseDate(elem.attribs.href.split('/')[3].split('.')[0].substring(0, 14), 'YYYYMMDDHHmmss'), 8),
+        }));
 }
 
 module.exports = async (ctx) => {
     let type = ctx.params.type || 'all';
-    if (!_.includes(_.keys(maps), type)) {
+    if (!Object.keys(maps).includes(type)) {
         type = 'all';
     }
 
@@ -46,14 +45,14 @@ module.exports = async (ctx) => {
     if (type === 'all') {
         const all = await Promise.all(
             Object.values(maps).map(async (link) => {
-                const response = await got.get(baseUrl + link);
+                const response = await got(baseUrl + link);
                 const news = getNews(response.data);
-                return Promise.resolve(news);
+                return news;
             })
         );
-        responseData.item = _.orderBy(_.flatMapDepth(all), (res) => new Date(res.pubDate), ['desc']);
+        responseData.item = all.flatMap((res) => res);
     } else {
-        const response = await got.get(baseUrl + maps[type]);
+        const response = await got(baseUrl + maps[type]);
         const news = getNews(response.data);
         responseData.item = news;
     }

package/lib/v2/utgd/category.js

@@ -0,0 +1,68 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'method';
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 20;
+
+    const rootUrl = 'https://utgd.net';
+    const apiUrl = `${rootUrl}/api/v2/pages/`;
+    const currentUrl = `${rootUrl}/category/s/${category}`;
+    const slugUrl = `${rootUrl}/api/v2/category/slug/${category}/`;
+
+    let response = await got({
+        method: 'get',
+        url: slugUrl,
+    });
+
+    const data = response.data;
+
+    response = await got({
+        method: 'get',
+        url: apiUrl,
+        searchParams: {
+            type: 'article.Article',
+            fields: `article_category(category_name),article_tag(tag_name),title,article_image,article_author,article_content,article_published_time`,
+            article_category: data.id,
+            order: '-article_published_time',
+            limit,
+        },
+    });
+
+    const items = await Promise.all(
+        response.data.items.map((item) =>
+            ctx.cache.tryGet(`untag-${item.id}`, async () => {
+                const authorResponse = await got({
+                    method: 'get',
+                    url: `${rootUrl}/api/v2/user/profile/${item.article_author.id}/`,
+                });
+
+                return {
+                    title: item.title,
+                    link: `${rootUrl}/article/${item.id}`,
+                    description: art(path.join(__dirname, 'templates/description.art'), {
+                        image: item.article_image,
+                        description: md.render(item.article_content),
+                    }),
+                    author: authorResponse.data.display_name,
+                    pubDate: timezone(parseDate(item.article_published_time), +8),
+                    category: [...item.article_category.map((c) => c.category_name), ...item.article_tag.map((t) => t.tag_name)],
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `UNTAG - ${data.category_name}`,
+        link: currentUrl,
+        item: items,
+        image: data.category_image,
+        description: data.category_description,
+    };
+};

package/lib/v2/utgd/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/timeline': ['nczitzk'],
+    '/topic/:topic?': ['nczitzk'],
+    '/:category?': ['nczitzk'],
+};

package/lib/v2/utgd/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'utgd.net': {
+        _name: 'UNTAG',
+        '.': [
+            {
+                title: '时间线',
+                docs: 'https://docs.rsshub.app/new-media.html#untag-shi-jian-xian',
+                source: ['/'],
+                target: '/utgd/timeline',
+            },
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/new-media.html#untag-fen-lei',
+                source: ['/category/s/:category', '/'],
+                target: '/utgd/:category',
+            },
+            {
+                title: '专题',
+                docs: 'https://docs.rsshub.app/new-media.html#untag-zhuan-ti',
+                source: ['/topic', '/'],
+                target: '/utgd/topic/:topic',
+            },
+        ],
+    },
+};

package/lib/v2/utgd/router.js

@@ -0,0 +1,5 @@
+module.exports = function (router) {
+    router.get('/timeline', require('./timeline'));
+    router.get('/topic/:topic?', require('./topic'));
+    router.get('/:category?', require('./category'));
+};

package/lib/v2/utgd/templates/description.art

@@ -0,0 +1,7 @@
+{{ if image }}
+<img src="{{ image }}">
+<br>
+{{ /if }}
+{{ if description }}
+{{@ description }}
+{{ /if }}

package/lib/v2/utgd/timeline.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 20;
+
+    const rootUrl = 'https://utgd.net';
+    const apiUrl = `${rootUrl}/api/v2/timeline/?page=1&page_size=${limit}`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    const items = await Promise.all(
+        response.data.results.slice(0, limit).map((item) =>
+            ctx.cache.tryGet(`untag-${item.id}`, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: `${rootUrl}/api/v2/pages/${item.id}/`,
+                    searchParams: {
+                        fields: 'article_content,article_category(category_name),article_tag(tag_name)',
+                    },
+                });
+
+                const data = detailResponse.data;
+
+                return {
+                    title: item.title,
+                    link: `${rootUrl}/article/${item.id}`,
+                    description: art(path.join(__dirname, 'templates/description.art'), {
+                        image: item.article_image,
+                        description: md.render(data.article_content),
+                    }),
+                    author: item.article_author_displayname,
+                    pubDate: timezone(parseDate(item.article_published_time), +8),
+                    category: [...data.article_category.map((c) => c.category_name), ...data.article_tag.map((t) => t.tag_name)],
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'UNTAG',
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/utgd/topic.js

@@ -0,0 +1,70 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const topic = ctx.params.topic ?? '在线阅读专栏';
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 20;
+
+    const rootUrl = 'https://utgd.net';
+    const currentUrl = `${rootUrl}/topic`;
+    const topicUrl = `${rootUrl}/api/v2/topic/`;
+
+    let response = await got({
+        method: 'get',
+        url: topicUrl,
+    });
+
+    const topicItems = response.data.filter((i) => i.title === topic);
+
+    if (!topicItems) {
+        throw Error(`No topic named ${topic}`);
+    }
+
+    const topicItem = topicItems[0];
+
+    const apiUrl = `${rootUrl}/api/v2/topic/${topicItem.id}/article/`;
+
+    response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    const items = await Promise.all(
+        response.data.slice(0, limit).map((item) =>
+            ctx.cache.tryGet(`untag-${item.id}`, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: `${rootUrl}/api/v2/pages/${item.id}/`,
+                    searchParams: {
+                        fields: 'article_content',
+                    },
+                });
+
+                return {
+                    title: item.title,
+                    link: `${rootUrl}/article/${item.id}`,
+                    description: art(path.join(__dirname, 'templates/description.art'), {
+                        image: item.article_image,
+                        description: md.render(detailResponse.data.article_content),
+                    }),
+                    author: item.article_author_displayname,
+                    pubDate: timezone(parseDate(item.article_published_time), +8),
+                    category: [...item.article_category.map((c) => c.name), ...item.article_tag.map((t) => t.name)],
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `UNTAG - ${topicItem.title}`,
+        link: currentUrl,
+        item: items,
+        description: topicItem.summary,
+    };
+};

package/lib/v2/v1tx/index.js

@@ -4,31 +4,50 @@ const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const baseUrl = 'https://www.v1tx.com';
-    const response = await got(`${baseUrl}/wp-json/wp/v2/posts`, {
-        searchParams: {
-            per_page: ctx.query.limit ?? 100,
-        },
-    });
-
-    const items = response.data.map((item) => {
-        const $ = cheerio.load(item.content.rendered, null, false);
-        $('figure > picture > source').remove();
-        $('img').each((_, img) => {
-            img.attribs.src = img.attribs.src.replace(/-1024x\d+\.jpg/, '.webp').replace('.jpg', '.webp');
-            delete img.attribs.srcset;
+    const { data: response } = await got(baseUrl);
+
+    const $ = cheerio.load(response);
+    const list = $('h2.entry-title a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
         });
-        return {
-            title: item.title.rendered,
-            link: item.link,
-            guid: item.guid.rendered,
-            description: $.html(),
-            pubDate: parseDate(item.date_gmt),
-        };
-    });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+
+                const data = JSON.parse($('script[type="application/ld+json"]').text());
+
+                $('.entry-content figure > picture > source, noscript').remove();
+                $('.entry-content img').each((_, img) => {
+                    if (img.attribs.src.startsWith('data:')) {
+                        img.attribs.src = img.attribs['data-lazy-src'];
+                        delete img.attribs['data-lazy-src'];
+                        delete img.attribs['data-lazy-srcset'];
+                    }
+                    img.attribs.src = img.attribs.src.replace(/-1024x\d+\.jpg/, '.webp').replace('.jpg', '.webp');
+                    delete img.attribs.srcset;
+                });
+
+                item.pubDate = parseDate(data.datePublished);
+                item.updated = parseDate(data.dateModified);
+                item.description = $('.entry-content').html();
+
+                return item;
+            })
+        )
+    );
 
     ctx.state.data = {
-        title: 'v1tx - 发现实用工具与软件',
-        description: 'v1tx.com 专注于发现新应用，推荐各种效率工具、软件、APP，包括Windows、Mac、Android、iOS、网页等多平台应用，让每个人找到适合的软件并掌握使用技巧',
+        title: $('head title').text(),
+        description: $('head meta[name="description"]').attr('content'),
         link: baseUrl,
         image: `${baseUrl}/wp-content/uploads/2018/10/cropped-Favicon.webp`,
         item: items,

package/lib/v2/wechat/feeddd.js

@@ -5,10 +5,19 @@ const { finishArticleItem } = require('@/utils/wechat-mp');
 module.exports = async (ctx) => {
     const id = ctx.params.id;
 
-    const baseUrl = 'https://api.feeddd.org';
-    const apiUrl = `${baseUrl}/feeds/${id}/json`;
+    const baseUrl = 'https://feed.hamibot.com';
+    const apiUrl = `${baseUrl}/api/feeds/${id}/json`;
 
-    const response = await got(apiUrl);
+    let response;
+
+    try {
+        response = await got(apiUrl);
+    } catch (error) {
+        if (error.name === 'HTTPError' && error.response.statusCode === 404) {
+            throw Error('该公众号不存在，有关如何获取公众号 id，详见 https://docs.rsshub.app/new-media.html#wei-xin-gong-zhong-hao-feeddd-lai-yuan');
+        }
+        throw error;
+    }
 
     let items = response.data.items.map((item) => ({
         title: item.title,

package/lib/v2/wechat/radar.js

@@ -32,9 +32,9 @@ module.exports = {
             },
         ],
     },
-    'feeddd.org': {
+    'hamibot.com': {
         _name: '微信',
-        '.': [
+        feed: [
             {
                 title: '公众号（feeddd 来源）',
                 docs: 'https://docs.rsshub.app/new-media.html#wei-xin',

package/lib/v2/weibo/group.js

@@ -0,0 +1,93 @@
+const querystring = require('querystring');
+const got = require('@/utils/got');
+const config = require('@/config').value;
+const weiboUtils = require('./utils');
+const { fallback, queryToBoolean } = require('@/utils/readable-social');
+
+module.exports = async (ctx) => {
+    if (!config.weibo_cookies) {
+        throw 'Weibo Group Timeline is not available due to the absense of [Weibo Cookies]. Check <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config tutorial</a>';
+    }
+
+    const gid = ctx.params.gid;
+    const groupName = ctx.params.gname || '微博分组';
+    let displayVideo = '1';
+    let displayArticle = '0';
+    let displayComments = '0';
+    if (ctx.params.routeParams) {
+        if (ctx.params.routeParams === '1' || ctx.params.routeParams === '0') {
+            displayVideo = ctx.params.routeParams;
+        } else {
+            const routeParams = querystring.parse(ctx.params.routeParams);
+            displayVideo = fallback(undefined, queryToBoolean(routeParams.displayVideo), true) ? '1' : '0';
+            displayArticle = fallback(undefined, queryToBoolean(routeParams.displayArticle), false) ? '1' : '0';
+            displayComments = fallback(undefined, queryToBoolean(routeParams.displayComments), false) ? '1' : '0';
+        }
+    }
+
+    const responseData = await ctx.cache.tryGet(
+        `weibo:group:index:${gid}`,
+        async () => {
+            const _r = await got({
+                method: 'get',
+                url: `https://m.weibo.cn/feed/group?gid=${gid}`,
+                headers: {
+                    Referer: `https://m.weibo.cn/`,
+                    'MWeibo-Pwa': 1,
+                    'X-Requested-With': 'XMLHttpRequest',
+                    Cookie: config.weibo.cookies,
+                },
+            });
+            return _r.data.data;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const resultItems = await Promise.all(
+        responseData.statuses.map(async (item) => {
+            const retweet = item.retweeted_status;
+            if (retweet && retweet.isLongText) {
+                const retweetData = await ctx.cache.tryGet(`weibo:retweeted:${retweet.user.id}:${retweet.bid}`, () => weiboUtils.getShowData(retweet.user.id, retweet.bid));
+                if (retweetData !== undefined && retweetData.text) {
+                    item.retweeted_status.text = retweetData.text;
+                }
+            }
+
+            const formatExtended = weiboUtils.formatExtended(ctx, item);
+            let description = formatExtended.description;
+
+            if (displayVideo === '1') {
+                if (item.retweeted_status) {
+                    description = weiboUtils.formatVideo(description, item.retweeted_status);
+                } else {
+                    description = weiboUtils.formatVideo(description, item);
+                }
+            }
+
+            if (displayComments === '1') {
+                description = await weiboUtils.formatComments(ctx, description, item);
+            }
+
+            if (displayArticle === '1') {
+                if (item.retweeted_status) {
+                    description = await weiboUtils.formatArticle(ctx, description, item.retweeted_status);
+                } else {
+                    description = await weiboUtils.formatArticle(ctx, description, item);
+                }
+            }
+
+            return {
+                ...formatExtended,
+                description,
+            };
+        })
+    );
+
+    ctx.state.data = weiboUtils.sinaimgTvax({
+        title: groupName,
+        link: `https://weibo.com/mygroups?gid=${gid}`,
+        description: '微博自定义分组',
+        item: resultItems,
+    });
+};