rsshub 1.0.0-master.d2b1df9 → 1.0.0-master.d2bf7b3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/lib/config.js +3 -0
  2. package/lib/middleware/cache/index.js +3 -0
  3. package/lib/router.js +5 -5
  4. package/lib/v2/bangumi/maintainer.js +1 -0
  5. package/lib/v2/bangumi/online.js +26 -0
  6. package/lib/v2/bangumi/radar.js +13 -2
  7. package/lib/v2/bangumi/router.js +1 -0
  8. package/lib/v2/bangumi/templates/image.art +1 -0
  9. package/lib/v2/bilibili/collection.js +34 -0
  10. package/lib/v2/bilibili/maintainer.js +1 -0
  11. package/lib/v2/bilibili/radar.js +9 -0
  12. package/lib/v2/bilibili/router.js +1 -0
  13. package/lib/v2/china/maintainer.js +4 -0
  14. package/lib/v2/china/news/highlights/news.js +40 -0
  15. package/lib/v2/china/news/military/news.js +31 -0
  16. package/lib/v2/china/radar.js +21 -0
  17. package/lib/v2/china/router.js +4 -0
  18. package/lib/v2/csdn/blog.js +39 -0
  19. package/lib/v2/csdn/maintainer.js +3 -0
  20. package/lib/v2/csdn/radar.js +13 -0
  21. package/lib/v2/csdn/router.js +3 -0
  22. package/lib/v2/douyu/post.js +9 -2
  23. package/lib/v2/github/comments.js +3 -4
  24. package/lib/v2/hupu/bbs.js +23 -23
  25. package/lib/v2/huxiu/maintainer.js +1 -0
  26. package/lib/v2/huxiu/moment.js +40 -0
  27. package/lib/v2/huxiu/radar.js +6 -0
  28. package/lib/v2/huxiu/router.js +1 -0
  29. package/lib/v2/huxiu/templates/moment.art +16 -0
  30. package/lib/v2/iwara/maintainer.js +1 -0
  31. package/lib/v2/iwara/radar.js +6 -0
  32. package/lib/v2/iwara/router.js +1 -0
  33. package/lib/v2/iwara/subscriptions.js +49 -0
  34. package/lib/v2/iwara/templates/subscriptions.art +1 -0
  35. package/lib/v2/leetcode/dailyquestion-solution-cn.js +183 -0
  36. package/lib/v2/leetcode/dailyquestion-solution-en.js +213 -0
  37. package/lib/v2/leetcode/maintainer.js +1 -0
  38. package/lib/v2/leetcode/radar.js +12 -0
  39. package/lib/v2/leetcode/router.js +2 -0
  40. package/lib/v2/lovelive-anime/maintainer.js +3 -0
  41. package/lib/v2/lovelive-anime/news.js +56 -0
  42. package/lib/v2/lovelive-anime/radar.js +13 -0
  43. package/lib/v2/lovelive-anime/router.js +3 -0
  44. package/lib/v2/lovelive-anime/templates/description.art +1 -0
  45. package/lib/v2/macfilos/blog.js +61 -0
  46. package/lib/v2/macfilos/maintainer.js +3 -0
  47. package/lib/v2/macfilos/radar.js +13 -0
  48. package/lib/v2/macfilos/router.js +3 -0
  49. package/lib/v2/meteor/boards.js +11 -0
  50. package/lib/v2/meteor/index.js +45 -0
  51. package/lib/v2/meteor/maintainer.js +4 -0
  52. package/lib/v2/meteor/radar.js +19 -0
  53. package/lib/v2/meteor/router.js +4 -0
  54. package/lib/v2/meteor/templates/desc.art +7 -0
  55. package/lib/v2/meteor/utils.js +90 -0
  56. package/lib/v2/nikkei-cn/index.js +4 -1
  57. package/lib/v2/nua/gra.js +56 -0
  58. package/lib/v2/nua/maintainer.js +3 -0
  59. package/lib/v2/nua/radar.js +13 -0
  60. package/lib/v2/nua/router.js +3 -0
  61. package/lib/v2/oschina/maintainer.js +6 -0
  62. package/lib/{routes → v2}/oschina/news.js +35 -39
  63. package/lib/v2/oschina/radar.js +33 -0
  64. package/lib/v2/oschina/router.js +6 -0
  65. package/lib/v2/oschina/topic.js +53 -0
  66. package/lib/v2/oschina/user.js +50 -0
  67. package/lib/v2/plurk/anonymous.js +23 -0
  68. package/lib/v2/plurk/hotlinks.js +20 -0
  69. package/lib/v2/plurk/maintainer.js +9 -0
  70. package/lib/v2/plurk/news.js +23 -0
  71. package/lib/v2/plurk/radar.js +56 -0
  72. package/lib/v2/plurk/router.js +9 -0
  73. package/lib/v2/plurk/search.js +27 -0
  74. package/lib/v2/plurk/top.js +29 -0
  75. package/lib/v2/plurk/topic.js +33 -0
  76. package/lib/v2/plurk/user.js +31 -0
  77. package/lib/v2/plurk/utils.js +56 -0
  78. package/lib/v2/segmentfault/blogs.js +38 -0
  79. package/lib/v2/segmentfault/maintainer.js +1 -0
  80. package/lib/v2/segmentfault/radar.js +8 -2
  81. package/lib/v2/segmentfault/router.js +1 -0
  82. package/lib/v2/sensortower/blog.js +69 -0
  83. package/lib/v2/sensortower/maintainer.js +3 -0
  84. package/lib/v2/sensortower/radar.js +13 -0
  85. package/lib/v2/sensortower/router.js +3 -0
  86. package/lib/v2/sensortower/templates/description.art +9 -0
  87. package/lib/v2/taptap/templates/imagePost.art +3 -0
  88. package/lib/v2/taptap/templates/videoPost.art +2 -1
  89. package/lib/v2/taptap/topic.js +33 -27
  90. package/lib/v2/taptap/utils.js +13 -8
  91. package/lib/v2/test/index.js +15 -0
  92. package/lib/v2/trending/allTrending.js +46 -19
  93. package/lib/v2/trending/templates/content.art +1 -0
  94. package/lib/v2/twitter/maintainer.js +1 -1
  95. package/lib/v2/twitter/router.js +1 -1
  96. package/lib/v2/twitter/web-api/twitter-api.js +1 -0
  97. package/lib/v2/warthunder/maintainer.js +3 -0
  98. package/lib/v2/warthunder/news.js +49 -0
  99. package/lib/v2/warthunder/radar.js +13 -0
  100. package/lib/v2/warthunder/router.js +3 -0
  101. package/lib/v2/warthunder/templates/description.art +1 -0
  102. package/lib/v2/xyzrank/index.js +111 -0
  103. package/lib/v2/xyzrank/maintainer.js +6 -0
  104. package/lib/v2/xyzrank/radar.js +31 -0
  105. package/lib/v2/xyzrank/router.js +3 -0
  106. package/lib/v2/xyzrank/templates/description.art +12 -0
  107. package/lib/v2/zaobao/util.js +6 -6
  108. package/lib/v2/zuvio/boards.js +14 -0
  109. package/lib/v2/zuvio/radar.js +13 -0
  110. package/lib/v2/zuvio/router.js +4 -0
  111. package/lib/v2/zuvio/student5.js +58 -0
  112. package/lib/v2/zuvio/templates/img.art +2 -0
  113. package/lib/v2/zuvio/templates/link.art +7 -0
  114. package/lib/v2/zuvio/templates/ref_article.art +8 -0
  115. package/lib/v2/zuvio/templates/rss.art +1 -0
  116. package/lib/v2/zuvio/templates/youtube.art +2 -0
  117. package/lib/v2/zuvio/utils.js +61 -0
  118. package/package.json +14 -15
  119. package/lib/routes/oschina/topic.js +0 -54
  120. package/lib/routes/oschina/u.js +0 -57
  121. package/lib/routes/oschina/user.js +0 -57
  122. package/lib/v2/hupu/templates/description.art +0 -6
@@ -0,0 +1,29 @@
1
+ const got = require('@/utils/got');
2
+ const { baseUrl, getPlurk } = require('./utils');
3
+
4
+ const categoryList = ['topReplurks', 'topFavorites', 'topResponded'];
5
+
6
+ module.exports = async (ctx) => {
7
+ const { category = 'topReplurks', lang = 'en' } = ctx.params;
8
+ if (!categoryList.includes(category)) {
9
+ throw Error(`Invalid category: ${category}`);
10
+ }
11
+
12
+ const { data: apiResponse } = await got(`${baseUrl}/Stats/${category}`, {
13
+ searchParams: {
14
+ period: 'day',
15
+ lang,
16
+ limit: ctx.query.limit ? Number(ctx.query.limit) : 90,
17
+ },
18
+ });
19
+
20
+ const items = await Promise.all(apiResponse.stats.map((item) => item[1]).map((item) => getPlurk(`plurk:${item.plurk_id}`, item, item.owner.display_name, ctx.cache.tryGet)));
21
+
22
+ ctx.state.data = {
23
+ title: 'Top Plurk - Plurk',
24
+ image: 'https://s.plurk.com/2c1574c02566f3b06e91.png',
25
+ link: `${baseUrl}/top#${category}`,
26
+ item: items,
27
+ language: lang,
28
+ };
29
+ };
@@ -0,0 +1,33 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const { baseUrl, fetchFriends, getPlurk } = require('./utils');
4
+
5
+ module.exports = async (ctx) => {
6
+ const { topic } = ctx.params;
7
+ const { data: pageResponse } = await got(`${baseUrl}/topic/${topic}`);
8
+ const { data: apiResponse } = await got(`${baseUrl}/topic/getPlurks`, {
9
+ searchParams: {
10
+ topic,
11
+ offset: 0,
12
+ limit: ctx.query.limit ? Number(ctx.query.limit) : 30,
13
+ },
14
+ });
15
+
16
+ const $ = cheerio.load(pageResponse);
17
+
18
+ delete apiResponse.pids;
19
+ delete apiResponse.count;
20
+
21
+ const userIds = Object.values(apiResponse).map((item) => item.user_id);
22
+ const names = await fetchFriends(userIds);
23
+
24
+ const items = await Promise.all(Object.values(apiResponse).map((item) => getPlurk(`plurk:${item.plurk_id}`, item, names[item.user_id].display_name, ctx.cache.tryGet)));
25
+
26
+ ctx.state.data = {
27
+ title: $('head title').text(),
28
+ description: $('meta[property=og:description]').attr('content'),
29
+ image: $('meta[property=og:image]').attr('content') || $('meta[name=msapplication-TileImage]').attr('content'),
30
+ link: `${baseUrl}/topic/${topic}`,
31
+ item: items,
32
+ };
33
+ };
@@ -0,0 +1,31 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const { baseUrl, fetchFriends, getPlurk } = require('./utils');
4
+
5
+ module.exports = async (ctx) => {
6
+ const { user } = ctx.params;
7
+ const { data: pageResponse } = await got(`${baseUrl}/${user}`);
8
+
9
+ const $ = cheerio.load(pageResponse);
10
+
11
+ const publicPlurks = JSON.parse(
12
+ $('body script[type]')
13
+ .text()
14
+ .match(/PUBLIC_PLURKS = (.*);\nPINNED_PLURK/)[1]
15
+ .replace(/new Date\((.*?)\)/g, '$1')
16
+ .replace(/null/g, '""')
17
+ );
18
+
19
+ const userIds = publicPlurks.map((item) => item.user_id);
20
+ const names = await fetchFriends(userIds);
21
+
22
+ const items = await Promise.all(publicPlurks.map((item) => getPlurk(`plurk:${item.plurk_id}`, item, names[item.user_id].display_name, ctx.cache.tryGet)));
23
+
24
+ ctx.state.data = {
25
+ title: $('head title').text(),
26
+ description: $('meta[property=og:description]').attr('content'),
27
+ image: $('meta[property=og:image]').attr('content') || $('meta[name=msapplication-TileImage]').attr('content'),
28
+ link: `${baseUrl}/${user}`,
29
+ item: items,
30
+ };
31
+ };
@@ -0,0 +1,56 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const { parseDate } = require('@/utils/parse-date');
4
+
5
+ const baseUrl = 'https://www.plurk.com';
6
+
7
+ /**
8
+ *
9
+ * @param {objects} userIds Array of user ids
10
+ * @returns
11
+ */
12
+ const fetchFriends = async (userIds) => {
13
+ const { data } = await got.post(`${baseUrl}/Users/fetchFriends`, {
14
+ form: {
15
+ ids: JSON.stringify(userIds),
16
+ r: 'gp',
17
+ },
18
+ });
19
+ return data;
20
+ };
21
+
22
+ /**
23
+ *
24
+ * @param {string} plurkGuid plurk guid, should start with 'plurk:'
25
+ * @param {object} item plurk object
26
+ * @param {string} author author name, can be `null`
27
+ * @param {function} tryGet cache get function
28
+ * @returns {object} item object
29
+ */
30
+ const getPlurk = (plurkGuid, item, author, tryGet) =>
31
+ tryGet(plurkGuid, () => {
32
+ const $ = cheerio.load(item.content || item.rendered, null, false);
33
+ $('img').each((_, e) => {
34
+ e = $(e);
35
+ e.removeAttr('height').removeAttr('width');
36
+ if (e.attr('alt') && e.attr('alt').startsWith('http')) {
37
+ e.attr('src', e.attr('alt'));
38
+ e.removeAttr('alt');
39
+ }
40
+ });
41
+
42
+ return {
43
+ title: item.content_raw ? item.content_raw : $.text() || plurkGuid,
44
+ description: $.html(),
45
+ guid: plurkGuid,
46
+ link: item.rendered ? item.link_url : null,
47
+ author,
48
+ pubDate: parseDate(item.posted),
49
+ };
50
+ });
51
+
52
+ module.exports = {
53
+ baseUrl,
54
+ fetchFriends,
55
+ getPlurk,
56
+ };
@@ -0,0 +1,38 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const { parseDate } = require('@/utils/parse-date');
4
+
5
+ const host = 'https://segmentfault.com';
6
+
7
+ module.exports = async (ctx) => {
8
+ const tag = ctx.params.tag;
9
+ const apiURL = `https://segmentfault.com/gateway/tag/${tag}/articles?loadMoreType=pagination&initData=true&page=1&sort=newest&pageSize=30`;
10
+ const response = await got(apiURL);
11
+ const data = response.data.rows;
12
+
13
+ const list = data.map((item) => ({
14
+ title: item.title,
15
+ link: new URL(item.url, host).href,
16
+ author: item.user.name,
17
+ }));
18
+
19
+ const items = await Promise.all(
20
+ list.map((item) =>
21
+ ctx.cache.tryGet(item.link, async () => {
22
+ const response = await got(item.link);
23
+ const content = cheerio.load(response.data);
24
+
25
+ item.description = content('article').html();
26
+ item.pubDate = parseDate(content('time').attr('datetime'));
27
+
28
+ return item;
29
+ })
30
+ )
31
+ );
32
+
33
+ ctx.state.data = {
34
+ title: `segmentfault-Blogs-${tag}`,
35
+ link: `${host}/t/${tag}/blogs`,
36
+ item: items,
37
+ };
38
+ };
@@ -1,4 +1,5 @@
1
1
  module.exports = {
2
+ '/blogs/:tag': ['shiluanzzz'],
2
3
  '/channel/:name': ['LogicJake', 'Fatpandac'],
3
4
  '/user/:name': ['leyuuu', 'Fatpandac'],
4
5
  };
@@ -6,13 +6,19 @@ module.exports = {
6
6
  title: '频道',
7
7
  docs: 'https://docs.rsshub.app/programming.html#segmentfault',
8
8
  source: ['/channel/:name'],
9
- target: '/channel/:name',
9
+ target: '/segmentfault/channel/:name',
10
10
  },
11
11
  {
12
12
  title: '用户',
13
13
  docs: 'https://docs.rsshub.app/programming.html#segmentfault',
14
14
  source: ['/u/:name'],
15
- target: '/user/:name',
15
+ target: '/segmentfault/user/:name',
16
+ },
17
+ {
18
+ title: '博客',
19
+ docs: 'https://docs.rsshub.app/programming.html#segmentfault',
20
+ source: ['/t/:tag/blogs'],
21
+ target: '/segmentfault/blogs/:tag',
16
22
  },
17
23
  ],
18
24
  },
@@ -1,4 +1,5 @@
1
1
  module.exports = function (router) {
2
+ router.get('/blogs/:tag', require('./blogs'));
2
3
  router.get('/channel/:name', require('./channel'));
3
4
  router.get('/user/:name', require('./user'));
4
5
  };
@@ -0,0 +1,69 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const { parseDate } = require('@/utils/parse-date');
4
+ const { art } = require('@/utils/render');
5
+ const path = require('path');
6
+
7
+ module.exports = async (ctx) => {
8
+ const language = ctx.params.language ?? '';
9
+
10
+ const rootUrl = 'https://sensortower.com';
11
+ const currentUrl = `${rootUrl}${language ? `/${language}` : ''}/blog`;
12
+
13
+ const response = await got({
14
+ method: 'get',
15
+ url: currentUrl,
16
+ });
17
+
18
+ const data = response.data.match(/"uri":"(\/blog\/.*?)"/g);
19
+
20
+ let items = data.map((item) => ({
21
+ link: `${rootUrl}${language ? `/${language}` : ''}${item.match(/"(\/blog\/.*?)"/)[1]}`,
22
+ }));
23
+
24
+ items = await Promise.all(
25
+ items.map((item) =>
26
+ ctx.cache.tryGet(item.link, async () => {
27
+ const detailResponse = await got({
28
+ method: 'get',
29
+ url: item.link,
30
+ });
31
+
32
+ const content = cheerio.load(detailResponse.data);
33
+ const detail = JSON.parse(`{${detailResponse.data.match(/("title":.*?),"body":/)[1]}}`);
34
+
35
+ content('h1').remove();
36
+ content('h5').parent().remove();
37
+ content('div[data-testid="Text-embedded-entry-block"]').remove();
38
+
39
+ content('img').each(function () {
40
+ const image = (content(this).attr('srcset') ?? content(this).attr('src')).split('?w=')[0];
41
+
42
+ content(this).replaceWith(
43
+ art(path.join(__dirname, 'templates/description.art'), {
44
+ image,
45
+ })
46
+ );
47
+ });
48
+
49
+ item.title = detail.title;
50
+ item.author = detail.author.name;
51
+ item.pubDate = parseDate(detail.pubDate, 'MMMM YYYY');
52
+ item.category = (detail.tags?.map((t) => t.title) ?? []).concat(detail.category?.map((c) => c.title) ?? []);
53
+ item.description = art(path.join(__dirname, 'templates/description.art'), {
54
+ header: content('header[data-csk-entry-type="blog"]').html(),
55
+ description: content('div[data-csk-entry-type="blog"] div[data-testid="Text-root"]').html(),
56
+ });
57
+
58
+ return item;
59
+ })
60
+ )
61
+ );
62
+
63
+ ctx.state.data = {
64
+ title: 'Sensor Tower - Blog',
65
+ link: currentUrl,
66
+ item: items,
67
+ language: language ? language : 'en-US',
68
+ };
69
+ };
@@ -0,0 +1,3 @@
1
+ module.exports = {
2
+ '/blog/:language?': ['nczitzk'],
3
+ };
@@ -0,0 +1,13 @@
1
+ module.exports = {
2
+ 'sensortower.com': {
3
+ _name: 'Sensor Tower',
4
+ '.': [
5
+ {
6
+ title: 'Blog',
7
+ docs: 'https://docs.rsshub.app/new-media.html#sensor-tower-blog',
8
+ source: ['/blog', '/zh-CN/blog', '/ja/blog', '/ko/blog', '/'],
9
+ target: '/sensortower/blog',
10
+ },
11
+ ],
12
+ },
13
+ };
@@ -0,0 +1,3 @@
1
+ module.exports = function (router) {
2
+ router.get('/blog/:language?', require('./blog'));
3
+ };
@@ -0,0 +1,9 @@
1
+ {{ if image }}
2
+ <img src="{{ image }}">
3
+ {{ /if }}
4
+ {{ if header }}
5
+ {{@ header }}
6
+ {{ /if }}
7
+ {{ if description }}
8
+ {{@ description }}
9
+ {{ /if }}
@@ -0,0 +1,3 @@
1
+ {{ each images image }}
2
+ <img src="{{ image.original_url || image.url }}">
3
+ {{ /each }}
@@ -1,2 +1,3 @@
1
1
  {{ if intro }}{{@ intro }}{{ /if }}
2
- <video controls src="{{@ videoUrl }}" {{ if posterUrl }}poster="{{@ posterUrl }}"{{ /if }}>Sorry, your browser doesn't support embedded videos.</video>
2
+ <p>Preview</p>
3
+ <p><img src="{{@ previewUrl }}" /></p>
@@ -1,6 +1,6 @@
1
1
  const got = require('@/utils/got');
2
2
  const { parseDate } = require('@/utils/parse-date');
3
- const { rootUrl, X_UA, appId2GroupId, textPost, videoPost } = require('./utils');
3
+ const { rootUrl, X_UA, appDetail, imagePost, topicPost, videoPost } = require('./utils');
4
4
 
5
5
  const typeMap = {
6
6
  feed: {
@@ -24,41 +24,47 @@ const typeMap = {
24
24
  module.exports = async (ctx) => {
25
25
  const lang = ctx.params.lang ?? 'zh_CN';
26
26
  const appId = ctx.params.id;
27
- const groupId = await appId2GroupId(appId, lang);
27
+ const appData = await appDetail(appId, lang);
28
28
  const type = ctx.params.type ?? 'feed';
29
29
  const sort = ctx.params.sort ?? 'created';
30
-
30
+ const groupId = appData.group.id;
31
+ const app_img = appData.app.icon.original_url || appData.app.icon.url;
32
+ const app_name = appData.app.title;
31
33
  const url = `${rootUrl}/webapiv2/feed/v6/by-group?group_id=${groupId}&type=${type}&sort=${sort}&${X_UA(lang)}`;
32
34
 
33
35
  const topics_list_response = await got(url);
34
36
  const topics_list = topics_list_response.data;
35
37
 
36
- const app_img = topics_list.data.list[0].moment.app.icon.url;
37
- const app_name = topics_list.data.list[0].moment.app.title;
38
+ const out = await Promise.all(
39
+ topics_list.data.list.map((list) => {
40
+ const link = list.moment.sharing?.url || list.moment.extended_entities?.topics?.[0].sharing.url || list.moment.extended_entities?.videos?.[0].sharing.url;
38
41
 
39
- const out = topics_list.data.list.map(async (list) => {
40
- const author = list.moment.author.user.name;
41
- const topicId = list.moment.extended_entities?.topics?.[0].id;
42
- // raw_text sometimes is "" so || is better than ??
43
- const title = list.moment.contents.raw_text || list.moment.extended_entities?.topics?.[0].title || list.moment.extended_entities?.videos?.[0].title;
44
- const createdTime = list.moment.created_time * 1000;
45
- const link = list.moment.sharing?.url || list.moment.extended_entities?.topics?.[0].sharing.url || list.moment.extended_entities?.videos?.[0].sharing.url;
46
- let description = '';
47
- try {
48
- description = await textPost(appId, topicId, lang);
49
- } catch (e) {
50
- // fallback if 400 bad request
51
- description = list.moment.extended_entities?.topics?.[0].summary || list.moment.contents.raw_text || videoPost(list.moment.extended_entities?.videos?.[0]);
52
- }
42
+ return ctx.cache.tryGet(link, async () => {
43
+ const author = list.moment.author.user.name;
44
+ const topicId = list.moment.extended_entities?.topics?.[0].id;
45
+ // raw_text sometimes is "" so || is better than ??
46
+ const title = list.moment.contents?.raw_text || list.moment.extended_entities?.topics?.[0].title || list.moment.extended_entities?.videos?.[0].title;
47
+ const createdTime = list.moment.created_time;
48
+ let description = '';
49
+ if (topicId) {
50
+ description = await topicPost(appId, topicId, lang);
51
+ } else {
52
+ description = list.moment.extended_entities?.topics?.[0].summary || list.moment.contents?.raw_text || videoPost(list.moment.extended_entities?.videos?.[0]);
53
+ if (list.moment.extended_entities?.images) {
54
+ description += imagePost(list.moment.extended_entities.images);
55
+ }
56
+ }
53
57
 
54
- return {
55
- title,
56
- description,
57
- author,
58
- link,
59
- pubDate: parseDate(createdTime),
60
- };
61
- });
58
+ return {
59
+ title,
60
+ description,
61
+ author,
62
+ link,
63
+ pubDate: parseDate(createdTime, 'X'),
64
+ };
65
+ });
66
+ })
67
+ );
62
68
 
63
69
  ctx.state.data = {
64
70
  title: `${app_name} - ${typeMap[type][lang]} - TapTap 论坛`,
@@ -7,16 +7,21 @@ const rootUrl = 'https://www.taptap.com';
7
7
  // Please do not change %26 to &
8
8
  const X_UA = (lang = 'zh_CN') => `X-UA=V=1%26PN=WebApp%26VN=0.1.0%26LANG=${lang}%26PLT=PC`;
9
9
 
10
- const appId2GroupId = async (appId, lang = 'zh_CN') => {
11
- const res = await got(`${rootUrl}/webapiv2/group/v1/detail?app_id=${appId}&${X_UA(lang)}`, {
10
+ const appDetail = async (appId, lang = 'zh_CN') => {
11
+ const { data } = await got(`${rootUrl}/webapiv2/group/v1/detail?app_id=${appId}&${X_UA(lang)}`, {
12
12
  headers: {
13
13
  Referer: `${rootUrl}/app/${appId}`,
14
14
  },
15
15
  });
16
- return res.data.data.group.id;
16
+ return data.data;
17
17
  };
18
18
 
19
- const textPost = async (appId, topicId, lang = 'zh_CN') => {
19
+ const imagePost = (images) =>
20
+ art(path.join(__dirname, 'templates/imagePost.art'), {
21
+ images,
22
+ });
23
+
24
+ const topicPost = async (appId, topicId, lang = 'zh_CN') => {
20
25
  const res = await got(`${rootUrl}/webapiv2/topic/v1/detail?id=${topicId}&${X_UA(lang)}`, {
21
26
  headers: {
22
27
  Referer: `${rootUrl}/app/${appId}`,
@@ -35,14 +40,14 @@ const textPost = async (appId, topicId, lang = 'zh_CN') => {
35
40
  const videoPost = (video) =>
36
41
  art(path.join(__dirname, 'templates/videoPost.art'), {
37
42
  intro: video?.intro?.text,
38
- videoUrl: video?.url,
39
- posterUrl: video?.raw_cover.url,
43
+ previewUrl: video?.video_resource.preview_animation.original_url,
40
44
  });
41
45
 
42
46
  module.exports = {
43
47
  rootUrl,
44
48
  X_UA,
45
- appId2GroupId,
46
- textPost,
49
+ appDetail,
50
+ imagePost,
51
+ topicPost,
47
52
  videoPost,
48
53
  };
@@ -75,6 +75,21 @@ module.exports = async (ctx) => {
75
75
  link: `https://github.com/DIYgod/RSSHub/issues/0`,
76
76
  author: `DIYgod0`,
77
77
  });
78
+ } else if (ctx.params.id === 'cacheUrlKey') {
79
+ const description = await ctx.cache.tryGet(
80
+ new URL('https://rsshub.app'),
81
+ () => ({
82
+ text: `Cache${++cacheIndex}`,
83
+ }),
84
+ config.cache.routeExpire * 2
85
+ );
86
+ item.push({
87
+ title: 'Cache Title',
88
+ description: description.text,
89
+ pubDate: new Date(`2019-3-1`).toUTCString(),
90
+ link: `https://github.com/DIYgod/RSSHub/issues/0`,
91
+ author: `DIYgod0`,
92
+ });
78
93
  } else if (ctx.params.id === 'complicated') {
79
94
  item.push({
80
95
  title: `Complicated Title`,
@@ -8,6 +8,7 @@ const config = require('@/config').value;
8
8
  const md5 = require('@/utils/md5');
9
9
 
10
10
  // Constants
11
+ const CACHE_KEY = 'trending-all-in-one';
11
12
  const DATA_REPO_BASE_URL = 'https://raw.githubusercontent.com/huqi-pr/trending-in-one/master/raw';
12
13
  const DATE_FORMAT = 'YYYY-MM-DD';
13
14
  // TODO: support custom data repo urls
@@ -35,7 +36,6 @@ const CHANNELS = {
35
36
  };
36
37
 
37
38
  // Helper Functions
38
- // TODO: integrate into CHANNELS
39
39
  const processRawDataByChannel = {
40
40
  'toutiao-search': ({ word: title }) => ({
41
41
  // 源 url 存在 encoding 问题,暂时不使用
@@ -61,38 +61,63 @@ const processRawDataByChannel = {
61
61
 
62
62
  const hasKeyword = (str, keywordList) => keywordList.some((keyword) => str.includes(keyword));
63
63
  const toShanghaiTimezone = (date) => dayjs.tz(date, 'Asia/Shanghai');
64
+ const processRawData = (channel) => (res) => res.map((item) => processRawDataByChannel[channel](item));
65
+ const filterKeyword = (keywordList) => (res) => res.filter(({ title }) => hasKeyword(title, keywordList));
64
66
 
65
67
  // Data Fetcher
66
68
  // TODO: support channel selection
67
69
  const fetchAllData = async (keywordList = [], dateList = [], cache) => {
68
- const cachedGetData = (url, dataMappingFn) =>
69
- cache.tryGet(
70
- url,
71
- () =>
72
- got(url)
73
- .json()
74
- // Normalize data to { title, url } format
75
- .then((res) => res.map((item) => dataMappingFn(item))),
76
- config.cache.contentExpire,
77
- false
78
- );
79
-
80
- const data = await Promise.all(
70
+ const cachedGetData = (url) => cache.tryGet(url, () => got(url).json(), config.cache.contentExpire, false);
71
+
72
+ let data = await Promise.all(
81
73
  dateList.map(async (dateTime) => ({
82
74
  dateTime,
83
75
  data: await Promise.all(
84
76
  Object.keys(CHANNELS).map(async (channel) => ({
85
77
  name: CHANNELS[channel].name,
86
- data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`, processRawDataByChannel[channel]).then((res) => res.filter(({ title }) => hasKeyword(title, keywordList))),
78
+ data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`)
79
+ .then(processRawData(channel))
80
+ .then(filterKeyword(keywordList)),
87
81
  }))
88
82
  ),
89
83
  }))
90
84
  );
85
+
91
86
  for (const i of data) {
92
87
  i.count = i.data.reduce((acc, { data }) => acc + data.length, 0);
93
88
  }
94
89
 
95
- return data.filter(({ count }) => count > 0);
90
+ data = data.filter(({ count }) => count > 0);
91
+
92
+ if (data.length === 0) {
93
+ return data;
94
+ }
95
+
96
+ const prev = cache.get(CACHE_KEY + ':latest-items');
97
+
98
+ const latest = data[0];
99
+ latest.newItemCount = 0;
100
+ if (latest.count > 0 && prev) {
101
+ // Mark new items in latest
102
+ for (const channel in latest.data) {
103
+ for (const i of latest.data[channel].data) {
104
+ i.new = !(i.url in prev);
105
+ latest.newItemCount += i.new ? 1 : 0;
106
+ }
107
+ }
108
+ // Save latest data to cache
109
+ const cachedItems = latest.data.reduce((acc, { data: channel }) => {
110
+ for (const item of channel) {
111
+ acc[item.url] = true;
112
+ }
113
+ return acc;
114
+ }, {});
115
+ cache.set(CACHE_KEY + ':latest-items', cachedItems, config.cache.contentExpire);
116
+
117
+ latest.count = Object.keys(cachedItems).length;
118
+ }
119
+
120
+ return data;
96
121
  };
97
122
 
98
123
  // Generate Feed Items
@@ -108,11 +133,13 @@ const searchLinkUrls = (keyword) => [
108
133
 
109
134
  const searchLinkNames = ['热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条'];
110
135
 
111
- const createItem = ({ dateTime, data, count }, keywords, isToday) => {
136
+ const createItem = ({ dateTime, data, count, newItemCount }, keywords, isToday) => {
112
137
  const EOD = dateTime.endOf('day');
113
138
  const pubDate = isToday ? new Date() : EOD.toDate();
139
+ const countStr = isToday && newItemCount ? newItemCount + '🆕' : count;
140
+
114
141
  return {
115
- title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${count})`,
142
+ title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${countStr})`,
116
143
  author: 'Trending All In One',
117
144
  pubDate,
118
145
  description: art(path.join(__dirname, 'templates/content.art'), {
@@ -147,7 +174,7 @@ module.exports = async (ctx) => {
147
174
  .map((keyword) => keyword.trim());
148
175
  const keywordStr = keywordList.join(', ');
149
176
 
150
- const data = await fetchAllData(keywordList, dateList, ctx.cache);
177
+ const data = await fetchAllData(keywordList, dateList, ctx.cache).catch(() => []);
151
178
  const item =
152
179
  data.length > 0
153
180
  ? data.map((i, index) => createItem(i, keywordList, index === 0))
@@ -7,6 +7,7 @@
7
7
  {{each channel.data item}}
8
8
  <li>
9
9
  <a href="{{item.url}}" rel="noopener noreferrer" target="_blank">{{item.title}}</a>
10
+ {{if item.new}}🆕{{/if}}
10
11
  </li>
11
12
  {{/each}}
12
13
  </ul>
@@ -1,7 +1,7 @@
1
1
  module.exports = {
2
2
  '/collection/:uid/:collectionId/:routeParams?': ['TonyRL'],
3
3
  '/followings/:id/:routeParams?': ['DIYgod'],
4
- '/keyword/:keyword/:routeParams?/:limit?': ['DIYgod', 'yindaheng98', 'Rongronggg9'],
4
+ '/keyword/:keyword/:routeParams?': ['DIYgod', 'yindaheng98', 'Rongronggg9'],
5
5
  '/likes/:id/:routeParams?': ['xyqfer'],
6
6
  '/list/:id/:name/:routeParams?': ['xyqfer'],
7
7
  '/media/:id/:routeParams?': ['yindaheng98', 'Rongronggg9'],
@@ -1,7 +1,7 @@
1
1
  module.exports = (router) => {
2
2
  router.get('/collection/:uid/:collectionId/:routeParams?', require('./collection'));
3
3
  router.get('/followings/:id/:routeParams?', require('./followings'));
4
- router.get('/keyword/:keyword/:routeParams?/:limit?', require('./keyword'));
4
+ router.get('/keyword/:keyword/:routeParams?', require('./keyword'));
5
5
  router.get('/likes/:id/:routeParams?', require('./likes'));
6
6
  router.get('/list/:id/:name/:routeParams?', require('./list'));
7
7
  router.get('/media/:id/:routeParams?', require('./media'));
@@ -106,6 +106,7 @@ const timelineKeywords = (keywords, params = {}) =>
106
106
  ..._params,
107
107
  ...params,
108
108
  q: keywords,
109
+ tweet_search_mode: 'live',
109
110
  query_source: 'typed_query',
110
111
  pc: 1,
111
112
  });
@@ -0,0 +1,3 @@
1
+ module.exports = {
2
+ '/news': ['axojhf'],
3
+ };