rsshub 1.0.0-master.f2c8f0c → 1.0.0-master.f2eee10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/lib/router.js +4 -3
  2. package/lib/utils/wechat-mp.js +1 -1
  3. package/lib/v2/2048/index.js +7 -5
  4. package/lib/v2/agirls/index.js +14 -42
  5. package/lib/v2/agirls/topic.js +16 -43
  6. package/lib/v2/agirls/topic_list.js +14 -14
  7. package/lib/v2/agirls/utils.js +37 -0
  8. package/lib/v2/apnews/topics.js +24 -30
  9. package/lib/v2/baozimh/index.js +9 -14
  10. package/lib/v2/bilibili/followers.js +12 -1
  11. package/lib/v2/bilibili/followings.js +16 -1
  12. package/lib/v2/bilibili/liveSearch.js +18 -10
  13. package/lib/v2/bilibili/maintainer.js +2 -2
  14. package/lib/v2/bilibili/router.js +2 -2
  15. package/lib/v2/blizzard/maintainer.js +3 -0
  16. package/lib/{routes → v2}/blizzard/news.js +14 -15
  17. package/lib/v2/blizzard/radar.js +13 -0
  18. package/lib/v2/blizzard/router.js +3 -0
  19. package/lib/v2/caixin/latest.js +5 -5
  20. package/lib/v2/copymanga/comic.js +74 -58
  21. package/lib/v2/copymanga/templates/comic.art +1 -1
  22. package/lib/v2/dhu/maintainer.js +1 -0
  23. package/lib/v2/dhu/news/xsxx.js +42 -0
  24. package/lib/v2/dhu/radar.js +8 -0
  25. package/lib/v2/dhu/router.js +1 -0
  26. package/lib/v2/globallawreview/index.js +55 -0
  27. package/lib/v2/globallawreview/maintainer.js +3 -0
  28. package/lib/v2/globallawreview/radar.js +13 -0
  29. package/lib/v2/globallawreview/router.js +3 -0
  30. package/lib/v2/gov/cac/index.js +4 -4
  31. package/lib/v2/gov/maintainer.js +1 -0
  32. package/lib/v2/gov/nifdc/index.js +2 -17
  33. package/lib/v2/gov/nmpa/generic.js +5 -27
  34. package/lib/v2/gov/nrta/dsj.js +59 -0
  35. package/lib/v2/gov/radar.js +13 -0
  36. package/lib/v2/gov/router.js +1 -0
  37. package/lib/v2/huoxian/maintainer.js +3 -0
  38. package/lib/v2/huoxian/radar.js +11 -0
  39. package/lib/v2/huoxian/router.js +3 -0
  40. package/lib/v2/huoxian/zone.js +19 -0
  41. package/lib/v2/javdb/utils.js +5 -4
  42. package/lib/v2/jike/radar.js +20 -0
  43. package/lib/v2/keylol/index.js +82 -0
  44. package/lib/v2/keylol/maintainer.js +3 -0
  45. package/lib/v2/keylol/radar.js +18 -0
  46. package/lib/v2/keylol/router.js +3 -0
  47. package/lib/v2/line/utils.js +18 -2
  48. package/lib/v2/macupdate/app.js +43 -0
  49. package/lib/v2/macupdate/maintainer.js +3 -0
  50. package/lib/v2/macupdate/radar.js +13 -0
  51. package/lib/v2/macupdate/router.js +3 -0
  52. package/lib/v2/radiofrance/templates/article.art +3 -1
  53. package/lib/v2/sec-in/index.js +19 -0
  54. package/lib/v2/sec-in/maintainer.js +3 -0
  55. package/lib/v2/sec-in/radar.js +11 -0
  56. package/lib/v2/sec-in/router.js +3 -0
  57. package/lib/v2/sec-wiki/maintainer.js +3 -0
  58. package/lib/v2/sec-wiki/radar.js +11 -0
  59. package/lib/v2/sec-wiki/router.js +3 -0
  60. package/lib/v2/sec-wiki/weekly.js +16 -0
  61. package/lib/v2/shu/index.js +48 -0
  62. package/lib/v2/shu/jwb.js +45 -0
  63. package/lib/v2/shu/maintainer.js +4 -0
  64. package/lib/v2/shu/radar.js +21 -0
  65. package/lib/v2/shu/router.js +5 -0
  66. package/lib/v2/stheadline/std/realtime.js +1 -1
  67. package/lib/v2/wikinews/index.js +28 -29
  68. package/lib/v2/wxkol/show.js +5 -34
  69. package/lib/v2/yangtzeu/dongke.js +60 -0
  70. package/lib/v2/yangtzeu/maintainer.js +3 -0
  71. package/lib/v2/yangtzeu/radar.js +18 -0
  72. package/lib/v2/yangtzeu/router.js +3 -0
  73. package/lib/v2/zaobao/util.js +38 -31
  74. package/package.json +3 -3
  75. package/lib/routes/universities/shu/index.js +0 -75
  76. package/lib/routes/universities/shu/jwc.js +0 -62
  77. package/lib/v2/agirls/templates/description.art +0 -1
@@ -1,22 +1,13 @@
1
1
  const got = require('@/utils/got');
2
2
  const cheerio = require('cheerio');
3
- const { fetchArticle } = require('@/utils/wechat-mp');
3
+ const { finishArticleItem } = require('@/utils/wechat-mp');
4
4
  const config = require('@/config').value;
5
- const asyncPool = require('tiny-async-pool');
6
5
 
7
6
  module.exports = async (ctx) => {
8
7
  const baseUrl = 'https://www.wxkol.com';
9
8
  const { id } = ctx.params;
10
9
  const url = `${baseUrl}/show/${id}.html`;
11
10
 
12
- const asyncPoolAll = async (...args) => {
13
- const results = [];
14
- for await (const result of asyncPool(...args)) {
15
- results.push(result);
16
- }
17
- return results;
18
- };
19
-
20
11
  const feedData = await ctx.cache.tryGet(
21
12
  url,
22
13
  async () => {
@@ -30,13 +21,13 @@ module.exports = async (ctx) => {
30
21
  const a = item.find('.title a');
31
22
  return {
32
23
  title: a.attr('title'),
33
- link: `${baseUrl}${a.attr('href')}`,
24
+ link: a.attr('href'),
34
25
  };
35
26
  });
36
27
 
37
28
  return {
38
29
  feedTitle: $('head title').text(),
39
- feedDescription: $('head description').text(),
30
+ feedDescription: $('.info_left .description i').text(),
40
31
  feedImage: $('.main .logo .avatar')
41
32
  .attr('style')
42
33
  .match(/url\('(.+)'\)/)[1],
@@ -47,34 +38,14 @@ module.exports = async (ctx) => {
47
38
  false
48
39
  );
49
40
 
50
- const urlList = await asyncPoolAll(2, feedData.feedItem, (item) =>
51
- ctx.cache.tryGet(item.link, async () => {
52
- const { data: response } = await got(item.link);
53
- const $ = cheerio.load(response);
54
- item.link = `${baseUrl}${$('.source a').attr('href')}`;
55
- return item;
56
- })
57
- );
58
-
59
- const items = await asyncPoolAll(4, urlList, async (item) => {
60
- const { title, author, description, summary, pubDate, mpName, link: itemLink } = await fetchArticle(ctx, item.link, true);
61
-
62
- item.title = title;
63
- item.author = author;
64
- item.description = description;
65
- item.summary = summary;
66
- item.pubDate = pubDate;
67
- item.author = mpName;
68
- item.link = itemLink;
69
-
70
- return item;
71
- });
41
+ const items = await Promise.all(feedData.feedItem.map((item) => finishArticleItem(ctx, item)));
72
42
 
73
43
  ctx.state.data = {
74
44
  title: feedData.feedTitle,
75
45
  description: feedData.feedDescription,
76
46
  link: url,
77
47
  image: feedData.feedImage,
48
+ icon: feedData.feedImage,
78
49
  item: items,
79
50
  };
80
51
  };
@@ -0,0 +1,60 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const timezone = require('@/utils/timezone');
4
+ const { parseDate } = require('@/utils/parse-date');
5
+
6
+ module.exports = async (ctx) => {
7
+ const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 10;
8
+
9
+ const rootUrl = 'https://dongke.yangtzeu.edu.cn';
10
+ const currentUrl = new URL(`${ctx.path.replace(/^\/dongke/, '') || '/yqzl/xyxw'}.htm`, rootUrl).href;
11
+
12
+ const { data: response } = await got(currentUrl);
13
+
14
+ const $ = cheerio.load(response);
15
+
16
+ let items = $('ul.list-item li a')
17
+ .slice(0, limit)
18
+ .toArray()
19
+ .map((item) => {
20
+ item = $(item);
21
+
22
+ return {
23
+ title: item.text(),
24
+ link: new URL(item.prop('href'), rootUrl).href,
25
+ };
26
+ });
27
+
28
+ items = await Promise.all(
29
+ items.map((item) =>
30
+ ctx.cache.tryGet(item.link, async () => {
31
+ const { data: detailResponse } = await got(item.link);
32
+
33
+ const content = cheerio.load(detailResponse);
34
+
35
+ item.title = content('title').text();
36
+ item.description = content('div.v_news_content').html();
37
+ item.category = content('meta[name="keywords"]').prop('content').split(',');
38
+ item.pubDate = timezone(
39
+ parseDate(
40
+ content('p.content-info')
41
+ .text()
42
+ .match(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/)[1]
43
+ ),
44
+ +8
45
+ );
46
+
47
+ return item;
48
+ })
49
+ )
50
+ );
51
+
52
+ ctx.state.data = {
53
+ item: items,
54
+ title: $('title').text(),
55
+ link: currentUrl,
56
+ language: 'zh-cn',
57
+ image: new URL($('#head-img a img').prop('src'), rootUrl).href,
58
+ author: '长江大学动物科学学院',
59
+ };
60
+ };
@@ -0,0 +1,3 @@
1
+ module.exports = {
2
+ '/dongke/:path+': ['nczitzk'],
3
+ };
@@ -0,0 +1,18 @@
1
+ module.exports = {
2
+ 'yangtzeu.edu.cn': {
3
+ _name: '长江大学',
4
+ '.': [
5
+ {
6
+ title: '动物科学学院',
7
+ docs: 'https://docs.rsshub.app/universities.html#chang-jiang-da-xue-dong-wu-ke-xue-xue-yuan',
8
+ source: ['/:category', '/'],
9
+ target: (params, url) => {
10
+ url = new URL(url);
11
+ const path = /\.edu\.cn(.*?)\.htm/.test(url.href) ? url.href.match(/\.edu\.cn(.*?)\.htm/)[1] : '';
12
+
13
+ return `/yangtzeu/dongke${path}`;
14
+ },
15
+ },
16
+ ],
17
+ },
18
+ };
@@ -0,0 +1,3 @@
1
+ module.exports = function (router) {
2
+ router.get(/dongke([\w-/]+)?/, require('./dongke'));
3
+ };
@@ -4,6 +4,7 @@ const date = require('@/utils/date');
4
4
  const timezone = require('@/utils/timezone');
5
5
  const { art } = require('@/utils/render');
6
6
  const path = require('path');
7
+ const { base32 } = require('rfc4648');
7
8
 
8
9
  const baseUrl = 'https://www.zaobao.com';
9
10
  const got_ins = got.extend({
@@ -35,22 +36,14 @@ const parseList = async (ctx, sectionUrl) => {
35
36
  data = $('.clearfix').find('.list-block');
36
37
  }
37
38
 
38
- const title = $('#breadcrumbs > a')
39
- .toArray()
40
- .reduce((acc, cV, cI) => {
41
- if (cI > 0) {
42
- return acc + '-' + $(cV).text();
43
- }
44
- return '';
45
- }, '');
39
+ const title = $('meta[property="og:title"]').attr('content');
40
+
46
41
  const resultList = await Promise.all(
47
- data.toArray().map(async (item) => {
42
+ data.toArray().map((item) => {
48
43
  const $item = $(item);
49
44
  // addBack: for HK version
50
45
  let link = $item.find('a').addBack('a')[0].attribs.href;
51
46
 
52
- let resultItem = {};
53
-
54
47
  if (link[0] !== '/') {
55
48
  // https://www.zaobao.com/interactive-graphics
56
49
  const title = $item.find('a').text();
@@ -71,31 +64,29 @@ const parseList = async (ctx, sectionUrl) => {
71
64
  pubDate = pubDate === dateString ? undefined : timezone(pubDate, +8);
72
65
  }
73
66
 
74
- resultItem = {
67
+ return {
75
68
  title,
76
69
  description: title,
77
70
  pubDate,
78
71
  link,
79
72
  };
80
- return Promise.resolve(resultItem);
81
- } else {
82
- link = baseUrl + link;
83
73
  }
84
74
 
85
- const value = await ctx.cache.get(link);
75
+ link = baseUrl + link;
86
76
 
87
- if (value) {
88
- resultItem = JSON.parse(value);
89
- } else {
77
+ return ctx.cache.tryGet(link, async () => {
90
78
  const article = await got_ins.get(link);
91
79
  const $1 = cheerio.load(article.data);
92
80
 
93
- let time = $1('.date-published', '.content').slice(0, 1).text().trim();
94
- if (!time) {
95
- // for HK version
96
- time = $1('.author-info').find('.mgt10').slice(0, 1).text().trim();
97
- }
98
- time = time.replace('年', '-').replace('月', '-').replace('日', '').replace('发布 /', '');
81
+ const time = (() => {
82
+ if ($1("head script[type='application/json']").text() !== '') {
83
+ // SG
84
+ return new Date(Number(JSON.parse($1("head script[type='application/json']").text())?.articleDetails?.created) * 1000);
85
+ } else {
86
+ // HK
87
+ return new Date(JSON.parse($1("head script[type='application/ld+json']").eq(1).text())?.datePublished);
88
+ }
89
+ })();
99
90
 
100
91
  $1('.overlay-microtransaction').remove();
101
92
  $1('#video-freemium-player').remove();
@@ -104,6 +95,7 @@ const parseList = async (ctx, sectionUrl) => {
104
95
  let articleBodyNode = $1('.article-content-rawhtml');
105
96
  if (articleBodyNode.length === 0) {
106
97
  // for HK version
98
+ orderContent($1('.article-body'));
107
99
  articleBodyNode = $1('.article-body');
108
100
  }
109
101
 
@@ -152,20 +144,17 @@ const parseList = async (ctx, sectionUrl) => {
152
144
  });
153
145
  }
154
146
 
155
- resultItem = {
147
+ return {
156
148
  // <- for SG version -> for HK version
157
149
  title: $1('h1', '.content').text().trim() || $1('h1.article-title').text(),
158
150
  description: art(path.join(__dirname, 'templates/zaobao.art'), {
159
151
  articleBody,
160
152
  imageDataArray,
161
153
  }),
162
- pubDate: timezone(time, +8), // zaobao seems always UTC+8
154
+ pubDate: time,
163
155
  link,
164
156
  };
165
- ctx.cache.set(link, JSON.stringify(resultItem));
166
- }
167
-
168
- return Promise.resolve(resultItem);
157
+ });
169
158
  })
170
159
  );
171
160
  return {
@@ -173,6 +162,24 @@ const parseList = async (ctx, sectionUrl) => {
173
162
  resultList,
174
163
  };
175
164
  };
165
+
166
+ const orderContent = (parent) => {
167
+ parent
168
+ .children()
169
+ .toArray()
170
+ .sort((a, b) => {
171
+ const index = Buffer.from(base32.parse('GM======')).toString(); // substring(3)
172
+ a = Buffer.from(base32.parse(parent.find(a).data('s').substring(index))).toString();
173
+ b = Buffer.from(base32.parse(parent.find(b).data('s').substring(index))).toString();
174
+ return a - b;
175
+ })
176
+ .forEach((e, i) => {
177
+ parent.find(e).attr('s', i);
178
+ parent.append(e);
179
+ });
180
+ };
181
+
176
182
  module.exports = {
177
183
  parseList,
184
+ orderContent,
178
185
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rsshub",
3
- "version": "1.0.0-master.f2c8f0c",
3
+ "version": "1.0.0-master.f2eee10",
4
4
  "description": "Make RSS Great Again!",
5
5
  "keywords": [
6
6
  "RSS"
@@ -111,6 +111,7 @@
111
111
  "rand-user-agent": "1.0.109",
112
112
  "re2": "1.19.1",
113
113
  "require-all": "3.0.0",
114
+ "rfc4648": "1.5.2",
114
115
  "rss-parser": "3.13.0",
115
116
  "showdown": "2.1.0",
116
117
  "simplecc-wasm": "0.1.5",
@@ -119,8 +120,7 @@
119
120
  "tiny-async-pool": "2.1.0",
120
121
  "tough-cookie": "4.1.3",
121
122
  "twitter-api-v2": "1.15.0",
122
- "winston": "3.9.0",
123
- "xml2js": "0.5.0"
123
+ "winston": "3.9.0"
124
124
  },
125
125
  "devDependencies": {
126
126
  "@microsoft/eslint-formatter-sarif": "3.0.0",
@@ -1,75 +0,0 @@
1
- const got = require('@/utils/got');
2
- const cheerio = require('cheerio');
3
- const url = require('url');
4
-
5
- const host = 'https://www.shu.edu.cn/';
6
-
7
- const config = {
8
- news: {
9
- link: 'https://www.shu.edu.cn/index/zhxw.htm',
10
- title: '综合新闻',
11
- },
12
- research: {
13
- link: 'https://www.shu.edu.cn/index/kydt1.htm',
14
- title: '科研动态',
15
- },
16
- notice: {
17
- link: 'https://www.shu.edu.cn/index/tzgg.htm',
18
- title: '通知公告',
19
- },
20
- };
21
-
22
- module.exports = async (ctx) => {
23
- const type = ctx.params.type ? ctx.params.type : 'news';
24
- const link = type === 'news' ? config.news.link : type === 'research' ? config.research.link : config.notice.link;
25
- const title = type === 'news' ? config.news.title : type === 'research' ? config.research.title : config.notice.title;
26
- const respond = await got.get(link);
27
- const $ = cheerio.load(respond.data);
28
- const list = $('.list-con')
29
- .find('li')
30
- .slice(0, 10)
31
- .map((index, ele) => ({
32
- title: $(ele).find('.list-txt-1').text(),
33
- link: $(ele).find('a').attr('href'),
34
- date: $(ele).find('.list-date').text(),
35
- }))
36
- .get();
37
-
38
- const all = await Promise.all(
39
- list.map(async (item) => {
40
- const itemUrl = url.resolve(host, item.link);
41
- const cache = await ctx.cache.get(itemUrl);
42
- if (cache) {
43
- return Promise.resolve(JSON.parse(cache));
44
- }
45
- let desc = '';
46
- const respond = await got.get(itemUrl);
47
- const $ = cheerio.load(respond.data);
48
- if (type === 'notice') {
49
- desc = $('.content-con').text();
50
- } else {
51
- desc = $('#vsb_content_2').text();
52
- if (desc.length > 256) {
53
- desc = desc.slice(0, 256) + '...';
54
- }
55
- }
56
- const date = new Date(item.date);
57
- const time_zone = 8;
58
- const server_offset = date.getTimezoneOffset() / 60;
59
- const single = {
60
- title: item.title,
61
- link: itemUrl,
62
- pubDate: new Date(date.getTime() - 60 * 60 * 1000 * (time_zone + server_offset)).toUTCString(),
63
- description: desc || item.title,
64
- };
65
- ctx.cache.set(itemUrl, JSON.stringify(single));
66
- return Promise.resolve(single);
67
- })
68
- );
69
-
70
- ctx.state.data = {
71
- title: title + ' - 上海大学',
72
- link: 'https://www.shu.edu.cn/',
73
- item: all,
74
- };
75
- };
@@ -1,62 +0,0 @@
1
- const got = require('@/utils/got');
2
- const cheerio = require('cheerio');
3
- const url = require('url');
4
-
5
- const host = 'http://www.jwc.shu.edu.cn';
6
-
7
- const config = {
8
- notice: {
9
- link: 'http://www.jwc.shu.edu.cn/nry.jsp?urltype=tree.TreeTempUrl&wbtreeid=1015',
10
- type: 'notice',
11
- title: '通知通告',
12
- },
13
- news: {
14
- link: 'http://www.jwc.shu.edu.cn/nry.jsp?urltype=tree.TreeTempUrl&wbtreeid=1016',
15
- type: 'news',
16
- title: '新闻',
17
- },
18
- };
19
-
20
- module.exports = async (ctx) => {
21
- let type = ctx.params.type;
22
- type = type ? type : 'notice';
23
- const link = type === 'news' ? config.news.link : config.notice.link;
24
- const title = type === 'news' ? config.news.title : config.notice.title;
25
- const respond = await got.get(link);
26
- const $ = cheerio.load(respond.data);
27
- const list = $('#dnn_ctr43516_ArticleList__ctl0_ArtDataList__ctl1_titleLink1')
28
- .slice(0, 10)
29
- .map((index, ele) => ({
30
- title: $(ele).attr('title'),
31
- link: $(ele).attr('href'),
32
- date: $('#dnn_ctr43516_ArticleList__ctl0_ArtDataList__ctl1_Label6').text(),
33
- }))
34
- .get();
35
-
36
- const all = await Promise.all(
37
- list.map(async (item) => {
38
- const itemUrl = url.resolve(host, item.link);
39
- const cache = await ctx.cache.get(itemUrl);
40
- if (cache) {
41
- return Promise.resolve(JSON.parse(cache));
42
- }
43
- const respond = await got.get(itemUrl);
44
- const $ = cheerio.load(respond.data);
45
- const single = {
46
- title: item.title,
47
- link: itemUrl,
48
- author: $('.normal').next().text().trim().slice(0, -3),
49
- guid: itemUrl,
50
- pubDate: new Date(new Date(...$('.normal').text().slice(0, 10).split('-')).getTime() - 60 * 60 * 24 * 30 * 1000).toUTCString(),
51
- description: item.title,
52
- };
53
- ctx.cache.set(itemUrl, JSON.stringify(single));
54
- return Promise.resolve(single);
55
- })
56
- );
57
- ctx.state.data = {
58
- title: '上海大学教务处--' + title,
59
- link: 'http://www.jwc.shu.edu.cn/',
60
- item: all,
61
- };
62
- };
@@ -1 +0,0 @@
1
- {{@ desc }}