rsshub 1.0.0-master.d799646 → 1.0.0-master.d82186b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/lib/middleware/cache/index.js +7 -1
  2. package/lib/router.js +20 -9
  3. package/lib/routes/asiantolick/index.js +64 -0
  4. package/lib/routes/copymanga/comic.js +67 -0
  5. package/lib/routes/coronavirus/yahoo-japan.js +77 -0
  6. package/lib/routes/dekudeals/index.js +1 -1
  7. package/lib/routes/gov/beijing/eea.js +12 -6
  8. package/lib/routes/initium/full.js +5 -7
  9. package/lib/routes/jandan/pic.js +1 -1
  10. package/lib/routes/micmicidol/article.js +28 -0
  11. package/lib/routes/micmicidol/latest.js +3 -0
  12. package/lib/routes/micmicidol/parse.js +22 -0
  13. package/lib/routes/micmicidol/search.js +7 -0
  14. package/lib/routes/netflix/newsroom.js +82 -0
  15. package/lib/routes/sbs/chinese.js +70 -0
  16. package/lib/routes/yuzu-emu/entry.js +1 -1
  17. package/lib/{routes/szse/rule.js → v2/bnu/bs.js} +16 -9
  18. package/lib/v2/bnu/maintainer.js +3 -0
  19. package/lib/v2/bnu/radar.js +13 -0
  20. package/lib/v2/bnu/router.js +3 -0
  21. package/lib/v2/ccac/maintainer.js +3 -0
  22. package/lib/v2/ccac/news.js +47 -0
  23. package/lib/v2/ccac/radar.js +13 -0
  24. package/lib/v2/ccac/router.js +3 -0
  25. package/lib/v2/ccac/utils.js +32 -0
  26. package/lib/v2/cebbank/all.js +48 -0
  27. package/lib/v2/cebbank/history.js +49 -0
  28. package/lib/v2/cebbank/maintainer.js +4 -0
  29. package/lib/v2/cebbank/radar.js +19 -0
  30. package/lib/v2/cebbank/router.js +4 -0
  31. package/lib/v2/cebbank/templates/allDes.art +2 -0
  32. package/lib/v2/cebbank/templates/historyDes.art +3 -0
  33. package/lib/v2/cebbank/utils.js +70 -0
  34. package/lib/v2/furstar/utils.js +1 -1
  35. package/lib/v2/hbr/maintainer.js +3 -0
  36. package/lib/v2/hbr/radar.js +13 -0
  37. package/lib/v2/hbr/router.js +3 -0
  38. package/lib/v2/hbr/topic.js +56 -0
  39. package/lib/v2/hket/index.js +114 -0
  40. package/lib/v2/hket/maintainer.js +3 -0
  41. package/lib/v2/hket/radar.js +45 -0
  42. package/lib/v2/hket/router.js +3 -0
  43. package/lib/v2/hket/templates/description.art +1 -0
  44. package/lib/v2/hket/templates/image.art +1 -0
  45. package/lib/v2/hkjunkcall/index.js +53 -0
  46. package/lib/v2/hkjunkcall/maintainer.js +3 -0
  47. package/lib/v2/hkjunkcall/radar.js +13 -0
  48. package/lib/v2/hkjunkcall/router.js +3 -0
  49. package/lib/v2/hotchina/index.js +55 -0
  50. package/lib/v2/hotchina/maintainer.js +5 -0
  51. package/lib/v2/hotchina/radar.js +25 -0
  52. package/lib/v2/hotchina/router.js +3 -0
  53. package/lib/v2/icac/news.js +2 -3
  54. package/lib/v2/ithome/maintainer.js +3 -0
  55. package/lib/v2/ithome/radar.js +13 -0
  56. package/lib/v2/ithome/router.js +3 -0
  57. package/lib/v2/ithome/zt.js +54 -0
  58. package/lib/v2/lfsyd/home.js +42 -0
  59. package/lib/v2/lfsyd/maintainer.js +7 -0
  60. package/lib/v2/lfsyd/old_home.js +29 -0
  61. package/lib/v2/lfsyd/radar.js +39 -0
  62. package/lib/v2/lfsyd/router.js +6 -0
  63. package/lib/{routes → v2}/lfsyd/tag.js +7 -7
  64. package/lib/v2/lfsyd/templates/card.art +1 -0
  65. package/lib/{routes → v2}/lfsyd/user.js +7 -7
  66. package/lib/{routes → v2}/lfsyd/utils.js +47 -29
  67. package/lib/v2/literotica/category.js +55 -0
  68. package/lib/v2/literotica/maintainer.js +4 -0
  69. package/lib/v2/literotica/new.js +59 -0
  70. package/lib/v2/literotica/radar.js +19 -0
  71. package/lib/v2/literotica/router.js +4 -0
  72. package/lib/v2/netease/maintainer.js +3 -0
  73. package/lib/v2/netease/radar.js +13 -0
  74. package/lib/{routes/netease/news → v2/netease}/rank.js +27 -20
  75. package/lib/v2/netease/router.js +3 -0
  76. package/lib/v2/ngocn2/index.js +56 -0
  77. package/lib/v2/ngocn2/maintainer.js +3 -0
  78. package/lib/v2/ngocn2/radar.js +13 -0
  79. package/lib/v2/ngocn2/router.js +3 -0
  80. package/lib/v2/people/liuyan.js +54 -0
  81. package/lib/v2/people/maintainer.js +3 -0
  82. package/lib/v2/people/radar.js +13 -0
  83. package/lib/v2/people/router.js +3 -0
  84. package/lib/v2/sciencedirect/journal.js +62 -0
  85. package/lib/v2/sciencedirect/maintainer.js +3 -0
  86. package/lib/v2/sciencedirect/radar.js +13 -0
  87. package/lib/v2/sciencedirect/router.js +3 -0
  88. package/lib/v2/secnews/index.js +27 -0
  89. package/lib/v2/secnews/maintainer.js +3 -0
  90. package/lib/v2/secnews/radar.js +13 -0
  91. package/lib/v2/secnews/router.js +3 -0
  92. package/lib/v2/szse/maintainer.js +3 -0
  93. package/lib/v2/szse/radar.js +13 -0
  94. package/lib/v2/szse/router.js +3 -0
  95. package/lib/v2/szse/rule.js +51 -0
  96. package/lib/v2/timednews/maintainer.js +3 -0
  97. package/lib/v2/timednews/news.js +106 -0
  98. package/lib/v2/timednews/radar.js +57 -0
  99. package/lib/v2/timednews/router.js +3 -0
  100. package/lib/v2/wangqiutiyu/anchor.js +42 -0
  101. package/lib/v2/wangqiutiyu/maintainer.js +3 -0
  102. package/lib/v2/wangqiutiyu/radar.js +13 -0
  103. package/lib/v2/wangqiutiyu/router.js +3 -0
  104. package/package.json +1 -1
  105. package/lib/routes/hket/index.js +0 -129
  106. package/lib/routes/lfsyd/index.js +0 -50
@@ -114,7 +114,13 @@ module.exports = function (app) {
114
114
 
115
115
  // Doesn't hit the cache? We need to let others know!
116
116
  await globalCache.set(controlKey, '1', config.cache.requestTimeout);
117
- await next();
117
+
118
+ try {
119
+ await next();
120
+ } catch (e) {
121
+ await globalCache.set(controlKey, '0', config.cache.requestTimeout);
122
+ throw e;
123
+ }
118
124
 
119
125
  if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) {
120
126
  ctx.state.data.lastBuildDate = new Date().toUTCString();
package/lib/router.js CHANGED
@@ -325,11 +325,6 @@ router.get('/nytimes/:lang?', lazyloadRouteHandler('./routes/nytimes/index'));
325
325
  router.get('/3dm/:name/:type', lazyloadRouteHandler('./routes/3dm/game'));
326
326
  router.get('/3dm/news', lazyloadRouteHandler('./routes/3dm/news_center'));
327
327
 
328
- // 旅法师营地
329
- router.get('/lfsyd/:typecode', lazyloadRouteHandler('./routes/lfsyd/index'));
330
- router.get('/lfsyd/user/:id', lazyloadRouteHandler('./routes/lfsyd/user'));
331
- router.get('/lfsyd/tag/:tag', lazyloadRouteHandler('./routes/lfsyd/tag'));
332
-
333
328
  // 喜马拉雅
334
329
  router.get('/ximalaya/album/:id/:all?', lazyloadRouteHandler('./routes/ximalaya/album'));
335
330
  router.get('/ximalaya/album/:id/:all/:shownote?', lazyloadRouteHandler('./routes/ximalaya/album'));
@@ -1137,6 +1132,9 @@ router.get('/manhuagui/comic/:id', lazyloadRouteHandler('./routes/manhuagui/comi
1137
1132
  router.get('/mhgui/comic/:id', lazyloadRouteHandler('./routes/mhgui/comic'));
1138
1133
  router.get('/twmanhuagui/comic/:id', lazyloadRouteHandler('./routes/twmanhuagui/comic'));
1139
1134
 
1135
+ // 拷贝漫画
1136
+ router.get('/copymanga/comic/:id', lazyloadRouteHandler('./routes/copymanga/comic'));
1137
+
1140
1138
  // 動漫狂
1141
1139
  router.get('/cartoonmad/comic/:id', lazyloadRouteHandler('./routes/cartoonmad/comic'));
1142
1140
  // Vol
@@ -2163,7 +2161,6 @@ router.get('/sse/disclosure/:query?', lazyloadRouteHandler('./routes/sse/disclos
2163
2161
  // 深圳证券交易所
2164
2162
  router.get('/szse/notice', lazyloadRouteHandler('./routes/szse/notice'));
2165
2163
  router.get('/szse/inquire/:type', lazyloadRouteHandler('./routes/szse/inquire'));
2166
- router.get('/szse/rule', lazyloadRouteHandler('./routes/szse/rule'));
2167
2164
  router.get('/szse/projectdynamic/:type?/:stage?/:status?', lazyloadRouteHandler('./routes/szse/projectdynamic'));
2168
2165
 
2169
2166
  // 前端艺术家每日整理&&飞冰早报
@@ -2566,6 +2563,7 @@ router.get('/coronavirus/nhc', lazyloadRouteHandler('./routes/coronavirus/nhc'))
2566
2563
  router.get('/coronavirus/mogov-2019ncov/:lang', lazyloadRouteHandler('./routes/coronavirus/mogov-2019ncov'));
2567
2564
  router.get('/coronavirus/qq/fact', lazyloadRouteHandler('./routes/tencent/factcheck'));
2568
2565
  router.get('/coronavirus/sg-moh', lazyloadRouteHandler('./routes/coronavirus/sg-moh'));
2566
+ router.get('/coronavirus/yahoo-japan/:tdfk?', lazyloadRouteHandler('./routes/coronavirus/yahoo-japan'));
2569
2567
 
2570
2568
  // 南京林业大学教务处
2571
2569
  router.get('/njfu/jwc/:category?', lazyloadRouteHandler('./routes/universities/njfu/jwc'));
@@ -3087,7 +3085,7 @@ router.get('/jike/topic/text/:id', lazyloadRouteHandler('./routes/jike/topicText
3087
3085
  router.get('/jike/user/:id', lazyloadRouteHandler('./routes/jike/user'));
3088
3086
 
3089
3087
  // 网易新闻
3090
- router.get('/netease/news/rank/:category?/:type?/:time?', lazyloadRouteHandler('./routes/netease/news/rank'));
3088
+ // router.get('/netease/news/rank/:category?/:type?/:time?', lazyloadRouteHandler('./routes/netease/news/rank'));
3091
3089
  router.get('/netease/news/special/:type?', lazyloadRouteHandler('./routes/netease/news/special'));
3092
3090
 
3093
3091
  // 网易 - 网易号
@@ -4208,8 +4206,12 @@ router.get('/x410/news', lazyloadRouteHandler('./routes/x410/news'));
4208
4206
  // 恩山无线论坛
4209
4207
  router.get('/right/forum/:id?', lazyloadRouteHandler('./routes/right/forum'));
4210
4208
 
4211
- // 香港經濟日報
4212
- router.get('/hket/:category?', lazyloadRouteHandler('./routes/hket/index'));
4209
+ // 香港經濟日報 migrated to v2
4210
+ // router.get('/hket/:category?', lazyloadRouteHandler('./routes/hket/index'));
4211
+
4212
+ // micmicidol
4213
+ router.get('/micmicidol', lazyloadRouteHandler('./routes/micmicidol/latest'));
4214
+ router.get('/micmicidol/search/:label', lazyloadRouteHandler('./routes/micmicidol/search'));
4213
4215
 
4214
4216
  // 香港高登
4215
4217
  router.get('/hkgolden/:id?/:limit?/:sort?', lazyloadRouteHandler('./routes/hkgolden'));
@@ -4226,6 +4228,15 @@ router.get('/now/news/rank', lazyloadRouteHandler('./routes/now/rank'));
4226
4228
  // s-hentai
4227
4229
  router.get('/s-hentai/:id?', lazyloadRouteHandler('./routes/s-hentai'));
4228
4230
 
4231
+ // Netflix
4232
+ router.get('/netflix/newsroom/:category?/:region?', lazyloadRouteHandler('./routes/netflix/newsroom'));
4233
+
4234
+ // SBS
4235
+ router.get('/sbs/chinese/:category?/:id?/:dialect?/:language?', lazyloadRouteHandler('./routes/sbs/chinese'));
4236
+
4237
+ // Asian to lick
4238
+ router.get('/asiantolick/:category?/:keyword?', lazyloadRouteHandler('./routes/asiantolick'));
4239
+
4229
4240
  // Deprecated: DO NOT ADD ROUTE HERE
4230
4241
 
4231
4242
  // Research Gate
@@ -0,0 +1,64 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const { parseDate } = require('@/utils/parse-date');
4
+
5
+ module.exports = async (ctx) => {
6
+ const category = ctx.params.category ?? '';
7
+ const keyword = ctx.params.keyword ?? '';
8
+
9
+ const rootUrl = 'https://asiantolick.com';
10
+ const currentUrl = `${rootUrl}${category === 'category' ? `/category-${keyword}` : ''}${category === 'tag' ? `/tag-${keyword}` : ''}${category === 'search' ? `/search/${keyword}` : ''}`;
11
+
12
+ const response = await got({
13
+ method: 'get',
14
+ url: currentUrl,
15
+ });
16
+
17
+ const $ = cheerio.load(response.data);
18
+
19
+ const list = $('.miniatura')
20
+ .map((_, item) => {
21
+ item = $(item);
22
+
23
+ return {
24
+ link: item.attr('href'),
25
+ title: item.find('.titulo_video').text(),
26
+ };
27
+ })
28
+ .get();
29
+
30
+ const items = await Promise.all(
31
+ list.map((item) =>
32
+ ctx.cache.tryGet(item.link, async () => {
33
+ const detailResponse = await got({
34
+ method: 'get',
35
+ url: item.link,
36
+ });
37
+
38
+ const downloadResponse = await got({
39
+ method: 'get',
40
+ url: `${rootUrl}/ajax/download_post.php?ver=1&dir=/down/3_750&post_id=${item.link.match(/post-\d+\//)[1]}&post_name=${item.title}`,
41
+ });
42
+
43
+ const content = cheerio.load(detailResponse.data);
44
+
45
+ item.enclosure_url = `${rootUrl}/temp_dl/${downloadResponse.data}`;
46
+ item.description = `<p>${content('meta[name="description"]').attr('content')}</p><a href=${item.enclosure_url}>Download ZIP</a>`;
47
+
48
+ content('.gallery_img').each(function () {
49
+ item.description += `<div><img src="${content(this).attr('data-src')}"></div>`;
50
+ });
51
+
52
+ item.pubDate = parseDate(detailResponse.data.match(/"pubDate": "(.*)",/)[1]);
53
+
54
+ return item;
55
+ })
56
+ )
57
+ );
58
+
59
+ ctx.state.data = {
60
+ title: $('title').text(),
61
+ link: currentUrl,
62
+ item: items,
63
+ };
64
+ };
@@ -0,0 +1,67 @@
1
+ const cheerio = require('cheerio');
2
+ const got = require('@/utils/got');
3
+
4
+ // 直接调用拷贝漫画的接口
5
+ module.exports = async (ctx) => {
6
+ const { id } = ctx.params;
7
+
8
+ // 获取漫画列表
9
+ let bHasNextPage = false;
10
+ const iReqLimit = (ctx.queries && ctx.queries.limit) || 100;
11
+ let iReqOffSet = 0;
12
+ const strBaseUrl = `https://api.copymanga.com/api/v3/comic/${id}/group/default/chapters?`;
13
+ let chapterArray = [];
14
+
15
+ do {
16
+ bHasNextPage = false;
17
+ // eslint-disable-next-line no-await-in-loop
18
+ const { data } = await got.get(strBaseUrl + 'limit=' + iReqLimit + '&offset=' + iReqOffSet);
19
+ const { code, results } = data;
20
+
21
+ if (code !== 200) {
22
+ break;
23
+ }
24
+
25
+ if (results.limit + results.offset < results.total) {
26
+ bHasNextPage = true;
27
+ }
28
+ iReqOffSet += iReqLimit;
29
+
30
+ chapterArray = chapterArray.concat(results.list);
31
+ } while (bHasNextPage);
32
+
33
+ chapterArray = chapterArray
34
+ .map(({ comic_path_word, uuid, name, size, datetime_created, index }) => ({
35
+ link: 'https://copymanga.com/comic/' + comic_path_word + '/chapter/' + uuid,
36
+ title: name,
37
+ num: size,
38
+ updateTime: datetime_created,
39
+ index,
40
+ }))
41
+ .reverse();
42
+
43
+ // 获取漫画标题、介绍
44
+ const { bookTitle, bookIntro } = await ctx.cache.tryGet(`https://copymanga.com/comic/${id}`, async () => {
45
+ const { data } = await got.get(`https://copymanga.com/comic/${id}`);
46
+ const $ = cheerio.load(data);
47
+ return {
48
+ bookTitle: $('.comicParticulars-title-right > ul > li > h6').text(),
49
+ bookIntro: $('.intro').text(),
50
+ };
51
+ });
52
+
53
+ const genResult = (chapter) => ({
54
+ link: chapter.link,
55
+ title: chapter.title,
56
+ description: `
57
+ <h1>${chapter.num}p</h1>
58
+ `.trim(),
59
+ });
60
+
61
+ ctx.state.data = {
62
+ title: `拷贝漫画 - ${bookTitle}`,
63
+ link: `https://copymanga.com/comic/${id}`,
64
+ description: bookIntro,
65
+ item: chapterArray.map(genResult),
66
+ };
67
+ };
@@ -0,0 +1,77 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const dayjs = require('dayjs');
4
+
5
+ module.exports = async (ctx) => {
6
+ const tdfk = ctx.params.tdfk || false;
7
+ const uri = tdfk ? `https://news.yahoo.co.jp/pages/article/covid19${tdfk}` : `https://news.yahoo.co.jp/pages/article/20200207`;
8
+ const req_header = { 'User-Agent': 'Mozilla/5.0 (Linux; Android 9; SM-G960F Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.157 Mobile Safari/537.36' };
9
+
10
+ const resp = await got({
11
+ method: 'get',
12
+ url: uri,
13
+ headers: req_header,
14
+ });
15
+
16
+ const $ = cheerio.load(resp.data);
17
+
18
+ const this_year = dayjs().year();
19
+ const this_month = dayjs().month() + 1;
20
+
21
+ const art_uri = [];
22
+ $('#layoutFooter ul.dlpThumbLink a').each((i, e) => {
23
+ const link = $(e).attr('href');
24
+ const text = $(e).find('.dlpThumbText span').eq(0).text();
25
+ const author = $(e).find('.dlpQuote').text();
26
+ const date = $(e).find('.dlpDate').text() + ' +9'; // explicit timezone
27
+
28
+ let date_obj = dayjs(date).year(this_year);
29
+
30
+ if (date_obj.month() + 1 > this_month) {
31
+ // if the article is from the last year
32
+ date_obj = date_obj.year(this_year - 1);
33
+ }
34
+
35
+ art_uri.push({ l: link, t: text, a: author, d: date_obj.toString() });
36
+ });
37
+
38
+ const getNews = async (uri) => {
39
+ const ret = {
40
+ link: uri.l,
41
+ title: uri.t,
42
+ author: uri.a,
43
+ pubDate: uri.d,
44
+ description: null,
45
+ };
46
+
47
+ if (!uri.l.includes('//news.yahoo.co.jp')) {
48
+ return ret; // do not process uncertain pages
49
+ }
50
+
51
+ const page_data = await ctx.cache.tryGet(uri.l, async () => {
52
+ const resp = await got({
53
+ method: 'get',
54
+ url: uri.l,
55
+ headers: req_header,
56
+ });
57
+ return resp.data;
58
+ });
59
+
60
+ const $doc = cheerio.load(page_data);
61
+
62
+ const iso_date = $doc('meta[name="pubdate"]').attr('content');
63
+ ret.pubDate = dayjs(iso_date).toString();
64
+ ret.description = $doc('div.article_body').html() || $doc('meta[name="description"]').attr('content');
65
+
66
+ return ret;
67
+ };
68
+
69
+ const items = await Promise.all(art_uri.map(getNews));
70
+
71
+ ctx.state.data = {
72
+ title: $('title').text(),
73
+ link: uri,
74
+ description: $('meta[name="description"]').attr('content'),
75
+ item: items,
76
+ };
77
+ };
@@ -12,7 +12,7 @@ module.exports = async (ctx) => {
12
12
  const $ = cheerio.load(response.data);
13
13
 
14
14
  const title = $('#search-title').text();
15
- const list = $('.search-main > .item-grid2 > div');
15
+ const list = $('.search-main > .item-grid2 > div.cell');
16
16
 
17
17
  const out = await Promise.all(
18
18
  list
@@ -17,7 +17,7 @@ async function load(link) {
17
17
  // 加载文章内容
18
18
  const $ = cheerio.load(response.data);
19
19
  // 提取文章内容
20
- const description = $('.sidNavText').html();
20
+ const description = $('div.info-txt').html();
21
21
  // 返回解析的结果
22
22
  return { description };
23
23
  }
@@ -37,7 +37,7 @@ module.exports = async (ctx) => {
37
37
  const $ = cheerio.load(response.data);
38
38
 
39
39
  // 获取当前页面的 list
40
- const list = $('div#info>ul>li');
40
+ const list = $('ul.com-list>li');
41
41
 
42
42
  const result = await Promise.all(
43
43
  // 遍历每一篇文章
@@ -57,7 +57,7 @@ module.exports = async (ctx) => {
57
57
 
58
58
  const $title = $('a').text(); // 获取每个的标题
59
59
 
60
- const date_txt = $.text().match(/[1-9]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])/); // 匹配 yyyy-mm-dd格式时间
60
+ const date_txt = $('span').text(); // 匹配 yyyy-mm-dd格式时间
61
61
  const $pubdate = new Date(date_txt); // 正则匹配发布时间 然后转换成时间
62
62
 
63
63
  // 列表上提取到的信息
@@ -67,12 +67,18 @@ module.exports = async (ctx) => {
67
67
  pubDate: $pubdate,
68
68
  link: $item_url,
69
69
  guid: $item_url,
70
+ description: $title,
70
71
  };
71
72
 
72
73
  // 对于列表的每一项, 单独获取 时间与详细内容
73
- const other = $is_interior ? await ctx.cache.tryGet($item_url, () => load($item_url)) : $title;
74
- // 合并解析后的结果集作为该篇文章最终的输出结果
75
- return Promise.resolve(Object.assign({}, single, other));
74
+ try {
75
+ // 文章可能存在404错误,错误则直接使用title当作内容
76
+ const other = $is_interior ? await ctx.cache.tryGet($item_url, () => load($item_url)) : $title;
77
+ // 合并解析后的结果集作为该篇文章最终的输出结果
78
+ return Promise.resolve(Object.assign({}, single, other));
79
+ } catch (e) {
80
+ return Promise.resolve(single);
81
+ }
76
82
  })
77
83
  .get()
78
84
  );
@@ -2,6 +2,7 @@ const got = require('@/utils/got');
2
2
  const config = require('@/config').value;
3
3
  const cheerio = require('cheerio');
4
4
  const resolve_url = require('url').resolve;
5
+ const timezone = require('@/utils/timezone');
5
6
 
6
7
  module.exports = async (ctx) => {
7
8
  // model是channel/tag/etc.,而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话,那就是model是follow,type是articles。
@@ -95,11 +96,6 @@ module.exports = async (ctx) => {
95
96
  // 如果model=author,那就是avatar;否则都是cover,要么就没封面
96
97
  const image = response.data[model] && (response.data[model].cover || response.data[model].avatar);
97
98
 
98
- function fixDate(date) {
99
- // The date is mistakenly declared as UTC, but actually it's UTC+08:00
100
- return date.replace(/Z$/, '+08:00');
101
- }
102
-
103
99
  const getFullText = async (slug) => {
104
100
  let content = await ctx.cache.get(slug + language);
105
101
  if (content) {
@@ -166,6 +162,8 @@ module.exports = async (ctx) => {
166
162
  .filter((a) => a.article)
167
163
  .slice(0, token === 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY=' && key.iapReceipt === undefined ? 10 : articles.length)
168
164
  .map(async (item) => {
165
+ item.article.date = timezone(item.article.date, +8);
166
+ item.article.updated = timezone(item.article.updated, +8);
169
167
  const description = await getFullText(item.article.slug);
170
168
  return {
171
169
  title: item.article.headline,
@@ -173,8 +171,8 @@ module.exports = async (ctx) => {
173
171
  category: item.article.channels.filter((x) => !x.homepage).map((x) => x.name),
174
172
  description,
175
173
  link: resolve_url('https://theinitium.com', item.article.url),
176
- pubDate: new Date(fixDate(item.article.date)),
177
- updated: new Date(fixDate(item.article.updated)),
174
+ pubDate: item.article.date,
175
+ updated: item.article.updated,
178
176
  // 如果遇到编辑部漏录入情况,则给uuid做个手脚,以便阅读器到时重抓。
179
177
  guid: description.endsWith('内容为空,请稍后再来') ? item.article.uuid + '-I-am-empty' : item.article.uuid,
180
178
  };
@@ -81,7 +81,7 @@ module.exports = async (ctx) => {
81
81
  author,
82
82
  pubDate: parseRelativeDate(timeInfo),
83
83
  guid: id,
84
- description: comment.find('.text > p').html(),
84
+ description: comment.find('.text > p').html().replace('thumb180', 'large').replace('mw1024', 'large'),
85
85
  };
86
86
  })
87
87
  .get(),
@@ -0,0 +1,28 @@
1
+ const Cheerio = require('cheerio');
2
+ const got = require('@/utils/got');
3
+ const { parseDate } = require('@/utils/parse-date');
4
+ const regStripAnchor = /(?:<br\/?> *)?<a name="more"\/?>(?:<\/a>)(?: *<br\/?>)?/i;
5
+
6
+ async function loadArticle(link) {
7
+ const resp = await got(link);
8
+ const $ = Cheerio.load(resp.body);
9
+ const matchResult = $('#post-header-top > script')
10
+ .html()
11
+ .match(/var timestamp="\w+,(.+)";/i)[1]
12
+ .trim();
13
+ // Ex. "August 20, 2021"
14
+ const pubDate = parseDate(matchResult + '+0000', 'MMMM D, YYYYZ');
15
+
16
+ const labels = $('.post-labels a')
17
+ .text()
18
+ .split('\n')
19
+ .filter((item) => item !== '');
20
+ return {
21
+ title: $('.entry-title').text().trim(),
22
+ description: $('.post-body.entry-content').html().replace(regStripAnchor, ''),
23
+ category: labels,
24
+ pubDate,
25
+ link,
26
+ };
27
+ }
28
+ module.exports = loadArticle;
@@ -0,0 +1,3 @@
1
+ const parse = require('./parse');
2
+
3
+ module.exports = (ctx) => parse(ctx, '', 'Latest');
@@ -0,0 +1,22 @@
1
+ const got = require('@/utils/got');
2
+ const Cheerio = require('cheerio');
3
+ const loadArticle = require('./article');
4
+
5
+ module.exports = async (ctx, urlParam, title) => {
6
+ const link = new URL(urlParam, 'https://www.micmicidol.com/').toString();
7
+ const response = await got(link);
8
+ ctx.state.data = {
9
+ title: `micmicidol-${title}`,
10
+ link,
11
+ item:
12
+ response.body &&
13
+ (await Promise.all(
14
+ Cheerio.load(response.body)('.post.hentry')
15
+ .map((_, entry) => {
16
+ const { href } = Cheerio.load(entry)('.post-title.entry-title a')[0].attribs;
17
+ return ctx.cache.tryGet(href, () => loadArticle(href));
18
+ })
19
+ .toArray()
20
+ )),
21
+ };
22
+ };
@@ -0,0 +1,7 @@
1
+ const parse = require('./parse');
2
+
3
+ module.exports = (ctx) => {
4
+ const max = ctx.query.limit || 50;
5
+ const { label } = ctx.params;
6
+ return parse(ctx, `search/label/${label}?max-results=${max}`, label);
7
+ };
@@ -0,0 +1,82 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const { parseDate } = require('@/utils/parse-date');
4
+
5
+ const languages = {
6
+ zh_cn: 'zh-hans',
7
+ zh_tw: 'zh-hant',
8
+ };
9
+
10
+ const categories = {
11
+ all: {
12
+ title: 'All News',
13
+ id: '0',
14
+ },
15
+ business: {
16
+ title: 'Business',
17
+ id: '1GnkLu7bxeOTxTRNCeu5qm',
18
+ },
19
+ entertainment: {
20
+ title: 'Entertainment',
21
+ id: '3SGbaxYYG5U05Z0G4piPV7',
22
+ },
23
+ innovation: {
24
+ title: 'Innovation',
25
+ id: '5TzuQELMABTu9jOPjXXlFU',
26
+ },
27
+ brazil: {
28
+ title: 'Made in Brazil',
29
+ id: '2tOmcnQB8PgkQSoQ1K4hfD',
30
+ },
31
+ impact: {
32
+ title: 'Social Impact',
33
+ id: '2bUcGjE2800LAsk3JDurGA',
34
+ },
35
+ };
36
+
37
+ module.exports = async (ctx) => {
38
+ const category = ctx.params.category ?? 'all';
39
+ const region = ctx.params.region ?? 'en';
40
+
41
+ const rootUrl = 'https://about.netflix.com';
42
+ const currentUrl = `${rootUrl}/api/data/articles?language=${languages.hasOwnProperty(region) ? languages[region] : region.replace(/_/g, '-')}${category === 'all' ? '' : `&category=${categories[category].id}`}`;
43
+
44
+ const response = await got({
45
+ method: 'get',
46
+ url: currentUrl,
47
+ });
48
+
49
+ const list = response.data.entities.articles.slice(0, ctx.params.limit ? parseInt(ctx.params.limit) : 15).map((item) => ({
50
+ title: item.title,
51
+ link: `${rootUrl}/${region}/news/${item.slug}`,
52
+ pubDate: parseDate(item.rawPublishedDate),
53
+ category: item.categories.map((category) => category.label),
54
+ }));
55
+
56
+ const items = await Promise.all(
57
+ list.map((item) =>
58
+ ctx.cache.tryGet(item.link, async () => {
59
+ const detailResponse = await got({
60
+ method: 'get',
61
+ url: item.link,
62
+ });
63
+
64
+ const content = cheerio.load(detailResponse.data);
65
+
66
+ if (!content('.article-contentstyles__ArticleCopy-pei0rm-6 ul li')) {
67
+ content('.article-contentstyles__ArticleCopy-pei0rm-6 p').slice(-3).remove();
68
+ }
69
+
70
+ item.description = content('.article-contentstyles__ArticleCopy-pei0rm-6').html();
71
+
72
+ return item;
73
+ })
74
+ )
75
+ );
76
+
77
+ ctx.state.data = {
78
+ title: `${categories[category].title} - Newsroom - Netflix`,
79
+ link: `${rootUrl}/${region}/newsroom`,
80
+ item: items,
81
+ };
82
+ };
@@ -0,0 +1,70 @@
1
+ const got = require('@/utils/got');
2
+ const cheerio = require('cheerio');
3
+ const { parseDate } = require('@/utils/parse-date');
4
+
5
+ module.exports = async (ctx) => {
6
+ const id = ctx.params.id ?? 'news';
7
+ const dialect = ctx.params.dialect ?? 'mandarin';
8
+ const category = ctx.params.category ?? 'news';
9
+ const language = ctx.params.language ?? dialect === 'mandarin' ? 'zh-hans' : 'zh-hant';
10
+
11
+ const rootUrl = 'https://www.sbs.com.au';
12
+ const currentUrl = `${rootUrl}/language/${dialect}/${category === 'podcast' ? `${language}/podcast/${id}` : `${id}/${language}`}`;
13
+
14
+ const response = await got({
15
+ method: 'get',
16
+ url: currentUrl,
17
+ });
18
+
19
+ const $ = cheerio.load(response.data);
20
+
21
+ const list = $('.headline, .audio-playlist__item-more')
22
+ .find('a')
23
+ .map((_, item) => {
24
+ item = $(item);
25
+
26
+ return {
27
+ link: `${rootUrl}${item.attr('href')}`,
28
+ };
29
+ })
30
+ .get();
31
+
32
+ const items = await Promise.all(
33
+ list.map((item) =>
34
+ ctx.cache.tryGet(item.link, async () => {
35
+ const detailResponse = await got({
36
+ method: 'get',
37
+ url: item.link,
38
+ });
39
+
40
+ const content = cheerio.load(detailResponse.data);
41
+
42
+ content('.block__container').remove();
43
+
44
+ const audio = content('a[download]')?.attr('href') ?? '';
45
+ const image = content('meta[itemprop="image"]')?.attr('content').split('/crop/')[0] ?? '';
46
+
47
+ item.title = content('meta[property="og:title"]').attr('content');
48
+ item.author = content('.article__meta-author').text().trim().replace('By', '');
49
+ item.pubDate = parseDate(content('meta[itemprop="datePublished"]').attr('content'));
50
+ item.description = `${image ? `<img src="${image}"><br>` : ''}${audio ? `<audio src="${audio}" controls></audio><br>` : ''}${content('.text-body').html()}`;
51
+
52
+ if (audio) {
53
+ item.itunes_item_image = image;
54
+ item.enclosure_url = audio;
55
+ item.enclosure_type = 'audio/mpeg';
56
+ }
57
+
58
+ return item;
59
+ })
60
+ )
61
+ );
62
+
63
+ ctx.state.data = {
64
+ title: `${$('meta[property="og:title"]').attr('content')} - SBS中文`,
65
+ link: currentUrl,
66
+ item: items,
67
+ itunes_author: 'SBS中文',
68
+ image: `${rootUrl}/language/img/logos/in-language/logo_chinese.svg`,
69
+ };
70
+ };
@@ -30,7 +30,7 @@ module.exports = async (ctx) => {
30
30
  method: 'get',
31
31
  url: item.link,
32
32
  });
33
- const content = cheerio.load(detailResponse.data);
33
+ const content = cheerio.load(detailResponse.data.replace(/<source src="\.\//g, `<source src="${item.link}`));
34
34
 
35
35
  const meta = content('.h3').text().trim().split(' on ');
36
36