rsshub 1.0.0-master.d799646 → 1.0.0-master.d82186b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/middleware/cache/index.js +7 -1
- package/lib/router.js +20 -9
- package/lib/routes/asiantolick/index.js +64 -0
- package/lib/routes/copymanga/comic.js +67 -0
- package/lib/routes/coronavirus/yahoo-japan.js +77 -0
- package/lib/routes/dekudeals/index.js +1 -1
- package/lib/routes/gov/beijing/eea.js +12 -6
- package/lib/routes/initium/full.js +5 -7
- package/lib/routes/jandan/pic.js +1 -1
- package/lib/routes/micmicidol/article.js +28 -0
- package/lib/routes/micmicidol/latest.js +3 -0
- package/lib/routes/micmicidol/parse.js +22 -0
- package/lib/routes/micmicidol/search.js +7 -0
- package/lib/routes/netflix/newsroom.js +82 -0
- package/lib/routes/sbs/chinese.js +70 -0
- package/lib/routes/yuzu-emu/entry.js +1 -1
- package/lib/{routes/szse/rule.js → v2/bnu/bs.js} +16 -9
- package/lib/v2/bnu/maintainer.js +3 -0
- package/lib/v2/bnu/radar.js +13 -0
- package/lib/v2/bnu/router.js +3 -0
- package/lib/v2/ccac/maintainer.js +3 -0
- package/lib/v2/ccac/news.js +47 -0
- package/lib/v2/ccac/radar.js +13 -0
- package/lib/v2/ccac/router.js +3 -0
- package/lib/v2/ccac/utils.js +32 -0
- package/lib/v2/cebbank/all.js +48 -0
- package/lib/v2/cebbank/history.js +49 -0
- package/lib/v2/cebbank/maintainer.js +4 -0
- package/lib/v2/cebbank/radar.js +19 -0
- package/lib/v2/cebbank/router.js +4 -0
- package/lib/v2/cebbank/templates/allDes.art +2 -0
- package/lib/v2/cebbank/templates/historyDes.art +3 -0
- package/lib/v2/cebbank/utils.js +70 -0
- package/lib/v2/furstar/utils.js +1 -1
- package/lib/v2/hbr/maintainer.js +3 -0
- package/lib/v2/hbr/radar.js +13 -0
- package/lib/v2/hbr/router.js +3 -0
- package/lib/v2/hbr/topic.js +56 -0
- package/lib/v2/hket/index.js +114 -0
- package/lib/v2/hket/maintainer.js +3 -0
- package/lib/v2/hket/radar.js +45 -0
- package/lib/v2/hket/router.js +3 -0
- package/lib/v2/hket/templates/description.art +1 -0
- package/lib/v2/hket/templates/image.art +1 -0
- package/lib/v2/hkjunkcall/index.js +53 -0
- package/lib/v2/hkjunkcall/maintainer.js +3 -0
- package/lib/v2/hkjunkcall/radar.js +13 -0
- package/lib/v2/hkjunkcall/router.js +3 -0
- package/lib/v2/hotchina/index.js +55 -0
- package/lib/v2/hotchina/maintainer.js +5 -0
- package/lib/v2/hotchina/radar.js +25 -0
- package/lib/v2/hotchina/router.js +3 -0
- package/lib/v2/icac/news.js +2 -3
- package/lib/v2/ithome/maintainer.js +3 -0
- package/lib/v2/ithome/radar.js +13 -0
- package/lib/v2/ithome/router.js +3 -0
- package/lib/v2/ithome/zt.js +54 -0
- package/lib/v2/lfsyd/home.js +42 -0
- package/lib/v2/lfsyd/maintainer.js +7 -0
- package/lib/v2/lfsyd/old_home.js +29 -0
- package/lib/v2/lfsyd/radar.js +39 -0
- package/lib/v2/lfsyd/router.js +6 -0
- package/lib/{routes → v2}/lfsyd/tag.js +7 -7
- package/lib/v2/lfsyd/templates/card.art +1 -0
- package/lib/{routes → v2}/lfsyd/user.js +7 -7
- package/lib/{routes → v2}/lfsyd/utils.js +47 -29
- package/lib/v2/literotica/category.js +55 -0
- package/lib/v2/literotica/maintainer.js +4 -0
- package/lib/v2/literotica/new.js +59 -0
- package/lib/v2/literotica/radar.js +19 -0
- package/lib/v2/literotica/router.js +4 -0
- package/lib/v2/netease/maintainer.js +3 -0
- package/lib/v2/netease/radar.js +13 -0
- package/lib/{routes/netease/news → v2/netease}/rank.js +27 -20
- package/lib/v2/netease/router.js +3 -0
- package/lib/v2/ngocn2/index.js +56 -0
- package/lib/v2/ngocn2/maintainer.js +3 -0
- package/lib/v2/ngocn2/radar.js +13 -0
- package/lib/v2/ngocn2/router.js +3 -0
- package/lib/v2/people/liuyan.js +54 -0
- package/lib/v2/people/maintainer.js +3 -0
- package/lib/v2/people/radar.js +13 -0
- package/lib/v2/people/router.js +3 -0
- package/lib/v2/sciencedirect/journal.js +62 -0
- package/lib/v2/sciencedirect/maintainer.js +3 -0
- package/lib/v2/sciencedirect/radar.js +13 -0
- package/lib/v2/sciencedirect/router.js +3 -0
- package/lib/v2/secnews/index.js +27 -0
- package/lib/v2/secnews/maintainer.js +3 -0
- package/lib/v2/secnews/radar.js +13 -0
- package/lib/v2/secnews/router.js +3 -0
- package/lib/v2/szse/maintainer.js +3 -0
- package/lib/v2/szse/radar.js +13 -0
- package/lib/v2/szse/router.js +3 -0
- package/lib/v2/szse/rule.js +51 -0
- package/lib/v2/timednews/maintainer.js +3 -0
- package/lib/v2/timednews/news.js +106 -0
- package/lib/v2/timednews/radar.js +57 -0
- package/lib/v2/timednews/router.js +3 -0
- package/lib/v2/wangqiutiyu/anchor.js +42 -0
- package/lib/v2/wangqiutiyu/maintainer.js +3 -0
- package/lib/v2/wangqiutiyu/radar.js +13 -0
- package/lib/v2/wangqiutiyu/router.js +3 -0
- package/package.json +1 -1
- package/lib/routes/hket/index.js +0 -129
- package/lib/routes/lfsyd/index.js +0 -50
|
@@ -114,7 +114,13 @@ module.exports = function (app) {
|
|
|
114
114
|
|
|
115
115
|
// Doesn't hit the cache? We need to let others know!
|
|
116
116
|
await globalCache.set(controlKey, '1', config.cache.requestTimeout);
|
|
117
|
-
|
|
117
|
+
|
|
118
|
+
try {
|
|
119
|
+
await next();
|
|
120
|
+
} catch (e) {
|
|
121
|
+
await globalCache.set(controlKey, '0', config.cache.requestTimeout);
|
|
122
|
+
throw e;
|
|
123
|
+
}
|
|
118
124
|
|
|
119
125
|
if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) {
|
|
120
126
|
ctx.state.data.lastBuildDate = new Date().toUTCString();
|
package/lib/router.js
CHANGED
|
@@ -325,11 +325,6 @@ router.get('/nytimes/:lang?', lazyloadRouteHandler('./routes/nytimes/index'));
|
|
|
325
325
|
router.get('/3dm/:name/:type', lazyloadRouteHandler('./routes/3dm/game'));
|
|
326
326
|
router.get('/3dm/news', lazyloadRouteHandler('./routes/3dm/news_center'));
|
|
327
327
|
|
|
328
|
-
// 旅法师营地
|
|
329
|
-
router.get('/lfsyd/:typecode', lazyloadRouteHandler('./routes/lfsyd/index'));
|
|
330
|
-
router.get('/lfsyd/user/:id', lazyloadRouteHandler('./routes/lfsyd/user'));
|
|
331
|
-
router.get('/lfsyd/tag/:tag', lazyloadRouteHandler('./routes/lfsyd/tag'));
|
|
332
|
-
|
|
333
328
|
// 喜马拉雅
|
|
334
329
|
router.get('/ximalaya/album/:id/:all?', lazyloadRouteHandler('./routes/ximalaya/album'));
|
|
335
330
|
router.get('/ximalaya/album/:id/:all/:shownote?', lazyloadRouteHandler('./routes/ximalaya/album'));
|
|
@@ -1137,6 +1132,9 @@ router.get('/manhuagui/comic/:id', lazyloadRouteHandler('./routes/manhuagui/comi
|
|
|
1137
1132
|
router.get('/mhgui/comic/:id', lazyloadRouteHandler('./routes/mhgui/comic'));
|
|
1138
1133
|
router.get('/twmanhuagui/comic/:id', lazyloadRouteHandler('./routes/twmanhuagui/comic'));
|
|
1139
1134
|
|
|
1135
|
+
// 拷贝漫画
|
|
1136
|
+
router.get('/copymanga/comic/:id', lazyloadRouteHandler('./routes/copymanga/comic'));
|
|
1137
|
+
|
|
1140
1138
|
// 動漫狂
|
|
1141
1139
|
router.get('/cartoonmad/comic/:id', lazyloadRouteHandler('./routes/cartoonmad/comic'));
|
|
1142
1140
|
// Vol
|
|
@@ -2163,7 +2161,6 @@ router.get('/sse/disclosure/:query?', lazyloadRouteHandler('./routes/sse/disclos
|
|
|
2163
2161
|
// 深圳证券交易所
|
|
2164
2162
|
router.get('/szse/notice', lazyloadRouteHandler('./routes/szse/notice'));
|
|
2165
2163
|
router.get('/szse/inquire/:type', lazyloadRouteHandler('./routes/szse/inquire'));
|
|
2166
|
-
router.get('/szse/rule', lazyloadRouteHandler('./routes/szse/rule'));
|
|
2167
2164
|
router.get('/szse/projectdynamic/:type?/:stage?/:status?', lazyloadRouteHandler('./routes/szse/projectdynamic'));
|
|
2168
2165
|
|
|
2169
2166
|
// 前端艺术家每日整理&&飞冰早报
|
|
@@ -2566,6 +2563,7 @@ router.get('/coronavirus/nhc', lazyloadRouteHandler('./routes/coronavirus/nhc'))
|
|
|
2566
2563
|
router.get('/coronavirus/mogov-2019ncov/:lang', lazyloadRouteHandler('./routes/coronavirus/mogov-2019ncov'));
|
|
2567
2564
|
router.get('/coronavirus/qq/fact', lazyloadRouteHandler('./routes/tencent/factcheck'));
|
|
2568
2565
|
router.get('/coronavirus/sg-moh', lazyloadRouteHandler('./routes/coronavirus/sg-moh'));
|
|
2566
|
+
router.get('/coronavirus/yahoo-japan/:tdfk?', lazyloadRouteHandler('./routes/coronavirus/yahoo-japan'));
|
|
2569
2567
|
|
|
2570
2568
|
// 南京林业大学教务处
|
|
2571
2569
|
router.get('/njfu/jwc/:category?', lazyloadRouteHandler('./routes/universities/njfu/jwc'));
|
|
@@ -3087,7 +3085,7 @@ router.get('/jike/topic/text/:id', lazyloadRouteHandler('./routes/jike/topicText
|
|
|
3087
3085
|
router.get('/jike/user/:id', lazyloadRouteHandler('./routes/jike/user'));
|
|
3088
3086
|
|
|
3089
3087
|
// 网易新闻
|
|
3090
|
-
router.get('/netease/news/rank/:category?/:type?/:time?', lazyloadRouteHandler('./routes/netease/news/rank'));
|
|
3088
|
+
// router.get('/netease/news/rank/:category?/:type?/:time?', lazyloadRouteHandler('./routes/netease/news/rank'));
|
|
3091
3089
|
router.get('/netease/news/special/:type?', lazyloadRouteHandler('./routes/netease/news/special'));
|
|
3092
3090
|
|
|
3093
3091
|
// 网易 - 网易号
|
|
@@ -4208,8 +4206,12 @@ router.get('/x410/news', lazyloadRouteHandler('./routes/x410/news'));
|
|
|
4208
4206
|
// 恩山无线论坛
|
|
4209
4207
|
router.get('/right/forum/:id?', lazyloadRouteHandler('./routes/right/forum'));
|
|
4210
4208
|
|
|
4211
|
-
// 香港經濟日報
|
|
4212
|
-
router.get('/hket/:category?', lazyloadRouteHandler('./routes/hket/index'));
|
|
4209
|
+
// 香港經濟日報 migrated to v2
|
|
4210
|
+
// router.get('/hket/:category?', lazyloadRouteHandler('./routes/hket/index'));
|
|
4211
|
+
|
|
4212
|
+
// micmicidol
|
|
4213
|
+
router.get('/micmicidol', lazyloadRouteHandler('./routes/micmicidol/latest'));
|
|
4214
|
+
router.get('/micmicidol/search/:label', lazyloadRouteHandler('./routes/micmicidol/search'));
|
|
4213
4215
|
|
|
4214
4216
|
// 香港高登
|
|
4215
4217
|
router.get('/hkgolden/:id?/:limit?/:sort?', lazyloadRouteHandler('./routes/hkgolden'));
|
|
@@ -4226,6 +4228,15 @@ router.get('/now/news/rank', lazyloadRouteHandler('./routes/now/rank'));
|
|
|
4226
4228
|
// s-hentai
|
|
4227
4229
|
router.get('/s-hentai/:id?', lazyloadRouteHandler('./routes/s-hentai'));
|
|
4228
4230
|
|
|
4231
|
+
// Netflix
|
|
4232
|
+
router.get('/netflix/newsroom/:category?/:region?', lazyloadRouteHandler('./routes/netflix/newsroom'));
|
|
4233
|
+
|
|
4234
|
+
// SBS
|
|
4235
|
+
router.get('/sbs/chinese/:category?/:id?/:dialect?/:language?', lazyloadRouteHandler('./routes/sbs/chinese'));
|
|
4236
|
+
|
|
4237
|
+
// Asian to lick
|
|
4238
|
+
router.get('/asiantolick/:category?/:keyword?', lazyloadRouteHandler('./routes/asiantolick'));
|
|
4239
|
+
|
|
4229
4240
|
// Deprecated: DO NOT ADD ROUTE HERE
|
|
4230
4241
|
|
|
4231
4242
|
// Research Gate
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const cheerio = require('cheerio');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
|
|
5
|
+
module.exports = async (ctx) => {
|
|
6
|
+
const category = ctx.params.category ?? '';
|
|
7
|
+
const keyword = ctx.params.keyword ?? '';
|
|
8
|
+
|
|
9
|
+
const rootUrl = 'https://asiantolick.com';
|
|
10
|
+
const currentUrl = `${rootUrl}${category === 'category' ? `/category-${keyword}` : ''}${category === 'tag' ? `/tag-${keyword}` : ''}${category === 'search' ? `/search/${keyword}` : ''}`;
|
|
11
|
+
|
|
12
|
+
const response = await got({
|
|
13
|
+
method: 'get',
|
|
14
|
+
url: currentUrl,
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
const $ = cheerio.load(response.data);
|
|
18
|
+
|
|
19
|
+
const list = $('.miniatura')
|
|
20
|
+
.map((_, item) => {
|
|
21
|
+
item = $(item);
|
|
22
|
+
|
|
23
|
+
return {
|
|
24
|
+
link: item.attr('href'),
|
|
25
|
+
title: item.find('.titulo_video').text(),
|
|
26
|
+
};
|
|
27
|
+
})
|
|
28
|
+
.get();
|
|
29
|
+
|
|
30
|
+
const items = await Promise.all(
|
|
31
|
+
list.map((item) =>
|
|
32
|
+
ctx.cache.tryGet(item.link, async () => {
|
|
33
|
+
const detailResponse = await got({
|
|
34
|
+
method: 'get',
|
|
35
|
+
url: item.link,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const downloadResponse = await got({
|
|
39
|
+
method: 'get',
|
|
40
|
+
url: `${rootUrl}/ajax/download_post.php?ver=1&dir=/down/3_750&post_id=${item.link.match(/post-\d+\//)[1]}&post_name=${item.title}`,
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const content = cheerio.load(detailResponse.data);
|
|
44
|
+
|
|
45
|
+
item.enclosure_url = `${rootUrl}/temp_dl/${downloadResponse.data}`;
|
|
46
|
+
item.description = `<p>${content('meta[name="description"]').attr('content')}</p><a href=${item.enclosure_url}>Download ZIP</a>`;
|
|
47
|
+
|
|
48
|
+
content('.gallery_img').each(function () {
|
|
49
|
+
item.description += `<div><img src="${content(this).attr('data-src')}"></div>`;
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
item.pubDate = parseDate(detailResponse.data.match(/"pubDate": "(.*)",/)[1]);
|
|
53
|
+
|
|
54
|
+
return item;
|
|
55
|
+
})
|
|
56
|
+
)
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
ctx.state.data = {
|
|
60
|
+
title: $('title').text(),
|
|
61
|
+
link: currentUrl,
|
|
62
|
+
item: items,
|
|
63
|
+
};
|
|
64
|
+
};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
const cheerio = require('cheerio');
|
|
2
|
+
const got = require('@/utils/got');
|
|
3
|
+
|
|
4
|
+
// 直接调用拷贝漫画的接口
|
|
5
|
+
module.exports = async (ctx) => {
|
|
6
|
+
const { id } = ctx.params;
|
|
7
|
+
|
|
8
|
+
// 获取漫画列表
|
|
9
|
+
let bHasNextPage = false;
|
|
10
|
+
const iReqLimit = (ctx.queries && ctx.queries.limit) || 100;
|
|
11
|
+
let iReqOffSet = 0;
|
|
12
|
+
const strBaseUrl = `https://api.copymanga.com/api/v3/comic/${id}/group/default/chapters?`;
|
|
13
|
+
let chapterArray = [];
|
|
14
|
+
|
|
15
|
+
do {
|
|
16
|
+
bHasNextPage = false;
|
|
17
|
+
// eslint-disable-next-line no-await-in-loop
|
|
18
|
+
const { data } = await got.get(strBaseUrl + 'limit=' + iReqLimit + '&offset=' + iReqOffSet);
|
|
19
|
+
const { code, results } = data;
|
|
20
|
+
|
|
21
|
+
if (code !== 200) {
|
|
22
|
+
break;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (results.limit + results.offset < results.total) {
|
|
26
|
+
bHasNextPage = true;
|
|
27
|
+
}
|
|
28
|
+
iReqOffSet += iReqLimit;
|
|
29
|
+
|
|
30
|
+
chapterArray = chapterArray.concat(results.list);
|
|
31
|
+
} while (bHasNextPage);
|
|
32
|
+
|
|
33
|
+
chapterArray = chapterArray
|
|
34
|
+
.map(({ comic_path_word, uuid, name, size, datetime_created, index }) => ({
|
|
35
|
+
link: 'https://copymanga.com/comic/' + comic_path_word + '/chapter/' + uuid,
|
|
36
|
+
title: name,
|
|
37
|
+
num: size,
|
|
38
|
+
updateTime: datetime_created,
|
|
39
|
+
index,
|
|
40
|
+
}))
|
|
41
|
+
.reverse();
|
|
42
|
+
|
|
43
|
+
// 获取漫画标题、介绍
|
|
44
|
+
const { bookTitle, bookIntro } = await ctx.cache.tryGet(`https://copymanga.com/comic/${id}`, async () => {
|
|
45
|
+
const { data } = await got.get(`https://copymanga.com/comic/${id}`);
|
|
46
|
+
const $ = cheerio.load(data);
|
|
47
|
+
return {
|
|
48
|
+
bookTitle: $('.comicParticulars-title-right > ul > li > h6').text(),
|
|
49
|
+
bookIntro: $('.intro').text(),
|
|
50
|
+
};
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
const genResult = (chapter) => ({
|
|
54
|
+
link: chapter.link,
|
|
55
|
+
title: chapter.title,
|
|
56
|
+
description: `
|
|
57
|
+
<h1>${chapter.num}p</h1>
|
|
58
|
+
`.trim(),
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
ctx.state.data = {
|
|
62
|
+
title: `拷贝漫画 - ${bookTitle}`,
|
|
63
|
+
link: `https://copymanga.com/comic/${id}`,
|
|
64
|
+
description: bookIntro,
|
|
65
|
+
item: chapterArray.map(genResult),
|
|
66
|
+
};
|
|
67
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const cheerio = require('cheerio');
|
|
3
|
+
const dayjs = require('dayjs');
|
|
4
|
+
|
|
5
|
+
module.exports = async (ctx) => {
|
|
6
|
+
const tdfk = ctx.params.tdfk || false;
|
|
7
|
+
const uri = tdfk ? `https://news.yahoo.co.jp/pages/article/covid19${tdfk}` : `https://news.yahoo.co.jp/pages/article/20200207`;
|
|
8
|
+
const req_header = { 'User-Agent': 'Mozilla/5.0 (Linux; Android 9; SM-G960F Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.157 Mobile Safari/537.36' };
|
|
9
|
+
|
|
10
|
+
const resp = await got({
|
|
11
|
+
method: 'get',
|
|
12
|
+
url: uri,
|
|
13
|
+
headers: req_header,
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
const $ = cheerio.load(resp.data);
|
|
17
|
+
|
|
18
|
+
const this_year = dayjs().year();
|
|
19
|
+
const this_month = dayjs().month() + 1;
|
|
20
|
+
|
|
21
|
+
const art_uri = [];
|
|
22
|
+
$('#layoutFooter ul.dlpThumbLink a').each((i, e) => {
|
|
23
|
+
const link = $(e).attr('href');
|
|
24
|
+
const text = $(e).find('.dlpThumbText span').eq(0).text();
|
|
25
|
+
const author = $(e).find('.dlpQuote').text();
|
|
26
|
+
const date = $(e).find('.dlpDate').text() + ' +9'; // explicit timezone
|
|
27
|
+
|
|
28
|
+
let date_obj = dayjs(date).year(this_year);
|
|
29
|
+
|
|
30
|
+
if (date_obj.month() + 1 > this_month) {
|
|
31
|
+
// if the article is from the last year
|
|
32
|
+
date_obj = date_obj.year(this_year - 1);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
art_uri.push({ l: link, t: text, a: author, d: date_obj.toString() });
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const getNews = async (uri) => {
|
|
39
|
+
const ret = {
|
|
40
|
+
link: uri.l,
|
|
41
|
+
title: uri.t,
|
|
42
|
+
author: uri.a,
|
|
43
|
+
pubDate: uri.d,
|
|
44
|
+
description: null,
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
if (!uri.l.includes('//news.yahoo.co.jp')) {
|
|
48
|
+
return ret; // do not process uncertain pages
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const page_data = await ctx.cache.tryGet(uri.l, async () => {
|
|
52
|
+
const resp = await got({
|
|
53
|
+
method: 'get',
|
|
54
|
+
url: uri.l,
|
|
55
|
+
headers: req_header,
|
|
56
|
+
});
|
|
57
|
+
return resp.data;
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
const $doc = cheerio.load(page_data);
|
|
61
|
+
|
|
62
|
+
const iso_date = $doc('meta[name="pubdate"]').attr('content');
|
|
63
|
+
ret.pubDate = dayjs(iso_date).toString();
|
|
64
|
+
ret.description = $doc('div.article_body').html() || $doc('meta[name="description"]').attr('content');
|
|
65
|
+
|
|
66
|
+
return ret;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const items = await Promise.all(art_uri.map(getNews));
|
|
70
|
+
|
|
71
|
+
ctx.state.data = {
|
|
72
|
+
title: $('title').text(),
|
|
73
|
+
link: uri,
|
|
74
|
+
description: $('meta[name="description"]').attr('content'),
|
|
75
|
+
item: items,
|
|
76
|
+
};
|
|
77
|
+
};
|
|
@@ -12,7 +12,7 @@ module.exports = async (ctx) => {
|
|
|
12
12
|
const $ = cheerio.load(response.data);
|
|
13
13
|
|
|
14
14
|
const title = $('#search-title').text();
|
|
15
|
-
const list = $('.search-main > .item-grid2 > div');
|
|
15
|
+
const list = $('.search-main > .item-grid2 > div.cell');
|
|
16
16
|
|
|
17
17
|
const out = await Promise.all(
|
|
18
18
|
list
|
|
@@ -17,7 +17,7 @@ async function load(link) {
|
|
|
17
17
|
// 加载文章内容
|
|
18
18
|
const $ = cheerio.load(response.data);
|
|
19
19
|
// 提取文章内容
|
|
20
|
-
const description = $('.
|
|
20
|
+
const description = $('div.info-txt').html();
|
|
21
21
|
// 返回解析的结果
|
|
22
22
|
return { description };
|
|
23
23
|
}
|
|
@@ -37,7 +37,7 @@ module.exports = async (ctx) => {
|
|
|
37
37
|
const $ = cheerio.load(response.data);
|
|
38
38
|
|
|
39
39
|
// 获取当前页面的 list
|
|
40
|
-
const list = $('
|
|
40
|
+
const list = $('ul.com-list>li');
|
|
41
41
|
|
|
42
42
|
const result = await Promise.all(
|
|
43
43
|
// 遍历每一篇文章
|
|
@@ -57,7 +57,7 @@ module.exports = async (ctx) => {
|
|
|
57
57
|
|
|
58
58
|
const $title = $('a').text(); // 获取每个的标题
|
|
59
59
|
|
|
60
|
-
const date_txt =
|
|
60
|
+
const date_txt = $('span').text(); // 匹配 yyyy-mm-dd格式时间
|
|
61
61
|
const $pubdate = new Date(date_txt); // 正则匹配发布时间 然后转换成时间
|
|
62
62
|
|
|
63
63
|
// 列表上提取到的信息
|
|
@@ -67,12 +67,18 @@ module.exports = async (ctx) => {
|
|
|
67
67
|
pubDate: $pubdate,
|
|
68
68
|
link: $item_url,
|
|
69
69
|
guid: $item_url,
|
|
70
|
+
description: $title,
|
|
70
71
|
};
|
|
71
72
|
|
|
72
73
|
// 对于列表的每一项, 单独获取 时间与详细内容
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
74
|
+
try {
|
|
75
|
+
// 文章可能存在404错误,错误则直接使用title当作内容
|
|
76
|
+
const other = $is_interior ? await ctx.cache.tryGet($item_url, () => load($item_url)) : $title;
|
|
77
|
+
// 合并解析后的结果集作为该篇文章最终的输出结果
|
|
78
|
+
return Promise.resolve(Object.assign({}, single, other));
|
|
79
|
+
} catch (e) {
|
|
80
|
+
return Promise.resolve(single);
|
|
81
|
+
}
|
|
76
82
|
})
|
|
77
83
|
.get()
|
|
78
84
|
);
|
|
@@ -2,6 +2,7 @@ const got = require('@/utils/got');
|
|
|
2
2
|
const config = require('@/config').value;
|
|
3
3
|
const cheerio = require('cheerio');
|
|
4
4
|
const resolve_url = require('url').resolve;
|
|
5
|
+
const timezone = require('@/utils/timezone');
|
|
5
6
|
|
|
6
7
|
module.exports = async (ctx) => {
|
|
7
8
|
// model是channel/tag/etc.,而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话,那就是model是follow,type是articles。
|
|
@@ -95,11 +96,6 @@ module.exports = async (ctx) => {
|
|
|
95
96
|
// 如果model=author,那就是avatar;否则都是cover,要么就没封面
|
|
96
97
|
const image = response.data[model] && (response.data[model].cover || response.data[model].avatar);
|
|
97
98
|
|
|
98
|
-
function fixDate(date) {
|
|
99
|
-
// The date is mistakenly declared as UTC, but actually it's UTC+08:00
|
|
100
|
-
return date.replace(/Z$/, '+08:00');
|
|
101
|
-
}
|
|
102
|
-
|
|
103
99
|
const getFullText = async (slug) => {
|
|
104
100
|
let content = await ctx.cache.get(slug + language);
|
|
105
101
|
if (content) {
|
|
@@ -166,6 +162,8 @@ module.exports = async (ctx) => {
|
|
|
166
162
|
.filter((a) => a.article)
|
|
167
163
|
.slice(0, token === 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY=' && key.iapReceipt === undefined ? 10 : articles.length)
|
|
168
164
|
.map(async (item) => {
|
|
165
|
+
item.article.date = timezone(item.article.date, +8);
|
|
166
|
+
item.article.updated = timezone(item.article.updated, +8);
|
|
169
167
|
const description = await getFullText(item.article.slug);
|
|
170
168
|
return {
|
|
171
169
|
title: item.article.headline,
|
|
@@ -173,8 +171,8 @@ module.exports = async (ctx) => {
|
|
|
173
171
|
category: item.article.channels.filter((x) => !x.homepage).map((x) => x.name),
|
|
174
172
|
description,
|
|
175
173
|
link: resolve_url('https://theinitium.com', item.article.url),
|
|
176
|
-
pubDate:
|
|
177
|
-
updated:
|
|
174
|
+
pubDate: item.article.date,
|
|
175
|
+
updated: item.article.updated,
|
|
178
176
|
// 如果遇到编辑部漏录入情况,则给uuid做个手脚,以便阅读器到时重抓。
|
|
179
177
|
guid: description.endsWith('内容为空,请稍后再来') ? item.article.uuid + '-I-am-empty' : item.article.uuid,
|
|
180
178
|
};
|
package/lib/routes/jandan/pic.js
CHANGED
|
@@ -81,7 +81,7 @@ module.exports = async (ctx) => {
|
|
|
81
81
|
author,
|
|
82
82
|
pubDate: parseRelativeDate(timeInfo),
|
|
83
83
|
guid: id,
|
|
84
|
-
description: comment.find('.text > p').html(),
|
|
84
|
+
description: comment.find('.text > p').html().replace('thumb180', 'large').replace('mw1024', 'large'),
|
|
85
85
|
};
|
|
86
86
|
})
|
|
87
87
|
.get(),
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
const Cheerio = require('cheerio');
|
|
2
|
+
const got = require('@/utils/got');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
const regStripAnchor = /(?:<br\/?> *)?<a name="more"\/?>(?:<\/a>)(?: *<br\/?>)?/i;
|
|
5
|
+
|
|
6
|
+
async function loadArticle(link) {
|
|
7
|
+
const resp = await got(link);
|
|
8
|
+
const $ = Cheerio.load(resp.body);
|
|
9
|
+
const matchResult = $('#post-header-top > script')
|
|
10
|
+
.html()
|
|
11
|
+
.match(/var timestamp="\w+,(.+)";/i)[1]
|
|
12
|
+
.trim();
|
|
13
|
+
// Ex. "August 20, 2021"
|
|
14
|
+
const pubDate = parseDate(matchResult + '+0000', 'MMMM D, YYYYZ');
|
|
15
|
+
|
|
16
|
+
const labels = $('.post-labels a')
|
|
17
|
+
.text()
|
|
18
|
+
.split('\n')
|
|
19
|
+
.filter((item) => item !== '');
|
|
20
|
+
return {
|
|
21
|
+
title: $('.entry-title').text().trim(),
|
|
22
|
+
description: $('.post-body.entry-content').html().replace(regStripAnchor, ''),
|
|
23
|
+
category: labels,
|
|
24
|
+
pubDate,
|
|
25
|
+
link,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
module.exports = loadArticle;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const Cheerio = require('cheerio');
|
|
3
|
+
const loadArticle = require('./article');
|
|
4
|
+
|
|
5
|
+
module.exports = async (ctx, urlParam, title) => {
|
|
6
|
+
const link = new URL(urlParam, 'https://www.micmicidol.com/').toString();
|
|
7
|
+
const response = await got(link);
|
|
8
|
+
ctx.state.data = {
|
|
9
|
+
title: `micmicidol-${title}`,
|
|
10
|
+
link,
|
|
11
|
+
item:
|
|
12
|
+
response.body &&
|
|
13
|
+
(await Promise.all(
|
|
14
|
+
Cheerio.load(response.body)('.post.hentry')
|
|
15
|
+
.map((_, entry) => {
|
|
16
|
+
const { href } = Cheerio.load(entry)('.post-title.entry-title a')[0].attribs;
|
|
17
|
+
return ctx.cache.tryGet(href, () => loadArticle(href));
|
|
18
|
+
})
|
|
19
|
+
.toArray()
|
|
20
|
+
)),
|
|
21
|
+
};
|
|
22
|
+
};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const cheerio = require('cheerio');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
|
|
5
|
+
const languages = {
|
|
6
|
+
zh_cn: 'zh-hans',
|
|
7
|
+
zh_tw: 'zh-hant',
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
const categories = {
|
|
11
|
+
all: {
|
|
12
|
+
title: 'All News',
|
|
13
|
+
id: '0',
|
|
14
|
+
},
|
|
15
|
+
business: {
|
|
16
|
+
title: 'Business',
|
|
17
|
+
id: '1GnkLu7bxeOTxTRNCeu5qm',
|
|
18
|
+
},
|
|
19
|
+
entertainment: {
|
|
20
|
+
title: 'Entertainment',
|
|
21
|
+
id: '3SGbaxYYG5U05Z0G4piPV7',
|
|
22
|
+
},
|
|
23
|
+
innovation: {
|
|
24
|
+
title: 'Innovation',
|
|
25
|
+
id: '5TzuQELMABTu9jOPjXXlFU',
|
|
26
|
+
},
|
|
27
|
+
brazil: {
|
|
28
|
+
title: 'Made in Brazil',
|
|
29
|
+
id: '2tOmcnQB8PgkQSoQ1K4hfD',
|
|
30
|
+
},
|
|
31
|
+
impact: {
|
|
32
|
+
title: 'Social Impact',
|
|
33
|
+
id: '2bUcGjE2800LAsk3JDurGA',
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
module.exports = async (ctx) => {
|
|
38
|
+
const category = ctx.params.category ?? 'all';
|
|
39
|
+
const region = ctx.params.region ?? 'en';
|
|
40
|
+
|
|
41
|
+
const rootUrl = 'https://about.netflix.com';
|
|
42
|
+
const currentUrl = `${rootUrl}/api/data/articles?language=${languages.hasOwnProperty(region) ? languages[region] : region.replace(/_/g, '-')}${category === 'all' ? '' : `&category=${categories[category].id}`}`;
|
|
43
|
+
|
|
44
|
+
const response = await got({
|
|
45
|
+
method: 'get',
|
|
46
|
+
url: currentUrl,
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
const list = response.data.entities.articles.slice(0, ctx.params.limit ? parseInt(ctx.params.limit) : 15).map((item) => ({
|
|
50
|
+
title: item.title,
|
|
51
|
+
link: `${rootUrl}/${region}/news/${item.slug}`,
|
|
52
|
+
pubDate: parseDate(item.rawPublishedDate),
|
|
53
|
+
category: item.categories.map((category) => category.label),
|
|
54
|
+
}));
|
|
55
|
+
|
|
56
|
+
const items = await Promise.all(
|
|
57
|
+
list.map((item) =>
|
|
58
|
+
ctx.cache.tryGet(item.link, async () => {
|
|
59
|
+
const detailResponse = await got({
|
|
60
|
+
method: 'get',
|
|
61
|
+
url: item.link,
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const content = cheerio.load(detailResponse.data);
|
|
65
|
+
|
|
66
|
+
if (!content('.article-contentstyles__ArticleCopy-pei0rm-6 ul li')) {
|
|
67
|
+
content('.article-contentstyles__ArticleCopy-pei0rm-6 p').slice(-3).remove();
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
item.description = content('.article-contentstyles__ArticleCopy-pei0rm-6').html();
|
|
71
|
+
|
|
72
|
+
return item;
|
|
73
|
+
})
|
|
74
|
+
)
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
ctx.state.data = {
|
|
78
|
+
title: `${categories[category].title} - Newsroom - Netflix`,
|
|
79
|
+
link: `${rootUrl}/${region}/newsroom`,
|
|
80
|
+
item: items,
|
|
81
|
+
};
|
|
82
|
+
};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const cheerio = require('cheerio');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
|
|
5
|
+
module.exports = async (ctx) => {
|
|
6
|
+
const id = ctx.params.id ?? 'news';
|
|
7
|
+
const dialect = ctx.params.dialect ?? 'mandarin';
|
|
8
|
+
const category = ctx.params.category ?? 'news';
|
|
9
|
+
const language = ctx.params.language ?? dialect === 'mandarin' ? 'zh-hans' : 'zh-hant';
|
|
10
|
+
|
|
11
|
+
const rootUrl = 'https://www.sbs.com.au';
|
|
12
|
+
const currentUrl = `${rootUrl}/language/${dialect}/${category === 'podcast' ? `${language}/podcast/${id}` : `${id}/${language}`}`;
|
|
13
|
+
|
|
14
|
+
const response = await got({
|
|
15
|
+
method: 'get',
|
|
16
|
+
url: currentUrl,
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
const $ = cheerio.load(response.data);
|
|
20
|
+
|
|
21
|
+
const list = $('.headline, .audio-playlist__item-more')
|
|
22
|
+
.find('a')
|
|
23
|
+
.map((_, item) => {
|
|
24
|
+
item = $(item);
|
|
25
|
+
|
|
26
|
+
return {
|
|
27
|
+
link: `${rootUrl}${item.attr('href')}`,
|
|
28
|
+
};
|
|
29
|
+
})
|
|
30
|
+
.get();
|
|
31
|
+
|
|
32
|
+
const items = await Promise.all(
|
|
33
|
+
list.map((item) =>
|
|
34
|
+
ctx.cache.tryGet(item.link, async () => {
|
|
35
|
+
const detailResponse = await got({
|
|
36
|
+
method: 'get',
|
|
37
|
+
url: item.link,
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
const content = cheerio.load(detailResponse.data);
|
|
41
|
+
|
|
42
|
+
content('.block__container').remove();
|
|
43
|
+
|
|
44
|
+
const audio = content('a[download]')?.attr('href') ?? '';
|
|
45
|
+
const image = content('meta[itemprop="image"]')?.attr('content').split('/crop/')[0] ?? '';
|
|
46
|
+
|
|
47
|
+
item.title = content('meta[property="og:title"]').attr('content');
|
|
48
|
+
item.author = content('.article__meta-author').text().trim().replace('By', '');
|
|
49
|
+
item.pubDate = parseDate(content('meta[itemprop="datePublished"]').attr('content'));
|
|
50
|
+
item.description = `${image ? `<img src="${image}"><br>` : ''}${audio ? `<audio src="${audio}" controls></audio><br>` : ''}${content('.text-body').html()}`;
|
|
51
|
+
|
|
52
|
+
if (audio) {
|
|
53
|
+
item.itunes_item_image = image;
|
|
54
|
+
item.enclosure_url = audio;
|
|
55
|
+
item.enclosure_type = 'audio/mpeg';
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return item;
|
|
59
|
+
})
|
|
60
|
+
)
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
ctx.state.data = {
|
|
64
|
+
title: `${$('meta[property="og:title"]').attr('content')} - SBS中文`,
|
|
65
|
+
link: currentUrl,
|
|
66
|
+
item: items,
|
|
67
|
+
itunes_author: 'SBS中文',
|
|
68
|
+
image: `${rootUrl}/language/img/logos/in-language/logo_chinese.svg`,
|
|
69
|
+
};
|
|
70
|
+
};
|
|
@@ -30,7 +30,7 @@ module.exports = async (ctx) => {
|
|
|
30
30
|
method: 'get',
|
|
31
31
|
url: item.link,
|
|
32
32
|
});
|
|
33
|
-
const content = cheerio.load(detailResponse.data);
|
|
33
|
+
const content = cheerio.load(detailResponse.data.replace(/<source src="\.\//g, `<source src="${item.link}`));
|
|
34
34
|
|
|
35
35
|
const meta = content('.h3').text().trim().split(' on ');
|
|
36
36
|
|