rsshub 1.0.0-master.df37260 → 1.0.0-master.dfcfa1f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.js +4 -0
- package/lib/middleware/anti-hotlink.js +90 -22
- package/lib/middleware/parameter.js +65 -32
- package/lib/middleware/templates/iframe.art +14 -0
- package/lib/radar-rules.js +0 -75
- package/lib/router.js +11 -11
- package/lib/routes/agefans/update.js +1 -1
- package/lib/routes/novel/biqugeinfo.js +2 -2
- package/lib/routes/universities/uestc/sice.js +12 -33
- package/lib/routes/weibo/user.js +10 -19
- package/lib/v2/19lou/index.js +74 -0
- package/lib/v2/19lou/maintainer.js +3 -0
- package/lib/v2/19lou/radar.js +29 -0
- package/lib/v2/19lou/router.js +3 -0
- package/lib/v2/apnews/maintainer.js +3 -0
- package/lib/v2/apnews/radar.js +13 -0
- package/lib/v2/apnews/router.js +3 -0
- package/lib/v2/apnews/templates/description.art +14 -0
- package/lib/v2/apnews/topics.js +44 -0
- package/lib/v2/ccac/news.js +12 -3
- package/lib/v2/cnjxol/index.js +94 -0
- package/lib/v2/cnjxol/maintainer.js +4 -0
- package/lib/v2/cnjxol/radar.js +19 -0
- package/lib/v2/cnjxol/router.js +3 -0
- package/lib/v2/cnjxol/templates/description.art +2 -0
- package/lib/v2/cscse/maintainer.js +3 -0
- package/lib/v2/cscse/radar.js +13 -0
- package/lib/v2/cscse/router.js +3 -0
- package/lib/v2/cscse/tzgg.js +54 -0
- package/lib/v2/domp4/detail.js +67 -0
- package/lib/v2/domp4/latest.js +31 -0
- package/lib/v2/domp4/maintainer.js +4 -0
- package/lib/v2/domp4/radar.js +25 -0
- package/lib/v2/domp4/router.js +4 -0
- package/lib/v2/domp4/utils.js +72 -0
- package/lib/v2/douyin/hashtag.js +4 -0
- package/lib/v2/douyin/user.js +4 -0
- package/lib/v2/elsevier/issue.js +62 -0
- package/lib/v2/elsevier/journal.js +72 -0
- package/lib/v2/elsevier/maintainer.js +2 -2
- package/lib/v2/elsevier/radar.js +5 -5
- package/lib/v2/elsevier/router.js +4 -2
- package/lib/v2/elsevier/templates/description.art +5 -5
- package/lib/v2/gov/beijing/jw/tzgg.js +58 -0
- package/lib/v2/gov/customs/list.js +21 -17
- package/lib/v2/gov/customs/utils.js +9 -5
- package/lib/v2/gov/maintainer.js +1 -0
- package/lib/v2/gov/pbc/goutongjiaoliu.js +20 -15
- package/lib/v2/gov/pbc/tradeAnnouncement.js +18 -14
- package/lib/v2/gov/radar.js +34 -26
- package/lib/v2/gov/router.js +1 -0
- package/lib/v2/hackertalk/index.js +23 -0
- package/lib/v2/hackertalk/maintainer.js +3 -0
- package/lib/v2/hackertalk/radar.js +13 -0
- package/lib/v2/hackertalk/router.js +3 -0
- package/lib/v2/hashnode/blog.js +62 -0
- package/lib/v2/hashnode/maintainer.js +3 -0
- package/lib/v2/hashnode/radar.js +13 -0
- package/lib/v2/hashnode/router.js +3 -0
- package/lib/v2/hashnode/templates/description.art +2 -0
- package/lib/v2/heu/gx/list.js +2 -2
- package/lib/v2/heu/job/list.js +2 -2
- package/lib/v2/heu/uae/list.js +2 -2
- package/lib/v2/heu/yjsy/list.js +2 -2
- package/lib/v2/hk01/channel.js +40 -0
- package/lib/{routes → v2}/hk01/hot.js +2 -1
- package/lib/v2/hk01/issue.js +41 -0
- package/lib/v2/hk01/maintainer.js +7 -0
- package/lib/v2/hk01/radar.js +37 -0
- package/lib/v2/hk01/router.js +7 -0
- package/lib/{routes → v2}/hk01/tag.js +2 -1
- package/lib/{routes → v2}/hk01/zone.js +3 -2
- package/lib/v2/hotukdeals/index.js +1 -1
- package/lib/v2/ieee/journal.js +78 -0
- package/lib/v2/ieee/maintainer.js +2 -2
- package/lib/v2/ieee/radar.js +4 -4
- package/lib/v2/ieee/{latestdate.js → recent.js} +34 -42
- package/lib/v2/ieee/router.js +4 -2
- package/lib/v2/ieee/templates/description.art +5 -5
- package/lib/v2/ielts/index.js +7 -2
- package/lib/v2/itch/devlog.js +62 -0
- package/lib/v2/itch/index.js +56 -0
- package/lib/v2/itch/maintainer.js +5 -0
- package/lib/v2/itch/posts.js +41 -0
- package/lib/v2/itch/radar.js +28 -0
- package/lib/v2/itch/router.js +5 -0
- package/lib/v2/itch/templates/description.art +4 -0
- package/lib/v2/keep/maintainer.js +3 -0
- package/lib/v2/keep/radar.js +13 -0
- package/lib/v2/keep/router.js +3 -0
- package/lib/v2/keep/templates/user.art +16 -0
- package/lib/{routes → v2}/keep/user.js +14 -4
- package/lib/v2/mirror/index.js +34 -0
- package/lib/v2/mirror/maintainer.js +3 -0
- package/lib/v2/mirror/radar.js +17 -0
- package/lib/v2/mirror/router.js +3 -0
- package/lib/v2/mysql/maintainer.js +3 -0
- package/lib/v2/mysql/radar.js +13 -0
- package/lib/v2/mysql/release.js +55 -0
- package/lib/v2/mysql/router.js +3 -0
- package/lib/v2/nbd/radar.js +2 -2
- package/lib/v2/ncwu/maintainer.js +3 -0
- package/lib/v2/ncwu/notice.js +27 -0
- package/lib/v2/ncwu/radar.js +13 -0
- package/lib/v2/ncwu/router.js +3 -0
- package/lib/v2/nju/rczp.js +34 -24
- package/lib/v2/njust/utils.js +4 -0
- package/lib/v2/nuaa/utils/pypasswaf.js +4 -5
- package/lib/v2/nytimes/daily_briefing_chinese.js +66 -41
- package/lib/v2/nytimes/index.js +3 -3
- package/lib/v2/nytimes/maintainer.js +1 -1
- package/lib/v2/nytimes/templates/image.art +4 -0
- package/lib/v2/nytimes/utils.js +7 -1
- package/lib/v2/oceanengine/arithmeticIndex.js +145 -0
- package/lib/v2/oceanengine/maintainer.js +3 -0
- package/lib/v2/oceanengine/radar.js +13 -0
- package/lib/v2/oceanengine/router.js +3 -0
- package/lib/v2/oceanengine/templates/content.art +14 -0
- package/lib/v2/openwrt/maintainer.js +3 -0
- package/lib/v2/openwrt/radar.js +13 -0
- package/lib/v2/openwrt/releases.js +25 -0
- package/lib/v2/openwrt/router.js +3 -0
- package/lib/v2/pincong/utils.js +7 -1
- package/lib/v2/prestige-av/maintainer.js +3 -0
- package/lib/v2/prestige-av/radar.js +18 -0
- package/lib/v2/prestige-av/router.js +3 -0
- package/lib/{routes → v2}/prestige-av/series.js +8 -5
- package/lib/v2/{jasa/latest.js → scitation/journal.js} +24 -27
- package/lib/v2/scitation/maintainer.js +4 -0
- package/lib/v2/scitation/radar.js +19 -0
- package/lib/v2/scitation/router.js +4 -0
- package/lib/v2/scitation/section.js +90 -0
- package/lib/v2/scitation/templates/description.art +11 -0
- package/lib/v2/springer/journal.js +72 -0
- package/lib/v2/springer/maintainer.js +3 -0
- package/lib/v2/springer/radar.js +13 -0
- package/lib/v2/springer/router.js +3 -0
- package/lib/v2/springer/templates/description.art +12 -0
- package/lib/v2/techflow520/index.js +34 -0
- package/lib/v2/techflow520/maintainer.js +4 -0
- package/lib/v2/techflow520/newsflash.js +32 -0
- package/lib/v2/techflow520/radar.js +19 -0
- package/lib/v2/techflow520/router.js +4 -0
- package/lib/v2/tencent/cloud/column.js +51 -0
- package/lib/v2/tencent/maintainer.js +1 -0
- package/lib/v2/tencent/radar.js +11 -0
- package/lib/v2/tencent/router.js +1 -0
- package/lib/v2/test/index.js +16 -1
- package/lib/v2/tiktok/user.js +47 -24
- package/lib/v2/trending/allTrending.js +169 -0
- package/lib/v2/trending/maintainer.js +3 -0
- package/lib/v2/trending/radar.js +35 -0
- package/lib/v2/trending/router.js +3 -0
- package/lib/v2/trending/templates/content.art +24 -0
- package/lib/v2/uraaka-joshi/uraaka-joshi-user.js +7 -1
- package/lib/v2/uraaka-joshi/uraaka-joshi.js +8 -1
- package/lib/v2/xiaohongshu/util.js +4 -0
- package/lib/v2/zhihu/activities.js +36 -0
- package/package.json +12 -11
- package/lib/routes/apnews/topics.js +0 -43
- package/lib/routes/hk01/channel.js +0 -22
- package/lib/routes/hk01/issue.js +0 -22
- package/lib/routes/mirror/entries.js +0 -80
- package/lib/routes/universities/ncwu/notice.js +0 -42
- package/lib/v2/elsevier/latest.js +0 -87
- package/lib/v2/elsevier/volume.js +0 -75
- package/lib/v2/ieee/latestvol.js +0 -87
- package/lib/v2/jasa/maintainer.js +0 -4
- package/lib/v2/jasa/radar.js +0 -19
- package/lib/v2/jasa/router.js +0 -4
- package/lib/v2/jasa/section.js +0 -95
- package/lib/v2/jasa/templates/description.art +0 -11
package/lib/config.js
CHANGED
|
@@ -91,6 +91,10 @@ const calculateValue = () => {
|
|
|
91
91
|
includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','),
|
|
92
92
|
excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','),
|
|
93
93
|
},
|
|
94
|
+
feature: {
|
|
95
|
+
allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
|
|
96
|
+
filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
|
|
97
|
+
},
|
|
94
98
|
suffix: envs.SUFFIX,
|
|
95
99
|
titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
|
|
96
100
|
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
const config = require('@/config').value;
|
|
2
2
|
const cheerio = require('cheerio');
|
|
3
3
|
const logger = require('@/utils/logger');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { art } = require('@/utils/render');
|
|
6
|
+
|
|
7
|
+
const templateRegex = /\$\{([^{}]+)}/g;
|
|
8
|
+
const allowedUrlProperties = ['hash', 'host', 'hostname', 'href', 'origin', 'password', 'pathname', 'port', 'protocol', 'search', 'searchParams', 'username'];
|
|
9
|
+
const IframeWrapperTemplate = path.join(__dirname, 'templates/iframe.art');
|
|
4
10
|
|
|
5
11
|
// match path or sub-path
|
|
6
12
|
const matchPath = (path, paths) => {
|
|
@@ -19,9 +25,16 @@ const filterPath = (path) => {
|
|
|
19
25
|
return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude));
|
|
20
26
|
};
|
|
21
27
|
|
|
22
|
-
const interpolate = (str, obj) =>
|
|
23
|
-
|
|
24
|
-
|
|
28
|
+
const interpolate = (str, obj) =>
|
|
29
|
+
str.replace(templateRegex, (_, prop) => {
|
|
30
|
+
let needEncode = false;
|
|
31
|
+
if (prop.endsWith('_ue')) {
|
|
32
|
+
// url encode
|
|
33
|
+
prop = prop.slice(0, -3);
|
|
34
|
+
needEncode = true;
|
|
35
|
+
}
|
|
36
|
+
return needEncode ? encodeURIComponent(obj[prop]) : obj[prop];
|
|
37
|
+
});
|
|
25
38
|
const parseUrl = (str) => {
|
|
26
39
|
let url;
|
|
27
40
|
try {
|
|
@@ -32,48 +45,103 @@ const parseUrl = (str) => {
|
|
|
32
45
|
|
|
33
46
|
return url;
|
|
34
47
|
};
|
|
35
|
-
const replaceUrls = (
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
const old_src = $(this).attr('src');
|
|
45
|
-
const url = parseUrl(old_src);
|
|
46
|
-
if (url) {
|
|
47
|
-
const new_src = interpolate(template, url);
|
|
48
|
-
$(this).attr('src', new_src);
|
|
48
|
+
const replaceUrls = ($, selector, template, attribute = 'src') => {
|
|
49
|
+
$(selector).each(function () {
|
|
50
|
+
const old_src = $(this).attr(attribute);
|
|
51
|
+
if (old_src) {
|
|
52
|
+
const url = parseUrl(old_src);
|
|
53
|
+
if (url) {
|
|
54
|
+
// Cheerio will do the right thing to prohibit XSS.
|
|
55
|
+
$(this).attr(attribute, interpolate(template, url));
|
|
56
|
+
}
|
|
49
57
|
}
|
|
50
58
|
});
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const wrapWithIframe = ($, selector) => {
|
|
62
|
+
$(selector).each((_, elem) => {
|
|
63
|
+
elem = $(elem);
|
|
64
|
+
elem.replaceWith(art(IframeWrapperTemplate, { content: elem.toString() }));
|
|
65
|
+
});
|
|
66
|
+
};
|
|
51
67
|
|
|
52
|
-
|
|
68
|
+
const process = (html, image_hotlink_template, multimedia_hotlink_template, wrap_multimedia_in_iframe) => {
|
|
69
|
+
const $ = cheerio.load(html, undefined, false);
|
|
70
|
+
if (image_hotlink_template) {
|
|
71
|
+
replaceUrls($, 'img, picture > source', image_hotlink_template);
|
|
72
|
+
replaceUrls($, 'video[poster]', image_hotlink_template, 'poster');
|
|
73
|
+
}
|
|
74
|
+
if (multimedia_hotlink_template) {
|
|
75
|
+
replaceUrls($, 'video, video > source, audio, audio > source', multimedia_hotlink_template);
|
|
76
|
+
if (!image_hotlink_template) {
|
|
77
|
+
replaceUrls($, 'video[poster]', multimedia_hotlink_template, 'poster');
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (wrap_multimedia_in_iframe) {
|
|
81
|
+
wrapWithIframe($, 'video, audio');
|
|
82
|
+
}
|
|
83
|
+
return $.html();
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
const validateTemplate = (template) => {
|
|
87
|
+
if (!template) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
[...template.matchAll(templateRegex)].forEach((match) => {
|
|
91
|
+
const prop = match[1].endsWith('_ue') ? match[1].slice(0, -3) : match[1];
|
|
92
|
+
if (!allowedUrlProperties.includes(prop)) {
|
|
93
|
+
throw new Error(`Invalid URL property: ${prop}`);
|
|
94
|
+
}
|
|
95
|
+
});
|
|
53
96
|
};
|
|
54
97
|
|
|
55
98
|
module.exports = async (ctx, next) => {
|
|
56
99
|
await next();
|
|
57
100
|
|
|
58
|
-
|
|
101
|
+
let image_hotlink_template = undefined;
|
|
102
|
+
let multimedia_hotlink_template = undefined;
|
|
103
|
+
const shouldWrapInIframe = ctx.query.wrap_multimedia_in_iframe === '1';
|
|
59
104
|
|
|
60
|
-
|
|
105
|
+
// Read params if enabled
|
|
106
|
+
if (config.feature.allow_user_hotlink_template) {
|
|
107
|
+
// By default, the config turns these features off. Set corresponding config to
|
|
108
|
+
// true to turn this feature on.
|
|
109
|
+
// A risk is that the media URLs will be replaced by user-supplied templates,
|
|
110
|
+
// so a user could literally take the control of "where are the media from",
|
|
111
|
+
// but only in their personal-use feed URL.
|
|
112
|
+
multimedia_hotlink_template = ctx.query.multimedia_hotlink_template;
|
|
113
|
+
image_hotlink_template = ctx.query.image_hotlink_template;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Force config hotlink template on conflict
|
|
117
|
+
if (config.hotlink.template) {
|
|
118
|
+
if (!filterPath(ctx.request.path)) {
|
|
119
|
+
image_hotlink_template = undefined;
|
|
120
|
+
} else {
|
|
121
|
+
image_hotlink_template = config.hotlink.template;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (!image_hotlink_template && !multimedia_hotlink_template && !shouldWrapInIframe) {
|
|
61
126
|
return;
|
|
62
127
|
}
|
|
63
128
|
|
|
129
|
+
validateTemplate(image_hotlink_template);
|
|
130
|
+
validateTemplate(multimedia_hotlink_template);
|
|
131
|
+
|
|
64
132
|
// Assume that only description include image link
|
|
65
133
|
// and here we will only check them in description.
|
|
66
134
|
// Use Cheerio to load the description as html and filter all
|
|
67
135
|
// image link
|
|
68
136
|
if (ctx.state.data) {
|
|
69
137
|
if (ctx.state.data.description) {
|
|
70
|
-
ctx.state.data.description =
|
|
138
|
+
ctx.state.data.description = process(ctx.state.data.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
|
|
71
139
|
}
|
|
72
140
|
|
|
73
141
|
ctx.state.data.item &&
|
|
74
142
|
ctx.state.data.item.forEach((item) => {
|
|
75
143
|
if (item.description) {
|
|
76
|
-
item.description =
|
|
144
|
+
item.description = process(item.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
|
|
77
145
|
}
|
|
78
146
|
});
|
|
79
147
|
}
|
|
@@ -3,9 +3,26 @@ const cheerio = require('cheerio');
|
|
|
3
3
|
const { simplecc } = require('simplecc-wasm');
|
|
4
4
|
const got = require('@/utils/got');
|
|
5
5
|
const config = require('@/config').value;
|
|
6
|
+
const RE2 = require('re2');
|
|
6
7
|
|
|
7
8
|
let mercury_parser;
|
|
8
9
|
|
|
10
|
+
const resolveRelativeLink = ($, elem, attr, baseUrl) => {
|
|
11
|
+
const $elem = $(elem);
|
|
12
|
+
|
|
13
|
+
if (baseUrl) {
|
|
14
|
+
try {
|
|
15
|
+
const oldAttr = $elem.attr(attr);
|
|
16
|
+
if (oldAttr) {
|
|
17
|
+
// e.g. <video><source src="https://example.com"></video> should leave <video> unchanged
|
|
18
|
+
$elem.attr(attr, new URL(oldAttr, baseUrl).href);
|
|
19
|
+
}
|
|
20
|
+
} catch (e) {
|
|
21
|
+
// no-empty
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
|
|
9
26
|
module.exports = async (ctx, next) => {
|
|
10
27
|
await next();
|
|
11
28
|
|
|
@@ -76,50 +93,49 @@ module.exports = async (ctx, next) => {
|
|
|
76
93
|
|
|
77
94
|
$('script').remove();
|
|
78
95
|
|
|
79
|
-
$('a').each((_, ele) => {
|
|
80
|
-
const $ele = $(ele);
|
|
81
|
-
|
|
82
|
-
// absolute link
|
|
83
|
-
if (baseUrl) {
|
|
84
|
-
try {
|
|
85
|
-
$ele.attr('href', new URL($ele.attr('href'), baseUrl).href);
|
|
86
|
-
} catch (e) {
|
|
87
|
-
// no-empty
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
});
|
|
91
|
-
|
|
92
96
|
$('img').each((_, ele) => {
|
|
93
97
|
const $ele = $(ele);
|
|
94
98
|
|
|
95
99
|
// fix lazyload
|
|
96
100
|
if (!$ele.attr('src')) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
101
|
+
const lazySrc = $ele.attr('data-src') || $ele.attr('data-original');
|
|
102
|
+
if (lazySrc) {
|
|
103
|
+
$ele.attr('src', lazySrc);
|
|
104
|
+
} else {
|
|
105
|
+
for (const key in ele.attribs) {
|
|
106
|
+
const value = ele.attribs[key].trim();
|
|
107
|
+
if (['.gif', '.png', '.jpg', '.webp'].some((suffix) => value.includes(suffix))) {
|
|
108
|
+
$ele.attr('src', value);
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
102
111
|
}
|
|
103
112
|
}
|
|
104
113
|
}
|
|
105
114
|
|
|
106
|
-
// absolute link
|
|
107
|
-
if (baseUrl) {
|
|
108
|
-
try {
|
|
109
|
-
$ele.attr('src', new URL($ele.attr('src'), baseUrl).href);
|
|
110
|
-
} catch (e) {
|
|
111
|
-
// no-empty
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
// referrerpolicy
|
|
116
|
-
$ele.attr('referrerpolicy', 'no-referrer');
|
|
117
|
-
|
|
118
115
|
// redundant attributes
|
|
119
116
|
['onclick', 'onerror', 'onload'].forEach((e) => {
|
|
120
117
|
$ele.removeAttr(e);
|
|
121
118
|
});
|
|
122
119
|
});
|
|
120
|
+
|
|
121
|
+
// resolve relative link & fix referrer policy
|
|
122
|
+
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
|
|
123
|
+
// https://www.w3schools.com/tags/att_href.asp
|
|
124
|
+
$('a, area').each((_, elem) => {
|
|
125
|
+
resolveRelativeLink($, elem, 'href', baseUrl);
|
|
126
|
+
// $(elem).attr('rel', 'noreferrer'); // currently no such a need
|
|
127
|
+
});
|
|
128
|
+
// https://www.w3schools.com/tags/att_src.asp
|
|
129
|
+
$('img, video, audio, source, iframe, embed, track').each((_, elem) => {
|
|
130
|
+
resolveRelativeLink($, elem, 'src', baseUrl);
|
|
131
|
+
});
|
|
132
|
+
$('video[poster]').each((_, elem) => {
|
|
133
|
+
resolveRelativeLink($, elem, 'poster', baseUrl);
|
|
134
|
+
});
|
|
135
|
+
$('img, iframe').each((_, elem) => {
|
|
136
|
+
$(elem).attr('referrerpolicy', 'no-referrer');
|
|
137
|
+
});
|
|
138
|
+
|
|
123
139
|
item.description = $('body').html() + '' + (config.suffix || '');
|
|
124
140
|
}
|
|
125
141
|
return item;
|
|
@@ -136,10 +152,25 @@ module.exports = async (ctx, next) => {
|
|
|
136
152
|
// filter
|
|
137
153
|
const makeRegex = (string) => {
|
|
138
154
|
// default: case_senstivie = true
|
|
155
|
+
const engine = config.feature.filter_regex_engine;
|
|
139
156
|
if (ctx.query.filter_case_sensitive === 'false') {
|
|
140
|
-
|
|
157
|
+
switch (engine) {
|
|
158
|
+
case 'regexp':
|
|
159
|
+
return new RegExp(string, 'i');
|
|
160
|
+
case 're2':
|
|
161
|
+
return new RE2(string, 'i');
|
|
162
|
+
default:
|
|
163
|
+
throw Error(`Invalid Engine Value: ${engine}, please check your config.`);
|
|
164
|
+
}
|
|
141
165
|
} else {
|
|
142
|
-
|
|
166
|
+
switch (engine) {
|
|
167
|
+
case 'regexp':
|
|
168
|
+
return new RegExp(string);
|
|
169
|
+
case 're2':
|
|
170
|
+
return new RE2(string);
|
|
171
|
+
default:
|
|
172
|
+
throw Error(`Invalid Engine Value: ${engine}, please check your config.`);
|
|
173
|
+
}
|
|
143
174
|
}
|
|
144
175
|
};
|
|
145
176
|
|
|
@@ -272,6 +303,8 @@ module.exports = async (ctx, next) => {
|
|
|
272
303
|
throw Error(`Invalid parameter <code>brief=${ctx.query.brief}</code>. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
|
|
273
304
|
}
|
|
274
305
|
}
|
|
306
|
+
|
|
307
|
+
// some parameters are processed in `anti-hotlink.js`
|
|
275
308
|
}
|
|
276
309
|
}
|
|
277
310
|
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
<iframe referrerpolicy="no-referrer" width=100% height=150vh frameborder=0 marginheight=0 marginwidth=0
|
|
2
|
+
style="border:0; margin:0; padding:0; width:100%; height:150vh;"
|
|
3
|
+
srcdoc="
|
|
4
|
+
<!DOCTYPE html>
|
|
5
|
+
<html>
|
|
6
|
+
<head>
|
|
7
|
+
<meta name="referrer" content="no-referrer">
|
|
8
|
+
</head>
|
|
9
|
+
<body>
|
|
10
|
+
{{ content }}
|
|
11
|
+
</body>
|
|
12
|
+
</html>
|
|
13
|
+
">
|
|
14
|
+
</iframe>
|
package/lib/radar-rules.js
CHANGED
|
@@ -1135,17 +1135,6 @@ module.exports = {
|
|
|
1135
1135
|
},
|
|
1136
1136
|
],
|
|
1137
1137
|
},
|
|
1138
|
-
'apnews.com': {
|
|
1139
|
-
_name: 'AP News',
|
|
1140
|
-
'.': [
|
|
1141
|
-
{
|
|
1142
|
-
title: '话题',
|
|
1143
|
-
docs: 'https://docs.rsshub.app/traditional-media.html#ap-news',
|
|
1144
|
-
source: '/:topic',
|
|
1145
|
-
target: '/apnews/topics/:topic',
|
|
1146
|
-
},
|
|
1147
|
-
],
|
|
1148
|
-
},
|
|
1149
1138
|
'csc.edu.cn': {
|
|
1150
1139
|
_name: '国家留学网',
|
|
1151
1140
|
www: [
|
|
@@ -1589,41 +1578,6 @@ module.exports = {
|
|
|
1589
1578
|
},
|
|
1590
1579
|
],
|
|
1591
1580
|
},
|
|
1592
|
-
'hk01.com': {
|
|
1593
|
-
_name: '香港01',
|
|
1594
|
-
www: [
|
|
1595
|
-
{
|
|
1596
|
-
title: '最 Hit',
|
|
1597
|
-
docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
|
|
1598
|
-
source: ['/hot', '/'],
|
|
1599
|
-
target: '/hk01/hot',
|
|
1600
|
-
},
|
|
1601
|
-
{
|
|
1602
|
-
title: 'zone',
|
|
1603
|
-
docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
|
|
1604
|
-
source: '/zone/:id/:title',
|
|
1605
|
-
target: '/hk01/zone/:id',
|
|
1606
|
-
},
|
|
1607
|
-
{
|
|
1608
|
-
title: 'channel',
|
|
1609
|
-
docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
|
|
1610
|
-
source: '/channel/:id/:title',
|
|
1611
|
-
target: '/hk01/channel/:id',
|
|
1612
|
-
},
|
|
1613
|
-
{
|
|
1614
|
-
title: 'issue',
|
|
1615
|
-
docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
|
|
1616
|
-
source: '/issue/:id/:title',
|
|
1617
|
-
target: '/hk01/issue/:id',
|
|
1618
|
-
},
|
|
1619
|
-
{
|
|
1620
|
-
title: 'tag',
|
|
1621
|
-
docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
|
|
1622
|
-
source: '/tag/:id/:title',
|
|
1623
|
-
target: '/hk01/tag/:id',
|
|
1624
|
-
},
|
|
1625
|
-
],
|
|
1626
|
-
},
|
|
1627
1581
|
'popiapp.cn': {
|
|
1628
1582
|
_name: 'Popi 提问箱',
|
|
1629
1583
|
www: [
|
|
@@ -1981,35 +1935,6 @@ module.exports = {
|
|
|
1981
1935
|
},
|
|
1982
1936
|
],
|
|
1983
1937
|
},
|
|
1984
|
-
'gcores.com': {
|
|
1985
|
-
_name: '机核网',
|
|
1986
|
-
www: [
|
|
1987
|
-
{
|
|
1988
|
-
title: '资讯',
|
|
1989
|
-
docs: 'https://docs.rsshub.app/program-update.html#eagle',
|
|
1990
|
-
source: '/news',
|
|
1991
|
-
target: '/gcores/category/news',
|
|
1992
|
-
},
|
|
1993
|
-
{
|
|
1994
|
-
title: '视频',
|
|
1995
|
-
docs: 'https://docs.rsshub.app/program-update.html#eagle',
|
|
1996
|
-
source: '/videos',
|
|
1997
|
-
target: '/gcores/category/videos',
|
|
1998
|
-
},
|
|
1999
|
-
{
|
|
2000
|
-
title: '电台',
|
|
2001
|
-
docs: 'https://docs.rsshub.app/program-update.html#eagle',
|
|
2002
|
-
source: '/radios',
|
|
2003
|
-
target: '/gcores/category/radios',
|
|
2004
|
-
},
|
|
2005
|
-
{
|
|
2006
|
-
title: '文章',
|
|
2007
|
-
docs: 'https://docs.rsshub.app/program-update.html#eagle',
|
|
2008
|
-
source: '/articles',
|
|
2009
|
-
target: '/gcores/category/articles',
|
|
2010
|
-
},
|
|
2011
|
-
],
|
|
2012
|
-
},
|
|
2013
1938
|
'bgm.tv': {
|
|
2014
1939
|
_name: 'Bangumi',
|
|
2015
1940
|
'.': [
|
package/lib/router.js
CHANGED
|
@@ -228,8 +228,8 @@ router.get('/pornhub/:language?/users/:username', lazyloadRouteHandler('./routes
|
|
|
228
228
|
router.get('/pornhub/:language?/model/:username/:sort?', lazyloadRouteHandler('./routes/pornhub/model'));
|
|
229
229
|
router.get('/pornhub/:language?/pornstar/:username/:sort?', lazyloadRouteHandler('./routes/pornhub/pornstar'));
|
|
230
230
|
|
|
231
|
-
// Prestige
|
|
232
|
-
router.get('/prestige-av/series/:mid/:sort?', lazyloadRouteHandler('./routes/prestige-av/series'));
|
|
231
|
+
// Prestige migrated to v2
|
|
232
|
+
// router.get('/prestige-av/series/:mid/:sort?', lazyloadRouteHandler('./routes/prestige-av/series'));
|
|
233
233
|
|
|
234
234
|
// yande.re
|
|
235
235
|
router.get('/yande.re/post/popular_recent', lazyloadRouteHandler('./routes/yande.re/post_popular_recent'));
|
|
@@ -336,7 +336,7 @@ router.get('/miui/:device/:type?/:region?', lazyloadRouteHandler('./routes/mi/mi
|
|
|
336
336
|
router.get('/mi/bbs/board/:boardId', lazyloadRouteHandler('./routes/mi/board'));
|
|
337
337
|
|
|
338
338
|
// Keep
|
|
339
|
-
router.get('/keep/user/:id', lazyloadRouteHandler('./routes/keep/user'));
|
|
339
|
+
// router.get('/keep/user/:id', lazyloadRouteHandler('./routes/keep/user'));
|
|
340
340
|
|
|
341
341
|
// 起点
|
|
342
342
|
router.get('/qidian/chapter/:id', lazyloadRouteHandler('./routes/qidian/chapter'));
|
|
@@ -759,7 +759,7 @@ router.get('/upc/main/:type?', lazyloadRouteHandler('./routes/universities/upc/m
|
|
|
759
759
|
router.get('/upc/jsj/:type?', lazyloadRouteHandler('./routes/universities/upc/jsj'));
|
|
760
760
|
|
|
761
761
|
// 华北水利水电大学
|
|
762
|
-
router.get('/ncwu/notice', lazyloadRouteHandler('./routes/universities/ncwu/notice'));
|
|
762
|
+
// router.get('/ncwu/notice', lazyloadRouteHandler('./routes/universities/ncwu/notice'));
|
|
763
763
|
|
|
764
764
|
// 太原师范学院
|
|
765
765
|
router.get('/tynu', lazyloadRouteHandler('./routes/universities/tynu/tynu'));
|
|
@@ -2201,11 +2201,11 @@ router.get('/mcbbs/post/:tid/:authorid?', lazyloadRouteHandler('./routes/mcbbs/p
|
|
|
2201
2201
|
router.get('/pocket/trending', lazyloadRouteHandler('./routes/pocket/trending'));
|
|
2202
2202
|
|
|
2203
2203
|
// HK01
|
|
2204
|
-
router.get('/hk01/zone/:id', lazyloadRouteHandler('./routes/hk01/zone'));
|
|
2205
|
-
router.get('/hk01/channel/:id', lazyloadRouteHandler('./routes/hk01/channel'));
|
|
2206
|
-
router.get('/hk01/issue/:id', lazyloadRouteHandler('./routes/hk01/issue'));
|
|
2207
|
-
router.get('/hk01/tag/:id', lazyloadRouteHandler('./routes/hk01/tag'));
|
|
2208
|
-
router.get('/hk01/hot', lazyloadRouteHandler('./routes/hk01/hot'));
|
|
2204
|
+
// router.get('/hk01/zone/:id', lazyloadRouteHandler('./routes/hk01/zone'));
|
|
2205
|
+
// router.get('/hk01/channel/:id', lazyloadRouteHandler('./routes/hk01/channel'));
|
|
2206
|
+
// router.get('/hk01/issue/:id', lazyloadRouteHandler('./routes/hk01/issue'));
|
|
2207
|
+
// router.get('/hk01/tag/:id', lazyloadRouteHandler('./routes/hk01/tag'));
|
|
2208
|
+
// router.get('/hk01/hot', lazyloadRouteHandler('./routes/hk01/hot'));
|
|
2209
2209
|
|
|
2210
2210
|
// 码农周刊
|
|
2211
2211
|
router.get('/manong-weekly', lazyloadRouteHandler('./routes/manong-weekly/issues'));
|
|
@@ -2364,7 +2364,7 @@ router.get('/xinquji/today/internal', lazyloadRouteHandler('./routes/xinquji/int
|
|
|
2364
2364
|
router.get('/gbcc/trust', lazyloadRouteHandler('./routes/gbcc/trust'));
|
|
2365
2365
|
|
|
2366
2366
|
// Associated Press
|
|
2367
|
-
router.get('/apnews/topics/:topic', lazyloadRouteHandler('./routes/apnews/topics'));
|
|
2367
|
+
// router.get('/apnews/topics/:topic', lazyloadRouteHandler('./routes/apnews/topics'));
|
|
2368
2368
|
|
|
2369
2369
|
// CBC
|
|
2370
2370
|
router.get('/cbc/topics/:topic?', lazyloadRouteHandler('./routes/cbc/topics'));
|
|
@@ -4065,7 +4065,7 @@ router.get('/secrss/author/:author?', lazyloadRouteHandler('./routes/secrss/auth
|
|
|
4065
4065
|
router.get('/fashionnetwork/headline/:country?', lazyloadRouteHandler('./routes/fashionnetwork/headline.js'));
|
|
4066
4066
|
|
|
4067
4067
|
// mirror.xyz
|
|
4068
|
-
router.get('/mirror/:id', lazyloadRouteHandler('./routes/mirror/entries'));
|
|
4068
|
+
// router.get('/mirror/:id', lazyloadRouteHandler('./routes/mirror/entries'));
|
|
4069
4069
|
|
|
4070
4070
|
// KBS migrated to v2
|
|
4071
4071
|
// router.get('/kbs/today/:language?', lazyloadRouteHandler('./routes/kbs/today'));
|
|
@@ -2,7 +2,7 @@ const got = require('@/utils/got');
|
|
|
2
2
|
const cheerio = require('cheerio');
|
|
3
3
|
|
|
4
4
|
module.exports = async (ctx) => {
|
|
5
|
-
const rootUrl = 'https://www.
|
|
5
|
+
const rootUrl = 'https://www.agemys.com';
|
|
6
6
|
const currentUrl = `${rootUrl}/update`;
|
|
7
7
|
const response = await got({
|
|
8
8
|
method: 'get',
|
|
@@ -35,7 +35,7 @@ module.exports = async (ctx) => {
|
|
|
35
35
|
|
|
36
36
|
const items = [];
|
|
37
37
|
for await (const item of asyncPool(3, chapter_item, (item) =>
|
|
38
|
-
ctx.cache.tryGet(item, async () => {
|
|
38
|
+
ctx.cache.tryGet(item.link.href, async () => {
|
|
39
39
|
const response = await got({
|
|
40
40
|
method: 'get',
|
|
41
41
|
url: item.link,
|
|
@@ -61,7 +61,7 @@ module.exports = async (ctx) => {
|
|
|
61
61
|
|
|
62
62
|
ctx.state.data = {
|
|
63
63
|
title: `笔趣阁 ${title}`,
|
|
64
|
-
link: pageUrl,
|
|
64
|
+
link: pageUrl.href,
|
|
65
65
|
image: cover_url,
|
|
66
66
|
description,
|
|
67
67
|
item: items,
|
|
@@ -1,40 +1,18 @@
|
|
|
1
1
|
const cheerio = require('cheerio');
|
|
2
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
3
|
+
const dayjs = require('dayjs');
|
|
2
4
|
|
|
3
5
|
module.exports = async (ctx) => {
|
|
4
6
|
const baseIndexUrl = 'https://www.sice.uestc.edu.cn/index.htm';
|
|
5
7
|
const host = 'https://www.sice.uestc.edu.cn/';
|
|
6
|
-
const browser = await require('@/utils/puppeteer')();
|
|
8
|
+
const browser = await require('@/utils/puppeteer')({ stealth: true });
|
|
7
9
|
const page = await browser.newPage();
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
// 在每个新页面打开前执行以下脚本
|
|
12
|
-
const newProto = navigator.__proto__;
|
|
13
|
-
delete newProto.webdriver; // 删除navigator.webdriver字段
|
|
14
|
-
navigator.__proto__ = newProto;
|
|
15
|
-
window.chrome = {}; // 添加window.chrome字段,为增加真实性还需向内部填充一些值
|
|
16
|
-
window.chrome.app = { InstallState: 'hehe', RunningState: 'haha', getDetails: 'xixi', getIsInstalled: 'ohno' };
|
|
17
|
-
window.chrome.csi = function () {};
|
|
18
|
-
window.chrome.loadTimes = function () {};
|
|
19
|
-
window.chrome.runtime = function () {};
|
|
20
|
-
Object.defineProperty(navigator, 'userAgent', {
|
|
21
|
-
// userAgent在无头模式下有headless字样,所以需覆写
|
|
22
|
-
get: () => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36',
|
|
23
|
-
});
|
|
24
|
-
Object.defineProperty(navigator, 'plugins', {
|
|
25
|
-
// 伪装真实的插件信息
|
|
26
|
-
get: () => [{ description: 'Portable Document Format', filename: 'internal-pdf-viewer', length: 1, name: 'Chrome PDF Plugin' }],
|
|
27
|
-
});
|
|
28
|
-
Object.defineProperty(navigator, 'languages', {
|
|
29
|
-
// 添加语言
|
|
30
|
-
get: () => ['zh-CN', 'zh', 'en'],
|
|
31
|
-
});
|
|
32
|
-
const originalQuery = window.navigator.permissions.query; // notification伪装
|
|
33
|
-
window.navigator.permissions.query = (parameters) => (parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters));
|
|
10
|
+
await page.setRequestInterception(true);
|
|
11
|
+
page.on('request', (request) => {
|
|
12
|
+
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
|
|
34
13
|
});
|
|
35
|
-
|
|
36
14
|
await page.goto(baseIndexUrl, {
|
|
37
|
-
waitUntil: '
|
|
15
|
+
waitUntil: 'networkidle2',
|
|
38
16
|
});
|
|
39
17
|
const content = await page.content();
|
|
40
18
|
await browser.close();
|
|
@@ -43,14 +21,15 @@ module.exports = async (ctx) => {
|
|
|
43
21
|
const out = $('.notice p')
|
|
44
22
|
.map((index, item) => {
|
|
45
23
|
item = $(item);
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
24
|
+
const now = dayjs();
|
|
25
|
+
let date = dayjs(now.year() + '-' + item.find('a.date').text());
|
|
26
|
+
if (now < date) {
|
|
27
|
+
date = dayjs(now.year() - 1 + '-' + item.find('a.date').text());
|
|
49
28
|
}
|
|
50
29
|
return {
|
|
51
30
|
title: item.find('a[href]').text(),
|
|
52
31
|
link: host + item.find('a[href]').attr('href'),
|
|
53
|
-
pubDate: date,
|
|
32
|
+
pubDate: parseDate(date),
|
|
54
33
|
};
|
|
55
34
|
})
|
|
56
35
|
.get();
|
package/lib/routes/weibo/user.js
CHANGED
|
@@ -62,7 +62,7 @@ module.exports = async (ctx) => {
|
|
|
62
62
|
false
|
|
63
63
|
);
|
|
64
64
|
|
|
65
|
-
|
|
65
|
+
let resultItems = await Promise.all(
|
|
66
66
|
cards
|
|
67
67
|
.filter((item) => item.mblog)
|
|
68
68
|
.map(async (item) => {
|
|
@@ -131,27 +131,18 @@ module.exports = async (ctx) => {
|
|
|
131
131
|
link,
|
|
132
132
|
pubDate,
|
|
133
133
|
author: item.mblog.user.screen_name,
|
|
134
|
+
isPinned: item.profile_type_id?.startsWith('proweibotop'),
|
|
134
135
|
};
|
|
135
136
|
})
|
|
136
137
|
);
|
|
137
138
|
|
|
138
|
-
// remove pinned weibo if
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
if (
|
|
144
|
-
|
|
145
|
-
resultItem[0].pubDate &&
|
|
146
|
-
resultItem[0].pubDate <
|
|
147
|
-
Math.min(
|
|
148
|
-
...resultItem
|
|
149
|
-
.slice(1)
|
|
150
|
-
.map((item) => item.pubDate)
|
|
151
|
-
.filter((item) => item)
|
|
152
|
-
)
|
|
153
|
-
) {
|
|
154
|
-
resultItem.shift();
|
|
139
|
+
// remove pinned weibo if they are too old (older than all the rest weibo)
|
|
140
|
+
// the character of pinned weibo is `card.profile_type_id.startsWith('proweibotop')`
|
|
141
|
+
// there can be 1 or 2 (WHAT A FANTASTIC BRAIN THE PM HAS?) pinned weibo at the same time
|
|
142
|
+
const pinnedItems = resultItems.filter((item) => item.isPinned);
|
|
143
|
+
const ordinaryItems = resultItems.filter((item) => !item.isPinned);
|
|
144
|
+
if (pinnedItems.length > 0 && ordinaryItems.length > 0 && Math.max(...pinnedItems.map((i) => i.pubDate).filter(Boolean)) < Math.min(...ordinaryItems.map((i) => i.pubDate).filter(Boolean))) {
|
|
145
|
+
resultItems = ordinaryItems;
|
|
155
146
|
}
|
|
156
147
|
|
|
157
148
|
ctx.state.data = {
|
|
@@ -159,6 +150,6 @@ module.exports = async (ctx) => {
|
|
|
159
150
|
link: `https://weibo.com/${uid}/`,
|
|
160
151
|
description,
|
|
161
152
|
image: profileImageUrl,
|
|
162
|
-
item:
|
|
153
|
+
item: resultItems,
|
|
163
154
|
};
|
|
164
155
|
};
|