rsshub 1.0.0-master.d77a039 → 1.0.0-master.d799646
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/lib/api_router.js +3 -5
- package/lib/app.js +10 -3
- package/lib/config.js +4 -0
- package/lib/core_router.js +17 -0
- package/lib/errors/RequestInProgress.js +3 -0
- package/lib/errors/index.js +3 -0
- package/lib/maintainer.js +27 -0
- package/lib/middleware/cache/index.js +129 -0
- package/lib/middleware/cache/memory.js +42 -0
- package/lib/middleware/cache/redis.js +48 -0
- package/lib/middleware/load-on-demand.js +36 -0
- package/lib/middleware/onerror.js +10 -69
- package/lib/middleware/template.js +12 -6
- package/lib/radar-rules.js +3127 -0
- package/lib/radar.js +22 -0
- package/lib/router.js +46 -48
- package/lib/routes/abc/documentId.js +7272 -0
- package/lib/routes/abc/id.js +7272 -0
- package/lib/routes/abc/index.js +42 -53
- package/lib/routes/anigamer/new_anime.js +1 -1
- package/lib/routes/apnews/topics.js +1 -1
- package/lib/routes/caixin/latest.js +8 -4
- package/lib/routes/cctv/lm.js +78 -0
- package/lib/routes/curseforge/files.js +23 -39
- package/lib/routes/dbaplus/activity.js +35 -0
- package/lib/routes/discuss/index.js +71 -0
- package/lib/routes/douban/people/status.js +1 -1
- package/lib/routes/dsndsht23/index.js +13 -6
- package/lib/routes/dykszx/news.js +57 -0
- package/lib/routes/fashionnetwork/news.js +70 -0
- package/lib/routes/firefox/addons.js +1 -1
- package/lib/routes/hkgolden/index.js +110 -0
- package/lib/routes/icable/category.js +20 -23
- package/lib/routes/infzm/news.js +3 -1
- package/lib/routes/instagram/utils.js +5 -1
- package/lib/routes/jandan/pic.js +43 -59
- package/lib/routes/lofter/tag.js +41 -37
- package/lib/routes/mail/imap.js +1 -1
- package/lib/routes/news/whxw.js +19 -11
- package/lib/routes/novel/ptwxz.js +43 -14
- package/lib/routes/now/rank.js +31 -0
- package/lib/routes/odaily/activity.js +47 -0
- package/lib/routes/pixiv/bookmarks.js +2 -8
- package/lib/routes/pixiv/illustfollow.js +3 -8
- package/lib/routes/pixiv/ranking.js +2 -8
- package/lib/routes/pixiv/search.js +2 -8
- package/lib/routes/pixiv/user.js +2 -8
- package/lib/routes/pixiv/utils.js +17 -0
- package/lib/routes/questmobile/report.js +122 -0
- package/lib/routes/researchgate/publications.js +60 -0
- package/lib/routes/rss3/blog.js +47 -0
- package/lib/routes/twitter/developer-api/search.js +22 -0
- package/lib/routes/twitter/developer-api/user.js +29 -0
- package/lib/routes/twitter/keyword.js +5 -19
- package/lib/routes/twitter/media.js +7 -0
- package/lib/routes/twitter/user.js +4 -47
- package/lib/routes/twitter/utils.js +46 -23
- package/lib/routes/twitter/web-api/media.js +20 -0
- package/lib/routes/twitter/web-api/search.js +16 -0
- package/lib/routes/twitter/web-api/twitter-api.js +123 -0
- package/lib/routes/twitter/web-api/twitter-got.js +87 -0
- package/lib/routes/twitter/web-api/user.js +32 -0
- package/lib/routes/universities/njfu/jwc.js +23 -13
- package/lib/routes/universities/scut/jwc/school.js +121 -0
- package/lib/routes/universities/thu/index.js +1 -1
- package/lib/routes/universities/ynnu/edu/base64.js +351 -0
- package/lib/routes/universities/ynnu/edu/news.js +109 -0
- package/lib/routes/uwants/index.js +70 -0
- package/lib/routes/who/news.js +43 -0
- package/lib/routes/who/speeches.js +45 -0
- package/lib/routes/woshipm/user_article.js +26 -38
- package/lib/routes/xiaoheihe/user.js +38 -27
- package/lib/routes/zimuku/index.js +1 -1
- package/lib/utils/render.js +7 -0
- package/lib/v2/2047/index.js +55 -0
- package/lib/v2/2047/maintainer.js +3 -0
- package/lib/v2/2047/radar.js +13 -0
- package/lib/v2/2047/router.js +3 -0
- package/lib/v2/8world/index.js +57 -0
- package/lib/v2/8world/maintainer.js +3 -0
- package/lib/v2/8world/radar.js +13 -0
- package/lib/v2/8world/router.js +3 -0
- package/lib/v2/aamacau/index.js +56 -0
- package/lib/v2/aamacau/maintainer.js +3 -0
- package/lib/v2/aamacau/radar.js +13 -0
- package/lib/v2/aamacau/router.js +3 -0
- package/lib/v2/biodiscover/index.js +44 -0
- package/lib/v2/biodiscover/maintainer.js +3 -0
- package/lib/v2/biodiscover/radar.js +13 -0
- package/lib/v2/biodiscover/router.js +3 -0
- package/lib/{routes → v2}/epicgames/index.js +0 -0
- package/lib/v2/epicgames/maintainer.js +3 -0
- package/lib/v2/epicgames/router.js +3 -0
- package/lib/{routes → v2}/epicgames/supportedList.js +0 -0
- package/lib/v2/eprice/maintainer.js +3 -0
- package/lib/v2/eprice/radar.js +24 -0
- package/lib/v2/eprice/router.js +3 -0
- package/lib/v2/eprice/rss.js +95 -0
- package/lib/v2/eprice/templates/description.art +1 -0
- package/lib/v2/fisher-spb/maintainer.js +3 -0
- package/lib/v2/fisher-spb/news.js +49 -0
- package/lib/v2/fisher-spb/radar.js +13 -0
- package/lib/v2/fisher-spb/router.js +3 -0
- package/lib/v2/fisher-spb/templates/image.art +1 -0
- package/lib/v2/fisher-spb/templates/video.art +1 -0
- package/lib/v2/focustaiwan/index.js +72 -0
- package/lib/v2/focustaiwan/maintainer.js +3 -0
- package/lib/v2/focustaiwan/radar.js +13 -0
- package/lib/v2/focustaiwan/router.js +3 -0
- package/lib/v2/focustaiwan/templates/article.art +1 -0
- package/lib/{routes → v2}/furstar/archive.js +0 -0
- package/lib/{routes → v2}/furstar/artists.js +0 -0
- package/lib/{routes → v2}/furstar/index.js +4 -0
- package/lib/v2/furstar/maintainer.js +5 -0
- package/lib/v2/furstar/radar.js +25 -0
- package/lib/v2/furstar/router.js +5 -0
- package/lib/v2/furstar/templates/author.art +8 -0
- package/lib/v2/furstar/templates/description.art +5 -0
- package/lib/{routes → v2}/furstar/utils.js +11 -10
- package/lib/v2/hotukdeals/index.js +34 -0
- package/lib/v2/hotukdeals/maintainer.js +3 -0
- package/lib/v2/hotukdeals/radar.js +31 -0
- package/lib/v2/hotukdeals/router.js +3 -0
- package/lib/v2/icac/maintainer.js +3 -0
- package/lib/v2/icac/news.js +47 -0
- package/lib/v2/icac/radar.js +13 -0
- package/lib/v2/icac/router.js +3 -0
- package/lib/v2/icac/utils.js +17 -0
- package/lib/v2/independent/maintainer.js +3 -0
- package/lib/v2/independent/ps5-stock-uk.js +27 -0
- package/lib/v2/independent/radar.js +13 -0
- package/lib/v2/independent/router.js +3 -0
- package/lib/v2/jump/discount.js +120 -0
- package/lib/v2/jump/maintainer.js +3 -0
- package/lib/v2/jump/radar.js +13 -0
- package/lib/v2/jump/router.js +3 -0
- package/lib/v2/jump/templates/discount.art +111 -0
- package/lib/v2/qq/live.js +36 -0
- package/lib/v2/qq/maintainer.js +3 -0
- package/lib/v2/qq/radar.js +13 -0
- package/lib/v2/qq/router.js +3 -0
- package/lib/v2/ruancan/index.js +54 -0
- package/lib/v2/ruancan/maintainer.js +6 -0
- package/lib/v2/ruancan/radar.js +25 -0
- package/lib/v2/ruancan/router.js +3 -0
- package/lib/{routes → v2}/test/index.js +0 -0
- package/lib/v2/test/maintainer.js +3 -0
- package/lib/v2/test/router.js +3 -0
- package/lib/v2/trow/maintainer.js +3 -0
- package/lib/v2/trow/portal.js +36 -0
- package/lib/v2/trow/radar.js +13 -0
- package/lib/v2/trow/router.js +3 -0
- package/lib/v2router.js +17 -0
- package/lib/views/error.art +9 -0
- package/package.json +10 -4
- package/lib/middleware/cache.js +0 -182
- package/lib/routes/biodiscover/index.js +0 -43
- package/lib/routes/epicgames/router.js +0 -9
- package/lib/routes/toutiao/today.js +0 -32
- package/lib/routes/toutiao/user.js +0 -38
package/README.md
CHANGED
|
@@ -32,7 +32,7 @@ RSSHub 是一个开源、简单易用、易于扩展的 RSS 生成器,可以
|
|
|
32
32
|
### Special Sponsors
|
|
33
33
|
|
|
34
34
|
<div>
|
|
35
|
-
<a href="https://rss3.io/" target="_blank"><img height="60px" src="https://i.imgur.com/v7TF0jp.jpg"></a
|
|
35
|
+
<a href="https://rss3.io/" target="_blank"><img height="60px" src="https://i.imgur.com/v7TF0jp.jpg"></a>
|
|
36
36
|
</div>
|
|
37
37
|
|
|
38
38
|
### Sponsors
|
|
@@ -85,6 +85,7 @@ You can support RSSHub via donations.
|
|
|
85
85
|
|
|
86
86
|
Recurring donors will be rewarded via express issue response, or even have your name displayed on our GitHub page and website.
|
|
87
87
|
|
|
88
|
+
- Become a Sponser on [GitHub](https://github.com/sponsors/DIYgod)
|
|
88
89
|
- Become a Sponser on [Patreon](https://www.patreon.com/DIYgod)
|
|
89
90
|
- Become a Sponser on [爱发电](https://afdian.net/@diygod)
|
|
90
91
|
- Contact us directly: i@diygod.me
|
package/lib/api_router.js
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
const Router = require('@koa/router');
|
|
2
2
|
const router = new Router();
|
|
3
|
-
const
|
|
3
|
+
const maintainer = require('./maintainer');
|
|
4
4
|
|
|
5
5
|
router.get('/routes/:name?', (ctx) => {
|
|
6
|
-
const allRoutes = Array.from(routes.stack);
|
|
7
|
-
allRoutes.shift();
|
|
8
6
|
const result = {};
|
|
9
7
|
let counter = 0;
|
|
10
8
|
|
|
11
|
-
|
|
12
|
-
const path = i
|
|
9
|
+
Object.keys(maintainer).forEach((i) => {
|
|
10
|
+
const path = i;
|
|
13
11
|
const top = path.split('/')[1];
|
|
14
12
|
|
|
15
13
|
if (!ctx.params.name || top === ctx.params.name) {
|
package/lib/app.js
CHANGED
|
@@ -16,8 +16,10 @@ const favicon = require('koa-favicon');
|
|
|
16
16
|
const debug = require('./middleware/debug');
|
|
17
17
|
const accessControl = require('./middleware/access-control');
|
|
18
18
|
const antiHotlink = require('./middleware/anti-hotlink');
|
|
19
|
+
const loadOnDemand = require('./middleware/load-on-demand');
|
|
19
20
|
|
|
20
21
|
const router = require('./router');
|
|
22
|
+
const core_router = require('./core_router');
|
|
21
23
|
const protected_router = require('./protected_router');
|
|
22
24
|
const mount = require('koa-mount');
|
|
23
25
|
|
|
@@ -70,16 +72,21 @@ app.use(antiHotlink);
|
|
|
70
72
|
// 3 filter content
|
|
71
73
|
app.use(parameter);
|
|
72
74
|
|
|
75
|
+
// No Cache routes
|
|
76
|
+
app.use(mount('/', core_router.routes())).use(core_router.allowedMethods());
|
|
77
|
+
// API router
|
|
78
|
+
app.use(mount('/api', api_router.routes())).use(api_router.allowedMethods());
|
|
79
|
+
|
|
73
80
|
// 2 cache
|
|
74
81
|
app.use(cache(app));
|
|
75
82
|
|
|
83
|
+
// 1 load on demand
|
|
84
|
+
app.use(loadOnDemand(app));
|
|
85
|
+
|
|
76
86
|
// router
|
|
77
87
|
app.use(mount('/', router.routes())).use(router.allowedMethods());
|
|
78
88
|
|
|
79
89
|
// routes the require authentication
|
|
80
90
|
app.use(mount('/protected', protected_router.routes())).use(protected_router.allowedMethods());
|
|
81
91
|
|
|
82
|
-
// API router
|
|
83
|
-
app.use(mount('/api', api_router.routes())).use(api_router.allowedMethods());
|
|
84
|
-
|
|
85
92
|
module.exports = app;
|
package/lib/config.js
CHANGED
|
@@ -33,6 +33,7 @@ const calculateValue = () => {
|
|
|
33
33
|
},
|
|
34
34
|
cache: {
|
|
35
35
|
type: typeof envs.CACHE_TYPE === 'undefined' ? 'memory' : envs.CACHE_TYPE, // 缓存类型,支持 'memory' 和 'redis',设为空可以禁止缓存
|
|
36
|
+
requestTimeout: parseInt(envs.CACHE_REQUEST_TIMEOUT) || 60,
|
|
36
37
|
routeExpire: parseInt(envs.CACHE_EXPIRE) || 5 * 60, // 路由缓存时间,单位为秒
|
|
37
38
|
contentExpire: parseInt(envs.CACHE_CONTENT_EXPIRE) || 1 * 60 * 60, // 不变内容缓存时间,单位为秒
|
|
38
39
|
},
|
|
@@ -54,6 +55,7 @@ const calculateValue = () => {
|
|
|
54
55
|
bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
|
|
55
56
|
bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
|
|
56
57
|
bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
|
|
58
|
+
imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
|
|
57
59
|
},
|
|
58
60
|
fanbox: {
|
|
59
61
|
session: envs.FANBOX_SESSION_ID,
|
|
@@ -65,6 +67,8 @@ const calculateValue = () => {
|
|
|
65
67
|
consumer_key: envs.TWITTER_CONSUMER_KEY,
|
|
66
68
|
consumer_secret: envs.TWITTER_CONSUMER_SECRET,
|
|
67
69
|
tokens: twitter_tokens,
|
|
70
|
+
authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
|
|
71
|
+
// 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
|
|
68
72
|
},
|
|
69
73
|
youtube: {
|
|
70
74
|
key: envs.YOUTUBE_KEY,
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
const Router = require('@koa/router');
|
|
2
|
+
const config = require('@/config').value;
|
|
3
|
+
const router = new Router();
|
|
4
|
+
|
|
5
|
+
// Load Core Route
|
|
6
|
+
router.get('/', require('./routes/index'));
|
|
7
|
+
|
|
8
|
+
router.get('/robots.txt', (ctx) => {
|
|
9
|
+
if (config.disallowRobot) {
|
|
10
|
+
ctx.set('Content-Type', 'text/plain');
|
|
11
|
+
ctx.body = 'User-agent: *\nDisallow: /';
|
|
12
|
+
} else {
|
|
13
|
+
ctx.throw(404, 'Not Found');
|
|
14
|
+
}
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
module.exports = router;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
const dirname = __dirname + '/v2';
|
|
2
|
+
|
|
3
|
+
// 遍历整个 routes 文件夹,收集模块 maintainer.js
|
|
4
|
+
const maintainerPath = require('require-all')({
|
|
5
|
+
dirname,
|
|
6
|
+
filter: /maintainer\.js$/,
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
const maintainers = {};
|
|
10
|
+
|
|
11
|
+
// 将收集到的自定义模块进行合并
|
|
12
|
+
for (const dir in maintainerPath) {
|
|
13
|
+
const routes = maintainerPath[dir]['maintainer.js']; // Do not merge other file
|
|
14
|
+
for (const key in routes) {
|
|
15
|
+
maintainers['/' + dir + key] = routes[key];
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// 兼容旧版路由
|
|
20
|
+
const router = require('./router');
|
|
21
|
+
router.stack.forEach((e) => {
|
|
22
|
+
if (!maintainers[e.path]) {
|
|
23
|
+
maintainers[e.path] = [];
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
module.exports = maintainers;
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
const md5 = require('@/utils/md5');
|
|
2
|
+
const config = require('@/config').value;
|
|
3
|
+
const logger = require('@/utils/logger');
|
|
4
|
+
const { RequestInProgressError } = require('@/errors');
|
|
5
|
+
|
|
6
|
+
const globalCache = {
|
|
7
|
+
get: () => null,
|
|
8
|
+
set: () => null,
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
let cacheModule = {
|
|
12
|
+
get: () => null,
|
|
13
|
+
set: () => null,
|
|
14
|
+
status: { available: false },
|
|
15
|
+
clients: {},
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
if (config.cache.type === 'redis') {
|
|
19
|
+
cacheModule = require('./redis');
|
|
20
|
+
const { redisClient } = cacheModule.clients;
|
|
21
|
+
globalCache.get = async (key) => {
|
|
22
|
+
if (key && cacheModule.status.available) {
|
|
23
|
+
const value = await redisClient.get(key);
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
globalCache.set = cacheModule.set;
|
|
28
|
+
} else if (config.cache.type === 'memory') {
|
|
29
|
+
cacheModule = require('./memory');
|
|
30
|
+
const { pageCache } = cacheModule.clients;
|
|
31
|
+
globalCache.get = (key) => {
|
|
32
|
+
if (key && cacheModule.status.available) {
|
|
33
|
+
return pageCache.get(key);
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
globalCache.set = (key, value, maxAge) => {
|
|
37
|
+
if (!value || value === 'undefined') {
|
|
38
|
+
value = '';
|
|
39
|
+
}
|
|
40
|
+
if (typeof value === 'object') {
|
|
41
|
+
value = JSON.stringify(value);
|
|
42
|
+
}
|
|
43
|
+
if (key) {
|
|
44
|
+
return pageCache.set(key, value, maxAge * 1000);
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
} else {
|
|
48
|
+
logger.error('Cache not available, concurrent requests are not limited. This could lead to bad behavior.');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// only give cache string, as the `!` condition tricky
|
|
52
|
+
// md5 is used to shrink key size
|
|
53
|
+
// plz, write these tips in comments!
|
|
54
|
+
module.exports = function (app) {
|
|
55
|
+
const { get, set, status } = cacheModule;
|
|
56
|
+
app.context.cache = {
|
|
57
|
+
...cacheModule,
|
|
58
|
+
tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire) => {
|
|
59
|
+
let v = await get(key);
|
|
60
|
+
if (!v) {
|
|
61
|
+
v = await getValueFunc();
|
|
62
|
+
set(key, v, maxAge);
|
|
63
|
+
} else {
|
|
64
|
+
let parsed;
|
|
65
|
+
try {
|
|
66
|
+
parsed = JSON.parse(v);
|
|
67
|
+
} catch (e) {
|
|
68
|
+
parsed = null;
|
|
69
|
+
}
|
|
70
|
+
if (parsed) {
|
|
71
|
+
v = parsed;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return v;
|
|
76
|
+
},
|
|
77
|
+
globalCache,
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
return async (ctx, next) => {
|
|
81
|
+
const key = 'koa-redis-cache:' + md5(ctx.request.path);
|
|
82
|
+
const controlKey = 'path-requested:' + md5(ctx.request.path);
|
|
83
|
+
|
|
84
|
+
if (!status.available) {
|
|
85
|
+
return next();
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const isRequesting = await globalCache.get(controlKey);
|
|
89
|
+
|
|
90
|
+
if (isRequesting === '1') {
|
|
91
|
+
throw new RequestInProgressError('This path is currently fetching, please come back later!');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
try {
|
|
95
|
+
const value = await globalCache.get(key);
|
|
96
|
+
|
|
97
|
+
if (value) {
|
|
98
|
+
ctx.response.status = 200;
|
|
99
|
+
if (config.cache.type === 'redis') {
|
|
100
|
+
ctx.response.set({
|
|
101
|
+
'X-Koa-Redis-Cache': 'true',
|
|
102
|
+
});
|
|
103
|
+
} else if (config.cache.type === 'memory') {
|
|
104
|
+
ctx.response.set({
|
|
105
|
+
'X-Koa-Memory-Cache': 'true',
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
ctx.state.data = JSON.parse(value);
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
} catch (e) {
|
|
112
|
+
//
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Doesn't hit the cache? We need to let others know!
|
|
116
|
+
await globalCache.set(controlKey, '1', config.cache.requestTimeout);
|
|
117
|
+
await next();
|
|
118
|
+
|
|
119
|
+
if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) {
|
|
120
|
+
ctx.state.data.lastBuildDate = new Date().toUTCString();
|
|
121
|
+
const body = JSON.stringify(ctx.state.data);
|
|
122
|
+
await globalCache.set(key, body, config.cache.routeExpire);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// We need to let it go, even no cache set.
|
|
126
|
+
// Wait to set cache so the next request could be handled correctly
|
|
127
|
+
await globalCache.set(controlKey, '0', config.cache.requestTimeout);
|
|
128
|
+
};
|
|
129
|
+
};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
const Lru = require('lru-cache');
|
|
2
|
+
const config = require('@/config').value;
|
|
3
|
+
|
|
4
|
+
const status = { available: false };
|
|
5
|
+
|
|
6
|
+
const pageCache = new Lru({
|
|
7
|
+
maxAge: config.cache.routeExpire * 1000,
|
|
8
|
+
max: Infinity,
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
const routeCache = new Lru({
|
|
12
|
+
maxAge: config.cache.routeExpire * 1000,
|
|
13
|
+
max: Infinity,
|
|
14
|
+
updateAgeOnGet: true,
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
status.available = true;
|
|
18
|
+
|
|
19
|
+
module.exports = {
|
|
20
|
+
get: (key, refresh = true) => {
|
|
21
|
+
if (key && status.available) {
|
|
22
|
+
let value = (refresh ? routeCache : pageCache).get(key);
|
|
23
|
+
if (value) {
|
|
24
|
+
value = value + '';
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
set: (key, value, maxAge = config.cache.contentExpire, refresh = true) => {
|
|
30
|
+
if (!value || value === 'undefined') {
|
|
31
|
+
value = '';
|
|
32
|
+
}
|
|
33
|
+
if (typeof value === 'object') {
|
|
34
|
+
value = JSON.stringify(value);
|
|
35
|
+
}
|
|
36
|
+
if (key && status.available) {
|
|
37
|
+
return (refresh ? routeCache : pageCache).set(key, value, maxAge * 1000);
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
clients: { pageCache, routeCache },
|
|
41
|
+
status,
|
|
42
|
+
};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
const config = require('@/config').value;
|
|
2
|
+
const Redis = require('ioredis');
|
|
3
|
+
const logger = require('@/utils/logger');
|
|
4
|
+
|
|
5
|
+
const redisClient = new Redis(config.redis.url);
|
|
6
|
+
|
|
7
|
+
const status = { available: false };
|
|
8
|
+
|
|
9
|
+
redisClient.on('error', (error) => {
|
|
10
|
+
status.available = false;
|
|
11
|
+
logger.error('Redis error: ', error);
|
|
12
|
+
});
|
|
13
|
+
redisClient.on('end', () => {
|
|
14
|
+
status.available = false;
|
|
15
|
+
});
|
|
16
|
+
redisClient.on('connect', () => {
|
|
17
|
+
status.available = true;
|
|
18
|
+
logger.info('Redis connected.');
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
module.exports = {
|
|
22
|
+
get: async (key, refresh = true) => {
|
|
23
|
+
if (key && status.available) {
|
|
24
|
+
let value = await redisClient.get(key);
|
|
25
|
+
if (value && refresh) {
|
|
26
|
+
redisClient.expire(key, config.cache.contentExpire);
|
|
27
|
+
value = value + '';
|
|
28
|
+
}
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
set: (key, value, maxAge = config.cache.contentExpire) => {
|
|
33
|
+
if (!status.available) {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
if (!value || value === 'undefined') {
|
|
37
|
+
value = '';
|
|
38
|
+
}
|
|
39
|
+
if (typeof value === 'object') {
|
|
40
|
+
value = JSON.stringify(value);
|
|
41
|
+
}
|
|
42
|
+
if (key) {
|
|
43
|
+
return redisClient.set(key, value, 'EX', maxAge); // setMode: https://redis.io/commands/set
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
clients: { redisClient },
|
|
47
|
+
status,
|
|
48
|
+
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
const mount = require('koa-mount');
|
|
2
|
+
const Router = require('@koa/router');
|
|
3
|
+
const routes = require('../v2router');
|
|
4
|
+
const loadedRoutes = new Set();
|
|
5
|
+
|
|
6
|
+
module.exports = function (app) {
|
|
7
|
+
return async function (ctx, next) {
|
|
8
|
+
const p = ctx.request.path.split('/').filter(Boolean);
|
|
9
|
+
let modName = null;
|
|
10
|
+
let mounted = false;
|
|
11
|
+
|
|
12
|
+
if (p.length > 0) {
|
|
13
|
+
modName = p[0];
|
|
14
|
+
if (!loadedRoutes.has(modName)) {
|
|
15
|
+
const mod = routes[modName];
|
|
16
|
+
// Mount module
|
|
17
|
+
if (mod) {
|
|
18
|
+
mounted = true;
|
|
19
|
+
loadedRoutes.add(modName);
|
|
20
|
+
const router = new Router();
|
|
21
|
+
mod(router);
|
|
22
|
+
app.use(mount(`/${modName}`, router.routes())).use(router.allowedMethods());
|
|
23
|
+
}
|
|
24
|
+
} else {
|
|
25
|
+
mounted = true;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
await next();
|
|
30
|
+
|
|
31
|
+
// We should only add it when koa router matched
|
|
32
|
+
if (mounted && ctx._matchedRoute) {
|
|
33
|
+
ctx._matchedRoute = `/${modName}${ctx._matchedRoute}`;
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
};
|
|
@@ -3,6 +3,8 @@ const config = require('@/config').value;
|
|
|
3
3
|
const art = require('art-template');
|
|
4
4
|
const path = require('path');
|
|
5
5
|
|
|
6
|
+
const { RequestInProgressError } = require('@/errors');
|
|
7
|
+
|
|
6
8
|
let Sentry;
|
|
7
9
|
|
|
8
10
|
if (config.sentry.dsn) {
|
|
@@ -33,77 +35,10 @@ module.exports = async (ctx, next) => {
|
|
|
33
35
|
let message = err;
|
|
34
36
|
if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
|
|
35
37
|
message = `${err.message}: target website might be blocking our access, you can <a href="https://docs.rsshub.app/install/">host your own RSSHub instance</a> for a better usability.`;
|
|
36
|
-
|
|
37
|
-
const templateZh = `<!--
|
|
38
|
-
请确保已阅读 [文档](https://docs.rsshub.app) 内相关部分,并按照模版提供信息,否则 issue 将被立即关闭。
|
|
39
|
-
|
|
40
|
-
由于部分源网站反爬缘故,演示地址一些 rss 会返回 status code 403,该问题不是 RSSHub 所致,请勿提交 issue。
|
|
41
|
-
-->
|
|
42
|
-
|
|
43
|
-
- 完整路由地址,包含所有必选与可选参数
|
|
44
|
-
\`${ctx.path}\`
|
|
45
|
-
- 预期是什么
|
|
46
|
-
|
|
47
|
-
- 实际发生了什么
|
|
48
|
-
|
|
49
|
-
- 部署相关信息
|
|
50
|
-
|
|
51
|
-
<!--
|
|
52
|
-
如果是演示地址(rsshub.app)有此问题请删除此部分。
|
|
53
|
-
|
|
54
|
-
请确保您部署的是[主线 master 分支](https://github.com/DIYgod/RSSHub/tree/master)最新版 RSSHub。
|
|
55
|
-
-->
|
|
56
|
-
|
|
57
|
-
| Env | Value |
|
|
58
|
-
| ------------------ | ------------- |
|
|
59
|
-
| OS | |
|
|
60
|
-
| Node version |${process.version} |
|
|
61
|
-
| if Docker, version | |
|
|
62
|
-
|
|
63
|
-
- 额外信息(日志、报错等)
|
|
64
|
-
\`\`\`
|
|
65
|
-
${err.message}
|
|
66
|
-
\`\`\` `;
|
|
67
|
-
|
|
68
|
-
const templateEn = `<!--
|
|
69
|
-
Please ensure you have read [documentation](https://docs.rsshub.app/en), and provide all the information required by this template, otherwise the issue will be closed immediately.
|
|
70
|
-
|
|
71
|
-
Due to the anti-crawling policy implemented by certain websites, some RSS routes provided by the demo will return status code 403. This is not an issue caused by RSSHub and please do not report it.
|
|
72
|
-
-->
|
|
73
|
-
|
|
74
|
-
- The involved route, with all required and optional parameters
|
|
75
|
-
\`${ctx.path}\`
|
|
76
|
-
- What is expected
|
|
77
|
-
|
|
78
|
-
- What is actually happening
|
|
79
|
-
|
|
80
|
-
- Self-deployment information
|
|
81
|
-
|
|
82
|
-
<!--
|
|
83
|
-
Delete this section if you are using [RSSHub demo](https://rsshub.app).
|
|
84
|
-
|
|
85
|
-
Please ensure you have deployed the [master branch](https://github.com/DIYgod/RSSHub/tree/master) of RSSHub.
|
|
86
|
-
-->
|
|
87
|
-
|
|
88
|
-
| Env | Value |
|
|
89
|
-
| ------------------ | ------------- |
|
|
90
|
-
| OS | |
|
|
91
|
-
| Node version |${process.version} |
|
|
92
|
-
| if Docker, version | |
|
|
93
|
-
|
|
94
|
-
- Additional info (logs errors etc)
|
|
95
|
-
\`\`\`
|
|
96
|
-
${err.message}
|
|
97
|
-
\`\`\` `;
|
|
98
|
-
|
|
99
|
-
message += `<br><br>如果您认为 RSSHub 导致了该错误,<a href="https://github.com/DIYgod/RSSHub/issues/new?template=bug_report_zh.md&title=[BUG]&body=${encodeURI(templateZh)}" target="_blank">请在 GitHub 报告</a>。`;
|
|
100
|
-
|
|
101
|
-
message += `<br><br>If you believe this is an error caused by RSSHub, please <a href="https://github.com/DIYgod/RSSHub/issues/new?template=bug_report_en.md&title=[BUG]&body=${encodeURI(
|
|
102
|
-
templateEn
|
|
103
|
-
)}" target="_blank">report me on GitHub</a>.`;
|
|
104
38
|
} else if (err instanceof Error) {
|
|
105
39
|
message = process.env.NODE_ENV === 'production' ? err.message : err.stack;
|
|
106
40
|
}
|
|
41
|
+
|
|
107
42
|
logger.error(`Error in ${ctx.request.path}: ${message}`);
|
|
108
43
|
|
|
109
44
|
if (config.isPackage) {
|
|
@@ -116,7 +51,11 @@ Please ensure you have deployed the [master branch](https://github.com/DIYgod/RS
|
|
|
116
51
|
ctx.set({
|
|
117
52
|
'Content-Type': 'text/html; charset=UTF-8',
|
|
118
53
|
});
|
|
119
|
-
|
|
54
|
+
|
|
55
|
+
if (err instanceof RequestInProgressError) {
|
|
56
|
+
ctx.status = 503;
|
|
57
|
+
message = err.message;
|
|
58
|
+
} else if (ctx.status === 403) {
|
|
120
59
|
message = err.message;
|
|
121
60
|
} else {
|
|
122
61
|
ctx.status = 404;
|
|
@@ -127,6 +66,8 @@ Please ensure you have deployed the [master branch](https://github.com/DIYgod/RS
|
|
|
127
66
|
ctx.body = art(path.resolve(__dirname, '../views/error.art'), {
|
|
128
67
|
requestPath,
|
|
129
68
|
message,
|
|
69
|
+
errorPath: ctx.path,
|
|
70
|
+
nodeVersion: process.version,
|
|
130
71
|
});
|
|
131
72
|
}
|
|
132
73
|
|
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
const art = require('art-template');
|
|
2
2
|
const path = require('path');
|
|
3
3
|
const config = require('@/config').value;
|
|
4
|
-
const typeRegex = /\.(atom|rss)$/;
|
|
5
|
-
const unsupportedRegex = /\.json$/;
|
|
4
|
+
const typeRegex = /\.(atom|rss|debug\.json)$/;
|
|
6
5
|
|
|
7
6
|
module.exports = async (ctx, next) => {
|
|
8
|
-
if (ctx.request.path.match(unsupportedRegex)) {
|
|
9
|
-
throw Error('<b>JSON output had been removed, see: <a href="https://github.com/DIYgod/RSSHub/issues/1114">https://github.com/DIYgod/RSSHub/issues/1114</a></b>');
|
|
10
|
-
}
|
|
11
|
-
|
|
12
7
|
if (ctx.headers['user-agent'] && ctx.headers['user-agent'].includes('Reeder')) {
|
|
13
8
|
ctx.request.path = ctx.request.path.replace(/.com$/, '');
|
|
14
9
|
}
|
|
@@ -18,6 +13,17 @@ module.exports = async (ctx, next) => {
|
|
|
18
13
|
|
|
19
14
|
await next();
|
|
20
15
|
|
|
16
|
+
if (ctx.state.type[1] === 'debug.json' && config.debugInfo) {
|
|
17
|
+
ctx.set({
|
|
18
|
+
'Content-Type': 'application/json; charset=UTF-8',
|
|
19
|
+
});
|
|
20
|
+
if (ctx.state.json) {
|
|
21
|
+
ctx.body = JSON.stringify(ctx.state.json, null, 4);
|
|
22
|
+
} else {
|
|
23
|
+
ctx.body = JSON.stringify({ message: 'plugin does not set json' });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
21
27
|
if (!ctx.body) {
|
|
22
28
|
let template;
|
|
23
29
|
|