rsshub 1.0.0-master.ef8e2e1 → 1.0.0-master.efd646d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/lib/config.js +9 -1
- package/lib/middleware/anti-hotlink.js +40 -14
- package/lib/radar-rules.js +0 -189
- package/lib/router.js +102 -121
- package/lib/routes/mastodon/utils.js +14 -14
- package/lib/routes/tencent/wechat/miniprogram/framework.js +4 -2
- package/lib/routes/twitter/developer-api/search.js +3 -3
- package/lib/routes/twitter/developer-api/user.js +2 -2
- package/lib/routes/twitter/followings.js +11 -13
- package/lib/routes/twitter/likes.js +2 -2
- package/lib/routes/twitter/list.js +4 -4
- package/lib/routes/twitter/trends.js +5 -4
- package/lib/routes/twitter/utils.js +73 -71
- package/lib/utils/parse-date.js +8 -7
- package/lib/utils/puppeteer.js +20 -3
- package/lib/utils/rand-user-agent.js +29 -0
- package/lib/utils/request-wrapper.js +5 -6
- package/lib/utils/wechat-mp.js +54 -3
- package/lib/v2/abskoop/index.js +15 -10
- package/lib/v2/abskoop/nsfw.js +52 -0
- package/lib/v2/abskoop/router.js +1 -0
- package/lib/v2/agirls/index.js +6 -0
- package/lib/v2/ally/maintainer.js +3 -0
- package/lib/v2/ally/radar.js +13 -0
- package/lib/v2/ally/rail.js +109 -0
- package/lib/v2/ally/router.js +3 -0
- package/lib/v2/arknights/announce.js +55 -0
- package/lib/{routes → v2}/arknights/japan.js +2 -1
- package/lib/v2/arknights/maintainer.js +5 -0
- package/lib/{routes → v2}/arknights/news.js +6 -5
- package/lib/v2/arknights/radar.js +40 -0
- package/lib/v2/arknights/router.js +8 -0
- package/lib/v2/bilibili/article.js +37 -15
- package/lib/v2/bilibili/page.js +2 -3
- package/lib/v2/bilibili/video.js +1 -0
- package/lib/v2/bjx/huanbao.js +78 -0
- package/lib/v2/bjx/maintainer.js +4 -0
- package/lib/v2/bjx/radar.js +21 -0
- package/lib/v2/bjx/router.js +4 -0
- package/lib/v2/bjx/types.js +33 -0
- package/lib/v2/bookfere/category.js +36 -0
- package/lib/v2/bookfere/maintainer.js +3 -0
- package/lib/v2/bookfere/radar.js +11 -0
- package/lib/v2/bookfere/router.js +3 -0
- package/lib/v2/c114/maintainer.js +3 -0
- package/lib/v2/c114/radar.js +13 -0
- package/lib/v2/c114/roll.js +57 -0
- package/lib/v2/c114/router.js +3 -0
- package/lib/v2/cankaoxiaoxi/index.js +81 -0
- package/lib/v2/cankaoxiaoxi/maintainer.js +3 -0
- package/lib/v2/cankaoxiaoxi/radar.js +61 -0
- package/lib/v2/cankaoxiaoxi/router.js +3 -0
- package/lib/{routes → v2}/cctv/category.js +1 -1
- package/lib/{routes → v2}/cctv/jx.js +6 -1
- package/lib/{routes → v2}/cctv/lm.js +1 -1
- package/lib/v2/cctv/maintainer.js +7 -0
- package/lib/v2/cctv/radar.js +41 -0
- package/lib/v2/cctv/router.js +7 -0
- package/lib/{routes → v2}/cctv/special.js +6 -4
- package/lib/{routes → v2}/cctv/utils/mzzlbg.js +8 -6
- package/lib/{routes → v2}/cctv/utils/news.js +25 -22
- package/lib/{routes → v2}/cctv/utils/xinwen1j1.js +20 -17
- package/lib/v2/cctv/xwlb.js +50 -0
- package/lib/v2/chinacef/experts.js +38 -0
- package/lib/v2/chinacef/hot.js +38 -0
- package/lib/v2/chinacef/index.js +39 -0
- package/lib/v2/chinacef/maintainer.js +5 -0
- package/lib/v2/chinacef/radar.js +25 -0
- package/lib/v2/chinacef/router.js +5 -0
- package/lib/v2/{wechat → chinacef}/templates/description.art +0 -0
- package/lib/v2/chinacef/utils.js +55 -0
- package/lib/v2/chinafactcheck/index.js +38 -0
- package/lib/v2/chinafactcheck/maintainer.js +3 -0
- package/lib/v2/chinafactcheck/radar.js +13 -0
- package/lib/v2/chinafactcheck/router.js +3 -0
- package/lib/v2/chinafactcheck/templates/description.art +1 -0
- package/lib/v2/chinafactcheck/utils.js +60 -0
- package/lib/v2/chinanews/index.js +65 -0
- package/lib/v2/chinanews/maintainer.js +3 -0
- package/lib/v2/chinanews/radar.js +13 -0
- package/lib/v2/chinanews/router.js +3 -0
- package/lib/v2/chinathinktanks/maintainer.js +3 -0
- package/lib/v2/chinathinktanks/radar.js +13 -0
- package/lib/v2/chinathinktanks/router.js +3 -0
- package/lib/v2/chinathinktanks/viewpoint.js +53 -0
- package/lib/v2/chinaventure/index.js +63 -0
- package/lib/v2/chinaventure/maintainer.js +3 -0
- package/lib/v2/chinaventure/radar.js +13 -0
- package/lib/v2/chinaventure/router.js +3 -0
- package/lib/v2/cnbeta/index.js +2 -1
- package/lib/v2/cnbeta/utils.js +1 -0
- package/lib/v2/coomer/artist.js +9 -0
- package/lib/v2/coomer/maintainer.js +4 -0
- package/lib/v2/coomer/posts.js +7 -0
- package/lib/v2/coomer/radar.js +19 -0
- package/lib/v2/coomer/router.js +4 -0
- package/lib/v2/coomer/utils.js +53 -0
- package/lib/v2/csu/cse.js +33 -0
- package/lib/v2/csu/fetch_article.js +12 -0
- package/lib/{routes/universities → v2}/csu/job.js +0 -0
- package/lib/v2/csu/maintainer.js +3 -0
- package/lib/v2/csu/radar.js +13 -0
- package/lib/v2/csu/router.js +4 -0
- package/lib/v2/dapenti/maintainer.js +4 -0
- package/lib/v2/dapenti/radar.js +27 -0
- package/lib/v2/dapenti/router.js +4 -0
- package/lib/{routes → v2}/dapenti/subject.js +1 -1
- package/lib/{routes → v2}/dapenti/tugua.js +1 -1
- package/lib/v2/dapenti/utils.js +75 -0
- package/lib/{routes → v2}/dongqiudi/daily.js +10 -13
- package/lib/v2/dongqiudi/maintainer.js +8 -0
- package/lib/{routes → v2}/dongqiudi/player_news.js +0 -0
- package/lib/v2/dongqiudi/radar.js +45 -0
- package/lib/{routes → v2}/dongqiudi/result.js +2 -1
- package/lib/v2/dongqiudi/router.js +8 -0
- package/lib/{routes → v2}/dongqiudi/special.js +0 -0
- package/lib/{routes → v2}/dongqiudi/team_news.js +0 -0
- package/lib/{routes → v2}/dongqiudi/top_news.js +12 -15
- package/lib/{routes → v2}/dongqiudi/utils.js +44 -33
- package/lib/v2/douyin/maintainer.js +3 -0
- package/lib/v2/douyin/radar.js +13 -0
- package/lib/v2/douyin/router.js +3 -0
- package/lib/v2/douyin/templates/cover.art +15 -0
- package/lib/v2/douyin/templates/desc.art +3 -0
- package/lib/v2/douyin/templates/embed.art +13 -0
- package/lib/v2/douyin/templates/iframe.art +14 -0
- package/lib/v2/douyin/user.js +125 -0
- package/lib/v2/dut/defaults.js +53 -0
- package/lib/v2/dut/index.js +78 -0
- package/lib/v2/dut/maintainer.js +14 -0
- package/lib/v2/dut/radar.js +109 -0
- package/lib/v2/dut/router.js +4 -0
- package/lib/v2/dut/shortcuts.js +77 -0
- package/lib/{routes → v2}/dx2025/index.js +0 -0
- package/lib/v2/dx2025/maintainer.js +3 -0
- package/lib/v2/dx2025/radar.js +19 -0
- package/lib/v2/dx2025/router.js +3 -0
- package/lib/v2/e-hentai/radar.js +21 -3
- package/lib/v2/eagle/blog.js +47 -0
- package/lib/{routes → v2}/eagle/changelog.js +3 -1
- package/lib/v2/eagle/maintainer.js +4 -0
- package/lib/v2/eagle/radar.js +101 -0
- package/lib/v2/eagle/router.js +4 -0
- package/lib/{routes → v2}/ehentai/ehapi.js +50 -21
- package/lib/{routes → v2}/ehentai/favorites.js +4 -2
- package/lib/v2/ehentai/maintainer.js +5 -0
- package/lib/v2/ehentai/router.js +5 -0
- package/lib/{routes → v2}/ehentai/search.js +5 -3
- package/lib/{routes → v2}/ehentai/tag.js +4 -2
- package/lib/v2/elsevier/templates/description.art +2 -0
- package/lib/v2/embassy/index.js +60 -0
- package/lib/v2/embassy/maintainer.js +3 -0
- package/lib/v2/embassy/radar.js +258 -0
- package/lib/v2/embassy/router.js +3 -0
- package/lib/{routes → v2}/embassy/supportedList.js +5 -5
- package/lib/v2/epicgames/index.js +27 -70
- package/lib/v2/epicgames/maintainer.js +1 -1
- package/lib/v2/epicgames/router.js +1 -1
- package/lib/v2/epicgames/templates/description.art +2 -0
- package/lib/v2/firefox/index.js +7 -2
- package/lib/v2/fortunechina/index.js +1 -1
- package/lib/v2/github/comments.js +7 -0
- package/lib/v2/gitpod/blog.js +8 -0
- package/lib/v2/gov/immiau/news.js +43 -0
- package/lib/v2/gov/maintainer.js +16 -11
- package/lib/v2/gov/moe/moe.js +89 -0
- package/lib/v2/gov/nrta/news.js +63 -0
- package/lib/v2/gov/radar.js +33 -0
- package/lib/v2/gov/router.js +17 -12
- package/lib/v2/gov/sichuan/deyang/govpulicinfo.js +1 -1
- package/lib/v2/guancha/index.js +1 -1
- package/lib/{routes/guokr/calendar.js → v2/guokr/channel.js} +2 -2
- package/lib/v2/guokr/maintainer.js +4 -0
- package/lib/v2/guokr/radar.js +19 -0
- package/lib/v2/guokr/router.js +4 -0
- package/lib/{routes → v2}/guokr/scientific.js +3 -3
- package/lib/v2/hkej/index.js +9 -15
- package/lib/v2/hkepc/index.js +8 -0
- package/lib/v2/hket/index.js +10 -2
- package/lib/v2/huangz/index.js +29 -0
- package/lib/v2/huangz/maintainer.js +3 -0
- package/lib/v2/huangz/radar.js +13 -0
- package/lib/v2/huangz/router.js +3 -0
- package/lib/v2/huanqiu/index.js +65 -0
- package/lib/v2/huanqiu/maintainer.js +3 -0
- package/lib/v2/huanqiu/radar.js +13 -0
- package/lib/v2/huanqiu/router.js +3 -0
- package/lib/v2/ieee/templates/description.art +2 -0
- package/lib/v2/iguoguo/index.js +75 -0
- package/lib/v2/iguoguo/maintainer.js +3 -0
- package/lib/v2/iguoguo/radar.js +11 -0
- package/lib/v2/iguoguo/router.js +3 -0
- package/lib/v2/jasa/templates/description.art +2 -0
- package/lib/v2/javdb/utils.js +4 -4
- package/lib/v2/jd/maintainer.js +3 -0
- package/lib/v2/jd/price.js +42 -0
- package/lib/v2/jd/radar.js +13 -0
- package/lib/v2/jd/router.js +3 -0
- package/lib/v2/jd/templates/description.art +3 -0
- package/lib/v2/jike/maintainer.js +5 -0
- package/lib/v2/jike/radar.js +34 -0
- package/lib/v2/jike/router.js +5 -0
- package/lib/v2/jike/topic.js +42 -0
- package/lib/v2/jike/topicText.js +21 -0
- package/lib/{routes → v2}/jike/user.js +9 -9
- package/lib/{routes/jike/common.js → v2/jike/utils.js} +60 -31
- package/lib/v2/news/whxw.js +2 -2
- package/lib/v2/njit/jwc.js +92 -0
- package/lib/v2/njit/maintainer.js +4 -0
- package/lib/v2/njit/radar.js +21 -0
- package/lib/v2/njit/router.js +4 -0
- package/lib/v2/njit/tzgg.js +74 -0
- package/lib/v2/njust/cwc.js +39 -0
- package/lib/v2/njust/dgxg.js +40 -0
- package/lib/v2/njust/eo.js +50 -0
- package/lib/v2/njust/eoe.js +39 -0
- package/lib/v2/njust/gs.js +41 -0
- package/lib/v2/njust/jwc.js +41 -0
- package/lib/v2/njust/maintainer.js +8 -0
- package/lib/v2/njust/radar.js +110 -0
- package/lib/v2/njust/router.js +8 -0
- package/lib/v2/njust/utils.js +31 -0
- package/lib/v2/npm/package.js +1 -1
- package/lib/v2/npm/router.js +1 -1
- package/lib/v2/people/index.js +1 -1
- package/lib/v2/pianyuan/app.js +34 -0
- package/lib/v2/pianyuan/maintainer.js +4 -0
- package/lib/v2/pianyuan/radar.js +13 -0
- package/lib/v2/pianyuan/router.js +4 -0
- package/lib/v2/pianyuan/search.js +45 -0
- package/lib/v2/pianyuan/utils.js +88 -0
- package/lib/{routes/universities → v2}/pku/bbs/hot.js +5 -10
- package/lib/{routes/universities → v2}/pku/cls/lecture.js +5 -7
- package/lib/v2/pku/eecs.js +64 -0
- package/lib/v2/pku/maintainer.js +6 -0
- package/lib/v2/pku/pkuyjs.js +29 -0
- package/lib/v2/pku/radar.js +109 -0
- package/lib/v2/pku/rccp/mzyt.js +23 -0
- package/lib/v2/pku/router.js +6 -0
- package/lib/v2/pku/scc/recruit.js +58 -0
- package/lib/v2/pku/utils.js +15 -0
- package/lib/v2/qbittorrent/maintainer.js +3 -0
- package/lib/v2/qbittorrent/news.js +59 -0
- package/lib/v2/qbittorrent/radar.js +13 -0
- package/lib/v2/qbittorrent/router.js +3 -0
- package/lib/v2/radio/index.js +48 -0
- package/lib/v2/radio/maintainer.js +3 -0
- package/lib/v2/radio/radar.js +13 -0
- package/lib/{routes → v2}/radio/radio.js +0 -0
- package/lib/v2/radio/router.js +3 -0
- package/lib/v2/radio/templates/description.art +5 -0
- package/lib/{routes → v2}/sciencenet/blog.js +17 -12
- package/lib/v2/sciencenet/maintainer.js +4 -0
- package/lib/v2/sciencenet/radar.js +19 -0
- package/lib/v2/sciencenet/router.js +4 -0
- package/lib/v2/sciencenet/user.js +65 -0
- package/lib/v2/sdu/cmse.js +59 -0
- package/lib/v2/sdu/cs.js +57 -0
- package/lib/v2/sdu/data.js +110 -0
- package/lib/v2/sdu/epe.js +55 -0
- package/lib/v2/sdu/extractor/index.js +15 -0
- package/lib/v2/sdu/extractor/sdrj.js +21 -0
- package/lib/v2/sdu/extractor/view.js +21 -0
- package/lib/v2/sdu/extractor/wh/jwc.js +24 -0
- package/lib/v2/sdu/extractor/wh/news.js +21 -0
- package/lib/v2/sdu/maintainer.js +9 -0
- package/lib/v2/sdu/mech.js +60 -0
- package/lib/v2/sdu/radar.js +159 -0
- package/lib/v2/sdu/router.js +9 -0
- package/lib/v2/sdu/sc.js +61 -0
- package/lib/v2/sdu/wh/jwc.js +42 -0
- package/lib/v2/sdu/wh/news.js +38 -0
- package/lib/{routes → v2}/sspai/activity.js +2 -1
- package/lib/{routes → v2}/sspai/author.js +15 -21
- package/lib/{routes → v2}/sspai/column.js +16 -20
- package/lib/v2/sspai/index.js +37 -0
- package/lib/v2/sspai/maintainer.js +12 -0
- package/lib/v2/sspai/matrix.js +38 -0
- package/lib/v2/sspai/radar.js +69 -0
- package/lib/v2/sspai/router.js +12 -0
- package/lib/{routes → v2}/sspai/series.js +2 -2
- package/lib/{routes → v2}/sspai/shortcutsGallery.js +3 -2
- package/lib/{routes → v2}/sspai/tag.js +15 -19
- package/lib/{routes → v2}/sspai/topic.js +15 -18
- package/lib/{routes → v2}/sspai/topics.js +12 -16
- package/lib/v2/syosetu/chapter.js +61 -0
- package/lib/v2/syosetu/maintainer.js +3 -0
- package/lib/v2/syosetu/radar.js +21 -0
- package/lib/v2/syosetu/router.js +1 -0
- package/lib/v2/szse/inquire.js +41 -0
- package/lib/v2/szse/maintainer.js +3 -0
- package/lib/{routes → v2}/szse/notice.js +0 -0
- package/lib/v2/szse/projectdynamic.js +90 -0
- package/lib/v2/szse/radar.js +21 -1
- package/lib/v2/szse/router.js +3 -0
- package/lib/v2/szse/rule.js +3 -3
- package/lib/v2/szse/templates/description.art +19 -0
- package/lib/v2/szse/templates/inquire.art +6 -0
- package/lib/{routes → v2}/taptap/changelog.js +2 -1
- package/lib/v2/taptap/maintainer.js +5 -0
- package/lib/v2/taptap/radar.js +25 -0
- package/lib/v2/taptap/review.js +69 -0
- package/lib/v2/taptap/router.js +5 -0
- package/lib/v2/taptap/templates/videoPost.art +2 -0
- package/lib/v2/taptap/topic.js +79 -0
- package/lib/v2/taptap/utils.js +48 -0
- package/lib/v2/test/index.js +1 -0
- package/lib/v2/thecover/channel.js +66 -0
- package/lib/v2/thecover/maintainer.js +3 -0
- package/lib/v2/thecover/radar.js +13 -0
- package/lib/v2/thecover/router.js +3 -0
- package/lib/v2/uraaka-joshi/maintainer.js +4 -0
- package/lib/v2/uraaka-joshi/radar.js +19 -0
- package/lib/v2/uraaka-joshi/router.js +4 -0
- package/lib/v2/uraaka-joshi/uraaka-joshi-user.js +89 -0
- package/lib/v2/uraaka-joshi/uraaka-joshi.js +71 -0
- package/lib/v2/ustc/eeis.js +59 -0
- package/lib/v2/ustc/gs.js +59 -0
- package/lib/v2/ustc/index.js +79 -0
- package/lib/{routes/universities → v2}/ustc/job.js +6 -5
- package/lib/{routes/universities/ustc/jwc/index.js → v2/ustc/jwc.js} +23 -26
- package/lib/v2/ustc/maintainer.js +8 -0
- package/lib/v2/ustc/radar.js +53 -0
- package/lib/v2/ustc/router.js +8 -0
- package/lib/v2/ustc/sist.js +59 -0
- package/lib/{routes/tencent → v2}/wechat/announce.js +3 -2
- package/lib/{routes/tencent → v2}/wechat/ce.js +0 -0
- package/lib/v2/wechat/data258.js +137 -0
- package/lib/{routes/tencent → v2}/wechat/ershcimi.js +0 -0
- package/lib/{routes/tencent → v2}/wechat/feeds.js +0 -0
- package/lib/v2/wechat/maintainer.js +11 -0
- package/lib/{routes/tencent → v2}/wechat/mp.js +0 -0
- package/lib/{routes/tencent → v2}/wechat/msgalbum.js +0 -0
- package/lib/v2/wechat/radar.js +76 -1
- package/lib/v2/wechat/router.js +11 -0
- package/lib/{routes/tencent → v2}/wechat/tgchannel.js +0 -0
- package/lib/{routes/tencent → v2}/wechat/uread.js +0 -0
- package/lib/{routes/tencent → v2}/wechat/wemp.js +0 -0
- package/lib/{routes/tencent → v2}/wechat/wxnmh.js +0 -0
- package/lib/v2/xueqiu/user.js +3 -2
- package/lib/v2/youzhiyouxing/maintainer.js +1 -1
- package/lib/v2/youzhiyouxing/materials.js +32 -22
- package/lib/v2/youzhiyouxing/radar.js +6 -6
- package/lib/v2/youzhiyouxing/router.js +1 -1
- package/lib/v2/yunspe/maintainer.js +3 -0
- package/lib/v2/yunspe/newsflash.js +42 -0
- package/lib/v2/yunspe/radar.js +13 -0
- package/lib/v2/yunspe/router.js +3 -0
- package/lib/v2/yyets/article.js +45 -0
- package/lib/v2/yyets/maintainer.js +4 -0
- package/lib/v2/yyets/radar.js +19 -0
- package/lib/v2/yyets/router.js +4 -0
- package/lib/v2/yyets/today.js +37 -0
- package/lib/{routes → v2}/yystv/category.js +8 -8
- package/lib/{routes → v2}/yystv/docs.js +2 -2
- package/lib/v2/yystv/maintainer.js +4 -0
- package/lib/v2/yystv/radar.js +49 -0
- package/lib/v2/yystv/router.js +4 -0
- package/lib/v2/zaobao/util.js +16 -4
- package/lib/{routes/universities → v2}/zju/career/index.js +14 -13
- package/lib/{routes/universities → v2}/zju/cst/custom.js +2 -1
- package/lib/{routes/universities → v2}/zju/cst/index.js +20 -24
- package/lib/v2/zju/grs/index.js +43 -0
- package/lib/v2/zju/list.js +66 -0
- package/lib/v2/zju/maintainer.js +8 -0
- package/lib/v2/zju/physics/index.js +52 -0
- package/lib/v2/zju/radar.js +139 -0
- package/lib/v2/zju/router.js +8 -0
- package/lib/v2/zotero/maintainer.js +3 -0
- package/lib/v2/zotero/radar.js +13 -0
- package/lib/v2/zotero/router.js +3 -0
- package/lib/v2/zotero/versions.js +36 -0
- package/lib/v2/zuel/maintainer.js +3 -0
- package/lib/v2/zuel/notice.js +52 -0
- package/lib/v2/zuel/radar.js +13 -0
- package/lib/v2/zuel/router.js +3 -0
- package/lib/views/atom.art +13 -1
- package/lib/views/welcome.art +4 -1
- package/package.json +29 -22
- package/lib/routes/bjx/huanbao.js +0 -78
- package/lib/routes/cctv/xwlb.js +0 -48
- package/lib/routes/dapenti/utils.js +0 -73
- package/lib/routes/dwnews/rank.js +0 -42
- package/lib/routes/dwnews/utils.js +0 -65
- package/lib/routes/dwnews/yaowen.js +0 -73
- package/lib/routes/embassy/index.js +0 -64
- package/lib/routes/gov/moe/moe.js +0 -84
- package/lib/routes/jike/topic.js +0 -65
- package/lib/routes/jike/topicText.js +0 -42
- package/lib/routes/pianyuan/app.js +0 -49
- package/lib/routes/rrys/review.js +0 -30
- package/lib/routes/sspai/matrix.js +0 -44
- package/lib/routes/szse/inquire.js +0 -87
- package/lib/routes/szse/projectdynamic.js +0 -103
- package/lib/routes/taptap/review.js +0 -39
- package/lib/routes/taptap/topic.js +0 -85
- package/lib/routes/tencent/wechat/_README +0 -1
- package/lib/routes/universities/dut/index.js +0 -60
- package/lib/routes/universities/dut/subsite.js +0 -50
- package/lib/routes/universities/njust/cwc/index.js +0 -37
- package/lib/routes/universities/njust/eo/index.js +0 -41
- package/lib/routes/universities/njust/eo/util.js +0 -28
- package/lib/routes/universities/njust/gs/index.js +0 -33
- package/lib/routes/universities/njust/jwc/index.js +0 -39
- package/lib/routes/universities/pku/eecs.js +0 -41
- package/lib/routes/universities/pku/pkuyjs.js +0 -23
- package/lib/routes/universities/pku/rccp/mzyt.js +0 -33
- package/lib/routes/universities/sdu/cmse.js +0 -55
- package/lib/routes/universities/sdu/cs.js +0 -55
- package/lib/routes/universities/sdu/epe.js +0 -54
- package/lib/routes/universities/sdu/mech.js +0 -58
- package/lib/routes/universities/sdu/sc.js +0 -56
- package/lib/routes/universities/ustc/index.js +0 -93
- package/lib/routes/universities/zju/grs/index.js +0 -44
- package/lib/routes/universities/zju/list.js +0 -78
- package/lib/routes/universities/zju/physics/index.js +0 -55
- package/lib/routes/uraaka-joshi/uraaka-joshi-user.js +0 -28
- package/lib/routes/uraaka-joshi/uraaka-joshi.js +0 -30
- package/lib/routes/yyets/todayfilelist.js +0 -34
- package/lib/v2/epicgames/supportedList.js +0 -23
- package/lib/v2/wechat/templates/image.art +0 -1
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const cheerio = require('cheerio');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
const timezone = require('@/utils/timezone');
|
|
5
|
+
|
|
6
|
+
const map = new Map([
|
|
7
|
+
['tzgg', { title: '中国科学技术大学研究生院 - 通知公告', id: '9' }],
|
|
8
|
+
['xwdt', { title: '中国科学技术大学研究生院 - 新闻动态', id: '10' }],
|
|
9
|
+
]);
|
|
10
|
+
|
|
11
|
+
const host = 'https://gradschool.ustc.edu.cn';
|
|
12
|
+
|
|
13
|
+
module.exports = async (ctx) => {
|
|
14
|
+
const type = ctx.params.type ?? 'tzgg';
|
|
15
|
+
const info = map.get(type);
|
|
16
|
+
if (!info) {
|
|
17
|
+
throw 'invalid type';
|
|
18
|
+
}
|
|
19
|
+
const id = info.id;
|
|
20
|
+
|
|
21
|
+
const response = await got(`${host}/column/${id}`);
|
|
22
|
+
const $ = cheerio.load(response.data);
|
|
23
|
+
let items = $('div.r-box > ul')
|
|
24
|
+
.find('li')
|
|
25
|
+
.toArray()
|
|
26
|
+
.map((item) => {
|
|
27
|
+
item = $(item);
|
|
28
|
+
const title = item.find('a').text().trim();
|
|
29
|
+
const link = item.find('a').attr('href').startsWith('/article') ? host + item.find('a').attr('href') : item.find('a').attr('href');
|
|
30
|
+
const pubDate = timezone(parseDate(item.find('time').text(), 'YYYY-MM-DD'), +8);
|
|
31
|
+
return {
|
|
32
|
+
title,
|
|
33
|
+
pubDate,
|
|
34
|
+
link,
|
|
35
|
+
};
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
items = await Promise.all(
|
|
39
|
+
items.map((item) =>
|
|
40
|
+
ctx.cache.tryGet(item.link, async () => {
|
|
41
|
+
let desc = '';
|
|
42
|
+
try {
|
|
43
|
+
const response = await got(item.link);
|
|
44
|
+
desc = cheerio.load(response.data)('article.article').html();
|
|
45
|
+
item.description = desc;
|
|
46
|
+
} catch (err) {
|
|
47
|
+
// intranet only contents
|
|
48
|
+
}
|
|
49
|
+
return item;
|
|
50
|
+
})
|
|
51
|
+
)
|
|
52
|
+
);
|
|
53
|
+
|
|
54
|
+
ctx.state.data = {
|
|
55
|
+
title: info.title,
|
|
56
|
+
link: `${host}/column/${id}`,
|
|
57
|
+
item: items,
|
|
58
|
+
};
|
|
59
|
+
};
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const cheerio = require('cheerio');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
const timezone = require('@/utils/timezone');
|
|
5
|
+
// const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'windows', device: 'desktop' });
|
|
6
|
+
|
|
7
|
+
const notice_type = {
|
|
8
|
+
jx: { title: '中国科学技术大学 - 教学类通知', url: 'https://ustc.edu.cn/tzgg/jxltz.htm' },
|
|
9
|
+
ky: { title: '中国科学技术大学 - 科研类通知', url: 'https://ustc.edu.cn/tzgg/kyltz.htm' },
|
|
10
|
+
gl: { title: '中国科学技术大学 - 管理类通知', url: 'https://ustc.edu.cn/tzgg/glltz.htm' },
|
|
11
|
+
fw: { title: '中国科学技术大学 - 服务类通知', url: 'https://ustc.edu.cn/tzgg/fwltz.htm' },
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
// 对防抓的措施
|
|
15
|
+
// function getCookie(ctx) {
|
|
16
|
+
// const cache_key = `ustc-cookie-${new Date().toLocaleDateString()}`;
|
|
17
|
+
// return ctx.cache.tryGet(cache_key, async () => {
|
|
18
|
+
// const { headers } = await got('https://ustc.edu.cn/system/resource/code/datainput.jsp', {
|
|
19
|
+
// headers: { 'user-agent': UA },
|
|
20
|
+
// });
|
|
21
|
+
// const cookie = headers['set-cookie']
|
|
22
|
+
// .filter((c) => c.match(/(user_trace_token|X_HTTP_TOKEN)/))
|
|
23
|
+
// .map((c) => c.split(';')[0])
|
|
24
|
+
// .join('; ');
|
|
25
|
+
// return cookie;
|
|
26
|
+
// });
|
|
27
|
+
// }
|
|
28
|
+
|
|
29
|
+
module.exports = async (ctx) => {
|
|
30
|
+
const type = ctx.params.type ?? 'gl';
|
|
31
|
+
// 发起 HTTP GET 请求
|
|
32
|
+
const response = await got({
|
|
33
|
+
method: 'get',
|
|
34
|
+
|
|
35
|
+
/* headers: {
|
|
36
|
+
'user-agent': UA,
|
|
37
|
+
cookie: await getCookie(ctx),
|
|
38
|
+
}, */
|
|
39
|
+
url: notice_type[type].url,
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
const $ = cheerio.load(response.data);
|
|
43
|
+
let items = $('table[portletmode=simpleList] > tbody > tr.light')
|
|
44
|
+
.map(function () {
|
|
45
|
+
const child = $(this).children();
|
|
46
|
+
const info = {
|
|
47
|
+
title: $(child[1]).find('a').attr('title'),
|
|
48
|
+
link: $(child[1]).find('a').attr('href').startsWith('../') ? new URL($(child[1]).find('a').attr('href'), notice_type[type].url).href : $(child[1]).find('a').attr('href'),
|
|
49
|
+
pubDate: timezone(parseDate($(child[2]).text(), 'YYYY-MM-DD'), +8),
|
|
50
|
+
};
|
|
51
|
+
return info;
|
|
52
|
+
})
|
|
53
|
+
.get();
|
|
54
|
+
|
|
55
|
+
items = await Promise.all(
|
|
56
|
+
items
|
|
57
|
+
.filter((item) => item.link)
|
|
58
|
+
.map((item) =>
|
|
59
|
+
ctx.cache.tryGet(item.link, async () => {
|
|
60
|
+
try {
|
|
61
|
+
const response = await got(item.link);
|
|
62
|
+
const $ = cheerio.load(response.data);
|
|
63
|
+
|
|
64
|
+
item.description = $('div.v_news_content').html();
|
|
65
|
+
} catch {
|
|
66
|
+
// intranet contents
|
|
67
|
+
}
|
|
68
|
+
return item;
|
|
69
|
+
})
|
|
70
|
+
)
|
|
71
|
+
);
|
|
72
|
+
|
|
73
|
+
ctx.state.data = {
|
|
74
|
+
title: notice_type[type].title,
|
|
75
|
+
description: notice_type[type].title,
|
|
76
|
+
link: notice_type[type].url,
|
|
77
|
+
item: items,
|
|
78
|
+
};
|
|
79
|
+
};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
const got = require('@/utils/got');
|
|
2
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
2
3
|
|
|
3
4
|
const titles = {
|
|
4
5
|
RecruitList: '专场招聘会',
|
|
@@ -8,7 +9,7 @@ const titles = {
|
|
|
8
9
|
};
|
|
9
10
|
|
|
10
11
|
module.exports = async (ctx) => {
|
|
11
|
-
const category = ctx.params.category
|
|
12
|
+
const category = ctx.params.category ?? 'joblist2';
|
|
12
13
|
|
|
13
14
|
const rand = 0.01234567890123456;
|
|
14
15
|
const apiRootUrl = 'http://ustc.ahbys.com';
|
|
@@ -65,26 +66,26 @@ module.exports = async (ctx) => {
|
|
|
65
66
|
case 'RecruitList':
|
|
66
67
|
return {
|
|
67
68
|
title: item.ID,
|
|
68
|
-
pubDate:
|
|
69
|
+
pubDate: parseDate(item.HoldDate),
|
|
69
70
|
link: `${rootUrl}/Recruit.html?id=${item.ID}`,
|
|
70
71
|
};
|
|
71
72
|
case 'Doublechoice':
|
|
72
73
|
return {
|
|
73
74
|
ID: item.ID,
|
|
74
75
|
title: item.Theme,
|
|
75
|
-
pubDate:
|
|
76
|
+
pubDate: parseDate(item.HoldDate),
|
|
76
77
|
link: `${rootUrl}/R2.html?id=${item.ID}`,
|
|
77
78
|
};
|
|
78
79
|
case 'Broadcast':
|
|
79
80
|
return {
|
|
80
81
|
title: item.ID,
|
|
81
|
-
pubDate:
|
|
82
|
+
pubDate: parseDate(item.HoldDate),
|
|
82
83
|
link: `${rootUrl}/R2.html?id=${item.ID}`,
|
|
83
84
|
};
|
|
84
85
|
case 'joblist2':
|
|
85
86
|
return {
|
|
86
87
|
title: item.JID,
|
|
87
|
-
pubDate:
|
|
88
|
+
pubDate: parseDate(item.UpdateDate),
|
|
88
89
|
link: `${rootUrl}/Job2.html?jid=${item.JID}&cid=${item.CompanyID}`,
|
|
89
90
|
};
|
|
90
91
|
default:
|
|
@@ -1,52 +1,49 @@
|
|
|
1
1
|
const got = require('@/utils/got');
|
|
2
2
|
const cheerio = require('cheerio');
|
|
3
|
-
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
const timezone = require('@/utils/timezone');
|
|
5
|
+
// const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'windows', device: 'desktop' });
|
|
4
6
|
const noticeUrl = 'https://www.teach.ustc.edu.cn/category/notice';
|
|
5
7
|
const noticeType = { teaching: '教学', info: '信息', exam: '考试', exchange: '交流' };
|
|
6
8
|
|
|
7
9
|
module.exports = async (ctx) => {
|
|
8
|
-
const type = ctx.params.type
|
|
10
|
+
const type = ctx.params.type ?? '';
|
|
9
11
|
// 发起 HTTP GET 请求
|
|
10
12
|
const response = await got({
|
|
11
13
|
method: 'get',
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
|
|
15
|
+
/* headers: {
|
|
16
|
+
'user-agent': UA,
|
|
17
|
+
}, */
|
|
15
18
|
url: `${noticeUrl}${type === '' ? '' : '-' + type}`,
|
|
16
19
|
});
|
|
17
20
|
|
|
18
21
|
const $ = cheerio.load(response.data);
|
|
19
|
-
|
|
22
|
+
let items = $(type === '' ? 'ul[class="article-list with-tag"] > li' : 'ul[class=article-list] > li')
|
|
20
23
|
.map(function () {
|
|
21
24
|
const child = $(this).children();
|
|
22
25
|
const info = {
|
|
23
26
|
title: type === '' ? $(child[0]).find('a').text() + ' - ' + $(child[1]).find('a').text() : $(child[0]).find('a').text(),
|
|
24
27
|
link: type === '' ? $(child[1]).find('a').attr('href') : $(child[0]).find('a').attr('href'),
|
|
28
|
+
pubDate: timezone(parseDate($(this).find('.date').text().trim(), 'YYYY-MM-DD'), +8),
|
|
25
29
|
};
|
|
26
30
|
return info;
|
|
27
31
|
})
|
|
28
32
|
.get();
|
|
29
33
|
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
items = await Promise.all(
|
|
35
|
+
items
|
|
32
36
|
.filter((item) => item.link)
|
|
33
|
-
.map(
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
link: itemUrl,
|
|
44
|
-
description: $('main[class=single]').html(),
|
|
45
|
-
pubDate: new Date($('li[class=meta-date]').text()).toUTCString(),
|
|
46
|
-
};
|
|
47
|
-
ctx.cache.set(itemUrl, JSON.stringify(single));
|
|
48
|
-
return Promise.resolve(single);
|
|
49
|
-
})
|
|
37
|
+
.map((item) =>
|
|
38
|
+
ctx.cache.tryGet(item.link, async () => {
|
|
39
|
+
const response = await got(item.link);
|
|
40
|
+
const $ = cheerio.load(response.data);
|
|
41
|
+
// www.teach ?? pms.cmet ?? news
|
|
42
|
+
item.description = $('main[class=single]').html() ?? $('.card-footer').html() ?? $('.v_news_content').html();
|
|
43
|
+
item.pubDate = $('li[class=meta-date]').text() ? timezone(parseDate($('li[class=meta-date]').text(), 'YYYY-MM-DD HH:mm'), +8) : item.pubDate;
|
|
44
|
+
return item;
|
|
45
|
+
})
|
|
46
|
+
)
|
|
50
47
|
);
|
|
51
48
|
|
|
52
49
|
const desc = type === '' ? '中国科学技术大学教务处 - 通知新闻' : `中国科学技术大学教务处 - ${noticeType[type]}类通知`;
|
|
@@ -55,6 +52,6 @@ module.exports = async (ctx) => {
|
|
|
55
52
|
title: desc,
|
|
56
53
|
description: desc,
|
|
57
54
|
link: `${noticeUrl}${type === '' ? '' : '-' + type}`,
|
|
58
|
-
item:
|
|
55
|
+
item: items,
|
|
59
56
|
};
|
|
60
57
|
};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
module.exports = {
|
|
2
|
+
'ustc.edu.cn': {
|
|
3
|
+
_name: '中国科学技术大学',
|
|
4
|
+
'.': [
|
|
5
|
+
{
|
|
6
|
+
title: '官网通知公告',
|
|
7
|
+
docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
|
|
8
|
+
source: '/',
|
|
9
|
+
target: '/ustc/news',
|
|
10
|
+
},
|
|
11
|
+
],
|
|
12
|
+
'www.teach': [
|
|
13
|
+
{
|
|
14
|
+
title: '教务处通知新闻',
|
|
15
|
+
docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
|
|
16
|
+
source: '/',
|
|
17
|
+
target: '/ustc/jwc',
|
|
18
|
+
},
|
|
19
|
+
],
|
|
20
|
+
job: [
|
|
21
|
+
{
|
|
22
|
+
title: '就业信息网',
|
|
23
|
+
docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
|
|
24
|
+
source: '/',
|
|
25
|
+
target: '/ustc/job',
|
|
26
|
+
},
|
|
27
|
+
],
|
|
28
|
+
gradschool: [
|
|
29
|
+
{
|
|
30
|
+
title: '研究生院',
|
|
31
|
+
docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
|
|
32
|
+
source: '/',
|
|
33
|
+
target: '/ustc/gs',
|
|
34
|
+
},
|
|
35
|
+
],
|
|
36
|
+
sist: [
|
|
37
|
+
{
|
|
38
|
+
title: '信息科学技术学院',
|
|
39
|
+
docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
|
|
40
|
+
source: '/',
|
|
41
|
+
target: '/ustc/sist',
|
|
42
|
+
},
|
|
43
|
+
],
|
|
44
|
+
eeis: [
|
|
45
|
+
{
|
|
46
|
+
title: '电子工程与信息科学系',
|
|
47
|
+
docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
|
|
48
|
+
source: '/',
|
|
49
|
+
target: '/ustc/eeis',
|
|
50
|
+
},
|
|
51
|
+
],
|
|
52
|
+
},
|
|
53
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
module.exports = function (router) {
|
|
2
|
+
router.get('/eeis/:type?', require('./eeis'));
|
|
3
|
+
router.get('/gs/:type?', require('./gs'));
|
|
4
|
+
router.get('/job/:category?', require('./job'));
|
|
5
|
+
router.get('/jwc/:type?', require('./jwc'));
|
|
6
|
+
router.get('/news/:type?', require('./index'));
|
|
7
|
+
router.get('/sist/:type?', require('./sist'));
|
|
8
|
+
};
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const cheerio = require('cheerio');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
const timezone = require('@/utils/timezone');
|
|
5
|
+
|
|
6
|
+
const map = new Map([
|
|
7
|
+
['tzgg', { title: '中国科学技术大学信息科学技术学院 - 通知公告', id: '5142' }],
|
|
8
|
+
['zsgz', { title: '中国科学技术大学信息科学技术学院 - 招生工作', id: '5108' }],
|
|
9
|
+
]);
|
|
10
|
+
|
|
11
|
+
const host = 'https://sist.ustc.edu.cn';
|
|
12
|
+
|
|
13
|
+
module.exports = async (ctx) => {
|
|
14
|
+
const type = ctx.params.type ?? 'tzgg';
|
|
15
|
+
const info = map.get(type);
|
|
16
|
+
if (!info) {
|
|
17
|
+
throw 'invalid type';
|
|
18
|
+
}
|
|
19
|
+
const id = info.id;
|
|
20
|
+
|
|
21
|
+
const response = await got(`${host}/${id}/list.htm`);
|
|
22
|
+
const $ = cheerio.load(response.data);
|
|
23
|
+
let items = $('div#wp_news_w12')
|
|
24
|
+
.find('li')
|
|
25
|
+
.toArray()
|
|
26
|
+
.map((item) => {
|
|
27
|
+
item = $(item);
|
|
28
|
+
const title = item.find('a').attr('title').trim();
|
|
29
|
+
const link = item.find('a').attr('href').startsWith('/') ? host + item.find('a').attr('href') : item.find('a').attr('href');
|
|
30
|
+
const pubDate = timezone(parseDate(item.find('span').text(), 'YYYY-MM-DD'), +8);
|
|
31
|
+
return {
|
|
32
|
+
title,
|
|
33
|
+
pubDate,
|
|
34
|
+
link,
|
|
35
|
+
};
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
items = await Promise.all(
|
|
39
|
+
items.map((item) =>
|
|
40
|
+
ctx.cache.tryGet(item.link, async () => {
|
|
41
|
+
let desc = '';
|
|
42
|
+
try {
|
|
43
|
+
const response = await got(item.link);
|
|
44
|
+
desc = cheerio.load(response.data)('div.wp_articlecontent').html();
|
|
45
|
+
item.description = desc;
|
|
46
|
+
} catch (err) {
|
|
47
|
+
// intranet only contents
|
|
48
|
+
}
|
|
49
|
+
return item;
|
|
50
|
+
})
|
|
51
|
+
)
|
|
52
|
+
);
|
|
53
|
+
|
|
54
|
+
ctx.state.data = {
|
|
55
|
+
title: info.title,
|
|
56
|
+
link: `${host}/${id}/list.htm`,
|
|
57
|
+
item: items,
|
|
58
|
+
};
|
|
59
|
+
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const got = require('@/utils/got');
|
|
2
2
|
const cheerio = require('cheerio');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
3
4
|
|
|
4
5
|
module.exports = async (ctx) => {
|
|
5
6
|
const { data: htmlString } = await got({
|
|
@@ -20,13 +21,13 @@ module.exports = async (ctx) => {
|
|
|
20
21
|
title: `${time} ${title}`,
|
|
21
22
|
link: `https://mp.weixin.qq.com${$link.attr('href')}`,
|
|
22
23
|
description: title,
|
|
23
|
-
pubDate:
|
|
24
|
+
pubDate: parseDate(time),
|
|
24
25
|
});
|
|
25
26
|
});
|
|
26
27
|
|
|
27
28
|
ctx.state.data = {
|
|
28
29
|
title: '微信公众平台-系统公告栏目',
|
|
29
|
-
link: 'https://mp.weixin.qq.com/cgi-bin/announce?action=getannouncementlist&lang=zh_CN
|
|
30
|
+
link: 'https://mp.weixin.qq.com/cgi-bin/announce?action=getannouncementlist&lang=zh_CN',
|
|
30
31
|
item: announceList,
|
|
31
32
|
};
|
|
32
33
|
};
|
|
File without changes
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
const got = require('@/utils/got');
|
|
2
|
+
const cheerio = require('cheerio');
|
|
3
|
+
const { parseDate } = require('@/utils/parse-date');
|
|
4
|
+
const timezone = require('@/utils/timezone');
|
|
5
|
+
const { finishArticleItem } = require('@/utils/wechat-mp');
|
|
6
|
+
const { RequestInProgressError } = require('@/errors');
|
|
7
|
+
const wait = require('@/utils/wait');
|
|
8
|
+
|
|
9
|
+
const parsePage = ($item, hyperlinkSelector, timeSelector) => {
|
|
10
|
+
const hyperlink = $item.find(hyperlinkSelector);
|
|
11
|
+
const title = hyperlink.text();
|
|
12
|
+
const link = hyperlink.attr('href');
|
|
13
|
+
const pubDate = timezone(parseDate($item.find(timeSelector).text(), 'YYYY-MM-DD HH:mm'), 8);
|
|
14
|
+
return {
|
|
15
|
+
title,
|
|
16
|
+
link,
|
|
17
|
+
pubDate,
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
module.exports = async (ctx) => {
|
|
22
|
+
// !!! here we must use a lock to prevent other requests to break the anti-anti-crawler workarounds !!!
|
|
23
|
+
if ((await ctx.cache.get('data258:lock', false)) === '1') {
|
|
24
|
+
throw new RequestInProgressError('Another request is in progress, please try again later.');
|
|
25
|
+
}
|
|
26
|
+
// !!! here no need to acquire the lock, because the MP/category page has no crawler detection !!!
|
|
27
|
+
|
|
28
|
+
const id = ctx.params.id;
|
|
29
|
+
|
|
30
|
+
const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 5;
|
|
31
|
+
|
|
32
|
+
const rootUrl = 'https://mp.data258.com';
|
|
33
|
+
const pageUrl = id ? `${rootUrl}/article/category/${id}` : rootUrl;
|
|
34
|
+
|
|
35
|
+
const response = await got(pageUrl);
|
|
36
|
+
const $ = cheerio.load(response.data);
|
|
37
|
+
|
|
38
|
+
const title = $('head title').text();
|
|
39
|
+
// title = title.endsWith('-微阅读') ? title.slice(0, title.length - 4) : title;
|
|
40
|
+
const description = $('meta[name="description"]').attr('content');
|
|
41
|
+
|
|
42
|
+
const categoryPage = $('ul.fly-list');
|
|
43
|
+
|
|
44
|
+
let items;
|
|
45
|
+
if (categoryPage && categoryPage.length) {
|
|
46
|
+
// got a category page
|
|
47
|
+
items = $(categoryPage)
|
|
48
|
+
.find('li')
|
|
49
|
+
.map((_, item) => parsePage($(item), 'h2 a', '.fly-list-info span'))
|
|
50
|
+
.get();
|
|
51
|
+
} else {
|
|
52
|
+
// got an MP page
|
|
53
|
+
items = $('ul.jie-row li')
|
|
54
|
+
.map((_, item) => parsePage($(item), 'a.jie-title', '.layui-hide-xs'))
|
|
55
|
+
.get();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
items = items.slice(0, limit); // limit to avoid being anti-crawled
|
|
59
|
+
|
|
60
|
+
// !!! double-check !!!
|
|
61
|
+
if ((await ctx.cache.get('data258:lock', false)) === '1') {
|
|
62
|
+
throw new RequestInProgressError('Another request is in progress, please try again later.');
|
|
63
|
+
} else {
|
|
64
|
+
// !!! here we acquire the lock because the jump page has crawler detection !!!
|
|
65
|
+
await ctx.cache.set('data258:lock', '1', 60);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// !!! here we must use a for-loop to ensure the concurrency is 1 !!!
|
|
69
|
+
// !!! please do note that if you try to increase the concurrency, your IP will be banned for a long time !!!
|
|
70
|
+
|
|
71
|
+
let err; // !!! let RSSHub throw an anti-crawler prompt if the route is empty !!!
|
|
72
|
+
|
|
73
|
+
/* eslint-disable no-await-in-loop */
|
|
74
|
+
for (const item of items) {
|
|
75
|
+
// https://mp.data258.com/wx?id=${id}&t={token}, id is a permanent hex, token is a temporary base64
|
|
76
|
+
const cacheId = item.link.match(/id=([\da-f]+)/)[1];
|
|
77
|
+
item.link = item.link.startsWith('http') ? item.link : `${rootUrl}${item.link}`;
|
|
78
|
+
const realLink = await ctx.cache.tryGet(`data258:${cacheId}`, async () => {
|
|
79
|
+
try {
|
|
80
|
+
// !!! here we must sleep 1s to avoid being anti-crawled !!!
|
|
81
|
+
// !!! please do note that if the interval is less than 1s, your IP will be banned for a long time !!!
|
|
82
|
+
await wait(1000);
|
|
83
|
+
|
|
84
|
+
const response = await got.get(item.link, {
|
|
85
|
+
headers: {
|
|
86
|
+
Referer: pageUrl, // essential
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
if (response.data.includes('今日浏览次数已达上限')) {
|
|
90
|
+
// !!! as long as cache hits, the link will not be crawled and consume the limit !!!
|
|
91
|
+
// !!! so that's not a big problem if the RSSHub instance is self-hosted !!!
|
|
92
|
+
err = new got.RequestError(response.data, {}, response.request);
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
const $ = cheerio.load(response.data);
|
|
96
|
+
const jmpJS = $('script')
|
|
97
|
+
.filter((_, e) => $(e).html().includes('location.href'))
|
|
98
|
+
.html();
|
|
99
|
+
return jmpJS.match(/location\.href='([^']+)'/)[1];
|
|
100
|
+
} catch (e) {
|
|
101
|
+
err = e;
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
if (realLink) {
|
|
106
|
+
item.link = realLink;
|
|
107
|
+
} else {
|
|
108
|
+
break; // being anti-crawled, immediately cancel following operations
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/* eslint-enable no-await-in-loop */
|
|
112
|
+
|
|
113
|
+
// !!! release the lock, let it expire immediately since no need to keep it in cache !!!
|
|
114
|
+
await ctx.cache.set('data258:lock', '0', 1);
|
|
115
|
+
|
|
116
|
+
// jump links are valid only for a short period of time, drop those un-jumped items
|
|
117
|
+
// http://mp.weixin.qq.com/s
|
|
118
|
+
items = items.filter((item) => item.link.match(/^https?:\/\/mp\.weixin\.qq\.com\/s/));
|
|
119
|
+
|
|
120
|
+
if (items.length === 0 && err) {
|
|
121
|
+
// !!! if each request is anti-crawled, the filtered items array will be empty !!!
|
|
122
|
+
// !!! let RSSHub throw an anti-crawler prompt !!!
|
|
123
|
+
throw err;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item, !!categoryPage)));
|
|
127
|
+
|
|
128
|
+
ctx.state.data = {
|
|
129
|
+
title,
|
|
130
|
+
link: pageUrl,
|
|
131
|
+
description,
|
|
132
|
+
item: items,
|
|
133
|
+
};
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
// TODO: login? the valid time for cookies seems to be short, and abusing account will probably get banned...
|
|
137
|
+
// TODO: fetch full article for the official RSS feed? unless someone who is VIP contributes their RSS feed for test...
|
|
File without changes
|
|
File without changes
|
|
@@ -1,3 +1,14 @@
|
|
|
1
1
|
module.exports = {
|
|
2
|
+
'/announce': ['xyqfer'],
|
|
3
|
+
'/ce/:id': ['HenryQW'],
|
|
4
|
+
'/ershicimi/:id': ['sanmmm'],
|
|
5
|
+
'/data258/:id?': ['Rongronggg9'],
|
|
2
6
|
'/feeddd/:id': ['TonyRL', 'Rongronggg9'],
|
|
7
|
+
'/feeds': ['tylinux'],
|
|
8
|
+
'/mp/homepage/:biz/:hid/:cid?': ['MisteryMonster'],
|
|
9
|
+
'/mp/msgalbum/:biz/:aid': ['MisteryMonster'],
|
|
10
|
+
'/tgchannel/:id/:mpName?/:searchQueryType?': ['LogicJake', 'Rongronggg9'],
|
|
11
|
+
'/uread/:userid': ['kt286'],
|
|
12
|
+
'/wemp/:id': ['HenryQW'],
|
|
13
|
+
'/wxnmh/:id': ['laampui'],
|
|
3
14
|
};
|
|
File without changes
|
|
File without changes
|