rsshub 1.0.0-master.e6d4e1e → 1.0.0-master.e6d53dc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api/radar/rules/all.ts +3 -6
- package/lib/api/radar/rules/one.ts +3 -6
- package/lib/config.ts +12 -22
- package/lib/errors/index.tsx +1 -1
- package/lib/index.ts +1 -1
- package/lib/middleware/anti-hotlink.test.ts +191 -6
- package/lib/middleware/anti-hotlink.ts +28 -0
- package/lib/middleware/parameter.test.ts +22 -5
- package/lib/middleware/parameter.ts +47 -17
- package/lib/registry.test.ts +8 -0
- package/lib/registry.ts +2 -0
- package/lib/router.js +0 -3
- package/lib/routes/141jav/index.ts +25 -21
- package/lib/routes/141ppv/index.ts +24 -20
- package/lib/routes/51read/article.ts +82 -0
- package/lib/routes/51read/namespace.ts +6 -0
- package/lib/routes/5eplay/utils.ts +5 -3
- package/lib/routes/agri/index.ts +307 -0
- package/lib/routes/agri/namespace.ts +8 -0
- package/lib/routes/agri/templates/description.art +21 -0
- package/lib/routes/aljazeera/index.ts +2 -2
- package/lib/routes/apnews/sitemap.ts +98 -0
- package/lib/routes/apnews/utils.ts +7 -5
- package/lib/routes/autocentre/index.ts +29 -0
- package/lib/routes/autocentre/namespace.ts +7 -0
- package/lib/routes/bilibili/api-interface.d.ts +1 -0
- package/lib/routes/bilibili/cache.ts +3 -1
- package/lib/routes/bilibili/dynamic.ts +7 -9
- package/lib/routes/binance/announcement.ts +168 -0
- package/lib/routes/binance/types.ts +26 -0
- package/lib/routes/bing/daily-wallpaper.ts +1 -1
- package/lib/routes/bjnews/cat.ts +43 -0
- package/lib/routes/bjnews/column.ts +43 -0
- package/lib/routes/bjnews/namespace.ts +6 -0
- package/lib/routes/bjnews/utils.ts +19 -0
- package/lib/routes/bjx/fd.ts +35 -26
- package/lib/routes/buaa/jiaowu.ts +1 -1
- package/lib/routes/buaa/lib/space/newbook.ts +171 -0
- package/lib/routes/buaa/lib/space/templates/newbook.art +44 -0
- package/lib/routes/cccfna/index.ts +2 -2
- package/lib/routes/cctv/category.ts +16 -12
- package/lib/routes/cctv/xwlb.ts +32 -10
- package/lib/routes/cdu/jwgg.ts +79 -0
- package/lib/routes/cdu/namespace.ts +6 -0
- package/lib/routes/cffex/announcement.ts +73 -0
- package/lib/routes/cffex/namespace.ts +6 -0
- package/lib/routes/chlinlearn/daily-blog.ts +55 -0
- package/lib/routes/chlinlearn/namespcae.ts +6 -0
- package/lib/routes/copymanga/comic.ts +3 -1
- package/lib/routes/devolverdigital/blog.ts +59 -42
- package/lib/routes/digitalcameraworld/news.ts +1 -1
- package/lib/routes/discord/channel.ts +10 -10
- package/lib/routes/discord/discord-api.ts +27 -32
- package/lib/routes/discord/templates/message.art +15 -0
- package/lib/routes/discourse/notifications.ts +2 -1
- package/lib/routes/dongqiudi/result.ts +1 -1
- package/lib/routes/eastmoney/report/index.ts +63 -15
- package/lib/routes/eastmoney/templates/stock_description.art +36 -0
- package/lib/routes/eastmoney/utils.ts +35 -0
- package/lib/routes/fediverse/timeline.ts +35 -18
- package/lib/routes/flashcat/blog.ts +61 -0
- package/lib/routes/flashcat/namespace.ts +6 -0
- package/lib/routes/ftm/index.ts +60 -0
- package/lib/routes/ftm/namespace.ts +6 -0
- package/lib/routes/galxe/index.ts +94 -0
- package/lib/routes/galxe/namespace.ts +9 -0
- package/lib/routes/gamersky/ent.ts +57 -0
- package/lib/routes/gamersky/namespace.ts +10 -0
- package/lib/routes/gamersky/news.ts +91 -0
- package/lib/routes/gamersky/review.ts +80 -0
- package/lib/routes/gamersky/utils.ts +94 -0
- package/lib/routes/gdut/oa-news.ts +12 -17
- package/lib/routes/gisreportsonline/index.ts +60 -0
- package/lib/routes/gisreportsonline/namespace.ts +6 -0
- package/lib/routes/github/activity.ts +4 -3
- package/lib/routes/github/search.ts +20 -18
- package/lib/routes/google/research.ts +45 -0
- package/lib/routes/gov/cbirc/index.ts +371 -0
- package/lib/routes/gov/chongqing/sydwgkzp.ts +4 -4
- package/lib/routes/gov/lswz/index.ts +289 -0
- package/lib/routes/gov/zj/ningbogzw-notice.ts +50 -0
- package/lib/routes/gov/zj/ningborsjnotice.ts +50 -0
- package/lib/routes/gq/news.ts +1 -1
- package/lib/routes/gs/developer/blog.ts +58 -0
- package/lib/routes/gs/namespace.ts +9 -0
- package/lib/routes/healthz.ts +8 -0
- package/lib/routes/hket/index.ts +80 -76
- package/lib/routes/hottoys/index.ts +63 -0
- package/lib/routes/hottoys/namespace.ts +6 -0
- package/lib/routes/hust/gs.ts +281 -0
- package/lib/routes/hust/namespace.ts +1 -1
- package/lib/routes/imhcg/blog.ts +47 -0
- package/lib/routes/imhcg/namespace.ts +7 -0
- package/lib/routes/imop/namespace.ts +10 -0
- package/lib/routes/imop/tianshu.ts +53 -0
- package/lib/routes/infzm/hot.ts +39 -0
- package/lib/routes/infzm/index.ts +3 -30
- package/lib/routes/infzm/utils.ts +34 -0
- package/lib/routes/inspirehep/author.ts +46 -0
- package/lib/routes/inspirehep/literature.ts +42 -0
- package/lib/routes/inspirehep/namespace.ts +7 -0
- package/lib/routes/inspirehep/types.ts +210 -0
- package/lib/routes/inspirehep/utils.ts +15 -0
- package/lib/routes/iwara/subscriptions.ts +1 -1
- package/lib/routes/javdb/actors.ts +6 -2
- package/lib/routes/joins/chinese.ts +218 -0
- package/lib/routes/joins/namespace.ts +8 -0
- package/lib/routes/joins/templates/description.art +17 -0
- package/lib/routes/juejin/dynamic.ts +114 -0
- package/lib/routes/kaopu/news.ts +3 -3
- package/lib/routes/kemono/index.ts +21 -0
- package/lib/routes/keylol/index.ts +56 -11
- package/lib/routes/kurogames/namespace.ts +7 -0
- package/lib/routes/kurogames/wutheringwaves/news.ts +59 -0
- package/lib/routes/link3/events.ts +86 -0
- package/lib/routes/link3/namespace.ts +6 -0
- package/lib/routes/link3/profile.ts +143 -0
- package/lib/routes/linkedin/cn/renderer.ts +1 -1
- package/lib/routes/lmu/jobs.ts +96 -0
- package/lib/routes/lmu/namespace.ts +17 -0
- package/lib/routes/lmu/templates/jobPosting.art +11 -0
- package/lib/routes/mcmod/index.ts +105 -0
- package/lib/routes/mcmod/namespace.ts +6 -0
- package/lib/routes/mcmod/templates/mod.art +13 -0
- package/lib/routes/misskey/user-timeline.ts +42 -0
- package/lib/routes/misskey/utils.ts +41 -1
- package/lib/routes/nationalgeographic/latest-stories.ts +1 -1
- package/lib/routes/nationalgeographic/templates/stories.art +1 -1
- package/lib/routes/newyorker/news.ts +24 -17
- package/lib/routes/oschina/news.ts +2 -0
- package/lib/routes/people/index.ts +4 -2
- package/lib/routes/picnob/user.ts +11 -3
- package/lib/routes/pixiv/search.ts +20 -1
- package/lib/routes/pixiv/utils.ts +3 -3
- package/lib/routes/psyche/topic.ts +1 -1
- package/lib/routes/psyche/type.ts +1 -1
- package/lib/routes/psyche/utils.ts +1 -1
- package/lib/routes/questn/community.ts +76 -0
- package/lib/routes/questn/events.ts +101 -0
- package/lib/routes/questn/namespace.ts +6 -0
- package/lib/routes/questn/util.ts +16 -0
- package/lib/routes/rebase/geekdaily.ts +35 -0
- package/lib/routes/rebase/namespace.ts +6 -0
- package/lib/routes/rss3/index.ts +22 -13
- package/lib/routes/rss3/interfaces/metadata.ts +67 -0
- package/lib/routes/rsshub/routes.ts +18 -12
- package/lib/routes/samd/news.ts +1 -1
- package/lib/routes/sara/index.ts +1 -1
- package/lib/routes/scut/gzic/media.ts +71 -0
- package/lib/routes/scut/gzic/news.ts +65 -0
- package/lib/routes/scut/gzic/notice.ts +88 -0
- package/lib/routes/shiep/config.ts +4 -10
- package/lib/routes/shiep/index.ts +9 -9
- package/lib/routes/showstart/artist.ts +5 -4
- package/lib/routes/showstart/brand.ts +5 -4
- package/lib/routes/showstart/event.ts +8 -7
- package/lib/routes/showstart/search.ts +43 -9
- package/lib/routes/showstart/service.ts +81 -53
- package/lib/routes/showstart/site.ts +41 -0
- package/lib/routes/showstart/utils.ts +9 -8
- package/lib/routes/sis001/author.ts +52 -0
- package/lib/routes/sis001/common.ts +38 -0
- package/lib/routes/sis001/forum.ts +2 -29
- package/lib/routes/sjtu/jwc.ts +4 -1
- package/lib/routes/sjtu/namespace.ts +1 -1
- package/lib/routes/sjtu/seiee/index.ts +97 -0
- package/lib/routes/smzdm/product.ts +86 -0
- package/lib/routes/spotify/show.ts +1 -1
- package/lib/routes/springer/journal.ts +10 -12
- package/lib/routes/sse/sselawsrules.ts +370 -0
- package/lib/routes/sspai/author.ts +1 -1
- package/lib/routes/sspai/column.ts +1 -1
- package/lib/routes/sspai/index.ts +1 -1
- package/lib/routes/sspai/matrix.ts +1 -1
- package/lib/routes/sspai/series-update.ts +1 -1
- package/lib/routes/sspai/tag.ts +1 -1
- package/lib/routes/sspai/topic.ts +1 -1
- package/lib/routes/sustainabilitymag/articles.ts +2 -0
- package/lib/routes/szse/rule.ts +315 -42
- package/lib/routes/t66y/index.ts +29 -7
- package/lib/routes/taoguba/blog.ts +1 -1
- package/lib/routes/taoguba/index.ts +1 -1
- package/lib/routes/techcrunch/news.ts +20 -29
- package/lib/routes/techcrunch/templates/description.art +7 -0
- package/lib/routes/telegram/channel.ts +34 -1
- package/lib/routes/test/index.ts +50 -43
- package/lib/routes/the/index.ts +210 -0
- package/lib/routes/the/namespace.ts +8 -0
- package/lib/routes/the/templates/description.art +27 -0
- package/lib/routes/the/util.ts +307 -0
- package/lib/routes/theblockbeats/index.ts +10 -5
- package/lib/routes/towardsdatascience/latest.ts +67 -0
- package/lib/routes/towardsdatascience/namespace.ts +6 -0
- package/lib/routes/tvb/news.ts +1 -0
- package/lib/routes/twitter/api/index.ts +2 -0
- package/lib/routes/twitter/api/web-api/api.ts +18 -0
- package/lib/routes/twitter/api/web-api/constants.ts +2 -0
- package/lib/routes/twitter/api/web-api/login.ts +66 -0
- package/lib/routes/twitter/api/web-api/utils.ts +53 -27
- package/lib/routes/twitter/home-latest.ts +60 -0
- package/lib/routes/twitter/home.ts +1 -1
- package/lib/routes/twitter/user.ts +1 -1
- package/lib/routes/twitter/utils.ts +4 -1
- package/lib/routes/uber/blog.ts +1 -1
- package/lib/routes/ustc/scms.ts +97 -0
- package/lib/routes/v2ex/xna.ts +55 -0
- package/lib/routes/wchscu/namespace.ts +6 -0
- package/lib/routes/wchscu/recruit.ts +73 -0
- package/lib/routes/xiaohongshu/notes.ts +10 -2
- package/lib/routes/xiaohongshu/user.ts +23 -2
- package/lib/routes/xidian/cs.ts +143 -0
- package/lib/routes/xidian/gr.ts +289 -0
- package/lib/routes/xidian/jwc.ts +1 -0
- package/lib/routes/xidian/namespace.ts +1 -1
- package/lib/routes/yahoo/news/index.ts +1 -0
- package/lib/routes/yahoo/news/listid.ts +67 -0
- package/lib/routes/yahoo/news/utils.ts +8 -2
- package/lib/routes/yicai/carousel.ts +48 -0
- package/lib/routes/yicai/utils.ts +23 -17
- package/lib/routes/youtube/subscriptions.ts +1 -0
- package/lib/routes/yystv/docs.ts +28 -19
- package/lib/routes/zhihu/answers.ts +5 -1
- package/lib/routes/zhihu/execlib/x-zse-96-v3.ts +2 -0
- package/lib/routes/zhihu/namespace.ts +3 -0
- package/lib/routes/zhihu/utils.ts +15 -2
- package/lib/routes/zsxq/types.ts +9 -0
- package/lib/routes/zsxq/utils.ts +4 -3
- package/lib/setup.test.ts +1 -1
- package/lib/types.ts +6 -0
- package/lib/utils/cache/index.ts +8 -0
- package/lib/utils/camelcase-keys.spec.ts +127 -0
- package/lib/utils/camelcase-keys.ts +27 -0
- package/lib/utils/common-utils.ts +1 -0
- package/lib/views/error.tsx +3 -3
- package/lib/views/index.tsx +4 -3
- package/lib/views/layout.tsx +1 -1
- package/package.json +46 -44
- package/lib/routes/sse/lawandrules.ts +0 -68
- package/lib/routes-deprecated/bjnews/news.js +0 -44
- package/lib/routes-deprecated/gamersky/ent.js +0 -89
- package/lib/routes-deprecated/gamersky/news.js +0 -34
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { Route, ViewType } from '@/types';
|
|
2
|
+
import { fetchArticle } from './utils';
|
|
3
|
+
import ofetch from '@/utils/ofetch';
|
|
4
|
+
import { load } from 'cheerio';
|
|
5
|
+
import { parseDate } from '@/utils/parse-date';
|
|
6
|
+
import asyncPool from 'tiny-async-pool';
|
|
7
|
+
import timezone from '@/utils/timezone';
|
|
8
|
+
const HOME_PAGE = 'https://apnews.com';
|
|
9
|
+
|
|
10
|
+
export const route: Route = {
|
|
11
|
+
path: '/sitemap/:route',
|
|
12
|
+
categories: ['traditional-media'],
|
|
13
|
+
example: '/apnews/sitemap/ap-sitemap-latest',
|
|
14
|
+
view: ViewType.Articles,
|
|
15
|
+
parameters: {
|
|
16
|
+
route: {
|
|
17
|
+
description: 'Route for sitemap, excluding the `.xml` extension',
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
features: {
|
|
21
|
+
requireConfig: false,
|
|
22
|
+
requirePuppeteer: false,
|
|
23
|
+
antiCrawler: false,
|
|
24
|
+
supportBT: false,
|
|
25
|
+
supportPodcast: false,
|
|
26
|
+
supportScihub: false,
|
|
27
|
+
},
|
|
28
|
+
radar: [
|
|
29
|
+
{
|
|
30
|
+
source: ['apnews.com/'],
|
|
31
|
+
},
|
|
32
|
+
],
|
|
33
|
+
name: 'Sitemap',
|
|
34
|
+
maintainers: ['zoenglinghou', 'mjysci', 'TonyRL', 'dzx-dzx'],
|
|
35
|
+
handler,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
async function handler(ctx) {
|
|
39
|
+
const route = ctx.req.param('route');
|
|
40
|
+
const url = `${HOME_PAGE}/${route}.xml`;
|
|
41
|
+
const response = await ofetch(url);
|
|
42
|
+
const $ = load(response);
|
|
43
|
+
|
|
44
|
+
const list = $('urlset url')
|
|
45
|
+
.toArray()
|
|
46
|
+
.map((e) => {
|
|
47
|
+
const LANGUAGE_MAP = new Map([
|
|
48
|
+
['eng', 'en'],
|
|
49
|
+
['spa', 'es'],
|
|
50
|
+
]);
|
|
51
|
+
|
|
52
|
+
const title = $(e)
|
|
53
|
+
.find(String.raw`news\:title`)
|
|
54
|
+
.text();
|
|
55
|
+
const pubDate = parseDate(
|
|
56
|
+
$(e)
|
|
57
|
+
.find(String.raw`news\:publication_date`)
|
|
58
|
+
.text()
|
|
59
|
+
);
|
|
60
|
+
const lastmod = timezone(parseDate($(e).find(`lastmod`).text()), -4);
|
|
61
|
+
const language = LANGUAGE_MAP.get(
|
|
62
|
+
$(e)
|
|
63
|
+
.find(String.raw`news\:language`)
|
|
64
|
+
.text()
|
|
65
|
+
);
|
|
66
|
+
let res = { link: $(e).find('loc').text() };
|
|
67
|
+
if (title) {
|
|
68
|
+
res = Object.assign(res, { title });
|
|
69
|
+
}
|
|
70
|
+
if (pubDate.toString() !== 'Invalid Date') {
|
|
71
|
+
res = Object.assign(res, { pubDate });
|
|
72
|
+
}
|
|
73
|
+
if (language) {
|
|
74
|
+
res = Object.assign(res, { language });
|
|
75
|
+
}
|
|
76
|
+
if (lastmod.toString() !== 'Invalid Date') {
|
|
77
|
+
res = Object.assign(res, { lastmod });
|
|
78
|
+
}
|
|
79
|
+
return res;
|
|
80
|
+
})
|
|
81
|
+
.filter((e) => Boolean(e.link) && !new URL(e.link).pathname.split('/').includes('hub'))
|
|
82
|
+
.sort((a, b) => (a.pubDate && b.pubDate ? b.pubDate - a.pubDate : b.lastmod - a.lastmod))
|
|
83
|
+
.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20);
|
|
84
|
+
|
|
85
|
+
const items = await asyncPoolAll(20, list, (item) => fetchArticle(item));
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
title: `AP News sitemap:${route}`,
|
|
89
|
+
item: items,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
|
|
93
|
+
const results: Awaited<OUT[]> = [];
|
|
94
|
+
for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
|
|
95
|
+
results.push(result);
|
|
96
|
+
}
|
|
97
|
+
return results;
|
|
98
|
+
}
|
|
@@ -16,16 +16,18 @@ export function fetchArticle(item) {
|
|
|
16
16
|
}
|
|
17
17
|
const rawLdjson = JSON.parse($('#link-ld-json').text());
|
|
18
18
|
let ldjson;
|
|
19
|
-
if (Array.isArray(rawLdjson)) {
|
|
20
|
-
// Regular
|
|
21
|
-
ldjson = rawLdjson[
|
|
19
|
+
if (rawLdjson['@type'] === 'NewsArticle' || (Array.isArray(rawLdjson) && rawLdjson.some((e) => e['@type'] === 'NewsArticle'))) {
|
|
20
|
+
// Regular(Articles, Videos)
|
|
21
|
+
ldjson = Array.isArray(rawLdjson) ? rawLdjson.find((e) => e['@type'] === 'NewsArticle') : rawLdjson;
|
|
22
22
|
|
|
23
23
|
$('div.Enhancement').remove();
|
|
24
|
+
const section = $("meta[property='article:section']").attr('content');
|
|
24
25
|
return {
|
|
26
|
+
title: ldjson.headline,
|
|
25
27
|
pubDate: parseDate(ldjson.datePublished),
|
|
26
28
|
updated: parseDate(ldjson.dateModified),
|
|
27
|
-
description: $('div.RichTextStoryBody').html(),
|
|
28
|
-
category: [
|
|
29
|
+
description: $('div.RichTextStoryBody').html() || $(':is(.VideoLead, .VideoPage-pageSubHeading)').html(),
|
|
30
|
+
category: [...(section ? [section] : []), ...(ldjson.keywords ?? [])],
|
|
29
31
|
guid: $("meta[name='brightspot.contentId']").attr('content'),
|
|
30
32
|
author: ldjson.author,
|
|
31
33
|
...item,
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { Data, Route } from '@/types';
|
|
2
|
+
import parser from '@/utils/rss-parser';
|
|
3
|
+
|
|
4
|
+
export const route: Route = {
|
|
5
|
+
path: '/',
|
|
6
|
+
name: 'Автомобільний сайт N1 в Україні',
|
|
7
|
+
categories: ['new-media'],
|
|
8
|
+
maintainers: ['driversti'],
|
|
9
|
+
example: '/autocentre',
|
|
10
|
+
handler,
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
const createItem = (item) => ({
|
|
14
|
+
title: item.title,
|
|
15
|
+
link: item.link,
|
|
16
|
+
description: item.contentSnippet,
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
async function handler(): Promise<Data> {
|
|
20
|
+
const feed = await parser.parseURL('https://www.autocentre.ua/rss');
|
|
21
|
+
|
|
22
|
+
return {
|
|
23
|
+
title: feed.title as string,
|
|
24
|
+
link: feed.link,
|
|
25
|
+
description: feed.description,
|
|
26
|
+
language: 'uk',
|
|
27
|
+
item: await Promise.all(feed.items.map((item) => createItem(item))),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -13,7 +13,9 @@ const getCookie = () => {
|
|
|
13
13
|
}
|
|
14
14
|
const key = 'bili-cookie';
|
|
15
15
|
return cache.tryGet(key, async () => {
|
|
16
|
-
const browser = await puppeteer(
|
|
16
|
+
const browser = await puppeteer({
|
|
17
|
+
stealth: true,
|
|
18
|
+
});
|
|
17
19
|
const page = await browser.newPage();
|
|
18
20
|
const waitForRequest = new Promise<string>((resolve) => {
|
|
19
21
|
page.on('requestfinished', async (request) => {
|
|
@@ -232,14 +232,8 @@ async function handler(ctx) {
|
|
|
232
232
|
|
|
233
233
|
const cookie = await cacheIn.getCookie();
|
|
234
234
|
|
|
235
|
-
const
|
|
236
|
-
|
|
237
|
-
url: `https://api.bilibili.com/x/polymer/web-dynamic/v1/feed/space`,
|
|
238
|
-
searchParams: {
|
|
239
|
-
host_mid: uid,
|
|
240
|
-
platform: 'web',
|
|
241
|
-
features: 'itemOpusStyle,listOnlyfans,opusBigCover,onlyfansVote',
|
|
242
|
-
},
|
|
235
|
+
const params = utils.addDmVerifyInfo(`host_mid=${uid}&platform=web&features=itemOpusStyle,listOnlyfans,opusBigCover,onlyfansVote`, utils.getDmImgList());
|
|
236
|
+
const response = await got(`https://api.bilibili.com/x/polymer/web-dynamic/v1/feed/space?${params}`, {
|
|
243
237
|
headers: {
|
|
244
238
|
Referer: `https://space.bilibili.com/${uid}/`,
|
|
245
239
|
Cookie: cookie,
|
|
@@ -315,6 +309,10 @@ async function handler(ctx) {
|
|
|
315
309
|
}
|
|
316
310
|
}
|
|
317
311
|
}
|
|
312
|
+
if (data.module_dynamic?.topic?.name) {
|
|
313
|
+
// 将话题作为 category
|
|
314
|
+
category.push(data.module_dynamic.topic.name);
|
|
315
|
+
}
|
|
318
316
|
|
|
319
317
|
if (item.type === 'DYNAMIC_TYPE_ARTICLE' && displayArticle) {
|
|
320
318
|
// 抓取专栏全文
|
|
@@ -365,7 +363,7 @@ async function handler(ctx) {
|
|
|
365
363
|
pubDate: data.module_author?.pub_ts ? parseDate(data.module_author.pub_ts, 'X') : undefined,
|
|
366
364
|
link,
|
|
367
365
|
author,
|
|
368
|
-
category: category.length ? category : undefined,
|
|
366
|
+
category: category.length ? [...new Set(category)] : undefined,
|
|
369
367
|
};
|
|
370
368
|
})
|
|
371
369
|
);
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { DataItem, Route } from '@/types';
|
|
2
|
+
import cache from '@/utils/cache';
|
|
3
|
+
import ofetch from '@/utils/ofetch';
|
|
4
|
+
import { parseDate } from '@/utils/parse-date';
|
|
5
|
+
import * as cheerio from 'cheerio';
|
|
6
|
+
import { AnnouncementCatalog, AnnouncementsConfig } from './types';
|
|
7
|
+
|
|
8
|
+
interface AnnouncementFragment {
|
|
9
|
+
reactRoot: [{ id: 'Fragment'; children: { id: string; props: object }[]; props: object }];
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const ROUTE_PARAMETERS_CATALOGID_MAPPING = {
|
|
13
|
+
'new-cryptocurrency-listing': 48,
|
|
14
|
+
'latest-binance-news': 49,
|
|
15
|
+
'latest-activities': 93,
|
|
16
|
+
'new-fiat-listings': 50,
|
|
17
|
+
'api-updates': 51,
|
|
18
|
+
'crypto-airdrop': 128,
|
|
19
|
+
'wallet-maintenance-updates': 157,
|
|
20
|
+
delisting: 161,
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
function assertAnnouncementsConfig(playlist: unknown): playlist is AnnouncementFragment {
|
|
24
|
+
if (!playlist || typeof playlist !== 'object') {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
if (!('reactRoot' in (playlist as { reactRoot: unknown[] }))) {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
if (!Array.isArray((playlist as { reactRoot: unknown[] }).reactRoot)) {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
if ((playlist as { reactRoot: { id: string }[] }).reactRoot?.[0]?.id !== 'Fragment') {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function assertAnnouncementsConfigList(props: unknown): props is { config: { list: AnnouncementsConfig[] } } {
|
|
40
|
+
if (!props || typeof props !== 'object') {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
if (!('config' in props)) {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
if (!('list' in (props.config as { list: AnnouncementsConfig[] }))) {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const handler: Route['handler'] = async (ctx) => {
|
|
53
|
+
const baseUrl = 'https://www.binance.com';
|
|
54
|
+
const announcementCategoryUrl = `${baseUrl}/support/announcement`;
|
|
55
|
+
const { type } = ctx.req.param<'/binance/announcement/:type'>();
|
|
56
|
+
const language = ctx.req.header('Accept-Language');
|
|
57
|
+
const headers = {
|
|
58
|
+
Referer: baseUrl,
|
|
59
|
+
'Accept-Language': language ?? 'en-US,en;q=0.9',
|
|
60
|
+
};
|
|
61
|
+
const announcementsConfig = (await cache.tryGet(`binance:announcements:${language}`, async () => {
|
|
62
|
+
const announcementRes = await ofetch<string>(announcementCategoryUrl, { headers });
|
|
63
|
+
const $ = cheerio.load(announcementRes);
|
|
64
|
+
|
|
65
|
+
const appData = JSON.parse($('#__APP_DATA').text());
|
|
66
|
+
|
|
67
|
+
const announcements = Object.values(appData.appState.loader.dataByRouteId as Record<string, object>).find((value) => 'playlist' in value) as { playlist: unknown };
|
|
68
|
+
|
|
69
|
+
if (!assertAnnouncementsConfig(announcements.playlist)) {
|
|
70
|
+
throw new Error('Get announcement config failed');
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const listConfigProps = announcements.playlist.reactRoot[0].children.find((i) => i.id === 'TopicCardList')?.props;
|
|
74
|
+
|
|
75
|
+
if (!assertAnnouncementsConfigList(listConfigProps)) {
|
|
76
|
+
throw new Error("Can't get announcement config list");
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return listConfigProps.config.list;
|
|
80
|
+
})) as AnnouncementsConfig[];
|
|
81
|
+
|
|
82
|
+
const announcementCatalogId = ROUTE_PARAMETERS_CATALOGID_MAPPING[type];
|
|
83
|
+
|
|
84
|
+
if (!announcementCatalogId) {
|
|
85
|
+
throw new Error(`${type} is not supported`);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const targetItem = announcementsConfig.find((i) => i.url.includes(`c-${announcementCatalogId}`));
|
|
89
|
+
|
|
90
|
+
if (!targetItem) {
|
|
91
|
+
throw new Error('Unexpected announcements config');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const link = new URL(targetItem.url, baseUrl).toString();
|
|
95
|
+
|
|
96
|
+
const response = await ofetch<string>(link, { headers });
|
|
97
|
+
|
|
98
|
+
const $ = cheerio.load(response);
|
|
99
|
+
const appData = JSON.parse($('#__APP_DATA').text());
|
|
100
|
+
|
|
101
|
+
const values = Object.values(appData.appState.loader.dataByRouteId as Record<string, object>);
|
|
102
|
+
const catalogs = values.find((value) => 'catalogs' in value) as { catalogs: AnnouncementCatalog[] };
|
|
103
|
+
const catalog = catalogs.catalogs.find((catalog) => catalog.catalogId === announcementCatalogId);
|
|
104
|
+
|
|
105
|
+
const item = await Promise.all(
|
|
106
|
+
catalog!.articles.map((i) => {
|
|
107
|
+
const link = `${announcementCategoryUrl}/${i.code}`;
|
|
108
|
+
const item = {
|
|
109
|
+
title: i.title,
|
|
110
|
+
link,
|
|
111
|
+
description: i.title,
|
|
112
|
+
pubDate: parseDate(i.releaseDate),
|
|
113
|
+
} as DataItem;
|
|
114
|
+
return cache.tryGet(`binance:announcement:${i.code}:${language}`, async () => {
|
|
115
|
+
const res = await ofetch(link, { headers });
|
|
116
|
+
const $ = cheerio.load(res);
|
|
117
|
+
const descriptionEl = $('#support_article > div').first();
|
|
118
|
+
descriptionEl.find('style').remove();
|
|
119
|
+
item.description = descriptionEl.html() ?? '';
|
|
120
|
+
return item;
|
|
121
|
+
}) as Promise<DataItem>;
|
|
122
|
+
})
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
title: targetItem.title,
|
|
127
|
+
link,
|
|
128
|
+
description: targetItem.description,
|
|
129
|
+
item,
|
|
130
|
+
};
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
export const route: Route = {
|
|
134
|
+
path: '/announcement/:type',
|
|
135
|
+
categories: ['finance'],
|
|
136
|
+
example: '/binance/announcement/new-cryptocurrency-listing',
|
|
137
|
+
parameters: {
|
|
138
|
+
type: {
|
|
139
|
+
description: 'Binance Announcement type',
|
|
140
|
+
default: 'new-cryptocurrency-listing',
|
|
141
|
+
options: [
|
|
142
|
+
{ value: 'new-cryptocurrency-listing', label: 'New Cryptocurrency Listing' },
|
|
143
|
+
{ value: 'latest-binance-news', label: 'Latest Binance News' },
|
|
144
|
+
{ value: 'latest-activities', label: 'Latest Activities' },
|
|
145
|
+
{ value: 'new-fiat-listings', label: 'New Fiat Listings' },
|
|
146
|
+
{ value: 'api-updates', label: 'API Updates' },
|
|
147
|
+
{ value: 'crypto-airdrop', label: 'Crypto Airdrop' },
|
|
148
|
+
{ value: 'wallet-maintenance-updates', label: 'Wallet Maintenance Updates' },
|
|
149
|
+
{ value: 'delisting', label: 'Delisting' },
|
|
150
|
+
],
|
|
151
|
+
},
|
|
152
|
+
},
|
|
153
|
+
name: 'Announcement',
|
|
154
|
+
description: `
|
|
155
|
+
Type category
|
|
156
|
+
|
|
157
|
+
- new-cryptocurrency-listing => New Cryptocurrency Listing
|
|
158
|
+
- latest-binance-news => Latest Binance News
|
|
159
|
+
- latest-activities => Latest Activities
|
|
160
|
+
- new-fiat-listings => New Fiat Listings
|
|
161
|
+
- api-updates => API Updates
|
|
162
|
+
- crypto-airdrop => Crypto Airdrop
|
|
163
|
+
- wallet-maintenance-updates => Wallet Maintenance Updates
|
|
164
|
+
- delisting => Delisting
|
|
165
|
+
`,
|
|
166
|
+
maintainers: ['enpitsulin'],
|
|
167
|
+
handler,
|
|
168
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export interface AnnouncementsConfig {
|
|
2
|
+
title: string;
|
|
3
|
+
description: string;
|
|
4
|
+
url: string;
|
|
5
|
+
imgUrl: string;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface AnnouncementCatalog {
|
|
9
|
+
articles: AnnouncementArticle[];
|
|
10
|
+
catalogId: number;
|
|
11
|
+
catalogName: string;
|
|
12
|
+
catalogType: 1;
|
|
13
|
+
catalogs: [];
|
|
14
|
+
description: null;
|
|
15
|
+
icon: string;
|
|
16
|
+
parentCatalogId: null;
|
|
17
|
+
total: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface AnnouncementArticle {
|
|
21
|
+
id: number;
|
|
22
|
+
code: string;
|
|
23
|
+
title: string;
|
|
24
|
+
type: number;
|
|
25
|
+
releaseDate: number;
|
|
26
|
+
}
|
|
@@ -57,7 +57,7 @@ async function handler(ctx) {
|
|
|
57
57
|
const items = resp.MediaContents.map((item) => {
|
|
58
58
|
const ssd = item.Ssd;
|
|
59
59
|
const link = `${apiUrl}${item.ImageContent.Image.Url.match(/\/th\?id=[^_]+_[^_]+/)[0].replace(/(_\d+x\d+\.webp)$/i, '')}_${type}.jpg`;
|
|
60
|
-
let description = `<img src="${link}" alt="Article Cover Image" style="display: block; margin: 0 auto;"><br>`;
|
|
60
|
+
let description = `<img width="1920" height="1080" src="${link}" alt="Article Cover Image" style="display: block; margin: 0 auto;"><br>`;
|
|
61
61
|
if (story) {
|
|
62
62
|
description += `<b>${item.ImageContent.Headline}</b>`;
|
|
63
63
|
description += `<i>${item.ImageContent.QuickFact.MainText}</i><br>`;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { Route } from '@/types';
|
|
2
|
+
import { load } from 'cheerio';
|
|
3
|
+
import ofetch from '@/utils/ofetch';
|
|
4
|
+
|
|
5
|
+
import { fetchArticle } from './utils';
|
|
6
|
+
|
|
7
|
+
export const route: Route = {
|
|
8
|
+
path: '/cat/:cat',
|
|
9
|
+
categories: ['traditional-media'],
|
|
10
|
+
example: '/bjnews/cat/depth',
|
|
11
|
+
parameters: { cat: '分类, 可从URL中找到' },
|
|
12
|
+
features: {},
|
|
13
|
+
radar: [
|
|
14
|
+
{
|
|
15
|
+
source: ['www.bjnews.com.cn/:cat'],
|
|
16
|
+
},
|
|
17
|
+
],
|
|
18
|
+
name: '分类',
|
|
19
|
+
maintainers: ['dzx-dzx'],
|
|
20
|
+
handler,
|
|
21
|
+
url: 'www.bjnews.com.cn',
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
async function handler(ctx) {
|
|
25
|
+
const url = `https://www.bjnews.com.cn/${ctx.req.param('cat')}`;
|
|
26
|
+
const res = await ofetch(url);
|
|
27
|
+
const $ = load(res);
|
|
28
|
+
const list = $('#waterfall-container .pin_demo > a')
|
|
29
|
+
.toArray()
|
|
30
|
+
.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 15)
|
|
31
|
+
.map((a) => ({
|
|
32
|
+
title: $(a).text(),
|
|
33
|
+
link: $(a).attr('href'),
|
|
34
|
+
category: $(a).parent().find('.source').text().trim(),
|
|
35
|
+
}));
|
|
36
|
+
|
|
37
|
+
const out = await Promise.all(list.map((item) => fetchArticle(item)));
|
|
38
|
+
return {
|
|
39
|
+
title: `新京报 - 分类 - ${$('.cur').text().trim()}`,
|
|
40
|
+
link: url,
|
|
41
|
+
item: out,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { Route } from '@/types';
|
|
2
|
+
import ofetch from '@/utils/ofetch';
|
|
3
|
+
import { parseDate } from '@/utils/parse-date';
|
|
4
|
+
import timezone from '@/utils/timezone';
|
|
5
|
+
|
|
6
|
+
import { fetchArticle } from './utils';
|
|
7
|
+
|
|
8
|
+
export const route: Route = {
|
|
9
|
+
path: '/column/:column',
|
|
10
|
+
categories: ['traditional-media'],
|
|
11
|
+
example: '/bjnews/column/204',
|
|
12
|
+
parameters: { column: '栏目ID, 可从手机版网页URL中找到' },
|
|
13
|
+
features: {},
|
|
14
|
+
radar: [
|
|
15
|
+
{
|
|
16
|
+
source: ['m.bjnews.com.cn/column/:column.htm'],
|
|
17
|
+
},
|
|
18
|
+
],
|
|
19
|
+
name: '分类',
|
|
20
|
+
maintainers: ['dzx-dzx'],
|
|
21
|
+
handler,
|
|
22
|
+
url: 'www.bjnews.com.cn',
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
async function handler(ctx) {
|
|
26
|
+
const columnID = ctx.req.param('column');
|
|
27
|
+
const url = `https://api.bjnews.com.cn/api/v101/news/column_news.php?column_id=${columnID}`;
|
|
28
|
+
const res = await ofetch(url);
|
|
29
|
+
const list = res.data.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 15).map((e) => ({
|
|
30
|
+
title: e.row.title,
|
|
31
|
+
guid: e.uuid,
|
|
32
|
+
pubDate: timezone(parseDate(e.row.publish_time), +8),
|
|
33
|
+
updated: timezone(parseDate(e.row.update_time), +8),
|
|
34
|
+
link: `https://www.bjnews.com.cn/detail/${e.uuid}.html`,
|
|
35
|
+
}));
|
|
36
|
+
|
|
37
|
+
const out = await Promise.all(list.map((item) => fetchArticle(item)));
|
|
38
|
+
return {
|
|
39
|
+
title: `新京报 - 栏目 - ${res.data[0].row.column_info[0].column_name}`,
|
|
40
|
+
link: `https://m.bjnews.com.cn/column/${columnID}.html`,
|
|
41
|
+
item: out,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import cache from '@/utils/cache';
|
|
2
|
+
import { parseDate } from '@/utils/parse-date';
|
|
3
|
+
import timezone from '@/utils/timezone';
|
|
4
|
+
import ofetch from '@/utils/ofetch';
|
|
5
|
+
import { load } from 'cheerio';
|
|
6
|
+
|
|
7
|
+
export function fetchArticle(item) {
|
|
8
|
+
return cache.tryGet(item.link, async () => {
|
|
9
|
+
const responses = await ofetch(item.link);
|
|
10
|
+
const $d = load(responses);
|
|
11
|
+
// $d('img').each((i, e) => $(e).attr('referrerpolicy', 'no-referrer'));
|
|
12
|
+
|
|
13
|
+
item.pubDate = timezone(parseDate($d('.left-info .timer').text()), +8);
|
|
14
|
+
item.author = $d('.left-info .reporter').text();
|
|
15
|
+
item.description = $d('#contentStr').html();
|
|
16
|
+
|
|
17
|
+
return item;
|
|
18
|
+
});
|
|
19
|
+
}
|
package/lib/routes/bjx/fd.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { DataItem, Route } from '@/types';
|
|
2
|
-
import got from '@/utils/got';
|
|
3
2
|
import { load } from 'cheerio';
|
|
4
3
|
import { parseDate } from '@/utils/parse-date';
|
|
4
|
+
import ofetch from '@/utils/ofetch';
|
|
5
|
+
import cache from '@/utils/cache';
|
|
5
6
|
|
|
6
7
|
export const route: Route = {
|
|
7
8
|
path: '/fd/:type',
|
|
8
9
|
categories: ['traditional-media'],
|
|
9
10
|
example: '/bjx/fd/yw',
|
|
10
|
-
parameters: { type: '
|
|
11
|
+
parameters: { type: '文章分类,详见下表' },
|
|
11
12
|
features: {
|
|
12
13
|
requireConfig: false,
|
|
13
14
|
requirePuppeteer: false,
|
|
@@ -18,37 +19,45 @@ export const route: Route = {
|
|
|
18
19
|
},
|
|
19
20
|
name: '风电',
|
|
20
21
|
maintainers: ['hualiong'],
|
|
22
|
+
description: `\`:type\` 类型可选如下
|
|
23
|
+
|
|
24
|
+
| 要闻 | 政策 | 数据 | 市场 | 企业 | 招标 | 技术 | 报道 |
|
|
25
|
+
| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
|
|
26
|
+
| yw | zc | sj | sc | mq | zb | js | bd |`,
|
|
21
27
|
handler: async (ctx) => {
|
|
22
28
|
const type = ctx.req.param('type');
|
|
23
|
-
const response = await
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
const response = await ofetch(`https://fd.bjx.com.cn/${type}/`);
|
|
30
|
+
|
|
31
|
+
const $ = load(response);
|
|
32
|
+
const typeName = $('div.box2 em:last-child').text();
|
|
33
|
+
const list = $('div.cc-list-content ul li:nth-child(-n+20)')
|
|
34
|
+
.toArray()
|
|
35
|
+
.map((item): DataItem => {
|
|
36
|
+
const each = $(item);
|
|
37
|
+
return {
|
|
38
|
+
title: each.find('a').attr('title')!,
|
|
39
|
+
link: each.find('a').attr('href'),
|
|
40
|
+
pubDate: parseDate(each.find('span').text()),
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
const items = await Promise.all(
|
|
45
|
+
list.map((item) =>
|
|
46
|
+
cache.tryGet(item.link!, async () => {
|
|
47
|
+
const response = await ofetch(item.link!);
|
|
48
|
+
const $ = load(response);
|
|
49
|
+
|
|
50
|
+
item.description = $('#article_cont').html()!;
|
|
51
|
+
return item;
|
|
52
|
+
})
|
|
53
|
+
)
|
|
54
|
+
);
|
|
31
55
|
|
|
32
56
|
return {
|
|
33
57
|
title: `北极星风力发电网${typeName}`,
|
|
34
58
|
description: $('meta[name="Description"]').attr('content'),
|
|
35
59
|
link: `https://fd.bjx.com.cn/${type}/`,
|
|
36
|
-
item:
|
|
37
|
-
.map((index, item) => {
|
|
38
|
-
const each = $(item);
|
|
39
|
-
return {
|
|
40
|
-
title: each.find('a').attr('title'),
|
|
41
|
-
description: each.html(),
|
|
42
|
-
link: each.find('a').attr('href'),
|
|
43
|
-
pubDate: parseDate(each.find('span').text()),
|
|
44
|
-
};
|
|
45
|
-
})
|
|
46
|
-
.get() as DataItem[],
|
|
60
|
+
item: items as DataItem[],
|
|
47
61
|
};
|
|
48
62
|
},
|
|
49
|
-
description: `\`:type\` 类型可选如下
|
|
50
|
-
|
|
51
|
-
| 要闻 | 政策 | 数据 | 市场 | 企业 | 招标 | 技术 | 报道 |
|
|
52
|
-
| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
|
|
53
|
-
| yw | zc | sj | sc | mq | zb | js | bd |`,
|
|
54
63
|
};
|
|
@@ -85,7 +85,7 @@ function getArticleUrl(onclick?: string) {
|
|
|
85
85
|
async function getList(url: string | URL, form: Record<string, string> = {}) {
|
|
86
86
|
const { body } = await got.post(url, { form });
|
|
87
87
|
const $ = load(body);
|
|
88
|
-
const title = $('#main > div.dqwz > a').last().text();
|
|
88
|
+
const title = $('#main > div.dqwz > a').last().text() || '北京航空航天大学教务部';
|
|
89
89
|
const list = $('#main div.news_list > ul > li')
|
|
90
90
|
.toArray()
|
|
91
91
|
.map((item) => {
|