rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/keylol/index.ts

@@ -1,55 +1,147 @@
 import { Route } from '@/types';
-import { getSubPath } from '@/utils/common-utils';
 import cache from '@/utils/cache';
+import { config } from '@/config';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import timezone from '@/utils/timezone';
-import { parseDate } from '@/utils/parse-date';
+import { parseDate, parseRelativeDate } from '@/utils/parse-date';
+import parser from '@/utils/rss-parser';
+import queryString from 'query-string';
+
+const threadIdRegex = /(\d+)-\d+-\d+/;
+const header = {
+    Cookie: config.keylol.cookie ?? undefined,
+};
 
 export const route: Route = {
-    path: '*',
-    name: 'Unknown',
-    maintainers: [],
+    path: '/:path',
+    name: '论坛',
+    parameters: { path: '路径，默认为热点聚焦' },
+    categories: ['game'],
+    example: '/keylol/f161-1',
+    features: {
+        requireConfig: [
+            {
+                name: 'KEYLOL_COOKIE',
+                optional: true,
+                description: `配置后可抓取具有阅读权限的帖子內容`,
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['keylol.com/:path'],
+            target: (params, url) => url.replaceAll('forum.php?', ''),
+        },
+    ],
+    maintainers: ['nczitzk', 'kennyfong19931'],
     handler,
+    description: `:::tip
+  若订阅 [热点聚焦](https://keylol.com/f161-1)，网址为 \`https://keylol.com/f161-1\`。截取 \`https://keylol.com/\` 到末尾的部分 \`f161-1\` 作为参数，此时路由为 [\`/keylol/f161-1\`](https://rsshub.app/keylol/f161-1)。
+  若订阅子分类 [试玩免费 - 热点聚焦](https://keylol.com/forum.php?mod=forumdisplay&fid=161&filter=typeid&typeid=459)，网址为 \`https://keylol.com/forum.php?mod=forumdisplay&fid=161&filter=typeid&typeid=459\`。提取\`fid\`及\`typeid\` 作为参数，此时路由为 [\`/keylol/fid=161&typeid=459\`](https://rsshub.app/keylol/fid=161&typeid=459)。注意不要包括\`filter\`，会调用[全局的内容过滤](https://docs.rsshub.app/guide/parameters#filtering)。
+  :::`,
 };
 
 async function handler(ctx) {
-    let thePath = getSubPath(ctx).replace(/^\//, '');
-
-    if (/^f\d+-\d+/.test(thePath)) {
-        thePath = `fid=${thePath.match(/^f(\d+)-\d+/)[1]}`;
+    let queryParams = {};
+    const path = ctx.req.param('path');
+    if (/^f\d+-\d+/.test(path)) {
+        queryParams.fid = path.match(/^f(\d+)-\d+/)[1];
+    } else {
+        queryParams = queryString.parse(path);
+    }
+    queryParams.mod = 'forumdisplay';
+    queryParams.orderby = 'dateline';
+    queryParams.filter = 'author';
+
+    // get real author name from official rss feed
+    let authorNameMap;
+    try {
+        const feed = await parser.parseURL(`https://keylol.com/forum.php?mod=rss&fid=${queryParams.fid}&auth=0`);
+        authorNameMap = feed.items.map((item) => ({
+            threadId: item.link.match(threadIdRegex)[1],
+            author: item.author,
+        }));
+    } catch {
+        authorNameMap = [];
     }
 
     const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 30;
 
     const rootUrl = 'https://keylol.com';
-    const currentUrl = new URL(`forum.php?mod=forumdisplay&${thePath.replaceAll(/mod=\w+&/g, '')}`, rootUrl).href;
+    const currentUrl = queryString.stringifyUrl({ url: `${rootUrl}/forum.php`, query: queryParams });
 
-    const { data: response } = await got(currentUrl);
+    const { data: response } = await got({
+        method: 'get',
+        url: currentUrl,
+        headers: header,
+    });
 
     const $ = load(response);
 
-    let items = $('a.xst')
+    let items = $('tbody[id^="normalthread_"]')
         .slice(0, limit)
         .toArray()
         .map((item) => {
             item = $(item);
 
             return {
-                title: item.text(),
-                link: new URL(item.prop('href').split('&extra=')[0], rootUrl).href,
+                title: item.find('a.xst').text(),
+                link: new URL(item.find(' a.xst').prop('href').split('&extra=')[0], rootUrl).href,
+                author: item.find('td.by-author cite').text(),
+                pubDate: parseRelativeDate(item.find('td.by-author em').text().replaceAll(' 发表', '')),
             };
         });
 
     items = await Promise.all(
         items.map((item) =>
             cache.tryGet(item.link, async () => {
-                const { data: detailResponse } = await got(item.link);
+                const threadId = threadIdRegex.test(item.link) ? item.link.match(threadIdRegex)[1] : queryString.parseUrl(item.link).query.tid;
+                const { data: detailResponse } = await got({
+                    method: 'get',
+                    url: item.link,
+                    headers: header,
+                });
 
                 const content = load(detailResponse);
 
-                item.description = content('td.t_f').html();
-                item.author = content('a.xw1').first().text();
+                let descriptionList: string[] = [];
+                const indexDiv = content('div#threadindex');
+                if (indexDiv.length > 0) {
+                    // post with page
+                    const postId = content('div.t_fsz > script')
+                        .text()
+                        .match(/show_threadindex\((\d+),/)[1];
+                    descriptionList = await Promise.all(
+                        indexDiv.find('a').map((i, a) => {
+                            const pageTitle = $(a).text();
+                            const page = $(a).attr('page');
+                            return getPage(
+                                `${rootUrl}/forum.php?${queryString.stringify({
+                                    mod: 'viewthread',
+                                    tid: threadId,
+                                    viewpid: postId,
+                                    cp: page,
+                                })}`,
+                                pageTitle
+                            );
+                        })
+                    );
+                } else {
+                    // normal post
+                    descriptionList.push(getDescription(content));
+                }
+
+                item.description = descriptionList.join('<br/>');
+                const realAuthorName = authorNameMap.find((a) => a.threadId === threadId);
+                if (realAuthorName) {
+                    item.author = realAuthorName.author;
+                }
                 item.category = content('#keyloL_thread_tags a')
                     .toArray()
                     .map((c) => content(c).text());
@@ -90,3 +182,32 @@ async function handler(ctx) {
         author: $('meta[name="author"]').prop('content'),
     };
 }
+
+function getDescription($) {
+    const descriptionEl = $('td.t_f');
+    descriptionEl.find('div.rnd_ai_pr').remove(); // remove ad image
+
+    // handle lazyload image
+    descriptionEl.find('img').each((_, img) => {
+        img = $(img);
+        if (img.attr('src')?.endsWith('none.gif') && img.attr('file')) {
+            img.attr('src', img.attr('file'));
+            img.removeAttr('file');
+            img.removeAttr('zoomfile');
+        }
+    });
+
+    return descriptionEl.length > 0 ? descriptionEl.html() : $('div.alert_info').html();
+}
+
+async function getPage(url, pageTitle) {
+    const { data: detailResponse } = await got({
+        method: 'get',
+        url,
+        headers: header,
+    });
+
+    const $ = load(detailResponse, { xmlMode: true });
+    const content = $('root').text();
+    return '<h3>' + pageTitle + '</h3>' + getDescription(load(content));
+}

package/lib/routes/konachan/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Konachan.com Anime Wallpapers',
+    url: 'konachan.com',
+    description: `konachan post`,
+};

package/lib/routes/konachan/post.ts

@@ -0,0 +1,84 @@
+import { Route } from '@/types';
+import got from '@/utils/got';
+import queryString from 'query-string';
+
+export const route: Route = {
+    path: '/post/popular_recent/:period?',
+    categories: ['picture'],
+    example: '/konachan/post/popular_recent/1d',
+    parameters: {
+        period: '展示时间',
+    },
+    radar: [
+        {
+            source: ['konachan.com/post'],
+        },
+    ],
+    name: 'Popular Recent Posts',
+    maintainers: ['magic-akari', 'NekoAria'],
+    description: `| 最近 24 小时    | 最近一周     | 最近一月    | 最近一年     |
+  | ------- | -------- | ------- | -------- |
+  | 1d | 1w | 1m | 1y |`,
+    handler,
+};
+
+async function handler(ctx) {
+    const { period = '1d' } = ctx.req.param();
+
+    const response = await got({
+        url: 'https://konachan.com/post/popular_recent.json',
+        searchParams: queryString.stringify({
+            period,
+        }),
+    });
+
+    const posts = response.data;
+
+    const titles = {
+        '1d': 'Last 24 hours',
+        '1w': 'Last week',
+        '1m': 'Last month',
+        '1y': 'Last year',
+    };
+
+    const mime = {
+        jpg: 'jpeg',
+        png: 'png',
+    };
+
+    const title = titles[period];
+
+    return {
+        title: `${title} - konachan.com`,
+        link: `https://konachan.com/post/popular_recent?period=${period}`,
+        item: posts.map((post) => ({
+            title: post.tags,
+            id: `${ctx.path}#${post.id}`,
+            guid: `${ctx.path}#${post.id}`,
+            link: `https://konachan.com/post/show/${post.id}`,
+            author: post.author,
+            pubDate: new Date(post.created_at * 1e3).toUTCString(),
+            description: (() => {
+                const result = [`<img src="${post.sample_url}" />`];
+                result.push(`<p>Rating:${post.rating}</p> <p>Score:${post.score}</p>`);
+                if (post.source) {
+                    result.push(`<a href="${post.source}">Source</a>`);
+                }
+                if (post.parent_id) {
+                    result.push(`<a href="https://konachan.com/post/show/${post.parent_id}">Parent</a>`);
+                }
+                return result.join('');
+            })(),
+            media: {
+                content: {
+                    url: post.file_url,
+                    type: `image/${mime[post.file_ext]}`,
+                },
+                thumbnail: {
+                    url: post.preview_url,
+                },
+            },
+            category: post.tags.split(/\s+/),
+        })),
+    };
+}

package/lib/routes/kpmg/insights.ts

@@ -0,0 +1,143 @@
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import * as cheerio from 'cheerio';
+import { Context } from 'hono';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import path from 'node:path';
+import { getCurrentPath } from '@/utils/helpers';
+const __dirname = getCurrentPath(import.meta.url);
+
+const baseUrl = 'https://kpmg.com';
+const payload = {
+    query: '',
+    filters: {
+        all: [{ kpmg_tab_type: ['Insights'] }, { kpmg_article_type: ['Article-General'] }, { kpmg_template_type: ['article-details-template', 'insights-flexible-template', 'editable-flex-template', 'editable-campaign-template'] }],
+    },
+    result_fields: {
+        kpmg_description: { raw: {} },
+        kpmg_banner_flag: { raw: {} },
+        kpmg_primary_tag: { raw: {} },
+        kpmg_article_date: { raw: {} },
+        kpmg_contact_job_ttl: { raw: {} },
+        kpmg_title: { raw: {} },
+        kpmg_contact_city: { raw: {} },
+        kpmg_event_start_time: { raw: {} },
+        kpmg_article_date_time: { raw: {} },
+        kpmg_tab_type: { raw: {} },
+        kpmg_short_desc: { raw: {} },
+        kpmg_image_alt: { raw: {} },
+        kpmg_url: { raw: {} },
+        kpmg_template_type: { raw: {} },
+        kpmg_image: { raw: {} },
+        kpmg_non_decorative_alt_text: { raw: {} },
+        kpmg_article_readtime: { raw: {} },
+        kpmg_contact_fn: { raw: {} },
+        kpmg_contact_ln: { raw: {} },
+        kpmg_event_type: { raw: {} },
+        kpmg_contact_country: { raw: {} },
+        kpmg_is_rendition_optimized: { raw: {} },
+        kpmg_article_primary_format: { raw: {} },
+        kpmg_article_type: { raw: {} },
+        kpmg_event_startdate: { raw: {} },
+    },
+    page: { size: 20, current: 1 },
+    sort: { kpmg_filter_date: 'desc' },
+};
+const endpoints = {
+    en: {
+        title: 'Insights - KPMG ',
+        link: `${baseUrl}/xx/en/home/insights.html`,
+        api: `${baseUrl}/esearch/xx-en`,
+    },
+    zh: {
+        title: '洞察 - 毕马威',
+        link: `${baseUrl}/cn/zh/home/insights.html`,
+        api: `${baseUrl}/esearch/cn-zh`,
+    },
+};
+const render = (data) => art(path.join(__dirname, 'templates/description.art'), data);
+
+const handler = async (ctx: Context) => {
+    const { lang = 'en' } = ctx.req.param();
+    const endpoint = endpoints[lang];
+    if (!endpoint) {
+        throw new InvalidParameterError('Invalid language');
+    }
+    const link = endpoint.link;
+
+    const response = await ofetch(endpoint.api, {
+        method: 'POST',
+        body: payload,
+    });
+
+    const list = response.results.map((item) => ({
+        title: item.kpmg_title.raw,
+        description: item.kpmg_description.raw,
+        link: item.kpmg_url.raw,
+        pubDate: parseDate(item.kpmg_article_date_time.raw),
+        image: item.kpmg_image.raw,
+        imageAlt: item.kpmg_image_alt?.raw,
+    }));
+
+    const item = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await ofetch(item.link);
+                const $ = cheerio.load(response);
+
+                const content = $('.bodytext-data')
+                    .toArray()
+                    .map((item) => {
+                        const element = $(item);
+                        element.find('.hidden-xs, .sr-only').remove();
+                        return element.parent().html();
+                    })
+                    .join('');
+
+                const pdfDetails = $('.pdfdetails').parent().parent().parent();
+                pdfDetails.find('.hidden-xs, .sr-only').remove();
+
+                item.description = render({
+                    image: item.image,
+                    alt: item.imageAlt,
+                    content,
+                    pdf: pdfDetails.prop('outerHTML'),
+                });
+
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: 'KPMG Insights',
+        link,
+        item,
+    };
+};
+
+export const route: Route = {
+    path: '/insights/:lang?',
+    example: '/kpmg/insights',
+    parameters: { lang: 'Language, either `en` or `zh`' },
+    radar: [
+        {
+            source: ['kpmg.com/xx/en/home/insights.html'],
+            target: '/insights/en',
+        },
+        {
+            source: ['kpmg.com/cn/zh/home/insights.html'],
+            target: '/insights/zh',
+        },
+    ],
+    name: 'Insights',
+    maintainers: ['LogicJake'],
+    handler,
+    url: 'kpmg.com/xx/en/home/insights.html',
+    zh: {
+        name: '洞察',
+    },
+};

package/lib/routes/kpmg/namespace.ts

@@ -0,0 +1,10 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'KPMG',
+    url: 'kpmg.com',
+    categories: ['other'],
+    zh: {
+        name: '毕马威',
+    },
+};

package/lib/routes/kpmg/templates/description.art

@@ -0,0 +1,11 @@
+{{ if image }}
+<img src="https://assets.kpmg.com{{ image }}" {{ if alt }} alt="{{ alt }}" {{ /if }}>
+{{ /if }}
+
+{{ if content }}
+<br>{{@ content }}
+{{ /if }}
+
+{{ if pdf }}
+<br>{{@ pdf }}
+{{ /if }}

package/lib/routes/kurogames/namespace.ts

@@ -0,0 +1,7 @@
+import { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '库洛游戏',
+    url: 'www.kurogames.com',
+    categories: ['game'],
+};

package/lib/routes/kurogames/wutheringwaves/news.ts

@@ -0,0 +1,59 @@
+import { DataItem, Route } from '@/types';
+import cache from '@/utils/cache';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+import * as cheerio from 'cheerio';
+import ofetch from '@/utils/ofetch';
+
+interface NewsItem {
+    articleContent: string;
+    articleDesc: string;
+    articleId: number;
+    articleTitle: string;
+    articleType: number;
+    createTime: string;
+    sortingMark: number;
+    startTime: string;
+    suggestCover: string;
+    top: number;
+}
+
+export const route: Route = {
+    path: '/wutheringwaves/news',
+    categories: ['game'],
+    example: '/kurogames/wutheringwaves/news',
+    name: '鸣潮 — 游戏公告、新闻与活动',
+    radar: [
+        {
+            source: ['mc.kurogames.com/m/main/news', 'mc.kurogames.com/main'],
+        },
+    ],
+    maintainers: ['enpitsulin'],
+    description: '',
+    async handler() {
+        const res = await ofetch<NewsItem[]>('https://media-cdn-mingchao.kurogame.com/akiwebsite/website2.0/json/G152/zh/ArticleMenu.json', { query: { t: Date.now() } });
+        const item = await Promise.all(
+            res.map((i) => {
+                const contentUrl = `https://media-cdn-mingchao.kurogame.com/akiwebsite/website2.0/json/G152/zh/article/${i.articleId}.json`;
+                const item = {
+                    title: i.articleTitle,
+                    pubDate: timezone(parseDate(i.createTime), +8),
+                    link: `https://mc.kurogames.com/main/news/detail/${i.articleId}`,
+                } as DataItem;
+                return cache.tryGet(contentUrl, async () => {
+                    const data = await ofetch<NewsItem>(contentUrl, { query: { t: Date.now() } });
+                    const $ = cheerio.load(data.articleContent);
+
+                    item.description = $.html() ?? i.articleDesc ?? '';
+                    return item;
+                }) as Promise<DataItem>;
+            })
+        );
+        return {
+            title: '《鸣潮》— 游戏公告、新闻和活动',
+            link: 'https://mc.kurogames.com/main#news',
+            item,
+            language: 'zh-cn',
+        };
+    },
+};

package/lib/routes/kyodonews/index.ts

@@ -35,7 +35,7 @@ export const route: Route = {
 
 async function handler(ctx) {
     const language = ctx.req.param('language') ?? 'china';
-    const keyword = ctx.req.param('keyword') === 'RSS' ? 'rss' : ctx.req.param('keyword') ?? '';
+    const keyword = ctx.req.param('keyword') === 'RSS' ? 'rss' : (ctx.req.param('keyword') ?? '');
 
     // raise error for invalid languages
     if (!['china', 'tchina'].includes(language)) {

package/lib/routes/layoffs/index.ts

@@ -95,7 +95,7 @@ async function handler() {
             $('script')
                 .text()
                 .match(/urlWithParams: "(.*?)"/)[1]
-                .replaceAll('\\u002F', '/');
+                .replaceAll(String.raw`\u002F`, '/');
 
         // Cache it again
         cache.set(ENTRY_URL, dataSourceUrl);

package/lib/routes/leetcode/articles.ts

@@ -1,6 +1,6 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import MarkdownIt from 'markdown-it';
@@ -38,8 +38,8 @@ export const route: Route = {
 
 async function handler() {
     const link = new URL('/articles/', host).href;
-    const response = await got(link);
-    const $ = load(response.data);
+    const response = await ofetch(link, { parseResponse: (txt) => txt });
+    const $ = load(response);
 
     const list = $('a.list-group-item')
         .filter((i, e) => $(e).find('h4.media-heading i').length === 0)
@@ -59,37 +59,35 @@ async function handler() {
             cache.tryGet(info.link, async () => {
                 const titleSlug = info.link.split('/')[4];
 
-                const questionContent = await got
-                    .post(gqlEndpoint, {
-                        json: {
-                            operationName: 'questionContent',
-                            variables: { titleSlug },
-                            query: `query questionContent($titleSlug: String!) {
+                const questionContent = await ofetch(gqlEndpoint, {
+                    method: 'POST',
+                    body: {
+                        operationName: 'questionContent',
+                        variables: { titleSlug },
+                        query: `query questionContent($titleSlug: String!) {
                                 question(titleSlug: $titleSlug) {
                                     content
                                     mysqlSchemas
                                     dataSchemas
                                 }
                             }`,
-                        },
-                    })
-                    .json();
+                    },
+                });
 
-                const officialSolution = await got
-                    .post(gqlEndpoint, {
-                        json: {
-                            operationName: 'officialSolution',
-                            variables: { titleSlug },
-                            query: `query officialSolution($titleSlug: String!) {
+                const officialSolution = await ofetch(gqlEndpoint, {
+                    method: 'POST',
+                    body: {
+                        operationName: 'officialSolution',
+                        variables: { titleSlug },
+                        query: `query officialSolution($titleSlug: String!) {
                                 question(titleSlug: $titleSlug) {
                                     solution {
                                         content
                                     }
                                 }
                             }`,
-                        },
-                    })
-                    .json();
+                    },
+                });
 
                 const solution = md.render(officialSolution.data.question.solution.content);
 

package/lib/routes/lemmy/index.ts

@@ -10,9 +10,36 @@ import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 export const route: Route = {
     path: '/:community/:sort?',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/lemmy/technology@lemmy.world/Hot',
-    parameters: { community: 'Lemmmy community, for example technology@lemmy.world', sort: 'Sort by, defaut to Active' },
+    parameters: {
+        community: 'Lemmmy community, for example technology@lemmy.world',
+        sort: {
+            description: 'Sort by',
+            options: [
+                { value: 'Active', label: 'Active' },
+                { value: 'Hot', label: 'Hot' },
+                { value: 'New', label: 'New' },
+                { value: 'Old', label: 'Old' },
+                { value: 'TopDay', label: 'TopDay' },
+                { value: 'TopWeek', label: 'TopWeek' },
+                { value: 'TopMonth', label: 'TopMonth' },
+                { value: 'TopYear', label: 'TopYear' },
+                { value: 'TopAll', label: 'TopAll' },
+                { value: 'MostComments', label: 'MostComments' },
+                { value: 'NewComments', label: 'NewComments' },
+                { value: 'TopHour', label: 'TopHour' },
+                { value: 'TopSixHour', label: 'TopSixHour' },
+                { value: 'TopTwelveHour', label: 'TopTwelveHour' },
+                { value: 'TopThreeMonths', label: 'TopThreeMonths' },
+                { value: 'TopSixMonths', label: 'TopSixMonths' },
+                { value: 'TopNineMonths', label: 'TopNineMonths' },
+                { value: 'Controversial', label: 'Controversial' },
+                { value: 'Scaled', label: 'Scaled' },
+            ],
+            default: 'Active',
+        },
+    },
     features: {
         requireConfig: [
             {
@@ -27,7 +54,7 @@ export const route: Route = {
         supportScihub: false,
     },
     name: 'Community',
-    maintainers: ['wb14123'],
+    maintainers: ['wb14123', 'pseudoyu'],
     handler,
 };
 

package/lib/routes/lemmy/namespace.ts

@@ -2,4 +2,5 @@ import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
     name: 'Lemmy',
+    url: 'join-lemmy.org',
 };