rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/ecnu/jwc.ts

@@ -0,0 +1,67 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+const type = (filename) => filename.split('.').pop();
+
+export const route: Route = {
+    path: '/jwc',
+    categories: ['university'],
+    example: '/ecnu/jwc',
+    radar: [
+        {
+            source: ['www.jwc.ecnu.edu.cn', 'www.ecnu.edu.cn'],
+            target: '/tzgg',
+        },
+    ],
+    name: '教务处通知',
+    maintainers: ['markbang'],
+    handler: async () => {
+        const baseUrl = 'http://www.jwc.ecnu.edu.cn/';
+
+        const response = await got(`${baseUrl}tzggwwxsgg/list.htm`);
+        const $ = load(response.data);
+        const links = $('.col_news_con ul.news_list > li')
+            .map((_, el) => ({
+                pubDate: timezone(parseDate($(el).find('.news_date').text()), 8),
+                link: new URL($(el).find('a').attr('href'), baseUrl).toString(),
+                title: $(el).find('a').text(),
+            }))
+            .get();
+        const items = await Promise.all(
+            links.map((item) =>
+                cache.tryGet(item.link, async () => {
+                    if (type(item.link) === 'htm') {
+                        try {
+                            const { data } = await got(item.link, {
+                                https: {
+                                    rejectUnauthorized: false,
+                                },
+                            });
+                            const $ = load(data);
+                            item.description = $('div.article')?.html()?.replaceAll('src="/', `src="${baseUrl}/`)?.replaceAll('href="/', `href="${baseUrl}/`)?.trim();
+                            return item;
+                        } catch {
+                            // intranet
+                            item.description = '请进行统一身份认证之后再访问';
+                            return item;
+                        }
+                    } else {
+                        // file to download
+                        item.description = '点击认证后访问内容';
+                        return item;
+                    }
+                })
+            )
+        );
+
+        return {
+            title: '教务处通知',
+            link: 'http://www.jwc.ecnu.edu.cn/tzggwwxsgg/list.htm',
+            item: items,
+        };
+    },
+};

package/lib/routes/ecnu/namespace.ts

@@ -2,5 +2,5 @@ import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
     name: 'East China Normal University 华东师范大学',
-    url: 'acm.ecnu.edu.cn',
+    url: 'ecnu.edu.cn',
 };

package/lib/routes/economist/espresso.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
@@ -8,7 +8,8 @@ const link = 'https://www.economist.com/the-world-in-brief';
 
 export const route: Route = {
     path: '/espresso',
-    categories: ['traditional-media'],
+    categories: ['traditional-media', 'popular'],
+    view: ViewType.Articles,
     example: '/economist/espresso',
     parameters: {},
     features: {
@@ -42,7 +43,7 @@ async function handler() {
     );
 
     const nextData = JSON.parse($('script#__NEXT_DATA__').text());
-    const parts = nextData.props.pageProps.content.hasPart.parts[0].hasPart.parts.filter((part) => part.type.includes('Article'));
+    const parts = nextData.props.pageProps.content.hasPart.parts[0].hasPart.parts.filter((part) => part.type.includes('Article') && part.headline !== '');
 
     const renderHTML = (node) => {
         let el;

package/lib/routes/economist/full.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import parser from '@/utils/rss-parser';
 import got from '@/utils/got';
@@ -21,7 +21,8 @@ const getArticleDetail = (link) =>
 
 export const route: Route = {
     path: '/:endpoint',
-    categories: ['traditional-media'],
+    categories: ['traditional-media', 'popular'],
+    view: ViewType.Articles,
     example: '/economist/latest',
     parameters: { endpoint: 'Category name, can be found on the [official page](https://www.economist.com/rss). For example, https://www.economist.com/china/rss.xml to china' },
     features: {

package/lib/routes/epicgames/index.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
@@ -10,9 +10,19 @@ import dayjs from 'dayjs';
 
 export const route: Route = {
     path: '/freegames/:locale?/:country?',
-    categories: ['game'],
-    example: '/epicgames/freegames',
-    parameters: { locale: 'Locale, en_US by default', country: 'Country, US by default' },
+    categories: ['game', 'popular'],
+    view: ViewType.Notifications,
+    example: '/epicgames/freegames/en-US/US',
+    parameters: {
+        locale: {
+            description: 'Locale',
+            default: 'en-US',
+        },
+        country: {
+            description: 'Country',
+            default: 'US',
+        },
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -67,23 +77,25 @@ async function handler(ctx) {
                 isBundles = true;
                 contentUrl = `${contentBaseUrl}/bundles/`;
             }
-            const linkSlug =
+            let linkSlug =
                 item.catalogNs.mappings && item.catalogNs.mappings.length > 0
                     ? item.catalogNs.mappings[0].pageSlug
                     : item.offerMappings && item.offerMappings.length > 0
                       ? item.offerMappings[0].pageSlug
-                      : item.productSlug ?? item.urlSlug;
+                      : (item.productSlug ?? item.urlSlug);
+            if (item.offerType === 'ADD_ON') {
+                linkSlug = item.offerMappings[0].pageSlug;
+            }
             link += linkSlug;
             contentUrl += linkSlug;
             let description = item.description;
-            if (item.offerType !== 'BASE_GAME') {
+            if (isBundles) {
                 const contentResp = await got({
                     method: 'get',
                     url: contentUrl,
                 });
-                description = isBundles ? contentResp.data.data.about.shortDescription : contentResp.data.pages[0].data.about.shortDescription;
+                description = contentResp.data.data.about.shortDescription;
             }
-
             let image = item.keyImages[0].url;
             item.keyImages.some((keyImage) => {
                 if (keyImage.type === 'DieselStoreFrontWide') {

package/lib/routes/espn/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'ESPN',
+    url: 'espn.com',
+};

package/lib/routes/espn/news.ts

@@ -0,0 +1,120 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import * as cheerio from 'cheerio';
+import path from 'path';
+import { getCurrentPath } from '@/utils/helpers';
+import { art } from '@/utils/render';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+const renderMedia = (media) =>
+    art(path.join(__dirname, 'templates', 'media.art'), {
+        video: {
+            cover: media.posterImages?.full?.href || media.posterImages?.default?.href,
+            src: media.links?.source.mezzanine?.href || media.links?.source.HD?.href || media.links?.source.full?.href || media.links?.source.href,
+            title: media.title,
+            description: media.description,
+        },
+        image: {
+            src: media.url,
+            alt: media.alt,
+            caption: media.caption,
+            credit: media.credit,
+        },
+    });
+
+const junkPattern = /inline\d+|alsosee/;
+const mediaPattern = /(photo|video)(\d+)/;
+
+export const route: Route = {
+    path: '/news/:sport',
+    name: 'News',
+    maintainers: ['weijianduan0302'],
+    example: '/espn/news/nba',
+    categories: ['traditional-media'],
+    parameters: { sport: 'sport category, can be nba, nfl, mlb, nhl etc.' },
+    description: `Get the news feed of the sport you love on ESPN.
+| Sport                |  sport  |  Sport         |  sport  |
+|----------------------|---------|----------------|---------|
+| 🏀 NBA                | nba     | 🎾 Tennis       | tennis  |
+| 🏀 WNBA               | wnba    | ⛳️ Golf         | golf    |
+| 🏈 NFL                | nfl     | 🏏 Cricket      | cricket |
+| ⚾️ MLB                | mlb     | ⚽️ Soccer       | soccer  |
+| 🏒 NHL                | nhl     | 🏎️ F1           | f1      |
+| ⛹️ College Basketball | ncb      | 🥊 MMA          | mma     |
+| 🏟️️ College Football   | ncf     | 🏈 UFL          | ufl     |
+| 🏉 Rugby              | rugby   | 🃏 Poker        | poker   |`,
+    radar: [
+        {
+            source: ['espn.com/:sport*'],
+            target: '/news/:sport',
+        },
+    ],
+    handler: async (ctx) => {
+        const { sport = 'nba' } = ctx.req.param();
+        const response = await ofetch(`https://onefeed.fan.api.espn.com/apis/v3/cached/contentEngine/oneFeed/leagues/${sport}?offset=0`, {
+            headers: {
+                accept: 'application/json',
+            },
+        });
+
+        const handledTypes = new Set(['HeadlineNews', 'Story', 'Media', 'Shortstop']);
+        const list = response.feed
+            .filter((item) => handledTypes.has(item.data.now[0].type))
+            .map((item) => {
+                const itemDetail = item.data.now[0];
+                const itemType = itemDetail.type;
+
+                return {
+                    title: itemDetail.headline,
+                    link: itemDetail.links.web.href,
+                    author: itemDetail.byline,
+                    pubDate: item.date,
+                    // for videos and shortstops, no need to extract full text below
+                    description: itemType === 'Media' ? renderMedia(itemDetail.video[0]) : itemType === 'Shortstop' ? itemDetail.headline : '',
+                };
+            });
+
+        const items = await Promise.all(
+            list.map((item) =>
+                cache.tryGet(item.link, async () => {
+                    if (item.description === '') {
+                        const article = await ofetch(`${item.link}?xhr=1`, {
+                            headers: {
+                                accept: 'application/json',
+                            },
+                        });
+
+                        const $ = cheerio.load(article.content.story, null, false);
+                        $('*').each((_, ele) => {
+                            if (junkPattern.test(ele.name)) {
+                                $(ele).remove();
+                            }
+                            if (mediaPattern.test(ele.name)) {
+                                const mediaType = ele.name.match(mediaPattern)[1] === 'photo' ? 'images' : 'video';
+                                const mediaIndex = Number.parseInt(ele.name.match(mediaPattern)[2]) - 1;
+                                const media = article.content[mediaType][mediaIndex];
+                                if (media) {
+                                    $(ele).replaceWith(renderMedia(media));
+                                } else {
+                                    $(ele).remove();
+                                }
+                            }
+                        });
+
+                        item.description = $.html();
+                    }
+
+                    return item;
+                })
+            )
+        );
+
+        return {
+            title: `ESPN ${sport.toUpperCase()} News`,
+            link: `https://www.espn.com/espn/rss/${sport}/news`,
+            item: items,
+        };
+    },
+};

package/lib/routes/espn/templates/media.art

@@ -0,0 +1,19 @@
+{{ if video.src }}
+    <video controls preload="metadata" width="100%" height="auto" cover="{{ video.cover }}">
+        <source src="{{ video.src }}" type="video/mp4">
+    </video>
+    {{ if video.title || video.description }}
+        <div>
+            {{ if video.title }}<div><b>{{ video.title }}</b></div>{{ /if }}
+            {{ if video.description }}<p>{{ video.description }}</p>{{ /if }}
+        </div>
+    {{ /if }}
+{{ /if }}
+
+{{ if image.src }}
+<figure>
+    <img src="{{ image.src }}" alt="{{ image.alt }}">
+    {{ if image.caption }}<figcaption>{{ image.caption }}</figcaption>{{ /if }}
+    {{ if image.credit }}<cite>{{ image.credit }}</cite>{{ /if }}
+</figure>
+{{ /if }}

package/lib/routes/famitsu/category.ts

@@ -1,10 +1,15 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
-import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
+import * as cheerio from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
-import timezone from '@/utils/timezone';
+import { art } from '@/utils/render';
+import path from 'node:path';
+import { getCurrentPath } from '@/utils/helpers';
+import { config } from '@/config';
+import { ArticleDetail, Category, CategoryArticle } from './types';
 
+const __dirname = getCurrentPath(import.meta.url);
 const baseUrl = 'https://www.famitsu.com';
 
 export const route: Route = {
@@ -12,71 +17,133 @@ export const route: Route = {
     categories: ['game'],
     example: '/famitsu/category/new-article',
     parameters: { category: 'Category, see table below, `new-article` by default' },
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
+    radar: [
+        {
+            source: ['www.famitsu.com/category/:category/page/1'],
+        },
+    ],
     name: 'Category',
     maintainers: ['TonyRL'],
     handler,
-    description: `| 新着        | PS5 | Switch | PS4 | ニュース | ゲームニュース | PR TIMES | 動画   | 特集・企画記事  | インタビュー | 取材・リポート | レビュー | インディーゲーム |
-  | ----------- | --- | ------ | --- | -------- | -------------- | -------- | ------ | --------------- | ------------ | -------------- | -------- | ---------------- |
-  | new-article | ps5 | switch | ps4 | news     | news-game      | prtimes  | videos | special-article | interview    | event-report   | review   | indie-game       |`,
+    description: `| 新着        | Switch | PS5 | PS4 | PC ゲーム | ニュース | 動画   | 特集・企画記事  | インタビュー | 取材・リポート | レビュー | インディーゲーム |
+  | ----------- | ------ | --- | --- | --------- | -------- | ------ | --------------- | ------------ | -------------- | -------- | ---------------- |
+  | new-article | switch | ps5 | ps4 | pc-game   | news     | videos | special-article | interview    | event-report   | review   | indie-game       |`,
 };
 
+function getBuildId() {
+    return cache.tryGet(
+        'famitsu:buildId',
+        async () => {
+            const data = await ofetch(baseUrl);
+            const $ = cheerio.load(data);
+            const nextData = JSON.parse($('#__NEXT_DATA__').text());
+            return nextData.buildId;
+        },
+        config.cache.routeExpire,
+        false
+    );
+}
+
+function render(data) {
+    return art(path.join(__dirname, 'templates', 'description.art'), data);
+}
+
+function renderJSON(c) {
+    if (Array.isArray(c.content)) {
+        return c.content.map((con) => con.type && renderJSON(con)).join('');
+    }
+
+    switch (c.type) {
+        case 'B':
+        case 'INTERVIEWEE':
+        case 'STRONG':
+            return `<b>${c.content}</b>`;
+        case 'HEAD':
+            return `<h2>${c.content}</h2>`;
+        case 'SHEAD':
+            return `<h3>${c.content}</h3>`;
+        case 'LINK_B':
+        case 'LINK_B_TAB':
+            return `<a href="${c.url}"><b>${c.content}</b></a><br>`;
+        case 'IMAGE':
+            return `<img src="${c.path}">`;
+        case 'NEWS':
+            return `<a href="${c.url}">${c.content}<br>${c.description}</a><br>`;
+        case 'HTML':
+            return c.content;
+        case 'ANNOTATION':
+        case 'CAPTION':
+        case 'ITEMIZATION':
+        case 'ITEMIZATION_NUM':
+        case 'NOLINK':
+        case 'STRING':
+        case 'TWITTER':
+        case 'YOUTUBE':
+            return `<div><span>${c.content}</span></div>`;
+        case 'BUTTON':
+        case 'BUTTON_ANDROID':
+        case 'BUTTON_EC':
+        case 'BUTTON_IOS':
+        case 'BUTTON_TAB':
+        case 'LINK':
+        case 'LINK_TAB':
+            return `<a href="${c.url}">${c.content}</a><br>`;
+        default:
+            throw new Error(`Unhandle type: ${c.type}`);
+    }
+}
+
 async function handler(ctx) {
     const { category = 'new-article' } = ctx.req.param();
-    const url = `${baseUrl}/search/?category=${category}`;
-    const { data } = await got(url);
-    const $ = load(data);
+    const url = `${baseUrl}/category/${category}/page/1`;
+
+    const buildId = await getBuildId();
 
-    const list = $('.col-12 .card__body')
-        .toArray()
+    const data = await ofetch(`https://www.famitsu.com/_next/data/${buildId}/category/${category}/page/1.json`, {
+        query: {
+            categoryCode: category,
+            pageNumber: 1,
+        },
+    });
+
+    const list = (data.pageProps.categoryArticleData as CategoryArticle[])
+        .filter((item) => !item.advertiserName)
         .map((item) => {
-            item = $(item);
+            const publicationDate = item.publishedAt?.slice(0, 7).replace('-', '');
             return {
-                title: item.find('.card__title').text(),
-                link: new URL(item.find('.card__title a').attr('href'), baseUrl).href,
-                pubDate: timezone(parseDate(item.find('time').attr('datetime'), 'YYYY.MM.DDTHH:mm'), +9),
+                title: item.title,
+                link: `https://www.famitsu.com/article/${publicationDate}/${item.id}`,
+                pubDate: parseDate(item.publishedAt!),
+                category: [...new Set([item.mainCategory.nameJa, ...(item.subCategories?.map((c) => c.nameJa) ?? [])])],
+                publicationDate,
+                articleId: item.id,
             };
-        })
-        .filter((item) => item.link.startsWith('https://www.famitsu.com/news/'));
+        });
 
     const items = await Promise.all(
         list.map((item) =>
             cache.tryGet(item.link, async () => {
-                const { data } = await got(item.link);
-                const $ = load(data);
-
-                // remove ads
-                $('.article-body__contents-pr-primary').remove();
-
-                // fix header image
-                $('.article-body div.media-image').each((_, e) => {
-                    e.tagName = 'img';
-                    e.attribs.src = e.attribs.style.match(/url\((.+?)\);/)[1];
-                    delete e.attribs['data-src'];
-                    delete e.attribs.style;
+                const data = await ofetch(`https://www.famitsu.com/_next/data/${buildId}/article/${item.publicationDate}/${item.articleId}.json`, {
+                    query: {
+                        publicationDate: item.publicationDate,
+                        articleId: item.articleId,
+                    },
                 });
 
-                // remove white space
-                $('.article-body__contents-img-block, .article-body__contents-img-common-col').each((_, e) => {
-                    delete e.attribs.style;
+                const articleDetail = data.pageProps.articleDetailData as ArticleDetail;
+                item.author = articleDetail.authors?.map((a) => a.name_ja).join(', ') ?? articleDetail.user.name_ja;
+                item.description = render({
+                    bannerImage: articleDetail.ogpImageUrl ?? articleDetail.thumbnailUrl,
+                    content: articleDetail.content.flatMap((c) => c.contents.map((con) => renderJSON(con))).join(''),
                 });
 
-                item.description = $('.article-body').html();
                 return item;
             })
         )
     );
 
     return {
-        title: $('head title').text(),
-        description: $('head meta[name="description"]').attr('content'),
+        title: `${(data.pageProps.targetCategory as Category).nameJa}の最新記事 | ゲーム・エンタメ最新情報のファミ通.com`,
         image: 'https://www.famitsu.com/img/1812/favicons/apple-touch-icon.png',
         link: url,
         item: items,

package/lib/routes/famitsu/templates/description.art

@@ -0,0 +1,7 @@
+{{ if bannerImage }}
+<img src="{{ bannerImage }}"><br>
+{{ /if }}
+
+{{ if content }}
+{{@ content }}
+{{ /if }}

package/lib/routes/famitsu/types.ts

@@ -0,0 +1,117 @@
+export interface Category {
+    id: string;
+    status: string;
+    isMainCategory: boolean;
+    code: string;
+    nameJa: string;
+    priority: number;
+}
+
+export interface CategoryArticle {
+    description: string;
+    id: string;
+    mainCategory: Category;
+    publishedAt: string | null;
+    thumbnailUrl: string;
+    title: string;
+    isPr: boolean;
+    content?: string;
+    isVideoArticle?: boolean;
+    subCategories?: Category[];
+    redirectUrl?: string;
+    iconImage?: string;
+    advertiserName?: string;
+    linkedUrl?: string;
+}
+
+interface Content {
+    type: string;
+    url: string;
+    content: string | Content[] | string[];
+    image_id?: number;
+    path?: string;
+}
+
+export interface ArticleDetail {
+    articleId: number;
+    isR18: boolean;
+    isDisplayAd: boolean;
+    dept: {
+        id: number;
+        code: string;
+        name_ja: string;
+    };
+    description: string;
+    content: {
+        page_no: number;
+        contents: Content[];
+        text: string;
+    }[];
+    mainCategories: Category;
+    publishedAt: string;
+    subCategories: Category[];
+    thumbnailUrl: string;
+    ogpImageUrl: string;
+    title: string;
+    updatedAt: string;
+    user: {
+        id: number;
+        name_ja: string;
+    };
+    relatedArticles: {
+        count: number;
+        offset: number;
+        page: number;
+        limit: number;
+        results: {
+            id: string;
+            article_type: string;
+            main_category_ids: string[];
+            sub_category_ids: string[];
+            content_text: string;
+            creation_time: string;
+            creation_time_jst: string;
+            dept_id: string;
+            description: string;
+            hide_on_top_page: string;
+            is_pr: string;
+            publication_time: string;
+            publication_time_jst: string;
+            is_r18: string;
+            revision: string;
+            importance_degree: string;
+            thumbnail_caption: string;
+            thumbnail_url: string;
+            title: string;
+            update_time: string;
+            update_time_jst: string;
+            user_id: string;
+            status: string;
+            has_video: string;
+            tweet_id: string;
+            game_ids: string[];
+            author_ids: string[];
+        }[];
+    };
+    authors: {
+        id: number;
+        icon_url: string;
+        name_ja: string;
+        description: string;
+        relate_urls: any[];
+    }[];
+    redirectUrl: string;
+    copyright: string;
+    relatedLinks: {
+        title: string;
+        description: null;
+        url: string;
+        article: any[];
+    }[];
+    isToc: boolean;
+    items: {
+        status: string;
+        item_id: string;
+        item_type: string;
+    }[];
+}