rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/modb/topic.ts

@@ -1,7 +1,7 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
 import { load } from 'cheerio';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import logger from '@/utils/logger';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
@@ -27,14 +27,13 @@ export const route: Route = {
 async function handler(ctx) {
     const baseUrl = 'https://www.modb.pro';
     const topicId = ctx.req.param('id');
-    const response = await got({
-        url: `${baseUrl}/api/columns/getKnowledge`,
-        searchParams: {
+    const response = await ofetch(`${baseUrl}/api/columns/getKnowledge`, {
+        query: {
             pageNum: 1,
             pageSize: 20,
             columnId: topicId,
         },
-    }).json();
+    });
     const list = response.list.map((item) => {
         let doc = {};
         let baseLink = {};
@@ -63,7 +62,7 @@ async function handler(ctx) {
     const items = await Promise.all(
         list.map((item) =>
             cache.tryGet(item.link, async () => {
-                const { data: response } = await got(item.link);
+                const response = await ofetch(item.link);
                 const $ = load(response);
                 item.description = $('div.editor-content-styl.article-style').first().html();
                 return item;

package/lib/routes/my-formosa/index.ts

@@ -0,0 +1,78 @@
+import { Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import cache from '@/utils/cache';
+import timezone from '@/utils/timezone';
+
+export const route: Route = {
+    path: '/',
+    categories: ['new-media'],
+    example: '/my-formosa',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['my-formosa.com/'],
+        },
+    ],
+    name: '首页',
+    maintainers: ['dzx-dzx'],
+    handler,
+    url: 'my-formosa.com',
+};
+
+function fetch(url) {
+    return ofetch(url, { responseType: 'arrayBuffer' }).then((raw) => {
+        const decoder = new TextDecoder('big5');
+        return decoder.decode(raw);
+    });
+}
+
+async function handler() {
+    const rootUrl = 'http://www.my-formosa.com/';
+
+    const res = await fetch(rootUrl);
+
+    const $ = load(res);
+
+    const items = await Promise.all(
+        $('#featured-news h3 a')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+
+                const title = item.text();
+                const link = new URL(item.attr('href'), rootUrl).href;
+
+                return cache.tryGet(link, async () => {
+                    const res = await fetch(link);
+                    const $ = load(res);
+
+                    const isTV = /^\/TV/.test(new URL(link).pathname);
+
+                    return {
+                        title,
+                        link,
+                        author: $('.page-header~#featured-news h4').text(),
+                        category: $("meta[name='keywords']").attr('content').split(',').filter(Boolean),
+                        pubDate: timezone(parseDate((isTV ? $('.icon-calendar')[0].next.data : $('.date').text()).trim()), +8),
+                        description: (isTV ? $('.post-item').html() : $('.body').html()).replaceAll(/\/News.*?\.jpg/g, (match) => `http://my-formosa.com${match}`),
+                    };
+                });
+            })
+    );
+
+    return {
+        title: $('title').text(),
+        link: rootUrl,
+        item: items,
+    };
+}

package/lib/routes/my-formosa/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '美麗島電子報',
+    url: 'my-formosa.com',
+};

package/lib/routes/myfans/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'myfans',
+    url: 'myfans.jp',
+};

package/lib/routes/myfans/post.ts

@@ -0,0 +1,64 @@
+import { Route } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import { showByUsername, getPostByAccountId, baseUrl } from './utils';
+import path from 'node:path';
+import { art } from '@/utils/render';
+import { getCurrentPath } from '@/utils/helpers';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+export const route: Route = {
+    path: '/user/:username',
+    categories: ['multimedia'],
+    example: '/myfans/user/secret_japan',
+    parameters: { username: 'User handle' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['myfans.jp/:username', 'myfans.jp/:language/:username'],
+        },
+    ],
+    name: 'User Posts',
+    maintainers: ['TonyRL'],
+    handler,
+};
+
+const render = (postImages, body) =>
+    art(path.join(__dirname, 'templates', 'post.art'), {
+        postImages,
+        body,
+    });
+
+async function handler(ctx) {
+    const { username } = ctx.req.param();
+
+    const account = await showByUsername(username);
+    const posts = await getPostByAccountId(account.id);
+
+    const items = posts.map((p) => ({
+        title: p.body?.replaceAll('\r\n', ' ').trim().split(' ')[0],
+        description: render(p.post_images, p.body?.replaceAll('\r\n', '<br>')),
+        pubDate: parseDate(p.published_at),
+        link: `${baseUrl}/posts/${p.id}`,
+        author: p.user.name,
+    }));
+
+    return {
+        title: `${account.name} (@${account.username})`,
+        link: `${baseUrl}/${account.username}`,
+        description: `${account.posts_count} Post ${account.likes_count} Like ${account.followers_count}
+Followers ${account.followings_count} Follow ${account.about.replaceAll('\r\n', ' ')}`,
+        image: account.avatar_url,
+        icon: account.avatar_url,
+        logo: account.avatar_url,
+        language: 'ja-JP',
+        item: items,
+    };
+}

package/lib/routes/myfans/templates/post.art

@@ -0,0 +1,8 @@
+{{ if postImages }}
+    {{ each postImages img }}
+        <img src="{{ img.file_url }}"><br>
+    {{ /each }}
+{{ /if }}
+{{ if body }}
+    {{@ body }}
+{{ /if }}

package/lib/routes/myfans/types.ts

@@ -0,0 +1,104 @@
+export interface UserProfile {
+    about: string;
+    active: boolean;
+    avatar_url: string;
+    banner_url: string;
+    id: string | null;
+    is_following: boolean;
+    likes_count: number;
+    name: string;
+    username: string;
+    is_official_creator: boolean;
+    is_official: boolean;
+    label: null;
+    back_number_post_images_count: number;
+    back_number_post_videos_count: number;
+    cant_receive_message: boolean;
+    current_back_number_plan: null;
+    followers_count: number;
+    followings_count: number;
+    has_approved_user_identification: boolean;
+    is_bought_back_number: boolean;
+    is_followed: boolean;
+    is_subscribed: boolean;
+    limited_posts_count: number;
+    link_instagram_id: string;
+    link_instagram_url: null;
+    link_tiktok_id: string;
+    link_tiktok_url: null;
+    link_twitter_id: string;
+    link_twitter_url: string;
+    link_youtube_url: string;
+    post_images_count: number;
+    post_videos_count: number;
+    posts_count: number;
+    sns_link1: string;
+    sns_link2: string;
+}
+
+export interface Post {
+    id: string;
+    kind: string;
+    status: string;
+    status_label: null;
+    body: string | null;
+    bookmarked: boolean;
+    humanized_publish_start_at: string;
+    deleted_at_i18n: null;
+    liked: boolean;
+    likes_count: number;
+    user: UserProfile;
+    post_images: {
+        file_url: string;
+        square_thumbnail_url: string;
+        raw_image_height: number;
+        raw_image_width: number;
+    }[];
+    visible: boolean;
+    publish_end_at: null;
+    published_at: string;
+    publish_start_at: string | null;
+    pinned_at: string | null;
+    attachment: null;
+    plan: null;
+    current_single_plan: {
+        id: string;
+        amount: number;
+        auto_message_body: string;
+    } | null;
+    plans: {
+        id: string;
+        product_name: string;
+        monthly_price: number;
+        status: null;
+        is_limited_access: boolean;
+        disallow_new_subscriber: boolean;
+        active_discount: {
+            id: string;
+            discount_rate: number;
+            start_at: string | null;
+            end_at: string | null;
+            limited_number: null;
+            status: string;
+        } | null;
+    }[];
+    video_processing: boolean | null;
+    video_duration: {
+        hours: string | null;
+        minutes: string;
+        seconds: string;
+    } | null;
+    free: boolean;
+    limited: boolean;
+    available: boolean;
+    metadata: {
+        video: {
+            duration: number;
+            resolutions: string[];
+        };
+        image: {
+            count: number;
+        };
+    };
+    thumbnail_url: string;
+}

package/lib/routes/myfans/utils.ts

@@ -0,0 +1,39 @@
+import ofetch from '@/utils/ofetch';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+import cache from '@/utils/cache';
+import { Post, UserProfile } from './types';
+
+const apiBaseUrl = 'https://api.myfans.jp';
+export const baseUrl = 'https://myfans.jp';
+
+const headers = {
+    'google-ga-data': 'event328',
+};
+
+export const showByUsername = (username: string) =>
+    cache.tryGet(`myfans:account:${username}`, async () => {
+        const accountInfo = await ofetch<UserProfile>(`${apiBaseUrl}/api/v2/users/show_by_username`, {
+            headers,
+            query: {
+                username,
+            },
+        });
+
+        if (!accountInfo.id) {
+            throw new InvalidParameterError('This creator does not exist.');
+        }
+
+        return accountInfo;
+    }) as Promise<UserProfile>;
+
+export const getPostByAccountId = async (accountId) => {
+    const post = await ofetch(`${apiBaseUrl}/api/v2/users/${accountId}/posts`, {
+        headers,
+        query: {
+            sort_key: 'publish_start_at',
+            page: 1,
+        },
+    });
+
+    return post.data as Promise<Post[]>;
+};

package/lib/routes/nasa/apod.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
@@ -7,7 +7,8 @@ import timezone from '@/utils/timezone';
 
 export const route: Route = {
     path: '/apod',
-    categories: ['picture'],
+    categories: ['picture', 'popular'],
+    view: ViewType.Pictures,
     example: '/nasa/apod',
     parameters: {},
     features: {
@@ -23,7 +24,7 @@ export const route: Route = {
             source: ['apod.nasa.govundefined'],
         },
     ],
-    name: 'NASA',
+    name: 'Astronomy Picture of the Day',
     maintainers: ['nczitzk', 'williamgateszhao'],
     handler,
     url: 'apod.nasa.govundefined',

package/lib/routes/nasa/namespace.ts

@@ -1,6 +1,6 @@
 import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
-    name: 'NASA Astronomy Picture of the Day',
+    name: 'NASA',
     url: 'apod.nasa.gov',
 };

package/lib/routes/natgeo/dailyphoto.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
@@ -12,7 +12,8 @@ import { config } from '@/config';
 
 export const route: Route = {
     path: '/dailyphoto',
-    categories: ['picture'],
+    categories: ['picture', 'popular'],
+    view: ViewType.Pictures,
     example: '/natgeo/dailyphoto',
     parameters: {},
     features: {
@@ -28,8 +29,8 @@ export const route: Route = {
             source: ['nationalgeographic.com/photo-of-the-day/*', 'nationalgeographic.com/'],
         },
     ],
-    name: '每日一图',
-    maintainers: ['LogicJake', 'OrangeEd1t', 'TonyRL'],
+    name: 'Daily Photo',
+    maintainers: ['LogicJake', 'OrangeEd1t', 'TonyRL', 'pseudoyu'],
     handler,
     url: 'nationalgeographic.com/photo-of-the-day/*',
 };

package/lib/routes/natgeo/dailyselection.ts

@@ -1,12 +1,27 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/dailyselection',
-    name: 'Unknown',
-    maintainers: ['OrangeEd1t'],
+    name: 'Daily Selection',
+    categories: ['picture'],
+    view: ViewType.Pictures,
+    example: '/natgeo/dailyselection',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+    },
+    radar: [
+        {
+            source: ['nationalgeographic.com/'],
+        },
+    ],
+    maintainers: ['OrangeEd1t', 'pseudoyu'],
     handler,
 };
 

package/lib/routes/nationalgeographic/latest-stories.ts

@@ -58,7 +58,7 @@ async function handler() {
                 cache.tryGet(item.link, async () => {
                     const response = await got(item.link);
                     const $ = load(response.data);
-                    const mods = findNatgeo($).page.content.article.frms.find((f) => f.cmsType === 'ArticleBodyFrame').mods;
+                    const mods = findNatgeo($).page.content.prismarticle.frms.find((f) => f.cmsType === 'ArticleBodyFrame').mods;
                     const bodyTile = mods.find((m) => m.edgs[0].cmsType === 'ArticleBodyTile').edgs[0];
 
                     item.author = bodyTile.cntrbGrp

package/lib/routes/nationalgeographic/templates/stories.art

@@ -13,7 +13,7 @@
     {{ if b.type === 'p' }}
         <p>{{@ b.cntnt.mrkup }}</p>
     {{ else if b.type === 'inline' }}
-        {{ if b.cntnt.cmsType === 'image' }}
+        {{ if b.cntnt.cmsType === 'image' && b.cntnt.image?.src }}
             <figure>
                 <img src="{{ b.cntnt.image.src }}" alt="{{ b.cntnt.image.altText }}">
                 <figcaption>{{@ b.cntnt.caption }}</figcaption>

package/lib/routes/nber/all.ts

@@ -0,0 +1,19 @@
+import { Route } from '@/types';
+import { handler } from './common';
+
+export const route: Route = {
+    name: 'All Papers',
+    maintainers: ['5upernova-heng'],
+    path: '/papers',
+    example: '/nber/papers',
+    features: {
+        supportScihub: true,
+    },
+    radar: [
+        {
+            source: ['nber.org/papers'],
+        },
+    ],
+    handler,
+    url: 'nber.org/papers',
+};

package/lib/routes/nber/{index.ts → common.ts}

@@ -1,10 +1,9 @@
-import { Route } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
 import { getSubPath } from '@/utils/common-utils';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import path from 'node:path';
 import { art } from '@/utils/render';
@@ -12,37 +11,10 @@ import { parseDate } from '@/utils/parse-date';
 import { config } from '@/config';
 
 async function getData(url) {
-    const response = await got(url).json();
+    const response = await ofetch(url);
     return response.results;
 }
-
-export const route: Route = {
-    path: ['/papers', '/news'],
-    categories: ['journal'],
-    example: '/nber/papers',
-    parameters: {},
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: true,
-    },
-    radar: [
-        {
-            source: ['nber.org/papers'],
-        },
-    ],
-    name: 'All Papers',
-    maintainers: [],
-    handler,
-    url: 'nber.org/papers',
-    description: `Papers that are published in this week.`,
-    url: 'nber.org/papers',
-};
-
-async function handler(ctx) {
+export async function handler(ctx) {
     const url = 'https://www.nber.org/api/v1/working_page_listing/contentType/working_paper/_/_/search';
     const baseUrl = 'https://www.nber.org';
     const data = await cache.tryGet(url, () => getData(url), config.cache.routeExpire, false);
@@ -52,8 +24,8 @@ async function handler(ctx) {
             .map((article) => {
                 const link = `${baseUrl}${article.url}`;
                 return cache.tryGet(link, async () => {
-                    const response = await got(link);
-                    const $ = load(response.data);
+                    const response = await ofetch(link);
+                    const $ = load(response);
                     const downloadLink = $('meta[name="citation_pdf_url"]').attr('content');
                     const fullAbstract = $('.page-header__intro-inner').html();
                     return {
@@ -76,6 +48,5 @@ async function handler(ctx) {
         link: 'https://www.nber.org/papers',
         item: items,
         description: `National Bureau of Economic Research Working Papers articles`,
-        language: $('html').attr('lang'),
     };
 }

package/lib/routes/nber/namespace.ts

@@ -3,4 +3,5 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: 'National Bureau of Economic Research',
     url: 'nber.org',
+    categories: ['journal'],
 };

package/lib/routes/nber/new.ts

@@ -0,0 +1,20 @@
+import { Route } from '@/types';
+import { handler } from './common';
+
+export const route: Route = {
+    name: 'New Papers',
+    maintainers: ['5upernova-heng'],
+    path: '/new',
+    example: '/nber/new',
+    features: {
+        supportScihub: true,
+    },
+    radar: [
+        {
+            source: ['nber.org/papers'],
+        },
+    ],
+    handler,
+    url: 'nber.org/papers',
+    description: 'Papers that are published in this week.',
+};