rsshub 1.0.0-master.f9003f9 → 1.0.0-master.f917bdc

package/lib/routes/mirrormedia/index.ts

@@ -0,0 +1,43 @@
+import { Route } from '@/types';
+
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import { getArticle } from './utils';
+
+export const route: Route = {
+    path: '/',
+    categories: ['traditional-media'],
+    example: '/mirrormedia',
+    parameters: {},
+    name: '首页',
+    maintainers: ['dzx-dzx'],
+    radar: [
+        {
+            source: ['mirrormedia.mg'],
+        },
+    ],
+    handler,
+};
+
+async function handler(ctx) {
+    const rootUrl = 'https://www.mirrormedia.mg';
+
+    const response = await ofetch('https://v3-statics.mirrormedia.mg/files/json/post_external01.json');
+
+    const items = [...response.choices.map((e) => ({ __from: 'choices', ...e })), ...response.latest.map((e) => ({ __from: 'latest', ...e }))]
+        .map((e) => ({
+            title: e.title,
+            pubDate: parseDate(e.publishedDate),
+            category: [...(e.sections ?? []).map((_) => _.name), e.__from],
+            link: `${rootUrl}/${e.style === '' ? 'external' : 'story'}/${e.slug}`,
+        }))
+        .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20);
+
+    const list = await Promise.all(items.map((item) => getArticle(item)));
+
+    return {
+        title: '鏡週刊 Mirror Media',
+        link: rootUrl,
+        item: list,
+    };
+}

package/lib/routes/mirrormedia/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '鏡週刊 Mirror Media',
+    url: 'mirrormedia.mg',
+};

package/lib/routes/mirrormedia/utils.ts

@@ -0,0 +1,19 @@
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+
+export function getArticle(item) {
+    return cache.tryGet(item.link, async () => {
+        const detailResponse = await ofetch(item.link);
+
+        const content = load(detailResponse);
+
+        item.description = item.link.includes('external')
+            ? content(':is([class^=external-article-brief],[class^=external-article-content])').html()
+            : content(':is([class^=brief__BriefContainer],[class^=article-content__Wrapper])').html();
+
+        item.category = [...item.category, ...(content("meta[name='keywords']").attr('content') ?? '').split(',')];
+
+        return item;
+    });
+}

package/lib/routes/misskey/user-timeline.ts

@@ -0,0 +1,42 @@
+import { Route } from '@/types';
+import utils from './utils';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+
+export const route: Route = {
+    path: '/users/notes/:username',
+    categories: ['social-media'],
+    example: '/misskey/users/notes/support@misskey.io',
+    parameters: { username: 'misskey username format, like support@misskey.io' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'User timeline',
+    maintainers: ['siygle'],
+    handler,
+};
+
+async function handler(ctx) {
+    const username = ctx.req.param('username');
+    const [, pureUsername, site] = username.match(/@?(\w+)@(\w+\.\w+)/) || [];
+    if (!pureUsername || !site) {
+        throw new InvalidParameterError('Provide a valid Misskey username');
+    }
+    if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) {
+        throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+    }
+
+    const { accountData } = await utils.getUserTimelineByUsername(pureUsername, site);
+
+    return {
+        title: `User timeline for ${username} on ${site}`,
+        link: `https://${site}/@${pureUsername}`,
+        item: utils.parseNotes(accountData, site),
+    };
+}

package/lib/routes/misskey/utils.ts

@@ -4,6 +4,8 @@ const __dirname = getCurrentPath(import.meta.url);
 import { art } from '@/utils/render';
 import { parseDate } from '@/utils/parse-date';
 import path from 'node:path';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
 
 const allowSiteList = ['misskey.io', 'madost.one', 'mk.nixnet.social'];
 
@@ -28,4 +30,42 @@ const parseNotes = (data, site) =>
         };
     });
 
-export default { parseNotes, allowSiteList };
+async function getUserTimelineByUsername(username, site) {
+    const searchUrl = `https://${site}/api/users/search-by-username-and-host`;
+    const cacheUid = `misskey_username/${site}/${username}`;
+
+    const accountId = await cache.tryGet(cacheUid, async () => {
+        const searchResponse = await got({
+            method: 'post',
+            url: searchUrl,
+            json: {
+                username,
+                host: site,
+                detail: true,
+                limit: 1,
+            },
+        });
+        const userData = searchResponse.data.find((item) => item.username === username);
+
+        if (userData.length === 0) {
+            throw new Error(`username ${username} not found`);
+        }
+        return userData.id;
+    });
+
+    const usernotesUrl = `https://${site}/api/users/notes`;
+    const usernotesResponse = await got({
+        method: 'post',
+        url: usernotesUrl,
+        json: {
+            userId: accountId,
+            withChannelNotes: true,
+            limit: 10,
+            offset: 0,
+        },
+    });
+    const accountData = usernotesResponse.data;
+    return { site, accountId, accountData };
+}
+
+export default { parseNotes, getUserTimelineByUsername, allowSiteList };

package/lib/routes/modb/topic.ts

@@ -1,7 +1,7 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
 import { load } from 'cheerio';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import logger from '@/utils/logger';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
@@ -27,14 +27,13 @@ export const route: Route = {
 async function handler(ctx) {
     const baseUrl = 'https://www.modb.pro';
     const topicId = ctx.req.param('id');
-    const response = await got({
-        url: `${baseUrl}/api/columns/getKnowledge`,
-        searchParams: {
+    const response = await ofetch(`${baseUrl}/api/columns/getKnowledge`, {
+        query: {
             pageNum: 1,
             pageSize: 20,
             columnId: topicId,
         },
-    }).json();
+    });
     const list = response.list.map((item) => {
         let doc = {};
         let baseLink = {};
@@ -63,7 +62,7 @@ async function handler(ctx) {
     const items = await Promise.all(
         list.map((item) =>
             cache.tryGet(item.link, async () => {
-                const { data: response } = await got(item.link);
+                const response = await ofetch(item.link);
                 const $ = load(response);
                 item.description = $('div.editor-content-styl.article-style').first().html();
                 return item;

package/lib/routes/my-formosa/index.ts

@@ -0,0 +1,78 @@
+import { Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import cache from '@/utils/cache';
+import timezone from '@/utils/timezone';
+
+export const route: Route = {
+    path: '/',
+    categories: ['new-media'],
+    example: '/my-formosa',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['my-formosa.com/'],
+        },
+    ],
+    name: '首页',
+    maintainers: ['dzx-dzx'],
+    handler,
+    url: 'my-formosa.com',
+};
+
+function fetch(url) {
+    return ofetch(url, { responseType: 'arrayBuffer' }).then((raw) => {
+        const decoder = new TextDecoder('big5');
+        return decoder.decode(raw);
+    });
+}
+
+async function handler() {
+    const rootUrl = 'http://www.my-formosa.com/';
+
+    const res = await fetch(rootUrl);
+
+    const $ = load(res);
+
+    const items = await Promise.all(
+        $('#featured-news h3 a')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+
+                const title = item.text();
+                const link = new URL(item.attr('href'), rootUrl).href;
+
+                return cache.tryGet(link, async () => {
+                    const res = await fetch(link);
+                    const $ = load(res);
+
+                    const isTV = /^\/TV/.test(new URL(link).pathname);
+
+                    return {
+                        title,
+                        link,
+                        author: $('.page-header~#featured-news h4').text(),
+                        category: $("meta[name='keywords']").attr('content').split(',').filter(Boolean),
+                        pubDate: timezone(parseDate((isTV ? $('.icon-calendar')[0].next.data : $('.date').text()).trim()), +8),
+                        description: (isTV ? $('.post-item').html() : $('.body').html()).replaceAll(/\/News.*?\.jpg/g, (match) => `http://my-formosa.com${match}`),
+                    };
+                });
+            })
+    );
+
+    return {
+        title: $('title').text(),
+        link: rootUrl,
+        item: items,
+    };
+}

package/lib/routes/my-formosa/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '美麗島電子報',
+    url: 'my-formosa.com',
+};

package/lib/routes/myfans/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'myfans',
+    url: 'myfans.jp',
+};

package/lib/routes/myfans/post.ts

@@ -0,0 +1,64 @@
+import { Route } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import { showByUsername, getPostByAccountId, baseUrl } from './utils';
+import path from 'node:path';
+import { art } from '@/utils/render';
+import { getCurrentPath } from '@/utils/helpers';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+export const route: Route = {
+    path: '/user/:username',
+    categories: ['multimedia'],
+    example: '/myfans/user/secret_japan',
+    parameters: { username: 'User handle' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['myfans.jp/:username', 'myfans.jp/:language/:username'],
+        },
+    ],
+    name: 'User Posts',
+    maintainers: ['TonyRL'],
+    handler,
+};
+
+const render = (postImages, body) =>
+    art(path.join(__dirname, 'templates', 'post.art'), {
+        postImages,
+        body,
+    });
+
+async function handler(ctx) {
+    const { username } = ctx.req.param();
+
+    const account = await showByUsername(username);
+    const posts = await getPostByAccountId(account.id);
+
+    const items = posts.map((p) => ({
+        title: p.body?.replaceAll('\r\n', ' ').trim().split(' ')[0],
+        description: render(p.post_images, p.body?.replaceAll('\r\n', '<br>')),
+        pubDate: parseDate(p.published_at),
+        link: `${baseUrl}/posts/${p.id}`,
+        author: p.user.name,
+    }));
+
+    return {
+        title: `${account.name} (@${account.username})`,
+        link: `${baseUrl}/${account.username}`,
+        description: `${account.posts_count} Post ${account.likes_count} Like ${account.followers_count}
+Followers ${account.followings_count} Follow ${account.about.replaceAll('\r\n', ' ')}`,
+        image: account.avatar_url,
+        icon: account.avatar_url,
+        logo: account.avatar_url,
+        language: 'ja-JP',
+        item: items,
+    };
+}

package/lib/routes/myfans/templates/post.art

@@ -0,0 +1,8 @@
+{{ if postImages }}
+    {{ each postImages img }}
+        <img src="{{ img.file_url }}"><br>
+    {{ /each }}
+{{ /if }}
+{{ if body }}
+    {{@ body }}
+{{ /if }}

package/lib/routes/myfans/types.ts

@@ -0,0 +1,104 @@
+export interface UserProfile {
+    about: string;
+    active: boolean;
+    avatar_url: string;
+    banner_url: string;
+    id: string | null;
+    is_following: boolean;
+    likes_count: number;
+    name: string;
+    username: string;
+    is_official_creator: boolean;
+    is_official: boolean;
+    label: null;
+    back_number_post_images_count: number;
+    back_number_post_videos_count: number;
+    cant_receive_message: boolean;
+    current_back_number_plan: null;
+    followers_count: number;
+    followings_count: number;
+    has_approved_user_identification: boolean;
+    is_bought_back_number: boolean;
+    is_followed: boolean;
+    is_subscribed: boolean;
+    limited_posts_count: number;
+    link_instagram_id: string;
+    link_instagram_url: null;
+    link_tiktok_id: string;
+    link_tiktok_url: null;
+    link_twitter_id: string;
+    link_twitter_url: string;
+    link_youtube_url: string;
+    post_images_count: number;
+    post_videos_count: number;
+    posts_count: number;
+    sns_link1: string;
+    sns_link2: string;
+}
+
+export interface Post {
+    id: string;
+    kind: string;
+    status: string;
+    status_label: null;
+    body: string | null;
+    bookmarked: boolean;
+    humanized_publish_start_at: string;
+    deleted_at_i18n: null;
+    liked: boolean;
+    likes_count: number;
+    user: UserProfile;
+    post_images: {
+        file_url: string;
+        square_thumbnail_url: string;
+        raw_image_height: number;
+        raw_image_width: number;
+    }[];
+    visible: boolean;
+    publish_end_at: null;
+    published_at: string;
+    publish_start_at: string | null;
+    pinned_at: string | null;
+    attachment: null;
+    plan: null;
+    current_single_plan: {
+        id: string;
+        amount: number;
+        auto_message_body: string;
+    } | null;
+    plans: {
+        id: string;
+        product_name: string;
+        monthly_price: number;
+        status: null;
+        is_limited_access: boolean;
+        disallow_new_subscriber: boolean;
+        active_discount: {
+            id: string;
+            discount_rate: number;
+            start_at: string | null;
+            end_at: string | null;
+            limited_number: null;
+            status: string;
+        } | null;
+    }[];
+    video_processing: boolean | null;
+    video_duration: {
+        hours: string | null;
+        minutes: string;
+        seconds: string;
+    } | null;
+    free: boolean;
+    limited: boolean;
+    available: boolean;
+    metadata: {
+        video: {
+            duration: number;
+            resolutions: string[];
+        };
+        image: {
+            count: number;
+        };
+    };
+    thumbnail_url: string;
+}

package/lib/routes/myfans/utils.ts

@@ -0,0 +1,39 @@
+import ofetch from '@/utils/ofetch';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+import cache from '@/utils/cache';
+import { Post, UserProfile } from './types';
+
+const apiBaseUrl = 'https://api.myfans.jp';
+export const baseUrl = 'https://myfans.jp';
+
+const headers = {
+    'google-ga-data': 'event328',
+};
+
+export const showByUsername = (username: string) =>
+    cache.tryGet(`myfans:account:${username}`, async () => {
+        const accountInfo = await ofetch<UserProfile>(`${apiBaseUrl}/api/v2/users/show_by_username`, {
+            headers,
+            query: {
+                username,
+            },
+        });
+
+        if (!accountInfo.id) {
+            throw new InvalidParameterError('This creator does not exist.');
+        }
+
+        return accountInfo;
+    }) as Promise<UserProfile>;
+
+export const getPostByAccountId = async (accountId) => {
+    const post = await ofetch(`${apiBaseUrl}/api/v2/users/${accountId}/posts`, {
+        headers,
+        query: {
+            sort_key: 'publish_start_at',
+            page: 1,
+        },
+    });
+
+    return post.data as Promise<Post[]>;
+};

package/lib/routes/nasa/apod.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
@@ -7,7 +7,8 @@ import timezone from '@/utils/timezone';
 
 export const route: Route = {
     path: '/apod',
-    categories: ['picture'],
+    categories: ['picture', 'popular'],
+    view: ViewType.Pictures,
     example: '/nasa/apod',
     parameters: {},
     features: {
@@ -23,7 +24,7 @@ export const route: Route = {
             source: ['apod.nasa.govundefined'],
         },
     ],
-    name: 'NASA',
+    name: 'Astronomy Picture of the Day',
     maintainers: ['nczitzk', 'williamgateszhao'],
     handler,
     url: 'apod.nasa.govundefined',

package/lib/routes/nasa/namespace.ts

@@ -1,6 +1,6 @@
 import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
-    name: 'NASA Astronomy Picture of the Day',
+    name: 'NASA',
     url: 'apod.nasa.gov',
 };

package/lib/routes/nationalgeographic/latest-stories.ts

@@ -58,7 +58,7 @@ async function handler() {
                 cache.tryGet(item.link, async () => {
                     const response = await got(item.link);
                     const $ = load(response.data);
-                    const mods = findNatgeo($).page.content.article.frms.find((f) => f.cmsType === 'ArticleBodyFrame').mods;
+                    const mods = findNatgeo($).page.content.prismarticle.frms.find((f) => f.cmsType === 'ArticleBodyFrame').mods;
                     const bodyTile = mods.find((m) => m.edgs[0].cmsType === 'ArticleBodyTile').edgs[0];
 
                     item.author = bodyTile.cntrbGrp

package/lib/routes/nationalgeographic/templates/stories.art

@@ -13,7 +13,7 @@
     {{ if b.type === 'p' }}
         <p>{{@ b.cntnt.mrkup }}</p>
     {{ else if b.type === 'inline' }}
-        {{ if b.cntnt.cmsType === 'image' }}
+        {{ if b.cntnt.cmsType === 'image' && b.cntnt.image?.src }}
             <figure>
                 <img src="{{ b.cntnt.image.src }}" alt="{{ b.cntnt.image.altText }}">
                 <figcaption>{{@ b.cntnt.caption }}</figcaption>