rsshub 1.0.0-master.fc1b99f → 1.0.0-master.fc2cf18

package/lib/routes/modb/topic.ts

@@ -1,7 +1,7 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
 import { load } from 'cheerio';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import logger from '@/utils/logger';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
@@ -27,14 +27,13 @@ export const route: Route = {
 async function handler(ctx) {
     const baseUrl = 'https://www.modb.pro';
     const topicId = ctx.req.param('id');
-    const response = await got({
-        url: `${baseUrl}/api/columns/getKnowledge`,
-        searchParams: {
+    const response = await ofetch(`${baseUrl}/api/columns/getKnowledge`, {
+        query: {
             pageNum: 1,
             pageSize: 20,
             columnId: topicId,
         },
-    }).json();
+    });
     const list = response.list.map((item) => {
         let doc = {};
         let baseLink = {};
@@ -63,7 +62,7 @@ async function handler(ctx) {
     const items = await Promise.all(
         list.map((item) =>
             cache.tryGet(item.link, async () => {
-                const { data: response } = await got(item.link);
+                const response = await ofetch(item.link);
                 const $ = load(response);
                 item.description = $('div.editor-content-styl.article-style').first().html();
                 return item;

package/lib/routes/modelscope/community.ts

@@ -5,7 +5,7 @@ const __dirname = getCurrentPath(import.meta.url);
 import cache from '@/utils/cache';
 import { load } from 'cheerio';
 import got from '@/utils/got';
-import * as path from 'node:path';
+import path from 'node:path';
 import timezone from '@/utils/timezone';
 import { art } from '@/utils/render';
 import { parseDate } from '@/utils/parse-date';

package/lib/routes/modelscope/datasets.ts

@@ -9,7 +9,7 @@ const md = MarkdownIt({
     html: true,
     linkify: true,
 });
-import * as path from 'node:path';
+import path from 'node:path';
 import { art } from '@/utils/render';
 import { parseDate } from '@/utils/parse-date';
 

package/lib/routes/modelscope/studios.ts

@@ -9,7 +9,7 @@ const md = MarkdownIt({
     html: true,
     linkify: true,
 });
-import * as path from 'node:path';
+import path from 'node:path';
 import { art } from '@/utils/render';
 import { parseDate } from '@/utils/parse-date';
 

package/lib/routes/modrinth/versions.ts

@@ -2,21 +2,22 @@ import { Route } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 import { config } from '@/config';
-import got from '@/utils/got';
+import _ofetch from '@/utils/ofetch';
 import MarkdownIt from 'markdown-it';
 import type { Author, Project, Version } from '@/routes/modrinth/api';
 import type { Context } from 'hono';
 
 const __dirname = getCurrentPath(import.meta.url);
 
-const customGot = got.extend({
+const ofetch = _ofetch.create({
     headers: {
         // https://docs.modrinth.com/#section/User-Agents
         'user-agent': config.trueUA,
     },
 });
+
 const md = MarkdownIt({
     html: true,
 });
@@ -37,23 +38,25 @@ export const route: Route = {
         supportPodcast: false,
         supportScihub: false,
     },
-    radar: {
-        source: [
-            'modrinth.com/mod/:id/*',
-            'modrinth.com/plugin/:id/*',
-            'modrinth.com/datapack/:id/*',
-            'modrinth.com/shader/:id/*',
-            'modrinth.com/resourcepack/:id/*',
-            'modrinth.com/modpack/:id/*',
-            'modrinth.com/mod/:id',
-            'modrinth.com/plugin/:id',
-            'modrinth.com/datapack/:id',
-            'modrinth.com/shader/:id',
-            'modrinth.com/resourcepack/:id',
-            'modrinth.com/modpack/:id',
-        ],
-        target: '/project/:id/versions',
-    },
+    radar: [
+        {
+            source: [
+                'modrinth.com/mod/:id/*',
+                'modrinth.com/plugin/:id/*',
+                'modrinth.com/datapack/:id/*',
+                'modrinth.com/shader/:id/*',
+                'modrinth.com/resourcepack/:id/*',
+                'modrinth.com/modpack/:id/*',
+                'modrinth.com/mod/:id',
+                'modrinth.com/plugin/:id',
+                'modrinth.com/datapack/:id',
+                'modrinth.com/shader/:id',
+                'modrinth.com/resourcepack/:id',
+                'modrinth.com/modpack/:id',
+            ],
+            target: '/project/:id/versions',
+        },
+    ],
     name: 'Project versions',
     maintainers: ['SettingDust'],
     handler,
@@ -81,25 +84,29 @@ async function handler(ctx: Context) {
      */
     const parsedQuery = new URLSearchParams(routeParams);
 
-    parsedQuery.set('loaders', parsedQuery.has('loaders') ? JSON.stringify(parsedQuery.getAll('loaders')) : '');
-    parsedQuery.set('game_versions', parsedQuery.has('game_versions') ? JSON.stringify(parsedQuery.getAll('game_versions')) : '');
-
     try {
-        const project = await customGot(`https://api.modrinth.com/v2/project/${id}`).json<Project>();
-        const versions = await customGot(`https://api.modrinth.com/v2/project/${id}/version`, {
-            searchParams: parsedQuery,
-        }).json<Version[]>();
-        const authors = await customGot(`https://api.modrinth.com/v2/users`, {
-            searchParams: {
-                ids: JSON.stringify(versions.map((it) => it.author_id)),
+        const project = await ofetch<Project>(`https://api.modrinth.com/v2/project/${id}`);
+        const versions = await ofetch<Version[]>(`https://api.modrinth.com/v2/project/${id}/version`, {
+            query: {
+                loaders: parsedQuery.has('loaders') ? JSON.stringify(parsedQuery.getAll('loaders')) : '',
+                game_versions: parsedQuery.has('game_versions') ? JSON.stringify(parsedQuery.getAll('game_versions')) : '',
+            },
+        });
+        const authors = await ofetch<Author[]>(`https://api.modrinth.com/v2/users`, {
+            query: {
+                ids: JSON.stringify([...new Set(versions.map((it) => it.author_id))]),
             },
-        }).json<Author[]>();
+        });
+        const groupedAuthors = <Record<string, Author>>{};
+        for (const author of authors) {
+            groupedAuthors[author.id] = author;
+        }
 
         return {
             title: `${project.title} Modrinth versions`,
             description: project.description,
             link: `https://modrinth.com/project/${id}`,
-            item: versions.map((it, index) => ({
+            item: versions.map((it) => ({
                 title: `${it.name} for ${it.loaders.join('/')} on ${[...new Set([it.game_versions[0], it.game_versions.at(-1)])].join('-')}`,
                 link: `https://modrinth.com/project/${id}/version/${it.version_number}`,
                 pubDate: parseDate(it.date_published),
@@ -108,7 +115,7 @@ async function handler(ctx: Context) {
                     changelog: md.render(it.changelog),
                 }),
                 guid: it.id,
-                author: authors[index].name,
+                author: groupedAuthors[it.author_id].username,
             })),
         };
     } catch (error: any) {

package/lib/routes/mox/index.ts

@@ -1,8 +1,8 @@
-import { Route } from '@/types';
+import { DataItem, Route } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
-import { parseDate } from '@/utils/parse-date';
+import { config } from '@/config';
 
 export const route: Route = {
     path: '/:category?',
@@ -10,12 +10,14 @@ export const route: Route = {
     example: '/mox',
     parameters: { category: '分类，可在对应分类页 URL 中找到' },
     features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
+        requireConfig: [
+            {
+                name: 'MOX_COOKIE',
+                optional: true,
+                description: `注册用户登录后的 Cookie, 可以从浏览器开发者工具Network面板中的mox页面请求获取，Cookie内容形如VOLSKEY=xxxxxx; VLIBSID=xxxxxx; VOLSESS=xxxxxx`,
+            },
+        ],
+        antiCrawler: true,
     },
     radar: [
         {
@@ -29,6 +31,10 @@ export const route: Route = {
   在首页将分类参数选择确定后跳转到的分类页面 URL 中，\`/l/\` 后的字段即为分类参数。
 
   如 [科幻 + 日語 + 日本 + 長篇 + 完結 + 最近更新](https://mox.moe/l/CAT%2A科幻,日本,完結,lastupdate,jpn,l,BL) 的 URL 为 [https://mox.moe/l/CAT%2A 科幻，日本，完結，lastupdate,jpn,l,BL](https://mox.moe/l/CAT%2A科幻,日本,完結,lastupdate,jpn,l,BL)，此时 \`/l/\` 后的字段为 \`CAT%2A科幻,日本,完結,lastupdate,jpn,l,BL\`。最终获得路由为 [\`/mox/CAT%2A科幻,日本,完結,lastupdate,jpn,l,BL\`](https://rsshub.app/mox/CAT%2A科幻,日本,完結,lastupdate,jpn,l,BL)
+  :::
+
+  :::warning
+  由于 mox.moe 对非登录用户屏蔽了部分漫画详情内容的获取，且极易触发反爬机制，导致访问ip被重定向至google.com，因此在未配置\`MOX_COOKIE\`参数的情况下路由只会返回漫画标题和封面，不会对详情内容进行抓取。
   :::`,
 };
 
@@ -38,46 +44,70 @@ async function handler(ctx) {
     const rootUrl = 'https://mox.moe';
     const currentUrl = `${rootUrl}${category ? `/l/${category}` : ''}`;
 
+    const cookie = config.mox.cookie;
+
     const response = await got({
         method: 'get',
         url: currentUrl,
+        headers: {
+            cookie,
+        },
     });
 
     const $ = load(response.data);
 
-    let items = $('.listbg td')
+    let items: DataItem[] = $('.listbg td')
         .toArray()
         .map((item) => {
-            item = $(item).find('a').last();
+            const lastItem = $(item).find('a').last();
 
-            const guid = item.attr('href').split('/').pop();
-            const pubDate = item.parent().find('.filesize').text();
+            const guid = lastItem.attr('href')?.split('/').pop();
+
+            const cover = $(item)
+                .find('a div div')
+                .attr('style')
+                ?.match(/background:url\((.*?)\)/)?.[1];
 
             return {
-                title: item.text(),
-                link: item.attr('href'),
-                pubDate: parseDate(pubDate),
-                guid: `${guid}-${pubDate}`,
+                title: lastItem.text(),
+                description: cover ? `<img src="${cover}">` : undefined,
+                link: lastItem.attr('href'),
+                guid,
             };
-        });
+        })
+        .filter((i) => i.guid);
+
+    if (cookie) {
+        items = await Promise.all(
+            items.map(
+                (item) =>
+                    cache.tryGet(item.guid!, async () => {
+                        const detailResponse = await got({
+                            method: 'get',
+                            url: item.link,
+                            headers: {
+                                cookie,
+                            },
+                        });
+
+                        const content = load(detailResponse.data);
 
-    items = await Promise.all(
-        items.map((item) =>
-            cache.tryGet(item.guid, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
+                        item.author = content('.author .text_bglight font a')
+                            .toArray()
+                            .map((i) => $(i).text())
+                            .filter(Boolean)
+                            .join('、');
 
-                const content = load(detailResponse.data);
+                        const infoBlock = content('.author .text_bglight').toArray();
 
-                item.author = content('.status font a').first().text();
-                item.description = `<img src="${content('.img_book').attr('src')}"><br>${content('.author').html()}<br>${content('#desc_text').html()}`;
+                        const desc = detailResponse.data?.match(/document\.getElementById\("div_desc_content"\)\.innerHTML = "(.*?)";/s)?.[1] ?? '';
+                        item.description = `<img src="${content('.img_book').attr('src')}"><br>${infoBlock.map((i) => $(i).html()).join('<br>')}<br>${desc}`;
 
-                return item;
-            })
-        )
-    );
+                        return item;
+                    }) as unknown as DataItem
+            )
+        );
+    }
 
     return {
         title: 'Mox.moe',

package/lib/routes/my-formosa/index.ts

@@ -0,0 +1,78 @@
+import { Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import cache from '@/utils/cache';
+import timezone from '@/utils/timezone';
+
+export const route: Route = {
+    path: '/',
+    categories: ['new-media'],
+    example: '/my-formosa',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['my-formosa.com/'],
+        },
+    ],
+    name: '首页',
+    maintainers: ['dzx-dzx'],
+    handler,
+    url: 'my-formosa.com',
+};
+
+function fetch(url) {
+    return ofetch(url, { responseType: 'arrayBuffer' }).then((raw) => {
+        const decoder = new TextDecoder('big5');
+        return decoder.decode(raw);
+    });
+}
+
+async function handler() {
+    const rootUrl = 'http://www.my-formosa.com/';
+
+    const res = await fetch(rootUrl);
+
+    const $ = load(res);
+
+    const items = await Promise.all(
+        $('#featured-news h3 a')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+
+                const title = item.text();
+                const link = new URL(item.attr('href'), rootUrl).href;
+
+                return cache.tryGet(link, async () => {
+                    const res = await fetch(link);
+                    const $ = load(res);
+
+                    const isTV = /^\/TV/.test(new URL(link).pathname);
+
+                    return {
+                        title,
+                        link,
+                        author: $('.page-header~#featured-news h4').text(),
+                        category: $("meta[name='keywords']").attr('content').split(',').filter(Boolean),
+                        pubDate: timezone(parseDate((isTV ? $('.icon-calendar')[0].next.data : $('.date').text()).trim()), +8),
+                        description: (isTV ? $('.post-item').html() : $('.body').html()).replaceAll(/\/News.*?\.jpg/g, (match) => `http://my-formosa.com${match}`),
+                    };
+                });
+            })
+    );
+
+    return {
+        title: $('title').text(),
+        link: rootUrl,
+        item: items,
+    };
+}

package/lib/routes/my-formosa/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '美麗島電子報',
+    url: 'my-formosa.com',
+};

package/lib/routes/mydrivers/index.ts

@@ -8,7 +8,7 @@ import { load } from 'cheerio';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 import { rootUrl, title, categories, convertToQueryString, getInfo, processItems } from './util';
 

package/lib/routes/myfans/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'myfans',
+    url: 'myfans.jp',
+};

package/lib/routes/myfans/post.ts

@@ -0,0 +1,64 @@
+import { Route } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import { showByUsername, getPostByAccountId, baseUrl } from './utils';
+import path from 'node:path';
+import { art } from '@/utils/render';
+import { getCurrentPath } from '@/utils/helpers';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+export const route: Route = {
+    path: '/user/:username',
+    categories: ['multimedia'],
+    example: '/myfans/user/secret_japan',
+    parameters: { username: 'User handle' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['myfans.jp/:username', 'myfans.jp/:language/:username'],
+        },
+    ],
+    name: 'User Posts',
+    maintainers: ['TonyRL'],
+    handler,
+};
+
+const render = (postImages, body) =>
+    art(path.join(__dirname, 'templates', 'post.art'), {
+        postImages,
+        body,
+    });
+
+async function handler(ctx) {
+    const { username } = ctx.req.param();
+
+    const account = await showByUsername(username);
+    const posts = await getPostByAccountId(account.id);
+
+    const items = posts.map((p) => ({
+        title: p.body?.replaceAll('\r\n', ' ').trim().split(' ')[0],
+        description: render(p.post_images, p.body?.replaceAll('\r\n', '<br>')),
+        pubDate: parseDate(p.published_at),
+        link: `${baseUrl}/posts/${p.id}`,
+        author: p.user.name,
+    }));
+
+    return {
+        title: `${account.name} (@${account.username})`,
+        link: `${baseUrl}/${account.username}`,
+        description: `${account.posts_count} Post ${account.likes_count} Like ${account.followers_count}
+Followers ${account.followings_count} Follow ${account.about.replaceAll('\r\n', ' ')}`,
+        image: account.avatar_url,
+        icon: account.avatar_url,
+        logo: account.avatar_url,
+        language: 'ja-JP',
+        item: items,
+    };
+}

package/lib/routes/myfans/templates/post.art

@@ -0,0 +1,8 @@
+{{ if postImages }}
+    {{ each postImages img }}
+        <img src="{{ img.file_url }}"><br>
+    {{ /each }}
+{{ /if }}
+{{ if body }}
+    {{@ body }}
+{{ /if }}

package/lib/routes/myfans/types.ts

@@ -0,0 +1,104 @@
+export interface UserProfile {
+    about: string;
+    active: boolean;
+    avatar_url: string;
+    banner_url: string;
+    id: string | null;
+    is_following: boolean;
+    likes_count: number;
+    name: string;
+    username: string;
+    is_official_creator: boolean;
+    is_official: boolean;
+    label: null;
+    back_number_post_images_count: number;
+    back_number_post_videos_count: number;
+    cant_receive_message: boolean;
+    current_back_number_plan: null;
+    followers_count: number;
+    followings_count: number;
+    has_approved_user_identification: boolean;
+    is_bought_back_number: boolean;
+    is_followed: boolean;
+    is_subscribed: boolean;
+    limited_posts_count: number;
+    link_instagram_id: string;
+    link_instagram_url: null;
+    link_tiktok_id: string;
+    link_tiktok_url: null;
+    link_twitter_id: string;
+    link_twitter_url: string;
+    link_youtube_url: string;
+    post_images_count: number;
+    post_videos_count: number;
+    posts_count: number;
+    sns_link1: string;
+    sns_link2: string;
+}
+
+export interface Post {
+    id: string;
+    kind: string;
+    status: string;
+    status_label: null;
+    body: string | null;
+    bookmarked: boolean;
+    humanized_publish_start_at: string;
+    deleted_at_i18n: null;
+    liked: boolean;
+    likes_count: number;
+    user: UserProfile;
+    post_images: {
+        file_url: string;
+        square_thumbnail_url: string;
+        raw_image_height: number;
+        raw_image_width: number;
+    }[];
+    visible: boolean;
+    publish_end_at: null;
+    published_at: string;
+    publish_start_at: string | null;
+    pinned_at: string | null;
+    attachment: null;
+    plan: null;
+    current_single_plan: {
+        id: string;
+        amount: number;
+        auto_message_body: string;
+    } | null;
+    plans: {
+        id: string;
+        product_name: string;
+        monthly_price: number;
+        status: null;
+        is_limited_access: boolean;
+        disallow_new_subscriber: boolean;
+        active_discount: {
+            id: string;
+            discount_rate: number;
+            start_at: string | null;
+            end_at: string | null;
+            limited_number: null;
+            status: string;
+        } | null;
+    }[];
+    video_processing: boolean | null;
+    video_duration: {
+        hours: string | null;
+        minutes: string;
+        seconds: string;
+    } | null;
+    free: boolean;
+    limited: boolean;
+    available: boolean;
+    metadata: {
+        video: {
+            duration: number;
+            resolutions: string[];
+        };
+        image: {
+            count: number;
+        };
+    };
+    thumbnail_url: string;
+}

package/lib/routes/myfans/utils.ts

@@ -0,0 +1,39 @@
+import ofetch from '@/utils/ofetch';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+import cache from '@/utils/cache';
+import { Post, UserProfile } from './types';
+
+const apiBaseUrl = 'https://api.myfans.jp';
+export const baseUrl = 'https://myfans.jp';
+
+const headers = {
+    'google-ga-data': 'event328',
+};
+
+export const showByUsername = (username: string) =>
+    cache.tryGet(`myfans:account:${username}`, async () => {
+        const accountInfo = await ofetch<UserProfile>(`${apiBaseUrl}/api/v2/users/show_by_username`, {
+            headers,
+            query: {
+                username,
+            },
+        });
+
+        if (!accountInfo.id) {
+            throw new InvalidParameterError('This creator does not exist.');
+        }
+
+        return accountInfo;
+    }) as Promise<UserProfile>;
+
+export const getPostByAccountId = async (accountId) => {
+    const post = await ofetch(`${apiBaseUrl}/api/v2/users/${accountId}/posts`, {
+        headers,
+        query: {
+            sort_key: 'publish_start_at',
+            page: 1,
+        },
+    });
+
+    return post.data as Promise<Post[]>;
+};

package/lib/routes/myfigurecollection/activity.ts

@@ -7,8 +7,9 @@ import { load } from 'cheerio';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 import { isValidHost } from '@/utils/valid-host';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
     path: '/activity/:category?/:language?/:latestAdditions?/:latestEdits?/:latestAlerts?/:latestPictures?',
@@ -75,7 +76,7 @@ async function handler(ctx) {
     const latestPictures = ctx.req.param('latestPictures') ?? '1';
 
     if (language && !isValidHost(language)) {
-        throw new Error('Invalid language');
+        throw new InvalidParameterError('Invalid language');
     }
 
     const rootUrl = `https://${language === 'en' || language === '' ? '' : `${language}.`}myfigurecollection.net`;