rsshub 1.0.0-master.f72af1b → 1.0.0-master.f7347d9

package/lib/routes/scmp/utils.ts

@@ -1,6 +1,6 @@
 import { load } from 'cheerio';
-import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
 
 export const renderHTML = (node) => {
     if (!node) {
@@ -60,7 +60,7 @@ export const renderHTML = (node) => {
 };
 
 export const parseItem = async (item) => {
-    const { data: response, url } = await got(item.link);
+    const { _data: response, url } = await ofetch.raw(item.link);
 
     if (new URL(url).hostname !== 'www.scmp.com') {
         // e.g., https://multimedia.scmp.com/

package/lib/routes/skeb/following-creators.ts

@@ -0,0 +1,52 @@
+import { Data, DataItem, Route } from '@/types';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import { getFollowingsItems } from './utils';
+
+export const route: Route = {
+    path: '/following_creators/:username',
+    categories: ['picture'],
+    example: '/skeb/following_creators/@brm2_1925',
+    parameters: { username: 'Skeb Username with @' },
+    features: {
+        requireConfig: [
+            {
+                name: 'SKEB_BEARER_TOKEN',
+                optional: false,
+                description: '在瀏覽器開發者工具（F12）的主控台中輸入 `localStorage.getItem("token")` 獲取',
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Following Creators',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: 'Following Creators',
+            source: ['skeb.jp/:username'],
+            target: '/following_creators/:username',
+        },
+    ],
+    description: 'Get the list of creators the specified user is following on Skeb.',
+};
+
+async function handler(ctx): Promise<Data> {
+    const username = ctx.req.param('username');
+
+    if (!config.skeb || !config.skeb.bearerToken) {
+        throw new ConfigNotFoundError('Skeb followings RSS is disabled due to the lack of relevant config');
+    }
+
+    const items = await getFollowingsItems(username, 'following_creators');
+
+    return {
+        title: `Skeb - ${username} - フォロー中のクリエイター`,
+        link: `https://skeb.jp/${username}`,
+        item: items as DataItem[],
+    };
+}

package/lib/routes/skeb/following-works.ts

@@ -0,0 +1,52 @@
+import { Data, DataItem, Route } from '@/types';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import { getFollowingsItems } from './utils';
+
+export const route: Route = {
+    path: '/following_works/:username',
+    categories: ['picture'],
+    example: '/skeb/following_works/@brm2_1925',
+    parameters: { username: 'Skeb Username with @' },
+    features: {
+        requireConfig: [
+            {
+                name: 'SKEB_BEARER_TOKEN',
+                optional: false,
+                description: '在瀏覽器開發者工具（F12）的主控台中輸入 `localStorage.getItem("token")` 獲取',
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Following Works',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: 'Following Works',
+            source: ['skeb.jp/:username'],
+            target: '/following_works/:username',
+        },
+    ],
+    description: "Get the latest works for the specified user's followings on Skeb.",
+};
+
+async function handler(ctx): Promise<Data> {
+    const username = ctx.req.param('username');
+
+    if (!config.skeb || !config.skeb.bearerToken) {
+        throw new ConfigNotFoundError('Skeb followings RSS is disabled due to the lack of relevant config');
+    }
+
+    const items = await getFollowingsItems(username, 'following_works');
+
+    return {
+        title: `Skeb - ${username} - フォロー中のクリエイターの新着作品`,
+        link: `https://skeb.jp/${username}`,
+        item: items as DataItem[],
+    };
+}

package/lib/routes/skeb/friend-works.ts

@@ -0,0 +1,52 @@
+import { Data, DataItem, Route } from '@/types';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import { getFollowingsItems } from './utils';
+
+export const route: Route = {
+    path: '/friend_works/:username',
+    categories: ['picture'],
+    example: '/skeb/friend_works/@brm2_1925',
+    parameters: { username: 'Skeb Username with @' },
+    features: {
+        requireConfig: [
+            {
+                name: 'SKEB_BEARER_TOKEN',
+                optional: false,
+                description: '在瀏覽器開發者工具（F12）的主控台中輸入 `localStorage.getItem("token")` 獲取',
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Friend Works',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: 'Friend Works',
+            source: ['skeb.jp/:username'],
+            target: '/friend_works/:username',
+        },
+    ],
+    description: "Get the latest requests for the specified user's followings on Skeb.",
+};
+
+async function handler(ctx): Promise<Data> {
+    const username = ctx.req.param('username');
+
+    if (!config.skeb || !config.skeb.bearerToken) {
+        throw new ConfigNotFoundError('Skeb followings RSS is disabled due to the lack of relevant config');
+    }
+
+    const items = await getFollowingsItems(username, 'friend_works');
+
+    return {
+        title: `Skeb - ${username} - フォロー中のクライアントの新着リクエスト`,
+        link: `https://skeb.jp/${username}`,
+        item: items as DataItem[],
+    };
+}

package/lib/routes/skeb/index.ts

@@ -0,0 +1,131 @@
+import { Route, Data, DataItem } from '@/types';
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
+import { baseUrl, processWork, processCreator } from './utils';
+import { config } from '@/config';
+
+const categoryMap = {
+    // Works categories
+    new_art_works: '新着作品 (Illust)',
+    new_voice_works: '新着作品 (Voice)',
+    new_novel_works: '新着作品 (Novel)',
+    new_video_works: '新着作品 (Video)',
+    new_music_works: '新着作品 (Music)',
+    new_correction_works: '新着作品 (Advice)',
+    new_comic_works: '新着作品 (Comic)',
+    popular_works: '人気の作品 (Popular)',
+    // Creators categories
+    popular_creators: '人気クリエイター',
+    new_creators: '新着クリエイター',
+};
+
+const workCategories = new Set(['new_art_works', 'new_voice_works', 'new_novel_works', 'new_video_works', 'new_music_works', 'new_correction_works', 'new_comic_works', 'popular_works']);
+
+export const route: Route = {
+    path: '/:category',
+    categories: ['picture'],
+    example: '/skeb/new_art_works',
+    parameters: { category: 'Category, the div id of the section title on the homepage.' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Skeb',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: '新着作品 (Illust)',
+            source: ['skeb.jp'],
+            target: '/new_art_works',
+        },
+        {
+            title: '新着作品 (Voice)',
+            source: ['skeb.jp'],
+            target: '/new_voice_works',
+        },
+        {
+            title: '新着作品 (Novel)',
+            source: ['skeb.jp'],
+            target: '/new_novel_works',
+        },
+        {
+            title: '新着作品 (Video)',
+            source: ['skeb.jp'],
+            target: '/new_video_works',
+        },
+        {
+            title: '新着作品 (Music)',
+            source: ['skeb.jp'],
+            target: '/new_music_works',
+        },
+        {
+            title: '新着作品 (Advice)',
+            source: ['skeb.jp'],
+            target: '/new_correction_works',
+        },
+        {
+            title: '新着作品 (Comic)',
+            source: ['skeb.jp'],
+            target: '/new_comic_works',
+        },
+        {
+            title: '人気の作品 (Popular)',
+            source: ['skeb.jp'],
+            target: '/popular_works',
+        },
+        {
+            title: '人気クリエイター',
+            source: ['skeb.jp'],
+            target: '/popular_creators',
+        },
+        {
+            title: '新着クリエイター',
+            source: ['skeb.jp'],
+            target: '/new_creators',
+        },
+    ],
+};
+
+async function handler(ctx): Promise<Data> {
+    const category = ctx.req.param('category') || 'new_art_works';
+
+    if (!(category in categoryMap)) {
+        throw new Error('Invalid category');
+    }
+
+    const url = `${baseUrl}/api`;
+
+    const apiData = await cache.tryGet(
+        url,
+        async () => {
+            const data = await ofetch(url);
+            return data;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    if (!apiData || typeof apiData !== 'object') {
+        throw new Error('Invalid data received from API');
+    }
+
+    const items = await cache.tryGet(category, async () => {
+        if (!(category in apiData) || !Array.isArray(apiData[category])) {
+            return [];
+        }
+
+        const processItem = workCategories.has(category) ? processWork : processCreator;
+        return (await Promise.all(apiData[category].map(async (item) => await processItem(item)).filter(Boolean))) as DataItem[];
+    });
+
+    return {
+        title: `Skeb - ${categoryMap[category]}`,
+        link: `${baseUrl}/#${category}`,
+        item: items as DataItem[],
+    };
+}

package/lib/routes/skeb/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Skeb',
+    url: 'skeb.jp',
+};

package/lib/routes/skeb/search.ts

@@ -0,0 +1,77 @@
+import { Data, DataItem, Route } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { processWork, baseUrl } from './utils';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+
+export const route: Route = {
+    path: '/search/:keyword',
+    categories: ['picture'],
+    example: '/skeb/search/初音ミク',
+    parameters: { keyword: 'Search keyword' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Search Results',
+    maintainers: ['SnowAgar25'],
+    handler,
+    description: 'Get the search results for works on Skeb',
+};
+
+async function handler(ctx): Promise<Data> {
+    const keyword = ctx.req.param('keyword');
+
+    if (!keyword) {
+        throw new InvalidParameterError('Invalid search keyword');
+    }
+
+    const url = 'https://hb1jt3kre9-dsn.algolia.net/1/indexes/*/queries';
+
+    const items = await cache.tryGet(`skeb:search:${keyword}`, async () => {
+        const data = await ofetch(url, {
+            method: 'POST',
+            headers: {
+                'x-algolia-application-id': 'HB1JT3KRE9',
+                'x-algolia-api-key': '9a4ce7d609e71bf29e977925e4c6740c',
+            },
+            body: {
+                requests: [
+                    {
+                        indexName: 'User',
+                        query: keyword,
+                        params: 'hitsPerPage=40',
+                        filters: 'genres:art OR genres:comic OR genres:voice OR genres:novel OR genres:video OR genres:music OR genres:correction',
+                    },
+                    {
+                        indexName: 'Request',
+                        query: keyword,
+                        params: 'hitsPerPage=40&filters=genre%3Aart%20OR%20genre%3Acomic%20OR%20genre%3Avoice%20OR%20genre%3Anovel%20OR%20genre%3Avideo%20OR%20genre%3Amusic%20OR%20genre%3Acorrection',
+                    },
+                ],
+            },
+        });
+
+        if (!data || !data.results || !Array.isArray(data.results) || data.results.length < 2) {
+            throw new Error('Invalid data received from API');
+        }
+
+        const works = data.results[1].hits;
+
+        if (!Array.isArray(works)) {
+            throw new TypeError('Invalid hits data received from API');
+        }
+
+        return works.map((item) => processWork(item)).filter(Boolean);
+    });
+
+    return {
+        title: `Skeb - Search Results for "${keyword}"`,
+        link: `${baseUrl}/search?q=${encodeURIComponent(keyword)}`,
+        item: items as DataItem[],
+    };
+}

package/lib/routes/skeb/templates/creator.art

@@ -0,0 +1,8 @@
+{{ if avatarUrl }}
+    <img src="{{ avatarUrl }}" />
+{{ /if }}
+<p>委託狀況（Accepting Commissions）：{{ acceptingCommissions }}</p>
+<p>NSFW：{{ nsfwAcceptable }}</p>
+{{ if skills }}
+    <p>類型（Genre）：{{ skills }}</p>
+{{ /if }}

package/lib/routes/skeb/templates/work.art

@@ -0,0 +1,10 @@
+{{ if imageUrl }}
+    <img src="{{ imageUrl }}" /><br>
+{{ /if }}
+{{ if audioUrl }}
+    <audio controls>
+    <source src="{{ audioUrl }}" type="audio/mp3">
+    Your browser does not support the audio element.
+    </audio><br>
+{{ /if }}
+{{ body }}

package/lib/routes/skeb/utils.ts

@@ -0,0 +1,155 @@
+import { config } from '@/config';
+import { DataItem } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { art } from '@/utils/render';
+import path from 'node:path';
+import { getCurrentPath } from '@/utils/helpers';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+export const baseUrl = 'https://skeb.jp';
+
+interface Work {
+    path: string;
+    private_thumbnail_image_urls: null | string;
+    private: boolean;
+    genre: string;
+    tipped: boolean;
+    creator_id: number;
+    client_id: number;
+    vtt_url: null | string;
+    thumbnail_image_urls: {
+        src: string;
+        srcset: string;
+    };
+    preview_url: null | string;
+    duration: null | number;
+    nsfw: boolean;
+    hardcore: boolean;
+    consored_thumbnail_image_urls: {
+        src: string;
+        srcset: string;
+    };
+    body: string;
+    nc: number;
+    word_count: number;
+    transcoder: string;
+    creator_acceptable_same_genre: boolean;
+}
+
+interface Creator {
+    id: number;
+    creator: boolean;
+    nsfw_acceptable: boolean;
+    acceptable: boolean;
+    name: string;
+    screen_name: string;
+    avatar_url: string;
+    header_url: string | null;
+    appeal_receivable: boolean;
+    popular_creator_rank: number | null;
+    request_master_rank: number | null;
+    first_requester_rank: number | null;
+    deleted_at: string | null;
+    tip_acceptable_by: string;
+    accept_expiration_days: number;
+    skills: { genre: string }[];
+    nc: number;
+}
+
+export function processWork(work: Work): DataItem | null {
+    if (!work || typeof work !== 'object' || work.private === true) {
+        return null;
+    }
+
+    const imageUrl = work.thumbnail_image_urls?.srcset?.split(',').pop()?.trim().split(' ')[0] || '';
+    const body = work.body || '';
+
+    const audioUrl = work.genre === 'music' || work.genre === 'voice' ? work.preview_url : null;
+
+    const renderedHtml = art(path.join(__dirname, 'templates/work.art'), {
+        imageUrl,
+        body,
+        audioUrl,
+    });
+
+    return {
+        title: work.path || '',
+        link: `${baseUrl}${work.path || ''}`,
+        description: renderedHtml,
+    };
+}
+
+const skillMap = {
+    art: 'Illust',
+    voice: 'Voice',
+    novel: 'Novel',
+    video: 'Video',
+    music: 'Music',
+    correction: 'Advice',
+    comic: 'Comic',
+};
+
+export function processCreator(creator: Creator): DataItem | null {
+    if (!creator || typeof creator !== 'object') {
+        return null;
+    }
+
+    const avatarUrl = creator.avatar_url || '';
+
+    let renderedHtml;
+
+    if (creator.creator) {
+        const acceptingCommissions = creator.acceptable ? 'Yes' : 'No';
+        const nsfwAcceptable = creator.nsfw_acceptable ? 'Yes' : 'No';
+
+        let skills = '';
+        if (Array.isArray(creator.skills) && creator.skills.length > 0) {
+            skills = creator.skills
+                .map((skill) => skillMap[skill.genre] || skill.genre)
+                .filter(Boolean)
+                .join(', ');
+        }
+
+        renderedHtml = art(path.join(__dirname, 'templates/creator.art'), {
+            avatarUrl,
+            acceptingCommissions,
+            nsfwAcceptable,
+            skills,
+        });
+    }
+
+    return {
+        title: creator.name || '',
+        link: `${baseUrl}/@${creator.screen_name || ''}`,
+        description: renderedHtml,
+    };
+}
+
+export async function getFollowingsItems(username: string, path: 'friend_works' | 'following_works' | 'following_creators'): Promise<DataItem[]> {
+    const url = `${baseUrl}/api/users/${username.replace('@', '')}/followings`;
+
+    const followings_data = await cache.tryGet(
+        `skeb:followings_data:${username}`,
+        async () => {
+            const data = await ofetch(url, {
+                headers: {
+                    Authorization: `Bearer ${config.skeb.bearerToken}`,
+                },
+            });
+            return data;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    if (!followings_data || typeof followings_data !== 'object') {
+        throw new Error('Failed to fetch followings data');
+    }
+
+    if (path === 'following_creators') {
+        return followings_data[path].map((item) => processCreator(item)).filter(Boolean) as DataItem[];
+    }
+    return followings_data[path].map((item) => processWork(item)).filter(Boolean) as DataItem[];
+}

package/lib/routes/skeb/works.ts

@@ -0,0 +1,88 @@
+import { Data, DataItem, Route } from '@/types';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import { baseUrl, processWork } from './utils';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+
+export const route: Route = {
+    path: '/works/:username',
+    categories: ['picture'],
+    example: '/skeb/works/@brm2_1925',
+    parameters: { username: 'Skeb Username with @' },
+    features: {
+        requireConfig: [
+            {
+                name: 'SKEB_BEARER_TOKEN',
+                optional: false,
+                description: '在瀏覽器開發者工具（F12）的主控台中輸入 `localStorage.getItem("token")` 獲取',
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Creator Works',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: 'Creator Works',
+            source: ['skeb.jp/:username'],
+            target: '/works/:username',
+        },
+    ],
+    description: 'Get the latest works of a specific creator on Skeb',
+};
+
+async function handler(ctx): Promise<Data> {
+    const username = ctx.req.param('username');
+
+    if (!config.skeb || !config.skeb.bearerToken) {
+        throw new ConfigNotFoundError('Skeb works RSS is disabled due to the lack of <a href="https://docs.rsshub.app/deploy/config#route-specific-configurations">relevant config</a>');
+    }
+
+    const url = `${baseUrl}/api/users/${username.replace('@', '')}/works`;
+
+    const items = await cache.tryGet(url, async () => {
+        const fetchData = async (retryCount = 0, maxRetries = 3) => {
+            const data = await ofetch(url, {
+                retry: 0,
+                method: 'GET',
+                query: { role: 'creator', sort: 'date', offset: '0' },
+                headers: {
+                    'User-Agent': config.ua,
+                    Cookie: `request_key=${cache.get('skeb:request_key')}`,
+                    Authorization: `Bearer ${config.skeb.bearerToken}`,
+                },
+            }).catch((error) => {
+                if (retryCount >= maxRetries) {
+                    throw new Error('Max retries reached');
+                }
+                const newRequestKey = error.response?._data?.match(/request_key=(.*?);/)?.[1];
+                if (newRequestKey) {
+                    cache.set('skeb:request_key', newRequestKey);
+                    return fetchData(retryCount + 1, maxRetries);
+                }
+                throw error;
+            });
+            return data;
+        };
+
+        const data = await fetchData();
+
+        if (!data || !Array.isArray(data)) {
+            throw new Error('Invalid data received from API');
+        }
+
+        return data.map((item) => processWork(item)).filter(Boolean);
+    });
+
+    return {
+        title: `Skeb - ${username}'s Works`,
+        link: `${baseUrl}/${username}`,
+        item: items as DataItem[],
+    };
+}