rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa04a11

package/lib/routes/pixiv/novel-api/user-novels/nsfw.ts

@@ -0,0 +1,82 @@
+import got from '../../pixiv-got';
+import { maskHeader } from '../../constants';
+import queryString from 'query-string';
+import { config } from '@/config';
+import pixivUtils from '../../utils';
+import { getNSFWNovelContent } from '../content/nsfw';
+import { parseDate } from '@/utils/parse-date';
+import { convertPixivProtocolExtended } from '../content/utils';
+import type { NSFWNovelsResponse, NovelList } from './types';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import cache from '@/utils/cache';
+import { getToken } from '../../token';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+
+function getNovels(user_id: string, token: string): Promise<NSFWNovelsResponse> {
+    return got('https://app-api.pixiv.net/v1/user/novels', {
+        headers: {
+            ...maskHeader,
+            Authorization: 'Bearer ' + token,
+        },
+        searchParams: queryString.stringify({
+            user_id,
+            filter: 'for_ios',
+        }),
+    });
+}
+
+export async function getNSFWUserNovels(id: string, fullContent: boolean = false, limit: number = 100): Promise<NovelList> {
+    if (!config.pixiv || !config.pixiv.refreshToken) {
+        throw new ConfigNotFoundError('This user is an R18 creator, PIXIV_REFRESHTOKEN is required.\npixiv RSS is disabled due to the lack of relevant config.\n該用戶爲 R18 創作者，需要 PIXIV_REFRESHTOKEN。');
+    }
+
+    const token = await getToken(cache.tryGet);
+    if (!token) {
+        throw new ConfigNotFoundError('pixiv not login');
+    }
+
+    const response = await getNovels(id, token);
+    const novels = limit ? response.data.novels.slice(0, limit) : response.data.novels;
+
+    if (novels.length === 0) {
+        throw new InvalidParameterError(`${id} is not a valid user ID, or the user has no novels.\n${id} 不是有效的用戶 ID，或者該用戶沒有小說作品。`);
+    }
+
+    const username = novels[0].user.name;
+
+    const items = await Promise.all(
+        novels.map(async (novel) => {
+            const baseItem = {
+                title: novel.series?.title ? `${novel.series.title} - ${novel.title}` : novel.title,
+                description: `
+                    <img src="${pixivUtils.getProxiedImageUrl(novel.image_urls.large)}" />
+                    <div>
+                    <p>${convertPixivProtocolExtended(novel.caption)}</p>
+                    </div>`,
+                author: novel.user.name,
+                pubDate: parseDate(novel.create_date),
+                link: `https://www.pixiv.net/novel/show.php?id=${novel.id}`,
+                category: novel.tags.map((t) => t.name),
+            };
+
+            if (!fullContent) {
+                return baseItem;
+            }
+
+            const { content } = await getNSFWNovelContent(novel.id, token);
+
+            return {
+                ...baseItem,
+                description: `${baseItem.description}<hr>${content}`,
+            };
+        })
+    );
+
+    return {
+        title: `${username}'s novels - pixiv`,
+        description: `${username} 的 pixiv 最新小说`,
+        image: pixivUtils.getProxiedImageUrl(novels[0].user.profile_image_urls.medium),
+        link: `https://www.pixiv.net/users/${id}/novels`,
+        item: items,
+    };
+}

package/lib/routes/pixiv/novel-api/user-novels/sfw.ts

@@ -0,0 +1,74 @@
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import pixivUtils from '../../utils';
+import { getSFWNovelContent } from '../content/sfw';
+import type { SFWNovelsResponse, NovelList } from './types';
+
+const baseUrl = 'https://www.pixiv.net';
+
+export async function getSFWUserNovels(id: string, fullContent: boolean = false, limit: number = 100): Promise<NovelList> {
+    const url = `${baseUrl}/users/${id}/novels`;
+    const { data: allData } = await got(`${baseUrl}/ajax/user/${id}/profile/all`, {
+        headers: {
+            referer: url,
+        },
+    });
+
+    const novels = Object.keys(allData.body.novels)
+        .sort((a, b) => Number(b) - Number(a))
+        .slice(0, Number.parseInt(String(limit), 10));
+
+    if (novels.length === 0) {
+        throw new Error('No novels found for this user, or is an R18 creator, fallback to ConfigNotFoundError');
+    }
+
+    const searchParams = new URLSearchParams();
+    for (const novel of novels) {
+        searchParams.append('ids[]', novel);
+    }
+
+    const { data } = (await got(`${baseUrl}/ajax/user/${id}/profile/novels`, {
+        headers: {
+            referer: url,
+        },
+        searchParams,
+    })) as SFWNovelsResponse;
+
+    const items = await Promise.all(
+        Object.values(data.body.works).map(async (item) => {
+            const baseItem = {
+                title: item.title,
+                description: `
+                    <img src=${pixivUtils.getProxiedImageUrl(item.url)} />
+                    <div>
+                    <p>${item.description}</p>
+                    </div>
+                `,
+                link: `${baseUrl}/novel/show.php?id=${item.id}`,
+                author: item.userName,
+                pubDate: parseDate(item.createDate),
+                updated: parseDate(item.updateDate),
+                category: item.tags,
+            };
+
+            if (!fullContent) {
+                return baseItem;
+            }
+
+            const { content } = await getSFWNovelContent(item.id);
+
+            return {
+                ...baseItem,
+                description: `${baseItem.description}<hr>${content}`,
+            };
+        })
+    );
+
+    return {
+        title: data.body.extraData.meta.title,
+        description: data.body.extraData.meta.ogp.description,
+        image: pixivUtils.getProxiedImageUrl(Object.values(data.body.works)[0].profileImageUrl),
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/pixiv/novel-api/user-novels/types.ts

@@ -0,0 +1,133 @@
+export interface SFWNovelsResponse {
+    data: {
+        error: boolean;
+        message: string;
+        body: {
+            works: Record<string, SFWNovelWork>;
+            extraData: {
+                meta: {
+                    title: string;
+                    description: string;
+                    canonical: string;
+                    ogp: {
+                        description: string;
+                        image: string;
+                        title: string;
+                        type: string;
+                    };
+                    twitter: {
+                        description: string;
+                        image: string;
+                        title: string;
+                        card: string;
+                    };
+                    alternateLanguages: {
+                        ja: string;
+                        en: string;
+                    };
+                    descriptionHeader: string;
+                };
+            };
+        };
+    };
+}
+
+export interface SFWNovelWork {
+    id: string;
+    title: string;
+    genre: string;
+    xRestrict: number;
+    restrict: number;
+    url: string;
+    tags: string[];
+    userId: string;
+    userName: string;
+    profileImageUrl: string;
+    textCount: number;
+    wordCount: number;
+    readingTime: number;
+    useWordCount: boolean;
+    description: string;
+    isBookmarkable: boolean;
+    bookmarkData: null;
+    bookmarkCount: number;
+    isOriginal: boolean;
+    marker: null;
+    titleCaptionTranslation: {
+        workTitle: null;
+        workCaption: null;
+    };
+    createDate: string;
+    updateDate: string;
+    isMasked: boolean;
+    aiType: number;
+    seriesId: string;
+    seriesTitle: string;
+    isUnlisted: boolean;
+}
+
+export interface NSFWNovelsResponse {
+    data: {
+        user: {
+            id: number;
+            name: string;
+            account: string;
+            profile_image_urls: {
+                medium: string;
+            };
+            is_followed: boolean;
+            is_access_blocking_user: boolean;
+        };
+        novels: NSFWNovelWork[];
+    };
+}
+
+export interface NSFWNovelWork {
+    id: string;
+    title: string;
+    caption: string;
+    restrict: number;
+    x_restrict: number;
+    image_urls: {
+        square_medium: string;
+        medium: string;
+        large: string;
+    };
+    create_date: string;
+    tags: Array<{
+        name: string;
+        translated_name: string | null;
+        added_by_uploaded_user: boolean;
+    }>;
+    text_length: number;
+    user: {
+        id: number;
+        name: string;
+        account: string;
+        profile_image_urls: {
+            medium: string;
+        };
+    };
+    series?: {
+        id?: number;
+        title?: string;
+    };
+    total_bookmarks: number;
+    total_view: number;
+    total_comments: number;
+}
+
+export interface NovelList {
+    title: string;
+    description: string;
+    image: string;
+    link: string;
+    item: Array<{
+        title: string;
+        description: string;
+        author: string;
+        pubDate: Date;
+        link: string;
+        category: string[];
+    }>;
+}

package/lib/routes/pixiv/novel-series.ts

@@ -0,0 +1,52 @@
+import { Data, Route } from '@/types';
+import { config } from '@/config';
+import { getNSFWSeriesNovels } from './novel-api/series/nsfw';
+import { getSFWSeriesNovels } from './novel-api/series/sfw';
+
+export const route: Route = {
+    path: '/novel/series/:id',
+    categories: ['social-media'],
+    example: '/pixiv/novel/series/11586857',
+    parameters: {
+        id: 'Series id, can be found in URL',
+    },
+    features: {
+        requireConfig: [
+            {
+                name: 'PIXIV_REFRESHTOKEN',
+                optional: true,
+                description: `
+refresh_token after Pixiv login, required for accessing R18 novels
+Pixiv 登錄後的 refresh_token，用於獲取 R18 小說
+[https://docs.rsshub.app/deploy/config#pixiv](https://docs.rsshub.app/deploy/config#pixiv)`,
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Novel Series',
+    maintainers: ['SnowAgar25', 'keocheung'],
+    handler,
+    radar: [
+        {
+            source: ['www.pixiv.net/novel/series/:id'],
+            target: '/novel/series/:id',
+        },
+    ],
+};
+
+const hasPixivAuth = () => Boolean(config.pixiv && config.pixiv.refreshToken);
+
+async function handler(ctx): Promise<Data> {
+    const id = ctx.req.param('id');
+    const { limit } = ctx.req.query();
+
+    if (hasPixivAuth()) {
+        return await getNSFWSeriesNovels(id, limit);
+    }
+
+    return await getSFWSeriesNovels(id, limit);
+}

package/lib/routes/pixiv/novels.ts

@@ -1,15 +1,37 @@
-import { Route } from '@/types';
-import got from '@/utils/got';
-import { parseDate } from '@/utils/parse-date';
-const baseUrl = 'https://www.pixiv.net';
+import { Data, Route, ViewType } from '@/types';
+import { fallback, queryToBoolean } from '@/utils/readable-social';
+import { config } from '@/config';
+import { getNSFWUserNovels } from './novel-api/user-novels/nsfw';
+import { getSFWUserNovels } from './novel-api/user-novels/sfw';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 export const route: Route = {
-    path: '/user/novels/:id',
+    path: '/user/novels/:id/:full_content?',
     categories: ['social-media'],
+    view: ViewType.Articles,
     example: '/pixiv/user/novels/27104704',
-    parameters: { id: "User id, available in user's homepage URL" },
+    parameters: {
+        id: "User id, available in user's homepage URL",
+        full_content: {
+            description: 'Enable or disable the display of full content. ',
+            options: [
+                { value: 'true', label: 'true' },
+                { value: 'false', label: 'false' },
+            ],
+            default: 'false',
+        },
+    },
     features: {
-        requireConfig: false,
+        requireConfig: [
+            {
+                name: 'PIXIV_REFRESHTOKEN',
+                optional: true,
+                description: `
+Pixiv 登錄後的 refresh_token，用於獲取 R18 小說
+refresh_token after Pixiv login, required for accessing R18 novels
+[https://docs.rsshub.app/deploy/config#pixiv](https://docs.rsshub.app/deploy/config#pixiv)`,
+            },
+        ],
         requirePuppeteer: false,
         antiCrawler: false,
         supportBT: false,
@@ -18,54 +40,57 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['www.pixiv.net/users/:id/novels'],
+            title: 'User Novels (簡介 Basic info)',
+            source: ['www.pixiv.net/users/:id/novels', 'www.pixiv.net/users/:id'],
+            target: '/user/novels/:id',
+        },
+        {
+            title: 'User Novels (全文 Full text)',
+            source: ['www.pixiv.net/users/:id/novels', 'www.pixiv.net/users/:id'],
+            target: '/user/novels/:id/true',
         },
     ],
     name: 'User Novels',
-    maintainers: ['TonyRL'],
+    maintainers: ['TonyRL', 'SnowAgar25'],
     handler,
+    description: `
+| 小說類型 Novel Type | full_content | PIXIV_REFRESHTOKEN | 返回內容 Content |
+|-------------------|--------------|-------------------|-----------------|
+| Non R18           | false        | 不需要 Not Required  | 簡介 Basic info |
+| Non R18           | true         | 不需要 Not Required  | 全文 Full text  |
+| R18               | false        | 需要 Required       | 簡介 Basic info |
+| R18               | true         | 需要 Required       | 全文 Full text  |
+
+Default value for \`full_content\` is \`false\` if not specified.
+
+Example:
+- \`/pixiv/user/novels/79603797\` → 簡介 Basic info
+- \`/pixiv/user/novels/79603797/true\` → 全文 Full text`,
 };
 
-async function handler(ctx) {
+const hasPixivAuth = () => Boolean(config.pixiv && config.pixiv.refreshToken);
+
+async function handler(ctx): Promise<Data> {
     const id = ctx.req.param('id');
-    const { limit = 100 } = ctx.req.query();
-    const url = `${baseUrl}/users/${id}/novels`;
-    const { data: allData } = await got(`${baseUrl}/ajax/user/${id}/profile/all`, {
-        headers: {
-            referer: url,
-        },
-    });
+    const fullContent = fallback(undefined, queryToBoolean(ctx.req.param('full_content')), false);
+    const { limit } = ctx.req.query();
 
-    const novels = Object.keys(allData.body.novels)
-        .sort((a, b) => b - a)
-        .slice(0, Number.parseInt(limit, 10));
-    const searchParams = new URLSearchParams();
-    for (const novel of novels) {
-        searchParams.append('ids[]', novel);
+    if (hasPixivAuth()) {
+        return await getNSFWUserNovels(id, fullContent, limit);
     }
 
-    const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/novels`, {
-        headers: {
-            referer: url,
-        },
-        searchParams,
+    const nonR18Result = await getSFWUserNovels(id, fullContent, limit).catch((error) => {
+        if (error.name === 'Error') {
+            return null;
+        }
+        throw error;
     });
 
-    const items = Object.values(data.body.works).map((item) => ({
-        title: item.seriesTitle || item.title,
-        description: item.description || item.title,
-        link: `${baseUrl}/novel/series/${item.id}`,
-        author: item.userName,
-        pubDate: parseDate(item.createDate),
-        updated: parseDate(item.updateDate),
-        category: item.tags,
-    }));
+    if (nonR18Result) {
+        return nonR18Result;
+    }
 
-    return {
-        title: data.body.extraData.meta.title,
-        description: data.body.extraData.meta.ogp.description,
-        image: Object.values(data.body.works)[0].profileImageUrl,
-        link: url,
-        item: items,
-    };
+    throw new ConfigNotFoundError(
+        'This user may not have any novel works, or is an R18 creator, PIXIV_REFRESHTOKEN is required.\npixiv RSS is disabled due to the lack of relevant config.\n該用戶可能沒有小說作品，或者該用戶爲 R18 創作者，需要 PIXIV_REFRESHTOKEN。'
+    );
 }

package/lib/routes/pixiv/user.ts

@@ -50,7 +50,7 @@ async function handler(ctx) {
     return {
         title: `${username} 的 pixiv 动态`,
         link: `https://www.pixiv.net/users/${id}`,
-        image: illusts[0].user.profile_image_urls.medium,
+        image: pixivUtils.getProxiedImageUrl(illusts[0].user.profile_image_urls.medium),
         description: `${username} 的 pixiv 最新动态`,
         item: illusts.map((illust) => {
             const images = pixivUtils.getImgs(illust);

package/lib/routes/pixiv/utils.ts

@@ -14,4 +14,7 @@ export default {
         }
         return images;
     },
+    getProxiedImageUrl(originalUrl: string): string {
+        return originalUrl.replace('https://i.pximg.net', config.pixiv.imgProxy || '');
+    },
 };

package/lib/routes/pixivision/index.ts

@@ -21,9 +21,9 @@ export const route: Route = {
     },
     name: 'Category',
     maintainers: ['SnowAgar25'],
-    description: `:::tip
+    description: `::: tip
   \`https://www.pixivision.net/zh-tw/c/interview\` → \`/pixivision/zh-tw/interview\`
-  :::`,
+:::`,
     radar: [
         {
             source: ['www.pixivision.net/:lang'],

package/lib/routes/pku/bbs/hot.ts

@@ -28,9 +28,9 @@ export const route: Route = {
     maintainers: ['wooddance'],
     handler,
     url: 'bbs.pku.edu.cn/v2/hot-topic.php',
-    description: `:::warning
+    description: `::: warning
   论坛部分帖子正文内容的获取需要用户登录后的 Cookie 值，详情见部署页面的配置模块。
-  :::`,
+:::`,
 };
 
 async function handler() {

package/lib/routes/pku/hr.ts

@@ -26,11 +26,11 @@ export const route: Route = {
     maintainers: ['nczitzk'],
     handler,
     url: 'hr.pku.edu.cn/',
-    description: `:::tip
+    description: `::: tip
   分类字段处填写的是对应北京大学人事处分类页网址中介于 **\`http://hr.pku.edu.cn/\`** 和 **/index.htm** 中间的一段，并将其中的 \`/\` 修改为 \`-\`。
 
   如 [北京大学人事处 - 人才招聘 - 教师 - 教学科研人员](https://hr.pku.edu.cn/rczp/js/jxkyry/index.htm) 的网址为 \`https://hr.pku.edu.cn/rczp/js/jxkyry/index.htm\` 其中介于 **\`http://hr.pku.edu.cn/\`** 和 **\`/index.ht\`** 中间的一段为 \`rczp/js/jxkyry\`。随后，并将其中的 \`/\` 修改为 \`-\`，可以得到 \`rczp-js-jxkyry\`。所以最终我们的路由为 [\`/pku/hr/rczp-js-jxkyry\`](https://rsshub.app/pku/hr/rczp-js-jxkyry)
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/playno1/av.ts

@@ -23,7 +23,7 @@ export const route: Route = {
     name: 'AV',
     maintainers: ['TonyRL'],
     handler,
-    description: `:::warning
+    description: `::: warning
 目前观测到该博客可能禁止日本 IP 访问。建议部署在日本区以外的服务器上。
 :::
 

package/lib/routes/plurk/anonymous.ts

@@ -5,7 +5,7 @@ import { baseUrl, getPlurk } from './utils';
 
 export const route: Route = {
     path: '/anonymous',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/plurk/anonymous',
     parameters: {},
     features: {

package/lib/routes/plurk/hotlinks.ts

@@ -5,7 +5,7 @@ import { baseUrl, getPlurk } from './utils';
 
 export const route: Route = {
     path: '/hotlinks',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/plurk/hotlinks',
     parameters: {},
     features: {

package/lib/routes/plurk/news.ts

@@ -5,7 +5,7 @@ import { baseUrl, fetchFriends, getPlurk } from './utils';
 
 export const route: Route = {
     path: '/news/:lang?',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/plurk/news/:lang?',
     parameters: { lang: 'Language, see the table above, `en` by default' },
     features: {

package/lib/routes/plurk/search.ts

@@ -6,7 +6,7 @@ import { baseUrl, getPlurk } from './utils';
 
 export const route: Route = {
     path: '/search/:keyword',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/plurk/search/FGO',
     parameters: { keyword: 'Search keyword' },
     features: {

package/lib/routes/psyche/topic.ts

@@ -5,7 +5,7 @@ import { getData } from './utils';
 
 export const route: Route = {
     path: '/topic/:topic',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/psyche/topic/therapeia',
     parameters: { topic: 'Topic' },
     radar: [

package/lib/routes/qbitai/category.ts

@@ -1,11 +1,13 @@
 import { Route } from '@/types';
 import parser from '@/utils/rss-parser';
-
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
+import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/category/:category',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/qbitai/category/资讯',
     parameters: { category: '分类名，见下表' },
     features: {
@@ -22,7 +24,7 @@ export const route: Route = {
         },
     ],
     name: '分类',
-    maintainers: ['FuryMartin'],
+    maintainers: ['FuryMartin, Geraldxm'],
     handler,
     description: `| 资讯 | 数码     | 智能车 | 智库  | 活动    |
 | ---- | -------- | ------ | ----- | ------- |
@@ -31,24 +33,36 @@ export const route: Route = {
 
 async function handler(ctx) {
     const category = ctx.req.param('category');
-    const link = encodeURI(`https://www.qbitai.com/category/${category}/feed`);
-    const feed = await parser.parseURL(link);
+    const url = encodeURI(`https://www.qbitai.com/category/${category}/feed`);
 
-    const items = feed.items.map((item) => ({
+    const feed = await parser.parseURL(url);
+    const entries = feed.items.map((item) => ({
         title: item.title,
         pubDate: parseDate(item.pubDate),
         link: item.link,
         author: '量子位',
         category: item.categories,
-        description: item['content:encoded'],
+        description: '', // Initialize description field
     }));
 
+    const resolvedEntries = await Promise.all(
+        entries.map((entry) =>
+            cache.tryGet(entry.link, async () => {
+                try {
+                    const response = await ofetch(entry.link);
+                    const $ = load(response);
+                    entry.description = $('.article').html() || 'No content found';
+                } catch {
+                    entry.description = 'Failed to fetch content';
+                }
+                return entry;
+            })
+        )
+    );
+
     return {
-        // 源标题
-        title: `量子位-${category}`,
-        // 源链接
+        title: `量子位 - ${category}`,
         link: `https://www.qbitai.com/category/${category}`,
-        // 源文章
-        item: items,
+        item: resolvedEntries,
     };
 }