rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa1dcc5

package/lib/routes/pixiv/novel-api/user-novels/nsfw.ts

@@ -0,0 +1,82 @@
+import got from '../../pixiv-got';
+import { maskHeader } from '../../constants';
+import queryString from 'query-string';
+import { config } from '@/config';
+import pixivUtils from '../../utils';
+import { getNSFWNovelContent } from '../content/nsfw';
+import { parseDate } from '@/utils/parse-date';
+import { convertPixivProtocolExtended } from '../content/utils';
+import type { NSFWNovelsResponse, NovelList } from './types';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import cache from '@/utils/cache';
+import { getToken } from '../../token';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+
+function getNovels(user_id: string, token: string): Promise<NSFWNovelsResponse> {
+    return got('https://app-api.pixiv.net/v1/user/novels', {
+        headers: {
+            ...maskHeader,
+            Authorization: 'Bearer ' + token,
+        },
+        searchParams: queryString.stringify({
+            user_id,
+            filter: 'for_ios',
+        }),
+    });
+}
+
+export async function getNSFWUserNovels(id: string, fullContent: boolean = false, limit: number = 100): Promise<NovelList> {
+    if (!config.pixiv || !config.pixiv.refreshToken) {
+        throw new ConfigNotFoundError('This user is an R18 creator, PIXIV_REFRESHTOKEN is required.\npixiv RSS is disabled due to the lack of relevant config.\n該用戶爲 R18 創作者，需要 PIXIV_REFRESHTOKEN。');
+    }
+
+    const token = await getToken(cache.tryGet);
+    if (!token) {
+        throw new ConfigNotFoundError('pixiv not login');
+    }
+
+    const response = await getNovels(id, token);
+    const novels = limit ? response.data.novels.slice(0, limit) : response.data.novels;
+
+    if (novels.length === 0) {
+        throw new InvalidParameterError(`${id} is not a valid user ID, or the user has no novels.\n${id} 不是有效的用戶 ID，或者該用戶沒有小說作品。`);
+    }
+
+    const username = novels[0].user.name;
+
+    const items = await Promise.all(
+        novels.map(async (novel) => {
+            const baseItem = {
+                title: novel.series?.title ? `${novel.series.title} - ${novel.title}` : novel.title,
+                description: `
+                    <img src="${pixivUtils.getProxiedImageUrl(novel.image_urls.large)}" />
+                    <div>
+                    <p>${convertPixivProtocolExtended(novel.caption)}</p>
+                    </div>`,
+                author: novel.user.name,
+                pubDate: parseDate(novel.create_date),
+                link: `https://www.pixiv.net/novel/show.php?id=${novel.id}`,
+                category: novel.tags.map((t) => t.name),
+            };
+
+            if (!fullContent) {
+                return baseItem;
+            }
+
+            const { content } = await getNSFWNovelContent(novel.id, token);
+
+            return {
+                ...baseItem,
+                description: `${baseItem.description}<hr>${content}`,
+            };
+        })
+    );
+
+    return {
+        title: `${username}'s novels - pixiv`,
+        description: `${username} 的 pixiv 最新小说`,
+        image: pixivUtils.getProxiedImageUrl(novels[0].user.profile_image_urls.medium),
+        link: `https://www.pixiv.net/users/${id}/novels`,
+        item: items,
+    };
+}

package/lib/routes/pixiv/novel-api/user-novels/sfw.ts

@@ -0,0 +1,74 @@
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import pixivUtils from '../../utils';
+import { getSFWNovelContent } from '../content/sfw';
+import type { SFWNovelsResponse, NovelList } from './types';
+
+const baseUrl = 'https://www.pixiv.net';
+
+export async function getSFWUserNovels(id: string, fullContent: boolean = false, limit: number = 100): Promise<NovelList> {
+    const url = `${baseUrl}/users/${id}/novels`;
+    const { data: allData } = await got(`${baseUrl}/ajax/user/${id}/profile/all`, {
+        headers: {
+            referer: url,
+        },
+    });
+
+    const novels = Object.keys(allData.body.novels)
+        .sort((a, b) => Number(b) - Number(a))
+        .slice(0, Number.parseInt(String(limit), 10));
+
+    if (novels.length === 0) {
+        throw new Error('No novels found for this user, or is an R18 creator, fallback to ConfigNotFoundError');
+    }
+
+    const searchParams = new URLSearchParams();
+    for (const novel of novels) {
+        searchParams.append('ids[]', novel);
+    }
+
+    const { data } = (await got(`${baseUrl}/ajax/user/${id}/profile/novels`, {
+        headers: {
+            referer: url,
+        },
+        searchParams,
+    })) as SFWNovelsResponse;
+
+    const items = await Promise.all(
+        Object.values(data.body.works).map(async (item) => {
+            const baseItem = {
+                title: item.title,
+                description: `
+                    <img src=${pixivUtils.getProxiedImageUrl(item.url)} />
+                    <div>
+                    <p>${item.description}</p>
+                    </div>
+                `,
+                link: `${baseUrl}/novel/show.php?id=${item.id}`,
+                author: item.userName,
+                pubDate: parseDate(item.createDate),
+                updated: parseDate(item.updateDate),
+                category: item.tags,
+            };
+
+            if (!fullContent) {
+                return baseItem;
+            }
+
+            const { content } = await getSFWNovelContent(item.id);
+
+            return {
+                ...baseItem,
+                description: `${baseItem.description}<hr>${content}`,
+            };
+        })
+    );
+
+    return {
+        title: data.body.extraData.meta.title,
+        description: data.body.extraData.meta.ogp.description,
+        image: pixivUtils.getProxiedImageUrl(Object.values(data.body.works)[0].profileImageUrl),
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/pixiv/novel-api/user-novels/types.ts

@@ -0,0 +1,133 @@
+export interface SFWNovelsResponse {
+    data: {
+        error: boolean;
+        message: string;
+        body: {
+            works: Record<string, SFWNovelWork>;
+            extraData: {
+                meta: {
+                    title: string;
+                    description: string;
+                    canonical: string;
+                    ogp: {
+                        description: string;
+                        image: string;
+                        title: string;
+                        type: string;
+                    };
+                    twitter: {
+                        description: string;
+                        image: string;
+                        title: string;
+                        card: string;
+                    };
+                    alternateLanguages: {
+                        ja: string;
+                        en: string;
+                    };
+                    descriptionHeader: string;
+                };
+            };
+        };
+    };
+}
+
+export interface SFWNovelWork {
+    id: string;
+    title: string;
+    genre: string;
+    xRestrict: number;
+    restrict: number;
+    url: string;
+    tags: string[];
+    userId: string;
+    userName: string;
+    profileImageUrl: string;
+    textCount: number;
+    wordCount: number;
+    readingTime: number;
+    useWordCount: boolean;
+    description: string;
+    isBookmarkable: boolean;
+    bookmarkData: null;
+    bookmarkCount: number;
+    isOriginal: boolean;
+    marker: null;
+    titleCaptionTranslation: {
+        workTitle: null;
+        workCaption: null;
+    };
+    createDate: string;
+    updateDate: string;
+    isMasked: boolean;
+    aiType: number;
+    seriesId: string;
+    seriesTitle: string;
+    isUnlisted: boolean;
+}
+
+export interface NSFWNovelsResponse {
+    data: {
+        user: {
+            id: number;
+            name: string;
+            account: string;
+            profile_image_urls: {
+                medium: string;
+            };
+            is_followed: boolean;
+            is_access_blocking_user: boolean;
+        };
+        novels: NSFWNovelWork[];
+    };
+}
+
+export interface NSFWNovelWork {
+    id: string;
+    title: string;
+    caption: string;
+    restrict: number;
+    x_restrict: number;
+    image_urls: {
+        square_medium: string;
+        medium: string;
+        large: string;
+    };
+    create_date: string;
+    tags: Array<{
+        name: string;
+        translated_name: string | null;
+        added_by_uploaded_user: boolean;
+    }>;
+    text_length: number;
+    user: {
+        id: number;
+        name: string;
+        account: string;
+        profile_image_urls: {
+            medium: string;
+        };
+    };
+    series?: {
+        id?: number;
+        title?: string;
+    };
+    total_bookmarks: number;
+    total_view: number;
+    total_comments: number;
+}
+
+export interface NovelList {
+    title: string;
+    description: string;
+    image: string;
+    link: string;
+    item: Array<{
+        title: string;
+        description: string;
+        author: string;
+        pubDate: Date;
+        link: string;
+        category: string[];
+    }>;
+}

package/lib/routes/pixiv/novel-series.ts

@@ -0,0 +1,52 @@
+import { Data, Route } from '@/types';
+import { config } from '@/config';
+import { getNSFWSeriesNovels } from './novel-api/series/nsfw';
+import { getSFWSeriesNovels } from './novel-api/series/sfw';
+
+export const route: Route = {
+    path: '/novel/series/:id',
+    categories: ['social-media'],
+    example: '/pixiv/novel/series/11586857',
+    parameters: {
+        id: 'Series id, can be found in URL',
+    },
+    features: {
+        requireConfig: [
+            {
+                name: 'PIXIV_REFRESHTOKEN',
+                optional: true,
+                description: `
+refresh_token after Pixiv login, required for accessing R18 novels
+Pixiv 登錄後的 refresh_token，用於獲取 R18 小說
+[https://docs.rsshub.app/deploy/config#pixiv](https://docs.rsshub.app/deploy/config#pixiv)`,
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Novel Series',
+    maintainers: ['SnowAgar25', 'keocheung'],
+    handler,
+    radar: [
+        {
+            source: ['www.pixiv.net/novel/series/:id'],
+            target: '/novel/series/:id',
+        },
+    ],
+};
+
+const hasPixivAuth = () => Boolean(config.pixiv && config.pixiv.refreshToken);
+
+async function handler(ctx): Promise<Data> {
+    const id = ctx.req.param('id');
+    const { limit } = ctx.req.query();
+
+    if (hasPixivAuth()) {
+        return await getNSFWSeriesNovels(id, limit);
+    }
+
+    return await getSFWSeriesNovels(id, limit);
+}

package/lib/routes/pixiv/novels.ts

@@ -1,15 +1,37 @@
-import { Route } from '@/types';
-import got from '@/utils/got';
-import { parseDate } from '@/utils/parse-date';
-const baseUrl = 'https://www.pixiv.net';
+import { Data, Route, ViewType } from '@/types';
+import { fallback, queryToBoolean } from '@/utils/readable-social';
+import { config } from '@/config';
+import { getNSFWUserNovels } from './novel-api/user-novels/nsfw';
+import { getSFWUserNovels } from './novel-api/user-novels/sfw';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 export const route: Route = {
-    path: '/user/novels/:id',
+    path: '/user/novels/:id/:full_content?',
     categories: ['social-media'],
+    view: ViewType.Articles,
     example: '/pixiv/user/novels/27104704',
-    parameters: { id: "User id, available in user's homepage URL" },
+    parameters: {
+        id: "User id, available in user's homepage URL",
+        full_content: {
+            description: 'Enable or disable the display of full content. ',
+            options: [
+                { value: 'true', label: 'true' },
+                { value: 'false', label: 'false' },
+            ],
+            default: 'false',
+        },
+    },
     features: {
-        requireConfig: false,
+        requireConfig: [
+            {
+                name: 'PIXIV_REFRESHTOKEN',
+                optional: true,
+                description: `
+Pixiv 登錄後的 refresh_token，用於獲取 R18 小說
+refresh_token after Pixiv login, required for accessing R18 novels
+[https://docs.rsshub.app/deploy/config#pixiv](https://docs.rsshub.app/deploy/config#pixiv)`,
+            },
+        ],
         requirePuppeteer: false,
         antiCrawler: false,
         supportBT: false,
@@ -18,54 +40,57 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['www.pixiv.net/users/:id/novels'],
+            title: 'User Novels (簡介 Basic info)',
+            source: ['www.pixiv.net/users/:id/novels', 'www.pixiv.net/users/:id'],
+            target: '/user/novels/:id',
+        },
+        {
+            title: 'User Novels (全文 Full text)',
+            source: ['www.pixiv.net/users/:id/novels', 'www.pixiv.net/users/:id'],
+            target: '/user/novels/:id/true',
         },
     ],
     name: 'User Novels',
-    maintainers: ['TonyRL'],
+    maintainers: ['TonyRL', 'SnowAgar25'],
     handler,
+    description: `
+| 小說類型 Novel Type | full_content | PIXIV_REFRESHTOKEN | 返回內容 Content |
+|-------------------|--------------|-------------------|-----------------|
+| Non R18           | false        | 不需要 Not Required  | 簡介 Basic info |
+| Non R18           | true         | 不需要 Not Required  | 全文 Full text  |
+| R18               | false        | 需要 Required       | 簡介 Basic info |
+| R18               | true         | 需要 Required       | 全文 Full text  |
+
+Default value for \`full_content\` is \`false\` if not specified.
+
+Example:
+- \`/pixiv/user/novels/79603797\` → 簡介 Basic info
+- \`/pixiv/user/novels/79603797/true\` → 全文 Full text`,
 };
 
-async function handler(ctx) {
+const hasPixivAuth = () => Boolean(config.pixiv && config.pixiv.refreshToken);
+
+async function handler(ctx): Promise<Data> {
     const id = ctx.req.param('id');
-    const { limit = 100 } = ctx.req.query();
-    const url = `${baseUrl}/users/${id}/novels`;
-    const { data: allData } = await got(`${baseUrl}/ajax/user/${id}/profile/all`, {
-        headers: {
-            referer: url,
-        },
-    });
+    const fullContent = fallback(undefined, queryToBoolean(ctx.req.param('full_content')), false);
+    const { limit } = ctx.req.query();
 
-    const novels = Object.keys(allData.body.novels)
-        .sort((a, b) => b - a)
-        .slice(0, Number.parseInt(limit, 10));
-    const searchParams = new URLSearchParams();
-    for (const novel of novels) {
-        searchParams.append('ids[]', novel);
+    if (hasPixivAuth()) {
+        return await getNSFWUserNovels(id, fullContent, limit);
     }
 
-    const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/novels`, {
-        headers: {
-            referer: url,
-        },
-        searchParams,
+    const nonR18Result = await getSFWUserNovels(id, fullContent, limit).catch((error) => {
+        if (error.name === 'Error') {
+            return null;
+        }
+        throw error;
     });
 
-    const items = Object.values(data.body.works).map((item) => ({
-        title: item.seriesTitle || item.title,
-        description: item.description || item.title,
-        link: `${baseUrl}/novel/series/${item.id}`,
-        author: item.userName,
-        pubDate: parseDate(item.createDate),
-        updated: parseDate(item.updateDate),
-        category: item.tags,
-    }));
+    if (nonR18Result) {
+        return nonR18Result;
+    }
 
-    return {
-        title: data.body.extraData.meta.title,
-        description: data.body.extraData.meta.ogp.description,
-        image: Object.values(data.body.works)[0].profileImageUrl,
-        link: url,
-        item: items,
-    };
+    throw new ConfigNotFoundError(
+        'This user may not have any novel works, or is an R18 creator, PIXIV_REFRESHTOKEN is required.\npixiv RSS is disabled due to the lack of relevant config.\n該用戶可能沒有小說作品，或者該用戶爲 R18 創作者，需要 PIXIV_REFRESHTOKEN。'
+    );
 }

package/lib/routes/pixiv/utils.ts

@@ -14,4 +14,7 @@ export default {
         }
         return images;
     },
+    getProxiedImageUrl(originalUrl: string): string {
+        return originalUrl.replace('https://i.pximg.net', config.pixiv.imgProxy || '');
+    },
 };

package/lib/routes/plurk/anonymous.ts

@@ -5,7 +5,7 @@ import { baseUrl, getPlurk } from './utils';
 
 export const route: Route = {
     path: '/anonymous',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/plurk/anonymous',
     parameters: {},
     features: {

package/lib/routes/plurk/hotlinks.ts

@@ -5,7 +5,7 @@ import { baseUrl, getPlurk } from './utils';
 
 export const route: Route = {
     path: '/hotlinks',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/plurk/hotlinks',
     parameters: {},
     features: {

package/lib/routes/plurk/news.ts

@@ -5,7 +5,7 @@ import { baseUrl, fetchFriends, getPlurk } from './utils';
 
 export const route: Route = {
     path: '/news/:lang?',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/plurk/news/:lang?',
     parameters: { lang: 'Language, see the table above, `en` by default' },
     features: {

package/lib/routes/plurk/search.ts

@@ -6,7 +6,7 @@ import { baseUrl, getPlurk } from './utils';
 
 export const route: Route = {
     path: '/search/:keyword',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/plurk/search/FGO',
     parameters: { keyword: 'Search keyword' },
     features: {

package/lib/routes/qstheory/index.ts

@@ -0,0 +1,121 @@
+import { Route } from '@/types';
+
+import ofetch from '@/utils/ofetch';
+import * as cheerio from 'cheerio';
+import cache from '@/utils/cache';
+import { baseUrl as rootUrl, getItem } from './utils';
+
+const config = {
+    toutiao: {
+        title: '头条',
+        url: `${rootUrl}/v9zhuanqu/toutiao/index.htm`,
+    },
+    qswp: {
+        title: '网评',
+        url: `${rootUrl}/qswp.htm`,
+    },
+    qssp: {
+        title: '视频',
+        url: `${rootUrl}/qssp/index.htm`,
+    },
+    qslgxd: {
+        title: '原创',
+        url: `${rootUrl}/qslgxd/index.htm`,
+    },
+    economy: {
+        title: '经济',
+        url: `${rootUrl}/economy/index.htm`,
+    },
+    politics: {
+        title: '政治',
+        url: `${rootUrl}/politics/index.htm`,
+    },
+    culture: {
+        title: '文化',
+        url: `${rootUrl}/culture/index.htm`,
+    },
+    society: {
+        title: '社会',
+        url: `${rootUrl}/society/index.htm`,
+    },
+    cpc: {
+        title: '党建',
+        url: `${rootUrl}/cpc/index.htm`,
+    },
+    science: {
+        title: '科教',
+        url: `${rootUrl}/science/index.htm`,
+    },
+    zoology: {
+        title: '生态',
+        url: `${rootUrl}/zoology/index.htm`,
+    },
+    defense: {
+        title: '国防',
+        url: `${rootUrl}/defense/index.htm`,
+    },
+    international: {
+        title: '国际',
+        url: `${rootUrl}/international/index.htm`,
+    },
+    books: {
+        title: '图书',
+        url: `${rootUrl}/books/index.htm`,
+    },
+    xxbj: {
+        title: '学习笔记',
+        url: `${rootUrl}/qszq/xxbj/index.htm`,
+    },
+    llwx: {
+        title: '理论文选',
+        url: `${rootUrl}/qszq/llwx/index.htm`,
+    },
+};
+
+export const route: Route = {
+    path: '/:category?',
+    categories: ['traditional-media'],
+    example: '/qstheory',
+    parameters: { industry: '分类，见下表' },
+    radar: [
+        {
+            source: ['www.qstheory.cn/v9zhuanqu/:category/index.htm', 'www.qstheory.cn/qszq/:category/index.htm', 'www.qstheory.cn/:category/index.htm'],
+        },
+    ],
+    name: '分类',
+    maintainers: ['nczitzk'],
+    handler,
+    description: `
+| 头条    | 网评 | 视频 | 原创   | 经济    | 政治     | 文化    | 社会    | 党建 | 科教    | 生态    | 国防    | 国际          | 图书  | 学习笔记 | 理论文选 |
+| ------- | ---- | ---- | ------ | ------- | -------- | ------- | ------- | ---- | ------- | ------- | ------- | ------------- | ----- | -------- | -------- |
+| toutiao | qswp | qssp | qslgxd | economy | politics | culture | society | cpc  | science | zoology | defense | international | books | xxbj     | llwx     |`,
+};
+
+async function handler(ctx) {
+    const { category = 'toutiao' } = ctx.req.param();
+    const limit = Number.parseInt(ctx.req.query('limit')) || 50;
+
+    const currentUrl = config[category].url;
+    const response = await ofetch(currentUrl);
+
+    const $ = cheerio.load(response);
+
+    const list = $('.list-style1 ul li a, .text h2 a, .no-pic ul li a')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            const $item = $(item);
+            return {
+                title: $item.text(),
+                link: $item.attr('href')!,
+            };
+        });
+
+    const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => getItem(item))));
+
+    return {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+}