rsshub 1.0.0-master.ff3a9ce → 1.0.0-master.ff3ea5a

package/lib/routes/paulgraham/article.ts

@@ -23,7 +23,7 @@ export const route: Route = {
         },
     ],
     name: 'Essays',
-    maintainers: ['Maecenas', 'nczitzk'],
+    maintainers: ['Maecenas', 'nczitzk', 'dvorak0'],
     handler,
     url: 'paulgraham.com/articles.html',
 };
@@ -31,7 +31,7 @@ export const route: Route = {
 async function handler(ctx) {
     const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 30;
 
-    const rootUrl = 'http://paulgraham.com';
+    const rootUrl = 'https://paulgraham.com';
     const currentUrl = new URL('articles.html', rootUrl).href;
 
     const { data: response } = await got(currentUrl);

package/lib/routes/picuki/profile.ts

@@ -104,6 +104,8 @@ async function handler(ctx) {
             throw new Error($('.error-p span').text().trim() || 'Profile not found');
         }
 
+        const username = $('.profile-info .username').text().trim();
+
         const items = $('.posts-video .posts__video-item')
             .toArray()
             .map((item) => {
@@ -112,6 +114,7 @@ async function handler(ctx) {
                 const img = $item.find('img');
                 return {
                     title: img.attr('alt') || '',
+                    author: username,
                     renderData: {
                         poster: img.attr('src'),
                         source: $item.find('.popup-open').data('source'),
@@ -134,6 +137,7 @@ async function handler(ctx) {
         image: string;
         items: {
             title: string;
+            author: string;
             renderData: {
                 poster: string;
                 source: string;

package/lib/routes/pixiv/bookmarks.ts

@@ -23,7 +23,7 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['www.pixiv.net/users/:id/bookmarks/artworks'],
+            source: ['www.pixiv.net/users/:id/bookmarks/artworks', 'www.pixiv.net/en/users/:id/bookmarks/artworks'],
         },
     ],
     name: 'User Bookmark',

package/lib/routes/pixiv/novel-api/series/sfw.ts

@@ -57,7 +57,7 @@ export async function getSFWSeriesNovels(seriesId: string, limit: number = 10):
                     category: novelContent.tags,
                 };
             })
-            .reverse()
+            .toReversed()
     );
 
     return {

package/lib/routes/pixiv/novels.ts

@@ -41,12 +41,12 @@ refresh_token after Pixiv login, required for accessing R18 novels
     radar: [
         {
             title: 'User Novels (簡介 Basic info)',
-            source: ['www.pixiv.net/users/:id/novels', 'www.pixiv.net/users/:id'],
+            source: ['www.pixiv.net/users/:id/novels', 'www.pixiv.net/users/:id', 'www.pixiv.net/en/users/:id/novels', 'www.pixiv.net/en/users/:id'],
             target: '/user/novels/:id',
         },
         {
             title: 'User Novels (全文 Full text)',
-            source: ['www.pixiv.net/users/:id/novels', 'www.pixiv.net/users/:id'],
+            source: ['www.pixiv.net/users/:id/novels', 'www.pixiv.net/users/:id', 'www.pixiv.net/en/users/:id/novels', 'www.pixiv.net/en/users/:id'],
             target: '/user/novels/:id/true',
         },
     ],

package/lib/routes/pixiv/user.ts

@@ -23,7 +23,7 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['www.pixiv.net/users/:id'],
+            source: ['www.pixiv.net/users/:id', 'www.pixiv.net/en/users/:id'],
         },
     ],
     name: 'User Activity',

package/lib/routes/playno1/av.ts

@@ -19,6 +19,7 @@ export const route: Route = {
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
+        nsfw: true,
     },
     name: 'AV',
     maintainers: ['TonyRL'],

package/lib/routes/pornhub/model.ts

@@ -2,7 +2,7 @@ import { Route, ViewType, Data } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { isValidHost } from '@/utils/valid-host';
-import { headers, parseItems } from './utils';
+import { headers, parseItems, getRadarDomin } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
@@ -18,13 +18,9 @@ export const route: Route = {
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
+        nsfw: true,
     },
-    radar: [
-        {
-            source: ['pornhub.com/model/:username/*'],
-            target: '/model/:username',
-        },
-    ],
+    radar: getRadarDomin('/model/:username'),
     name: 'Model',
     maintainers: ['I2IMk', 'queensferryme'],
     handler,

package/lib/routes/pornhub/pornstar.ts

@@ -2,7 +2,7 @@ import { Route, ViewType, Data } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { isValidHost } from '@/utils/valid-host';
-import { headers, parseItems } from './utils';
+import { headers, parseItems, getRadarDomin } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
@@ -62,12 +62,7 @@ export const route: Route = {
         supportPodcast: false,
         supportScihub: false,
     },
-    radar: [
-        {
-            source: ['pornhub.com/pornstar/:username/*'],
-            target: '/pornstar/:username',
-        },
-    ],
+    radar: getRadarDomin('/pornstar/:username'),
     name: 'Pornstar',
     maintainers: ['I2IMk', 'queensferryme'],
     handler,

package/lib/routes/pornhub/users.ts

@@ -2,7 +2,7 @@ import { Route, Data } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { isValidHost } from '@/utils/valid-host';
-import { headers, parseItems } from './utils';
+import { headers, parseItems, getRadarDomin } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
@@ -18,12 +18,7 @@ export const route: Route = {
         supportPodcast: false,
         supportScihub: false,
     },
-    radar: [
-        {
-            source: ['pornhub.com/users/:username/*'],
-            target: '/users/:username',
-        },
-    ],
+    radar: getRadarDomin('/users/:username'),
     name: 'Users',
     maintainers: ['I2IMk', 'queensferryme'],
     handler,

package/lib/routes/pornhub/utils.ts

@@ -27,4 +27,15 @@ const parseItems = (e) => ({
     pubDate: dayjs(extractDateFromImageUrl(e.find('img').data('mediumthumb'))).toDate() || parseRelativeDate(e.find('.added').text()),
 });
 
-export { defaultDomain, headers, renderDescription, parseItems };
+const getRadarDomin = (path: string) => [
+    {
+        source: [`www.pornhub.com${path}`, `www.pornhub.com${path}/*`],
+        target: path,
+    },
+    ...['de', 'fr', 'es', 'it', 'pt', 'pl', 'rt', 'jp', 'nl', 'cz', 'cn'].map((language) => ({
+        source: [`${language}.pornhub.com${path}`, `${language}.pornhub.com${path}/*`],
+        target: `${path}/${language}`,
+    })),
+];
+
+export { defaultDomain, headers, renderDescription, parseItems, getRadarDomin };

package/lib/routes/qingting/podcast.ts

@@ -42,6 +42,7 @@ function getMediaUrl(channelId: string, mediaId: string) {
 
 async function handler(ctx) {
     const channelId = ctx.req.param('id');
+    const pageSize = Number(ctx.req.query('limit')) || 30;
 
     const channelUrl = `https://i.qingting.fm/capi/v3/channel/${channelId}`;
     const response = await ofetch(channelUrl, {
@@ -54,7 +55,7 @@ async function handler(ctx) {
     const channel_img = response.data.thumbs['400_thumb'];
     const authors = response.data.podcasters.map((author) => author.nick_name).join(',');
     const desc = response.data.description;
-    const programUrl = `https://i.qingting.fm/capi/channel/${channelId}/programs/${response.data.v}?curpage=1&pagesize=10&order=asc`;
+    const programUrl = `https://i.qingting.fm/capi/channel/${channelId}/programs/${response.data.v}?curpage=1&pagesize=${pageSize}&order=asc`;
 
     const {
         data: { programs },

package/lib/routes/qipamaijia/index.ts

@@ -16,6 +16,7 @@ export const route: Route = {
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
+        nsfw: true,
     },
     radar: [
         {

package/lib/routes/questmobile/report.ts

@@ -173,7 +173,7 @@ async function handler(ctx) {
     const labels = parseTree(labelTree);
 
     const industryObj = industry ? industries.find((i) => i.key === industry || i.value === industry) : undefined;
-    const labelObj = label ? labels.find((i) => i.key === label || i.value === label) : (industryObj ? undefined : labels.find((i) => i.key === industry || i.value === industry));
+    const labelObj = label ? labels.find((i) => i.key === label || i.value === label) : industryObj ? undefined : labels.find((i) => i.key === industry || i.value === industry);
 
     const industryId = industryObj?.key ?? -1;
     const labelId = labelObj?.key ?? -1;

package/lib/routes/railway/index.ts

@@ -0,0 +1,73 @@
+import { Route, DataItem } from '@/types';
+
+import cache from '@/utils/cache';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import { Context } from 'hono';
+
+export const route: Route = {
+    path: '/blog',
+    categories: ['blog'],
+    example: '/railway/blog',
+    url: 'blog.railway.com',
+    name: 'Blog',
+    maintainers: ['jihuayu'],
+    handler,
+};
+
+async function handler(ctx: Context) {
+    const limit = Number.parseInt(ctx.req.query('limit') || '10');
+
+    const items = await fetchArticles(limit);
+
+    return {
+        title: 'Railway articles',
+        link: 'https://blog.railway.com',
+        item: items,
+    };
+}
+
+export const fetchArticles = async (limit: number): Promise<DataItem[]> => {
+    const page = await ofetch('https://blog.railway.com/rss.xml', {
+        responseType: 'text',
+    });
+
+    const $ = load(page, { xml: true });
+
+    return Promise.all(
+        $('item')
+            .toArray()
+            .slice(0, limit)
+            .map<Promise<DataItem>>((element) => {
+                const id = $(element).find('guid').text();
+
+                return cache.tryGet(`railway:blogs:${id}`, async () => {
+                    const title = $(element).find('title').text();
+                    const pubDate = $(element).find('pubDate').text();
+                    const link = $(element).find('link').text();
+
+                    const { content } = await fetchArticleDetails(link);
+
+                    return {
+                        guid: id,
+                        title,
+                        link,
+                        pubDate: parseDate(pubDate),
+                        description: content,
+                    } as DataItem;
+                }) as Promise<DataItem>;
+            })
+    );
+};
+
+export const fetchArticleDetails = async (url: string) => {
+    const page = await ofetch(url);
+    const $ = load(page);
+
+    const $article = $('article > section ');
+
+    return {
+        content: $article.html() ?? undefined,
+    };
+};

package/lib/routes/railway/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Railway',
+    url: 'railway.com',
+    description: '',
+    lang: 'en',
+};

package/lib/routes/reuters/common.ts

@@ -4,7 +4,6 @@ import cache from '@/utils/cache';
 import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
-import randUserAgent from '@/utils/rand-user-agent';
 import { art } from '@/utils/render';
 import path from 'node:path';
 
@@ -90,9 +89,7 @@ async function handler(ctx) {
 
     const section_id = `/${category}/${topic ? `${topic}/` : ''}`;
 
-    const ua = randUserAgent({ browser: 'chrome', os: 'windows', device: 'desktop' });
     const browserHeaders = {
-        'User-Agent': ua,
         Accept: 'application/json, text/plain, */*',
         'Accept-Language': 'en-US,en;q=0.9',
         Referer: 'https://www.reuters.com/',

package/lib/routes/rockthejvm/articles.ts

@@ -0,0 +1,167 @@
+import { type Data, type DataItem, type Route, ViewType } from '@/types';
+
+import { art } from '@/utils/render';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+
+import { type CheerioAPI, type Cheerio, load } from 'cheerio';
+import type { Element } from 'domhandler';
+import { type Context } from 'hono';
+import path from 'node:path';
+
+export const handler = async (ctx: Context): Promise<Data> => {
+    const limit: number = Number.parseInt(ctx.req.query('limit') ?? '30', 10);
+
+    const baseUrl: string = 'https://rockthejvm.com';
+    const targetUrl: string = new URL('articles/1', baseUrl).href;
+
+    const response = await ofetch(targetUrl);
+    const $: CheerioAPI = load(response);
+    const language = $('html').attr('lang') ?? 'en';
+
+    $('footer').remove();
+
+    let items: DataItem[] = [];
+
+    items = $('h2')
+        .slice(0, limit)
+        .toArray()
+        .map((el): Element => {
+            const $el: Cheerio<Element> = $(el).parent().parent();
+            const $aEl: Cheerio<Element> = $el.find('h2 a');
+
+            const title: string = $aEl.text();
+            const description: string | undefined = art(path.join(__dirname, 'templates/description.art'), {
+                intro: $el.find('p.text-content').first().text(),
+            });
+            const pubDateStr: string | undefined = $el.find('time').attr('datetime');
+            const linkUrl: string | undefined = $aEl.attr('href');
+            const categoryEls: Element[] = $el.find('a.bg-tag').toArray();
+            const categories: string[] = [...new Set(categoryEls.map((el) => $(el).text()).filter(Boolean))];
+            const authorEls: Element[] = $el.find('picture').toArray();
+            const authors: DataItem['author'] = authorEls.map((authorEl) => {
+                const $authorEl: Cheerio<Element> = $(authorEl).parent();
+
+                return {
+                    name: $authorEl.find('p.text-content').text(),
+                    url: $authorEl.find('p.text-content a').attr('href') ? new URL($authorEl.find('p.text-content a').attr('href') as string, baseUrl).href : undefined,
+                    avatar: $authorEl.find('img').attr('src') ? new URL($authorEl.find('img').attr('src') as string, baseUrl).href : undefined,
+                };
+            });
+            const upDatedStr: string | undefined = pubDateStr;
+
+            const processedItem: DataItem = {
+                title,
+                description,
+                pubDate: pubDateStr ? parseDate(pubDateStr) : undefined,
+                link: linkUrl ? new URL(linkUrl, baseUrl).href : undefined,
+                category: categories,
+                author: authors,
+                content: {
+                    html: description,
+                    text: description,
+                },
+                updated: upDatedStr ? parseDate(upDatedStr) : undefined,
+                language,
+            };
+
+            return processedItem;
+        });
+
+    items = await Promise.all(
+        items.map((item) => {
+            if (!item.link) {
+                return item;
+            }
+
+            return cache.tryGet(item.link, async (): Promise<DataItem> => {
+                const detailResponse = await ofetch(item.link);
+                const $$: CheerioAPI = load(detailResponse);
+
+                const title: string = $$('meta[property="og:title"]').attr('content') ?? item.title;
+                const description: string | undefined =
+                    item.description +
+                    art(path.join(__dirname, 'templates/description.art'), {
+                        description: $$('div.prose').html(),
+                    });
+                const pubDateStr: string | undefined = $$('meta[property="article:published_time"]').attr('content');
+                const categoryEls: Element[] = $$('meta[property="article:tag"]').toArray();
+                const categories: string[] = [...new Set(categoryEls.map((el) => $$(el).attr('content')).filter(Boolean))];
+                const authorEls: Element[] = $$('meta[property="article:author"]').toArray();
+                const authors: DataItem['author'] = authorEls.map((authorEl) => {
+                    const $$authorEl: Cheerio<Element> = $$(authorEl);
+
+                    return {
+                        name: $$authorEl.attr('content'),
+                        url: undefined,
+                        avatar: undefined,
+                    };
+                });
+                const image: string | undefined = $$('meta[property="og:image"]').attr('content');
+                const upDatedStr: string | undefined = pubDateStr;
+
+                const processedItem: DataItem = {
+                    title,
+                    description,
+                    pubDate: pubDateStr ? parseDate(pubDateStr) : item.pubDate,
+                    category: categories,
+                    author: authors,
+                    content: {
+                        html: description,
+                        text: description,
+                    },
+                    image,
+                    banner: image,
+                    updated: upDatedStr ? parseDate(upDatedStr) : item.updated,
+                    language,
+                };
+
+                return {
+                    ...item,
+                    ...processedItem,
+                };
+            });
+        })
+    );
+
+    return {
+        title: $('title').text(),
+        description: $('meta[property="og:description"]').attr('content'),
+        link: targetUrl,
+        item: items,
+        allowEmpty: true,
+        image: $('meta[property="og:image"]').attr('content'),
+        author: $('meta[property="og:site_name"]').attr('content'),
+        language,
+        id: $('meta[property="og:url"]').attr('content'),
+    };
+};
+
+export const route: Route = {
+    path: '/articles',
+    name: 'Article',
+    url: 'rockthejvm.com',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/rockthejvm/articles',
+    parameters: undefined,
+    description: undefined,
+    categories: ['programming'],
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['rockthejvm.com/articles'],
+            target: '/articles',
+        },
+    ],
+    view: ViewType.Articles,
+};

package/lib/routes/rockthejvm/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Rock the JVM',
+    url: 'rockthejvm.com',
+    categories: ['programming'],
+    description: '',
+    lang: 'en',
+};

package/lib/routes/rockthejvm/templates/description.art

@@ -0,0 +1,7 @@
+{{ if intro }}
+  <blockquote>{{ intro }}</blockquote>
+{{ /if }}
+
+{{ if description }}
+  {{@ description }}
+{{ /if }}

package/lib/routes/samrdprc/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '国家市场监督管理总局',
+    url: 'www.samrdprc.org.cn',
+    lang: 'zh-CN',
+};

package/lib/routes/samrdprc/news.ts

@@ -0,0 +1,79 @@
+import { Route, DataItem } from '@/types';
+import cache from '@/utils/cache';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+import got from '@/utils/got';
+
+export const route: Route = {
+    path: '/news/:type1/:type2',
+    categories: ['government'],
+    example: '/samrdprc/news/xfpzh/xfpgnzh',
+    parameters: {
+        type1: '召回类型ID1，见下表',
+        type2: '召回类型ID2，见下表',
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    description: `
+| 类型中文 | 召回类型ID1 | 召回类型ID2 |
+| --- | --- | --- |
+| 消费品召回 | xfpzh | xfpgnzh |
+| 汽车召回 | qczh | gnzhqc |
+`,
+    name: '召回信息',
+    maintainers: ['a180285'],
+    radar: [
+        {
+            source: ['www.samrdprc.org.cn/:type1/:type2'],
+            target: '/news/:type1/:type2',
+        },
+    ],
+    handler: async (ctx) => {
+        const baseURL = 'https://www.samrdprc.org.cn';
+        const type1 = ctx.req.param('type1');
+        const type2 = ctx.req.param('type2');
+
+        const url = `${baseURL}/${type1}/${type2}`;
+        const response = await got(url);
+        const $ = load(response.body);
+
+        const typeName = $('div.box_main > div.boxl.fl > div.boxl_tit > div > span.fl').text();
+
+        const listSelector = $('div.main > div.box1 > div.box_main > div.boxl.fl > div.boxl_ul > ul > li');
+        const items: DataItem[] = await Promise.all(
+            listSelector.toArray().map((el) => {
+                const item = $(el);
+
+                const title = item.find('a').text();
+                const link = url + '/' + item.find('a').attr('href');
+                const pubDate = parseDate(item.find('span').text().trim());
+
+                return cache.tryGet(link, async () => {
+                    const detail = await got(link);
+                    const $ = load(detail.body);
+
+                    const articleHTML = $('div.main > div.box1 > div.box_main > div.boxl.fl > div.show_txt > div.TRS_Editor > div').html() || '';
+
+                    return {
+                        title,
+                        link,
+                        pubDate,
+                        description: articleHTML,
+                    };
+                });
+            })
+        );
+
+        return {
+            title: `${typeName} - 国家市场监督管理总局`,
+            link: url,
+            item: items as DataItem[],
+        };
+    },
+};

package/lib/routes/sankei/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Sankei Shimbun 産経新聞',
+    url: 'sankei.com',
+    lang: 'ja',
+};