rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/podwise/episodes.ts

@@ -0,0 +1,119 @@
+import { Route } from '@/types';
+import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch'; // 统一使用的请求库
+import cache from '@/utils/cache';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+import dayjs from 'dayjs';
+import duration from 'dayjs/plugin/duration';
+dayjs.extend(duration);
+
+export const route: Route = {
+    path: '/explore/:type',
+    categories: ['multimedia'],
+    example: '/podwise/explore/latest',
+    parameters: { type: 'latest or all episodes.' },
+    radar: [
+        {
+            source: ['podwise.ai/explore/:type'],
+        },
+    ],
+    name: 'Episodes',
+    maintainers: ['lyling'],
+    handler: async (ctx) => {
+        const type = ctx.req.param('type');
+        const link = `https://podwise.ai/explore/${type}`;
+        const response = await ofetch(link);
+        const $ = load(response);
+        const content = $('#navigator').next();
+        // header/[div => content]/footer, content>div(2)>h1
+        const title = content.find('h1').first().text();
+        const description = content.find('p').eq(1).text();
+
+        const list = content
+            .find('.group')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+                const link = item.find('a').first().attr('href');
+                const description = item.find('p').first().text();
+                const pubDate = item.find('a').next().children('span').text();
+
+                return {
+                    link: `https://podwise.ai${link}`,
+                    description,
+                    pubDate: timezone(parseDate(pubDate, 'DD MMM YYYY', 'en'), 8),
+                };
+            });
+
+        const items = await Promise.all(
+            list.map((item) =>
+                cache.tryGet(item.link, async () => {
+                    const response = await ofetch(item.link);
+                    const $ = load(response);
+
+                    item.description = $('summary').first().html();
+
+                    // duration
+                    const $cover = $('img[alt="Podcast cover"]').eq(1);
+                    const $duration = $cover.next().find('span').eq(2);
+
+                    const nextData = JSON.parse(
+                        $('script:contains("podName")')
+                            .first()
+                            .text()
+                            .match(/self\.__next_f\.push\((.+)\)/)?.[1] ?? ''
+                    );
+                    const podcastData = JSON.parse(nextData[1].slice(2))[1][3].children[3].episode;
+
+                    // rss feed
+                    item.title = podcastData.title;
+                    item.author = podcastData.podName;
+
+                    // podcast feed
+                    item.itunes_item_image = podcastData.cover;
+                    item.itunes_duration = parseDuration($duration.text());
+                    item.enclosure_url = podcastData.link;
+                    // item.enclosure_length: nothing can convert to.
+                    item.enclosure_type = toEnclosureType(podcastData.linkType);
+
+                    return item;
+                })
+            )
+        );
+
+        return {
+            title,
+            description,
+            link,
+            item: items,
+        };
+    },
+};
+
+function parseDuration(durationStr) {
+    const matches = durationStr.match(/(\d+h)?(\d+m)?/);
+    const hours = matches[1] ? Number.parseInt(matches[1]) : 0;
+    const minutes = matches[2] ? Number.parseInt(matches[2]) : 0;
+
+    // 使用 dayjs 的 duration 创建持续时间对象
+    return dayjs
+        .duration({
+            hours,
+            minutes,
+        })
+        .asSeconds();
+}
+
+function toEnclosureType(linkType: string): string {
+    switch (linkType) {
+        case 'mp3':
+            return 'audio/mpeg';
+        case 'm4a':
+            return 'audio/x-m4a';
+        case 'mp4':
+            return 'video/mp4';
+        default:
+            return linkType;
+    }
+}

package/lib/routes/podwise/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Podwise',
+    url: 'podwise.ai',
+    zh: {
+        name: 'Podwise',
+    },
+};

package/lib/routes/pornhub/category-url.ts

@@ -6,7 +6,7 @@ import { headers, parseItems } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
-    path: '/:language?/category_url/:url?',
+    path: '/category_url/:url?/:language?',
     categories: ['multimedia'],
     example: '/pornhub/category_url/video%3Fc%3D15%26o%3Dmv%26t%3Dw%26cc%3Djp',
     parameters: { language: 'language, see below', url: 'relative path after `pornhub.com/`, need to be URL encoded' },

package/lib/routes/pornhub/category.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
@@ -7,7 +7,8 @@ import { config } from '@/config';
 
 export const route: Route = {
     path: '/category/:caty',
-    categories: ['multimedia'],
+    categories: ['multimedia', 'popular'],
+    view: ViewType.Videos,
     example: '/pornhub/category/popular-with-women',
     parameters: { caty: 'category, see [categories](https://www.pornhub.com/webmasters/categories)' },
     features: {

package/lib/routes/pornhub/model.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { isValidHost } from '@/utils/valid-host';
@@ -6,8 +6,9 @@ import { headers, parseItems } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
-    path: '/:language?/model/:username/:sort?',
-    categories: ['multimedia'],
+    path: '/model/:username/:language?/:sort?',
+    categories: ['multimedia', 'popular'],
+    view: ViewType.Videos,
     example: '/pornhub/model/stacy-starando',
     parameters: { language: 'language, see below', username: 'username, part of the url e.g. `pornhub.com/model/stacy-starando`', sort: 'sorting method, see below' },
     features: {
@@ -24,7 +25,7 @@ export const route: Route = {
             target: '/model/:username',
         },
     ],
-    name: 'Verified amateur / Model',
+    name: 'Model',
     maintainers: ['I2IMk', 'queensferryme'],
     handler,
 };
@@ -43,7 +44,7 @@ async function handler(ctx) {
         .map((e) => parseItems($(e)));
 
     return {
-        title: $('title').first().text(),
+        title: $('h1').first().text(),
         description: $('section.aboutMeSection').text().trim(),
         link,
         image: $('#coverPictureDefault').attr('src'),

package/lib/routes/pornhub/pornstar.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { isValidHost } from '@/utils/valid-host';
@@ -6,10 +6,54 @@ import { headers, parseItems } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
-    path: '/:language?/pornstar/:username/:sort?',
-    categories: ['multimedia'],
-    example: '/pornhub/pornstar/june-liu',
-    parameters: { language: 'language, see below', username: 'username, part of the url e.g. `pornhub.com/pornstar/june-liu`', sort: 'sorting method, see below' },
+    path: '/pornstar/:username/:language?/:sort?',
+    categories: ['multimedia', 'popular'],
+    view: ViewType.Videos,
+    example: '/pornhub/pornstar/june-liu/www/mr',
+    parameters: {
+        username: {
+            description: 'username, part of the url e.g. `pornhub.com/pornstar/june-liu`',
+        },
+        language: {
+            description: 'language',
+            options: [
+                { value: 'www', label: 'English' },
+                { value: 'de', label: 'Deutsch' },
+                { value: 'es', label: 'Español' },
+                { value: 'fr', label: 'Français' },
+                { value: 'it', label: 'Italiano' },
+                { value: 'ja', label: '日本語' },
+                { value: 'pt', label: 'Português' },
+                { value: 'pl', label: 'Polski' },
+                { value: 'rt', label: 'Русский' },
+                { value: 'nl', label: 'Dutch' },
+                { value: 'cs', label: 'Czech' },
+                { value: 'cn', label: '中文（简体）' },
+            ],
+            default: 'www',
+        },
+        sort: {
+            description: 'sorting method, leave empty for `Best`',
+            options: [
+                {
+                    label: 'Most Recent',
+                    value: 'mr',
+                },
+                {
+                    label: 'Most Viewed',
+                    value: 'mv',
+                },
+                {
+                    label: 'Top Rated',
+                    value: 'tr',
+                },
+                {
+                    label: 'Longest',
+                    value: 'lg',
+                },
+            ],
+        },
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -24,31 +68,37 @@ export const route: Route = {
             target: '/pornstar/:username',
         },
     ],
-    name: 'Verified model / Pornstar',
+    name: 'Pornstar',
     maintainers: ['I2IMk', 'queensferryme'],
     handler,
-    description: `**\`sort\`**
-
-  | Most Recent | Most Viewed | Top Rated | Longest | Best |
-  | ----------- | ----------- | --------- | ------- | ---- |
-  | mr          | mv          | tr        | lg      |      |`,
 };
 
 async function handler(ctx) {
     const { language = 'www', username, sort = 'mr' } = ctx.req.param();
-    const link = `https://${language}.pornhub.com/pornstar/${username}/videos?o=${sort}`;
+    let link = `https://${language}.pornhub.com/pornstar/${username}?o=${sort}`;
     if (!isValidHost(language)) {
         throw new InvalidParameterError('Invalid language');
     }
 
     const { data: response } = await got(link, { headers });
-    const $ = load(response);
-    const items = $('#mostRecentVideosSection .videoBox')
-        .toArray()
-        .map((e) => parseItems($(e)));
+    let $ = load(response);
+    let items;
+
+    if ($('.withBio').length === 0) {
+        link = `https://${language}.pornhub.com/pornstar/${username}/videos?o=${sort}`;
+        const { data: response } = await got(link, { headers });
+        $ = load(response);
+        items = $('#mostRecentVideosSection .videoBox')
+            .toArray()
+            .map((e) => parseItems($(e)));
+    } else {
+        items = $('#pornstarsVideoSection .videoBox')
+            .toArray()
+            .map((e) => parseItems($(e)));
+    }
 
     return {
-        title: $('title').first().text(),
+        title: $('h1').first().text(),
         description: $('section.aboutMeSection').text().trim(),
         link,
         image: $('#coverPictureDefault').attr('src'),

package/lib/routes/pornhub/search.ts

@@ -1,11 +1,12 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
 import { defaultDomain, renderDescription } from './utils';
 
 export const route: Route = {
     path: '/search/:keyword',
-    categories: ['multimedia'],
+    categories: ['multimedia', 'popular'],
+    view: ViewType.Videos,
     example: '/pornhub/search/stepsister',
     parameters: { keyword: 'keyword' },
     features: {

package/lib/routes/pornhub/templates/description.art

@@ -1,5 +1,5 @@
 {{ if previewVideo }}
-<video controls preload="none" poster="{{ poster }}">
+<video controls preload="metadata" poster="{{ poster }}">
     <source src="{{ previewVideo }}" type="video/webm">
 </video>
 {{ /if }}

package/lib/routes/pornhub/users.ts

@@ -6,7 +6,7 @@ import { headers, parseItems } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
-    path: '/:language?/users/:username',
+    path: '/users/:username/:language?',
     categories: ['multimedia'],
     example: '/pornhub/users/pornhubmodels',
     parameters: { language: 'language, see below', username: 'username, part of the url e.g. `pornhub.com/users/pornhubmodels`' },
@@ -43,7 +43,7 @@ async function handler(ctx) {
         .map((e) => parseItems($(e)));
 
     return {
-        title: $('title').first().text(),
+        title: $('.profileUserName a').text(),
         description: $('.aboutMeText').text().trim(),
         link,
         image: $('#coverPictureDefault').attr('src'),

package/lib/routes/pornhub/utils.ts

@@ -27,7 +27,7 @@ const parseItems = (e) => ({
         previewVideo: e.find('img').data('mediabook'),
     }),
     author: e.find('.usernameWrap a').text(),
-    pubDate: dayjs(extractDateFromImageUrl(e.find('img').data('mediumthumb'))) || parseRelativeDate(e.find('.added').text()),
+    pubDate: dayjs(extractDateFromImageUrl(e.find('img').data('mediumthumb'))).toDate() || parseRelativeDate(e.find('.added').text()),
 });
 
 export { defaultDomain, headers, renderDescription, parseItems };

package/lib/routes/ps/monthly-games.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
@@ -9,7 +9,8 @@ import { art } from '@/utils/render';
 
 export const route: Route = {
     path: '/monthly-games',
-    categories: ['game'],
+    categories: ['game', 'popular'],
+    view: ViewType.Notifications,
     example: '/ps/monthly-games',
     parameters: {},
     features: {

package/lib/routes/psyche/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Psyche',
+    url: 'psyche.co',
+};

package/lib/routes/psyche/templates/essay.art

@@ -0,0 +1,3 @@
+<img src="{{ banner }}" alt="">
+{{@ authorsBio }}
+{{@ content}}

package/lib/routes/psyche/templates/video.art

@@ -0,0 +1,10 @@
+{{ set video = article.hosterId }}
+{{ if article.hoster === 'vimeo' }}
+    {{ set video = "https://player.vimeo.com/video/" + video + "?dnt=1"}}
+{{ else if article.hoster == 'youtube' }}
+    {{ set video = "https://www.youtube-nocookie.com/embed/" + video }}
+{{ /if }}
+
+<iframe width="672" height="377" src="{{ video }}" frameborder="0" allowfullscreen></iframe>
+{{@ article.credits}}
+{{@ article.description}}

package/lib/routes/psyche/topic.ts

@@ -0,0 +1,46 @@
+import { Route } from '@/types';
+import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
+import { getData } from './utils';
+
+export const route: Route = {
+    path: '/topic/:topic',
+    categories: ['new-media'],
+    example: '/psyche/topic/therapeia',
+    parameters: { topic: 'Topic' },
+    radar: [
+        {
+            source: ['psyche.co/:topic'],
+        },
+    ],
+    name: 'Topics',
+    maintainers: ['emdoe'],
+    handler,
+    description: 'Supported categories: Therapeia, Eudaimonia, and Poiesis.',
+};
+
+async function handler(ctx) {
+    const url = `https://psyche.co/${ctx.req.param('topic')}`;
+    const response = await ofetch(url);
+    const $ = load(response);
+
+    const data = JSON.parse($('script#__NEXT_DATA__').text());
+    const articles = data.props.pageProps.articles;
+    const prefix = `https://psyche.co/_next/data/${data.buildId}`;
+    const list = Object.keys(articles).flatMap((type) =>
+        articles[type].edges.map((item) => ({
+            title: item.node.title,
+            link: `https://psyche.co/${type}/${item.node.slug}`,
+            json: `${prefix}/${type}/${item.node.slug}.json`,
+        }))
+    );
+
+    const items = await getData(list);
+
+    return {
+        title: `Psyche | ${data.props.pageProps.section.title}`,
+        link: url,
+        description: data.props.pageProps.section.metaDescription,
+        item: items,
+    };
+}

package/lib/routes/psyche/type.ts

@@ -0,0 +1,53 @@
+import { Route } from '@/types';
+import { load } from 'cheerio';
+import { getData } from './utils';
+import ofetch from '@/utils/ofetch';
+
+export const route: Route = {
+    path: '/type/:type',
+    categories: ['new-media'],
+    example: '/psyche/type/ideas',
+    parameters: { type: 'Type' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['psyche.co/:type'],
+        },
+    ],
+    name: 'Types',
+    maintainers: ['emdoe'],
+    handler,
+    description: `Supported types: Ideas, Guides, and Films.`,
+};
+
+async function handler(ctx) {
+    const type = ctx.req.param('type');
+    const capitalizedType = type.charAt(0).toUpperCase() + type.slice(1);
+
+    const url = `https://psyche.co/${type}`;
+    const response = await ofetch(url);
+    const $ = load(response);
+
+    const data = JSON.parse($('script#__NEXT_DATA__').text());
+    const prefix = `https://psyche.co/_next/data/${data.buildId}`;
+    const list = data.props.pageProps.articles.map((item) => ({
+        title: item.title,
+        link: `${url}/${item.slug}`,
+        json: `${prefix}/${type}/${item.slug}.json`,
+    }));
+
+    const items = await getData(list);
+
+    return {
+        title: `Psyche | ${capitalizedType}`,
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/psyche/utils.ts

@@ -0,0 +1,107 @@
+import { getCurrentPath } from '@/utils/helpers';
+const __dirname = getCurrentPath(import.meta.url);
+
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { art } from '@/utils/render';
+import path from 'node:path';
+
+const getImageById = async (id) => {
+    const response = await ofetch('https://api.aeonmedia.co/graphql', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+            query: 'query getImageById($id: ID!) { image(id: $id) { id url alt caption width height } }',
+            variables: { id, site: 'Aeon' },
+            operationName: 'getImageById',
+        }),
+    });
+
+    return response.data.image.url;
+};
+
+function format(article) {
+    const type = article.type.toLowerCase();
+
+    let block = '';
+    let banner = '';
+    let authorsBio = '';
+
+    switch (type) {
+        case 'film':
+            block = art(path.join(__dirname, 'templates/video.art'), { article });
+
+            break;
+
+        case 'guide': {
+            banner = article.imageSquare?.url;
+            authorsBio = article.authors.map((author) => author.bio).join(' ');
+
+            const sectionNames = ['Need To Know', 'What To Do', 'Key Points', 'Learn More', 'Links & Books'];
+            const sections = Object.keys(article).filter((key) => key.startsWith('section') && key !== 'section');
+            const content = sections
+                .map((section) => {
+                    const capture = load(article[section]);
+                    capture('p.pullquote').remove();
+                    const sectionName = sectionNames.shift();
+                    return `<h2>${sectionName}</h2>` + capture.html();
+                })
+                .join('');
+
+            block = art(path.join(__dirname, 'templates/essay.art'), { banner, authorsBio, content });
+
+            break;
+        }
+        case 'idea': {
+            banner = article.imageLandscape?.url;
+            authorsBio = article.authors.map((author) => author.bio).join(' ');
+
+            const capture = load(article.body);
+            capture('p.pullquote').remove();
+            block = art(path.join(__dirname, 'templates/essay.art'), { banner, authorsBio, content: capture.html() });
+
+            break;
+        }
+        default:
+            break;
+    }
+    return block;
+}
+
+const getData = async (list) => {
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const data = await ofetch(item.json);
+                const article = data.pageProps.article;
+                item.pubDate = new Date(article.publishedAt).toUTCString();
+                const content = format(article);
+                const capture = load(content);
+                await Promise.all(
+                    capture('dl > dt')
+                        .toArray()
+                        .map(async (item) => {
+                            const id = capture(item).text();
+                            const image = await getImageById(id);
+                            capture(item).replaceWith(`<img src="${image}" alt="${id}">`);
+                        })
+                );
+
+                let authors = '';
+                authors = article.type === 'film' ? article.creditsShort : article.authors.map((author) => author.name).join(', ');
+
+                item.description = capture.html();
+                item.author = authors;
+
+                return item;
+            })
+        )
+    );
+
+    return items;
+};
+
+export { getData };

package/lib/routes/pubscholar/explore.ts

@@ -0,0 +1,62 @@
+import { Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { baseUrl, uuidv4, getArticleLink, getSignedHeaders } from './utils';
+import md5 from '@/utils/md5';
+import { Resource } from './types';
+import sanitizeHtml from 'sanitize-html';
+
+export const route: Route = {
+    path: '/explore/:category?/:keyword?',
+    name: 'Explore',
+    maintainers: ['TonyRL'],
+    example: '/pubscholar/explore',
+    parameters: {
+        category: 'Category, see the table below, `articles` by default',
+        keyword: 'Search Keyword',
+    },
+    handler,
+    description: `| Articles / 论文 | Patents / 专利 | Reports / 领域快报 | Information / 动态快讯 | Datasets / 科学数据 | Books / 图书 |
+| --------------- | -------------- | ------------------ | ---------------------- | ------------------- | ------------ |
+| articles        | patents        | bulletins          | reports                | sciencedata         | books        |`,
+};
+
+async function handler(ctx) {
+    const { category = 'articles', keyword } = ctx.req.param();
+    const uuid = uuidv4();
+
+    const response = await ofetch<Resource>(`${baseUrl}/hky/open/resources/api/v1/${category}`, {
+        method: 'POST',
+        headers: {
+            ...getSignedHeaders(),
+            Cookie: `XSRF-TOKEN=${uuid}`,
+            'X-XSRF-TOKEN': uuid,
+        },
+        body: {
+            page: 1,
+            size: 10,
+            order_field: 'date',
+            order_direction: 'desc',
+            user_id: md5(Date.now().toString()),
+            lang: 'zh',
+            query: keyword,
+            strategy: null,
+            orderField: 'default',
+        },
+    });
+
+    const list = response.content.map((item) => ({
+        title: (item.is_free || item.links.some((l) => l.is_open_access) ? '「Open Access」' : '') + sanitizeHtml(item.title, { allowedTags: [], allowedAttributes: {} }),
+        description: item.abstracts + `<br>${item.links.map((link) => `<a href="${link.url}">${link.is_open_access ? '「Open Access」' : ''}${link.name}</a>`).join('<br>')}`,
+        author: item.author.join('; '),
+        pubDate: parseDate(item.date),
+        category: item.keywords.map((keyword) => sanitizeHtml(keyword, { allowedTags: [], allowedAttributes: {} })),
+        link: `${baseUrl}/${category}/${getArticleLink(item.id)}`,
+    }));
+
+    return {
+        title: 'PubScholar 公益学术平台',
+        link: `${baseUrl}/explore`,
+        item: list,
+    };
+}