rsshub 1.0.0-master.fe99193 → 1.0.0-master.fe9c82a

package/lib/v2/picnob/templates/desc.art

@@ -0,0 +1,13 @@
+{{ if item.type === 'video' }}
+    <video poster="{{ item.pic }}" controls>
+        <source src="{{ item.video }}" type="video/mp4">
+    </video>
+{{ else if item.type === 'img_multi' }}
+    {{ each images i }}
+        <a href="{{ i.ori }}"><img src="{{ i.url }}"></a>
+    {{ /each }}
+{{ else if item.type === 'img_sig' }}
+    <a href="{{ item.pic }}"><img src="{{ item.pic }}"></a>
+{{ /if }}
+<br>
+{{@ item.sum }}

package/lib/v2/picnob/user.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://www.picnob.com';
+    const { id } = ctx.params;
+    const url = `${baseUrl}/profile/${id}/`;
+    const { data: response } = await got(url);
+    const $ = cheerio.load(response);
+
+    const profileName = $('h1.fullname').text();
+    const userId = $('input[name=userid]').attr('value');
+
+    const { data } = await got(`${baseUrl}/api/posts`, {
+        searchParams: {
+            userid: userId,
+        },
+    });
+
+    const list = data.posts.items.map(async (item) => {
+        const { shortcode, type } = item;
+        const link = `${baseUrl}/post/${shortcode}/`;
+        let images = [];
+        if (type === 'img_multi') {
+            images = await ctx.cache.tryGet(link, async () => {
+                const { data } = await got(link);
+                const $ = cheerio.load(data);
+                return [
+                    ...new Set(
+                        $('.post_slide a')
+                            .toArray()
+                            .map((a) => {
+                                a = $(a);
+                                return {
+                                    ori: a.attr('href'),
+                                    url: a.find('img').attr('data-src'),
+                                };
+                            })
+                    ),
+                ];
+            });
+        }
+        return {
+            title: item.sum_pure,
+            description: art(path.join(__dirname, 'templates/desc.art'), { item, images }),
+            link,
+            pubDate: parseDate(item.time, 'X'),
+        };
+    });
+
+    ctx.state.data = {
+        title: `${profileName} (@${id}) - Picnob`,
+        description: $('.info .sum').text(),
+        link: url,
+        image: $('.ava .pic img').attr('src'),
+        item: list,
+    };
+};

package/lib/v2/picuki/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/profile/:id/:functionalFlag?': ['hoilc', 'Rongronggg9'],
+    '/profile/:id/:functionalFlag?': ['hoilc', 'Rongronggg9', 'devinmugen'],
 };

package/lib/v2/picuki/profile.js

@@ -4,8 +4,12 @@ const chrono = require('chrono-node');
 const { art } = require('@/utils/render');
 const path = require('path');
 const config = require('@/config').value;
+const { puppeteerGet } = require('./utils');
 
 module.exports = async (ctx) => {
+    // use Puppeteer due to the obstacle by cloudflare challenge
+    const browser = await require('@/utils/puppeteer')();
+
     const id = ctx.params.id;
     const displayVideo = ctx.params.functionalFlag !== '0';
     const includeStories = ctx.params.functionalFlag === '10';
@@ -15,8 +19,8 @@ module.exports = async (ctx) => {
     const data = await ctx.cache.tryGet(
         profileUrl,
         async () => {
-            const _r = await got(profileUrl);
-            return _r.data;
+            const _r = await puppeteerGet(profileUrl, browser);
+            return _r;
         },
         config.cache.routeExpire,
         false
@@ -31,12 +35,8 @@ module.exports = async (ctx) => {
 
     async function getMedia(url) {
         const getPost = async () => {
-            const response = await got(url, {
-                headers: {
-                    Referer: profileUrl,
-                },
-            });
-            return response.data;
+            const response = await puppeteerGet(url, browser);
+            return response;
         };
         let data = await ctx.cache.tryGet(url, getPost);
         if (Object.prototype.toString.call(data) === '[object Object]') {
@@ -178,6 +178,8 @@ module.exports = async (ctx) => {
         }
     }
 
+    await browser.close();
+
     ctx.state.data = {
         title: `${profileName} (@${id}) - Picuki`,
         link: profileUrl,

package/lib/v2/picuki/utils.js

@@ -0,0 +1,21 @@
+const puppeteerGet = async (url, browser) => {
+    const page = await browser.newPage();
+    // await page.setExtraHTTPHeaders({ referer: host });
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    await page.waitForSelector('.wrapper');
+
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    return html;
+};
+
+module.exports = {
+    puppeteerGet,
+};

package/lib/v2/pikabu/community.js

@@ -0,0 +1,37 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, fixImage, fixVideo } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { type, name, sort = 'new' } = ctx.params;
+    const sortString = sort === 'default' || type === 'tag' ? '' : `/${sort}`;
+    const { data: response } = await got(`${baseUrl}/ajax/${type}/${name}${sortString}`);
+
+    const items = response.data.stories.map((story) => {
+        const $ = cheerio.load(story.html, null, false);
+        const data = JSON.parse($('script[type="application/ld+json"]').text());
+
+        const content = $('.story__main');
+
+        fixImage(content);
+        content.find('.player').each((_, elem) => {
+            elem = $(elem);
+            fixVideo(elem);
+        });
+        return {
+            title: data.name,
+            description: content.find('.story__content-inner').html(),
+            pubDate: parseDate(data.dateCreated),
+            author: data.author.name,
+            link: data.url,
+        };
+    });
+
+    ctx.state.data = {
+        title: response.data.title,
+        link: `${baseUrl}/${type}/${name}`,
+        language: 'ru-RU',
+        item: items,
+    };
+};

package/lib/v2/pikabu/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/community/:name': ['TonyRL'],
+    '/tag/:name': ['TonyRL'],
+    '/user/:name': ['TonyRL'],
+};

package/lib/v2/pikabu/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'pikabu.ru': {
+        _name: 'Pikabu',
+        '.': [
+            {
+                title: 'Community',
+                docs: 'https://docs.rsshub.app/en/bbs.html#pikabu',
+                source: ['/community/:name'],
+                target: '/pikabu/community/:name',
+            },
+            {
+                title: 'Tag',
+                docs: 'https://docs.rsshub.app/en/bbs.html#pikabu',
+                source: ['/tag/:name'],
+                target: '/pikabu/tag/:name',
+            },
+            {
+                title: 'User',
+                docs: 'https://docs.rsshub.app/en/bbs.html#pikabu',
+                source: ['/:name'],
+                target: '/pikabu/user/:name',
+            },
+        ],
+    },
+};

package/lib/v2/pikabu/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/user/:name', require('./user'));
+    router.get('/:type/:name', require('./community'));
+};

package/lib/v2/pikabu/templates/video.art

@@ -0,0 +1,8 @@
+{{ if videoId }}
+    <iframe id="ytplayer" type="text/html" width="640" height="360" src="https://www.youtube-nocookie.com/embed/{{ videoId }}" frameborder="0" allowfullscreen></iframe>
+{{ else if webm || mp4 }}
+    <video controls preload="none" poster="{{ preview }}" width="{{ width }}">
+        {{ if webm }}<source src="{{ webm }}" type="video/webm">{{ /if }}
+        {{ if mp4 }}<source src="{{ mp4 }}" type="video/mp4">{{ /if }}
+    </video>
+{{ /if }}

package/lib/v2/pikabu/user.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const iconv = require('iconv-lite');
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, fixImage, fixVideo } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { name } = ctx.params;
+    const link = `${baseUrl}/${name}`;
+    const response = await got(link, {
+        responseType: 'buffer',
+    });
+
+    const charset = response.headers['content-type'].match(/charset=([\w-]+)/)[1]; // windows-1251
+    const $ = cheerio.load(iconv.decode(response.data, charset));
+
+    const items = $('.story__main')
+        .not('.story__placeholder')
+        .toArray()
+        .map((story) => {
+            story = $(story);
+
+            const a = story.find('.story__title a');
+            fixImage(story);
+            story.find('.player').each((_, elem) => {
+                elem = $(elem);
+                fixVideo(elem);
+            });
+            return {
+                title: a.text(),
+                link: a.attr('href'),
+                pubDate: parseDate(story.find('time').attr('datetime')),
+                description: story.find('.story__content-inner').html(),
+                author: story.find('.user__nick').text(),
+            };
+        });
+
+    ctx.state.data = {
+        title: $('meta[property="og:title"]').attr('content'),
+        description: $('.profile__user-about-content').text(),
+        image: $('meta[property="og:image"]').attr('content'),
+        language: 'ru-RU',
+        link,
+        item: items,
+    };
+};

package/lib/v2/pikabu/utils.js

@@ -0,0 +1,43 @@
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const baseUrl = 'https://pikabu.ru';
+
+const fixImage = (element) => {
+    element.find('.story-image__stretch').remove();
+    element.find('.story-image__image').each((_, img) => {
+        if (img.attribs['data-src'] && img.attribs['data-large-image']) {
+            img.attribs.src = img.attribs['data-large-image'];
+            delete img.attribs['data-src'];
+            delete img.attribs['data-large-image'];
+        }
+    });
+};
+
+const fixVideo = (element) => {
+    const preview = element
+        .find('.player__preview')
+        .attr('style')
+        .match(/url\((.+)\);/)[1];
+    const dataType = element.attr('data-type');
+    let videoHtml = '';
+
+    if (dataType === 'video') {
+        const videoId = element.attr('data-source').match(/\/embed\/(.+)$/)[1];
+        videoHtml = art(path.join(__dirname, 'templates/video.art'), { videoId });
+    } else if (dataType === 'video-file') {
+        const width = element.find('.player__svg-stretch').attr('width');
+        const mp4 = `${element.attr('data-source')}.mp4`;
+        const webm = element.attr('data-webm');
+        videoHtml = art(path.join(__dirname, 'templates/video.art'), { preview, width, mp4, webm });
+    } else {
+        throw Error(`Unknown video type: ${dataType}`);
+    }
+    element.replaceWith(videoHtml);
+};
+
+module.exports = {
+    baseUrl,
+    fixImage,
+    fixVideo,
+};

package/lib/v2/pixabay/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/search/:q/:order?': ['TonyRL'],
+};

package/lib/v2/pixabay/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'pixabay.com': {
+        _name: 'Pixabay',
+        '.': [
+            {
+                title: 'Search',
+                docs: 'https://docs.rsshub.app/en/picture.html#pixabay',
+                source: ['/:searchType/search/:q'],
+                target: '/pixabay/search/:q',
+            },
+        ],
+    },
+};

package/lib/v2/pixabay/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/search/:q/:order?', require('./search'));
+};

package/lib/v2/pixabay/search.js

@@ -0,0 +1,50 @@
+const got = require('@/utils/got');
+const config = require('@/config').value;
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { q, order = 'latest' } = ctx.params;
+    const key = config.pixabay?.key ?? '7329690-bbadad6d872ba577d5a358679';
+    const baseUrl = 'https://pixabay.com';
+
+    const data = await ctx.cache.tryGet(
+        `pixabay:search:${q}:${order}`,
+        async () => {
+            const { data } = await got(`${baseUrl}/api/`, {
+                searchParams: {
+                    key,
+                    q,
+                    order,
+                    per_page: ctx.query.limit ?? 200,
+                },
+            });
+            return data;
+        },
+        Math.max(config.cache.contentExpire, 24 * 60 * 60), // required by Pixabay API
+        false
+    );
+
+    const items = data.hits.map((item) => {
+        const { pageURL, tags, user } = item;
+        return {
+            title: pageURL
+                .substring(pageURL.lastIndexOf('/', pageURL.lastIndexOf('/') - 1) + 1, pageURL.lastIndexOf('/'))
+                .replace(/(-\d+)$/, '')
+                .replace(/-/g, ' '),
+            description: art(path.join(__dirname, 'templates/img.art'), { item }),
+            link: pageURL,
+            category: tags.split(', '),
+            author: user,
+        };
+    });
+
+    ctx.state.data = {
+        title: `Search ${q} - Pixabay`,
+        description: 'Download & use free nature stock photos in high resolution ✓ New free images everyday ✓ HD to 4K ✓ Best nature pictures for all devices on Pixabay',
+        link: `${baseUrl}/images/search/${q}/${order === 'latest' ? '?order=latest' : ''}`,
+        image: `https://pixabay.com/apple-touch-icon.png`,
+        language: 'en',
+        item: items,
+    };
+};

package/lib/v2/pixabay/templates/img.art

@@ -0,0 +1 @@
+<img src="{{ item.largeImageURL || item.webformatURL || item.previewURL }}">

package/lib/v2/pixiv/api/getRanking.js

@@ -3,7 +3,7 @@ const maskHeader = require('../constants').maskHeader;
 const assert = require('assert');
 const queryString = require('query-string');
 
-const allowMode = ['day', 'week', 'month', 'day_male', 'day_female', 'week_original', 'week_rookie', 'day_r18', 'day_male_r18', 'day_female_r18', 'week_r18', 'week_r18g'];
+const allowMode = ['day', 'week', 'month', 'day_male', 'day_female', 'day_ai', 'week_original', 'week_rookie', 'day_r18', 'day_r18_ai', 'day_male_r18', 'day_female_r18', 'week_r18', 'week_r18g'];
 
 /**
  * 获取某天的排行榜

package/lib/v2/pixiv/novels.js

@@ -4,32 +4,41 @@ const baseUrl = 'https://www.pixiv.net';
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/all`);
+    const url = `${baseUrl}/users/${id}/novels`;
+    const { data: allData } = await got(`${baseUrl}/ajax/user/${id}/profile/all`, {
+        headers: {
+            referer: url,
+        },
+    });
 
-    const items = [
-        ...data.body.novelSeries.map((item) => ({
-            title: item.title,
-            description: item.caption,
-            link: `https://www.pixiv.net/novel/series/${item.id}`,
-            author: item.userName,
-            pubDate: parseDate(item.createDate),
-            updated: parseDate(item.updateDate),
-            category: item.tags,
-        })),
-        ...data.body.pickup.map((item) => ({
-            title: item.title,
-            description: item.description,
-            link: item.contentUrl,
-            author: item.userName,
-            pubDate: parseDate(item.createDate),
-            updated: parseDate(item.updateDate),
-            category: item.tags,
-        })),
-    ];
+    const novels = Object.keys(allData.body.novels);
+    const searchParams = new URLSearchParams();
+    novels.forEach((novel) => {
+        searchParams.append('ids[]', novel);
+    });
+
+    const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/novels`, {
+        headers: {
+            referer: url,
+        },
+        searchParams,
+    });
+
+    const items = Object.values(data.body.works).map((item) => ({
+        title: item.seriesTitle || item.title,
+        description: item.description || item.title,
+        link: `${baseUrl}/novel/series/${item.id}`,
+        author: item.userName,
+        pubDate: parseDate(item.createDate),
+        updated: parseDate(item.updateDate),
+        category: item.tags,
+    }));
 
     ctx.state.data = {
-        title: `${data.body.novelSeries[0].userName || data.body.pickup[0].userName} 的小说`,
-        link: `${baseUrl}/users/${id}/novels`,
+        title: data.body.extraData.meta.title,
+        description: data.body.extraData.meta.ogp.description,
+        image: Object.values(data.body.works)[0].profileImageUrl,
+        link: url,
         item: items,
     };
 };

package/lib/v2/pixiv/ranking.js

@@ -13,10 +13,12 @@ const titles = {
     week_original: 'pixiv 原创作品排行',
     week_rookie: 'pixiv 新人排行',
     day_r18: 'pixiv R-18 日排行',
+    day_r18_ai: 'pixiv R-18 AI生成作品排行',
     day_male_r18: 'pixiv R-18 受男性欢迎排行',
     day_female_r18: 'pixiv R-18 受女性欢迎排行',
     week_r18: 'pixiv R-18 周排行',
     week_r18g: 'pixiv R-18G 排行',
+    day_ai: 'AI 生成作品排行榜',
 };
 
 const links = {
@@ -25,9 +27,11 @@ const links = {
     month: 'https://www.pixiv.net/ranking.php?mode=monthly',
     day_male: 'https://www.pixiv.net/ranking.php?mode=male',
     day_female: 'https://www.pixiv.net/ranking.php?mode=female',
+    day_ai: 'https://www.pixiv.net/ranking.php?mode=daily_ai',
     week_original: 'https://www.pixiv.net/ranking.php?mode=original',
     week_rookie: 'https://www.pixiv.net/ranking.php?mode=rookie',
     day_r18: 'https://www.pixiv.net/ranking.php?mode=daily_r18',
+    day_r18_ai: 'https://www.pixiv.net/ranking.php?mode=daily_r18_ai',
     day_male_r18: 'https://www.pixiv.net/ranking.php?mode=male_r18',
     day_female_r18: 'https://www.pixiv.net/ranking.php?mode=female_r18',
     week_r18: 'https://www.pixiv.net/ranking.php?mode=weekly_r18',
@@ -40,9 +44,11 @@ const alias = {
     monthly: 'month',
     male: 'day_male',
     female: 'day_female',
+    daily_ai: 'day_ai',
     original: 'week_original',
     rookie: 'week_rookie',
     daily_r18: 'day_r18',
+    daily_r18_ai: 'day_r18_ai',
     male_r18: 'day_male_r18',
     female_r18: 'day_female_r18',
     weekly_r18: 'week_r18',

package/lib/v2/piyao/jrpy.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const rootUrl = 'https://www.piyao.org.cn';
+
+module.exports = async (ctx) => {
+    const currentUrl = `${rootUrl}/jrpy/index.htm`;
+
+    const response = await got(currentUrl);
+    const $ = cheerio.load(response.data);
+    const list = $('ul#list li')
+        .map((_, item) => ({
+            title: $(item).find('a').text(),
+            link: new URL($(item).find('a').attr('href'), rootUrl).href,
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.title = content('div.con_tit > h2').text();
+                item.description = content('div.con_txt').html();
+                item.pubDate = parseDate(content('div.con_tit > p > span').text().split('时间：')[1]);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '今日辟谣',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/piyao/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/jrpy': ['Fatpandac'],
+};

package/lib/v2/piyao/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'piyao.org.cn': {
+        _name: '中国互联网联合辟谣平台',
+        '.': [
+            {
+                title: '今日辟谣',
+                docs: 'https://docs.rsshub.app/other.html#zhong-guo-hu-lian-wang-lian-he-pi-yao-ping-tai',
+                source: ['/jrpy/index.htm'],
+                target: '/piyao/jrpy',
+            },
+        ],
+    },
+};

package/lib/v2/piyao/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/jrpy', require('./jrpy'));
+};

package/lib/v2/pku/ss/admission.js

@@ -1,48 +1,10 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, getSingleRecord, getArticle } = require('./common');
 
-const link = 'http://www.ss.pku.edu.cn';
-const host = 'http://www.ss.pku.edu.cn/index.php/admission/admnotice';
-
-const getSingleRecord = async () => {
-    const res = await got(host);
-
-    const $ = cheerio.load(res.data);
-    const list = $('#info-list-ul').find('li');
-
-    return (
-        list &&
-        list
-            .map((index, item) => {
-                item = $(item);
-                const date = item.find('.time').text();
-                return {
-                    title: item.find('a').attr('title'),
-                    pubDate: parseDate(date),
-                    link: link + item.find('a').attr('href'),
-                };
-            })
-            .get()
-    );
-};
+const host = `${baseUrl}/admission/admnotice/`;
 
 module.exports = async (ctx) => {
-    const items = await getSingleRecord();
-    const out = await Promise.all(
-        items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const response = await got(item.link);
-                const $ = cheerio.load(response.data);
-                return {
-                    title: item.title,
-                    link: item.link,
-                    description: $('.article-content').html(),
-                    pubDate: item.pubDate,
-                };
-            })
-        )
-    );
+    const items = await getSingleRecord(host);
+    const out = await Promise.all(items.map((item) => getArticle(item, ctx.cache.tryGet)));
 
     ctx.state.data = {
         title: '北大软微-招生通知',