rsshub 1.0.0-master.fcdc18f → 1.0.0-master.fcff124

package/lib/v2/patagonia/templates/product-description.art

@@ -0,0 +1,4 @@
+<div>
+    <img src={{imgUrl}}>
+</div>
+

package/lib/v2/penguin-random-house/radar.js

@@ -4,13 +4,13 @@ module.exports = {
         '.': [
             {
                 title: 'Penguin Random House Book Lists',
-                docs: 'https://docs.rsshub.app/reading.html#penguin-random-house',
+                docs: 'https://docs.rsshub.app/routes/reading#penguin-random-house',
                 source: ['/the-read-down'],
                 target: '/penguin-random-house/the-read-down',
             },
             {
                 title: 'Penguin Random House Articles',
-                docs: 'https://docs.rsshub.app/reading.html#penguin-random-house',
+                docs: 'https://docs.rsshub.app/routes/reading#penguin-random-house',
                 source: ['/articles'],
                 target: '/penguin-random-house/articles',
             },

package/lib/v2/people/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '通用',
-                docs: 'https://docs.rsshub.app/traditional-media.html#ren-min-wang-tong-yong',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#ren-min-wang-tong-yong',
                 source: '/',
                 target: '/people/:site?/:category?',
             },
@@ -12,7 +12,7 @@ module.exports = {
         liuyan: [
             {
                 title: '领导留言板',
-                docs: 'https://docs.rsshub.app/traditional-media.html#ren-min-wang-ling-dao-liu-yan-ban',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#ren-min-wang-ling-dao-liu-yan-ban',
                 source: '/',
                 target: '/people/liuyan/:id/:state?',
             },

package/lib/v2/people/router.js

@@ -1,6 +1,6 @@
 module.exports = function (router) {
     router.get('/liuyan/:id/:state?', require('./liuyan'));
     router.get('/xjpjh/:keyword?/:year?', require('./xjpjh'));
-    router.get(/([\w|\d]+)\/([\w|\d|/]+)?/, require('./index'));
+    router.get(/([\w\d]+)\/([\w\d/]+)?/, require('./index'));
     router.get('/:0?', require('./index'));
 };

package/lib/v2/peopo/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '新聞分類',
-                docs: 'https://docs.rsshub.app/new-media.html#peopo-gong-min-xin-wen',
+                docs: 'https://docs.rsshub.app/routes/new-media#peopo-gong-min-xin-wen',
                 source: '/topic/:topicId',
                 target: '/peopo/topic/:topicId',
             },

package/lib/v2/phoronix/index.js

@@ -0,0 +1,107 @@
+const parser = require('@/utils/rss-parser');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+const baseUrl = 'https://www.phoronix.com';
+
+module.exports = async (ctx) => {
+    const { page, queryOrItem } = ctx.params;
+    const rssUrl = new URL('/rss.php', baseUrl);
+    rssUrl.searchParams.set('page', page);
+    if (queryOrItem) {
+        if (page === 'category') {
+            rssUrl.searchParams.set('item', queryOrItem);
+        } else {
+            rssUrl.searchParams.set('q', queryOrItem);
+        }
+    }
+
+    const feed = await parser.parseURL(rssUrl.toString());
+
+    const items = await Promise.all(
+        feed.items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const html = response.body;
+                const $ = cheerio.load(html);
+                const content = $('.content');
+
+                // Author
+                const authorSelector = $('.author > a');
+                // thel last 2 are the category and comments
+                const author = authorSelector
+                    .slice(0, authorSelector.length - 2)
+                    .toArray()
+                    .map((e) => $(e).text());
+
+                // Maybe it's paginated
+                const links = $('.pagination > a')
+                    .toArray()
+                    .map((pager) => `${baseUrl}${pager.attribs.href}`)
+                    .slice(0, -1); // the last one is "next page"
+
+                if (links.length) {
+                    const pages = await Promise.all(
+                        links.map((link) =>
+                            ctx.cache.tryGet(link, async () => {
+                                const response = await got(link);
+                                const html = response.data;
+                                const $$ = cheerio.load(html);
+                                const page = $$('.content');
+                                return page.html();
+                            })
+                        )
+                    );
+                    content.append(pages);
+                }
+
+                // Summary
+                const summary = $('.content > p:nth-child(1)');
+
+                // High res images
+                content.find('img').each((_, img) => {
+                    if (img.attribs.src.endsWith('_med')) {
+                        img.attribs.src = img.attribs.src.replace('_med', '_show');
+                    }
+                });
+
+                return {
+                    title: item.title,
+                    id: item.guid,
+                    pubDate: item.pubDate,
+                    author: author.join(', '),
+                    link: item.link,
+                    summary: summary.html(),
+                    description: content.html(),
+                    icon: 'https://www.phoronix.com/android-chrome-192x192.png',
+                    logo: 'https://www.phoronix.com/phxcms7-css/phoronix.png',
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: items,
+        language: feed.language,
+        icon: 'https://www.phoronix.com/android-chrome-192x192.png',
+        image: 'https://www.phoronix.com/android-chrome-192x192.png',
+        logo: 'https://www.phoronix.com/phxcms7-css/phoronix.png',
+        // Copied from thier web page metadata
+        category: [
+            'Linux Hardware Reviews',
+            'Linux hardware benchmarks',
+            'Linux Hardware',
+            'Linux benchmarking',
+            'Desktop Linux',
+            'GNU/Linux benchmarks',
+            'Open Source AMD',
+            'Linux How To',
+            'X.Org drivers',
+            'Ubuntu hardware',
+            'Phoronix Test Suite',
+        ],
+    };
+};

package/lib/v2/phoronix/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:page/:queryOrItem?': ['oppliate'],
+};

package/lib/v2/phoronix/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'phoronix.com': {
+        _name: 'Phoronix',
+        '.': [
+            {
+                title: '新闻与评测',
+                docs: 'https://docs.rsshub.app/routes/new-media#phoronix',
+                source: ['/*'],
+                target: '/phoronix/news',
+            },
+        ],
+    },
+};

package/lib/v2/phoronix/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:page/:queryOrItem?', require('./index'));
+};

package/lib/v2/pianyuan/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '电影和剧集',
-                docs: 'https://docs.rsshub.app/multimedia.html#pian-yuan',
+                docs: 'https://docs.rsshub.app/routes/multimedia#pian-yuan',
                 source: '/',
                 target: '/pianyuan/index',
             },

package/lib/v2/picnob/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/user/:id': ['TonyRL'],
+};

package/lib/v2/picnob/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'picnob.com': {
+        _name: 'Picnob',
+        '.': [
+            {
+                title: 'User profile',
+                docs: 'https://docs.rsshub.app/routes/en/social-media#picnob',
+                source: ['/profile/:id/*'],
+                target: '/picnob/user/:id',
+            },
+        ],
+    },
+};

package/lib/v2/picnob/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/user/:id', require('./user'));
+};

package/lib/v2/picnob/templates/desc.art

@@ -0,0 +1,13 @@
+{{ if item.type === 'video' }}
+    <video poster="{{ item.pic }}" controls>
+        <source src="{{ item.video }}" type="video/mp4">
+    </video>
+{{ else if item.type === 'img_multi' }}
+    {{ each images i }}
+        <a href="{{ i.ori }}"><img src="{{ i.url }}"></a>
+    {{ /each }}
+{{ else if item.type === 'img_sig' }}
+    <a href="{{ item.pic }}"><img src="{{ item.pic }}"></a>
+{{ /if }}
+<br>
+{{@ item.sum }}

package/lib/v2/picnob/user.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://www.picnob.com';
+    const { id } = ctx.params;
+    const url = `${baseUrl}/profile/${id}/`;
+    const { data: response } = await got(url);
+    const $ = cheerio.load(response);
+
+    const profileName = $('h1.fullname').text();
+    const userId = $('input[name=userid]').attr('value');
+
+    const { data } = await got(`${baseUrl}/api/posts`, {
+        searchParams: {
+            userid: userId,
+        },
+    });
+
+    const list = data.posts.items.map(async (item) => {
+        const { shortcode, type } = item;
+        const link = `${baseUrl}/post/${shortcode}/`;
+        let images = [];
+        if (type === 'img_multi') {
+            images = await ctx.cache.tryGet(link, async () => {
+                const { data } = await got(link);
+                const $ = cheerio.load(data);
+                return [
+                    ...new Set(
+                        $('.post_slide a')
+                            .toArray()
+                            .map((a) => {
+                                a = $(a);
+                                return {
+                                    ori: a.attr('href'),
+                                    url: a.find('img').attr('data-src'),
+                                };
+                            })
+                    ),
+                ];
+            });
+        }
+        return {
+            title: item.sum_pure,
+            description: art(path.join(__dirname, 'templates/desc.art'), { item, images }),
+            link,
+            pubDate: parseDate(item.time, 'X'),
+        };
+    });
+
+    ctx.state.data = {
+        title: `${profileName} (@${id}) - Picnob`,
+        description: $('.info .sum').text(),
+        link: url,
+        image: $('.ava .pic img').attr('src'),
+        item: list,
+    };
+};

package/lib/v2/picuki/profile.js

@@ -1,4 +1,3 @@
-const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const chrono = require('chrono-node');
 const { art } = require('@/utils/render');
@@ -6,6 +5,19 @@ const path = require('path');
 const config = require('@/config').value;
 const { puppeteerGet } = require('./utils');
 
+function deVideo(media) {
+    const $ = cheerio.load(media);
+    let media_deVideo = '';
+    $('img,video').each((_, medium) => {
+        const tag = medium.name;
+        medium = $(medium);
+        const poster = medium.attr('poster');
+        // 如果有 poster 属性，表明它是视频，将它替换为它的 poster；如果不是，就原样返回
+        media_deVideo += poster ? `<img src='${poster}' alt='video poster'>` : tag === 'img' ? medium.toString() : '';
+    });
+    return media_deVideo;
+}
+
 module.exports = async (ctx) => {
     // use Puppeteer due to the obstacle by cloudflare challenge
     const browser = await require('@/utils/puppeteer')();
@@ -17,9 +29,9 @@ module.exports = async (ctx) => {
     const profileUrl = `https://www.picuki.com/profile/${id}`;
 
     const data = await ctx.cache.tryGet(
-        profileUrl,
+        `picuki-${id}-profile-${includeStories}`,
         async () => {
-            const _r = await puppeteerGet(profileUrl, browser);
+            const _r = await puppeteerGet(profileUrl, browser, includeStories);
             return _r;
         },
         config.cache.routeExpire,
@@ -33,124 +45,39 @@ module.exports = async (ctx) => {
 
     const list = $('ul.box-photos [data-s="media"]').get();
 
-    async function getMedia(url) {
-        const getPost = async () => {
-            const response = await puppeteerGet(url, browser);
-            return response;
-        };
-        let data = await ctx.cache.tryGet(url, getPost);
-        if (Object.prototype.toString.call(data) === '[object Object]') {
-            // oops, it's a json, maybe it's an old cache from the old version of this route!
-            data = await getPost();
-            // re-cache it
-            await ctx.cache.set(url, data);
-        }
-        const $ = cheerio.load(data);
-        // Instagram 允许最多 10 条图片/视频任意混合于一条 post，picuki 在所有情况下都会将它（们）置于 .single-photo 内
-        let html = '';
-        $('.single-photo img').each((_, item) => {
-            html += $(item).toString();
-        });
-        $('.single-photo video').each((_, item) => {
-            item = $(item);
-            let videoSrc = item.attr('src');
-            if (videoSrc === undefined) {
-                videoSrc = item.children().attr('src');
-            }
-            const videoPoster = item.attr('poster');
-            html += art(path.join(__dirname, 'templates/video.art'), {
-                videoSrc,
-                videoPoster,
-            });
-        });
-        return html;
-    }
-
-    function deVideo(media) {
-        const $ = cheerio.load(media);
-        let media_deVideo = '';
-        $('img,video').each((_, medium) => {
-            const tag = medium.name;
-            medium = $(medium);
-            const poster = medium.attr('poster');
-            // 如果有 poster 属性，表明它是视频，将它替换为它的 poster；如果不是，就原样返回
-            media_deVideo += poster ? `<img src='${poster}' alt='video poster'>` : tag === 'img' ? medium.toString() : '';
-        });
-        return media_deVideo;
-    }
+    let items = [];
 
-    let items = await Promise.all(
-        list.map(async (post) => {
-            post = $(post);
-
-            const postLink = post.find('.photo > a').attr('href');
-            const postTime = post.find('.time');
-            const pubDate = postTime ? chrono.parseDate(postTime.text()) : new Date();
-            const media_displayVideo = await getMedia(postLink);
-            const postText = post
-                .find('.photo-description')
-                .text()
-                .trim()
-                .replace(/[^\S\n]+/g, ' ')
-                .replace(/((?<=\n|^)[^\S\n])|([^\S\n](?=\n|$))/g, '');
-            const title = postText.replace(/\n/g, ' ') || 'Untitled';
-            const description = art(path.join(__dirname, 'templates/post.art'), {
-                media: displayVideo ? media_displayVideo : deVideo(media_displayVideo),
-                desc: postText,
-                locationLink: post.find('.photo-location .icon-globe-alt a'),
-            });
-
-            return {
-                title,
-                author: `@${id}`,
-                description,
-                link: postLink,
-                pubDate,
-            };
-        })
-    );
-
-    const getStories = async (queryId) => {
-        const apiUrl = `https://www.picuki.com/app/controllers/ajax.php`;
-        const key = `${apiUrl}?query=${queryId}&type=story`;
-        const data = await ctx.cache.tryGet(key, async () => {
-            const response = await got.post(apiUrl, {
-                form: {
-                    query: queryId,
-                    type: 'story',
-                },
-                headers: {
-                    Referer: profileUrl,
-                    Origin: 'https://www.picuki.com',
-                    'X-Requested-With': 'XMLHttpRequest',
-                },
-            });
-            return response.data;
-        });
-        if (data && data.stories_container) {
-            const $ = cheerio.load(data.stories_container);
-            const storyItems = $('.item')
+    if (includeStories) {
+        const stories_wrapper = $('.stories_wrapper');
+        if (stories_wrapper.length) {
+            const storyItems = $(stories_wrapper)
+                .find('.item')
                 .get()
                 .map((item) => {
                     const $item = $(item);
-                    const postTime = $item.find('.stories_count');
-                    const pubDate = postTime.length ? chrono.parseDate(postTime.text()) : new Date();
+                    let title = $item.find('.stories_count');
+                    title = title.length ? title.text() : '';
+                    const pubDate = title ? chrono.parseDate(title) : new Date();
                     const postBox = $item.find('.launchLightbox');
                     const poster = postBox.attr('data-video-poster');
                     const href = postBox.attr('href');
                     const type = postBox.attr('data-post-type'); // video / image
+                    const origin = postBox.attr('data-origin');
                     const storiesBackground = $item.find('.stories_background');
                     const storiesBackgroundUrl = storiesBackground && storiesBackground.css('background-image').match(/url\('?(.*)'?\)/);
                     const storiesBackgroundUrlSrc = storiesBackgroundUrl && storiesBackgroundUrl[1];
                     let description;
                     if (type === 'video') {
-                        description = `<video controls src="${href}" poster="${poster || storiesBackgroundUrlSrc || ''}">`;
+                        description = art(path.join(__dirname, 'templates/video.art'), {
+                            videoSrcs: [href, origin].filter(Boolean),
+                            videoPoster: poster || storiesBackgroundUrlSrc || '',
+                        });
                     } else if (type === 'image') {
                         description = `<img src="${href || poster || storiesBackgroundUrlSrc || ''}" alt="Instagram Story">`;
                     }
 
                     return {
-                        title: 'Instagram Story',
+                        title: 'Instagram Story' + (pubDate ? '' : `: ${title}`),
                         author: `@${id}`,
                         description,
                         link: href,
@@ -158,26 +85,79 @@ module.exports = async (ctx) => {
                     };
                 });
             if (storyItems.length) {
-                items = items.concat(storyItems);
+                items = storyItems;
             }
         }
-    };
+    }
 
-    if (includeStories) {
-        let profileQueryId;
-        $('script:not([src])').each((_, script) => {
-            const queryMatch = $(script)
-                .html()
-                .match(/let query ?= ?['"](\d+)['"]/);
-            if (queryMatch) {
-                profileQueryId = queryMatch[1];
+    async function getMedia(url) {
+        const getPost = () => puppeteerGet(url, browser);
+        let data = await ctx.cache.tryGet(url, getPost);
+        if (Object.prototype.toString.call(data) === '[object Object]') {
+            // oops, it's a json, maybe it's an old cache from the old version of this route!
+            data = await getPost();
+            // re-cache it
+            await ctx.cache.set(url, data);
+        }
+        const $ = cheerio.load(data);
+        // Instagram 允许最多 10 条图片/视频任意混合于一条 post，picuki 在所有情况下都会将它（们）置于 .single-photo 内
+        let html = '';
+        $('.single-photo img').each((_, item) => {
+            html += $(item).toString();
+        });
+        $('.single-photo video').each((_, item) => {
+            item = $(item);
+            let videoSrc = item.attr('src');
+            if (videoSrc === undefined) {
+                videoSrc = item.children().attr('src');
+            }
+            const videoPoster = item.attr('poster');
+            let origin = item.parent().attr('onclick');
+            if (origin) {
+                origin = origin.match(/window\.open\('([^']*)'/);
+                origin = origin && origin[1];
             }
+            html += art(path.join(__dirname, 'templates/video.art'), {
+                videoSrcs: [videoSrc, origin].filter(Boolean),
+                videoPoster,
+            });
         });
-        if (profileQueryId) {
-            await getStories(profileQueryId);
-        }
+        return html;
     }
 
+    items = items.concat(
+        await Promise.all(
+            list.map(async (post) => {
+                post = $(post);
+
+                const postLink = post.find('.photo > a').attr('href');
+                const postTime = post.find('.time');
+                const pubDate = postTime ? chrono.parseDate(postTime.text()) : new Date();
+                const media_displayVideo = await getMedia(postLink);
+                const postText = post
+                    .find('.photo-description')
+                    .text()
+                    .trim()
+                    .replace(/[^\S\n]+/g, ' ')
+                    .replace(/((?<=\n|^)[^\S\n])|([^\S\n](?=\n|$))/g, '');
+                const title = postText.replace(/\n/g, ' ') || 'Untitled';
+                const description = art(path.join(__dirname, 'templates/post.art'), {
+                    media: displayVideo ? media_displayVideo : deVideo(media_displayVideo),
+                    desc: postText,
+                    locationLink: post.find('.photo-location .icon-globe-alt a'),
+                });
+
+                return {
+                    title,
+                    author: `@${id}`,
+                    description,
+                    link: postLink,
+                    pubDate,
+                };
+            })
+        )
+    );
+
     await browser.close();
 
     ctx.state.data = {

package/lib/v2/picuki/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         www: [
             {
                 title: '用户',
-                docs: 'https://docs.rsshub.app/social-media.html#picuki-yong-hu',
+                docs: 'https://docs.rsshub.app/routes/social-media#picuki-yong-hu',
                 source: '/profile/:id',
                 target: '/picuki/profile/:id',
             },

package/lib/v2/picuki/templates/video.art

@@ -1,3 +1,5 @@
 <video poster="{{ videoPoster }}" controls>
-    <source src="{{ videoSrc }}" type="video/mp4">
+  <% for (let src of videoSrcs) { %>
+    <source src="{{ src }}" type="video/mp4">
+  <% } %>
 </video>

package/lib/v2/picuki/utils.js

@@ -1,15 +1,20 @@
-const puppeteerGet = async (url, browser) => {
+const puppeteerGet = async (url, browser, includeStories = false) => {
     const page = await browser.newPage();
+    const expectResourceTypes = includeStories ? ['document', 'script', 'xhr'] : ['document'];
     // await page.setExtraHTTPHeaders({ referer: host });
     await page.setRequestInterception(true);
     page.on('request', (request) => {
-        request.resourceType() === 'document' ? request.continue() : request.abort();
+        expectResourceTypes.includes(request.resourceType()) ? request.continue() : request.abort();
     });
     await page.goto(url, {
         waitUntil: 'domcontentloaded',
     });
 
     await page.waitForSelector('.wrapper');
+    if (includeStories) {
+        await page.$eval('.show_stories_button', (btn) => btn.click());
+        await page.waitForSelector('.stories_container .content');
+    }
 
     const html = await page.evaluate(() => document.documentElement.innerHTML);
     await page.close();

package/lib/v2/pikabu/community.js

@@ -0,0 +1,37 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, fixImage, fixVideo } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { type, name, sort = 'new' } = ctx.params;
+    const sortString = sort === 'default' || type === 'tag' ? '' : `/${sort}`;
+    const { data: response } = await got(`${baseUrl}/ajax/${type}/${name}${sortString}`);
+
+    const items = response.data.stories.map((story) => {
+        const $ = cheerio.load(story.html, null, false);
+        const data = JSON.parse($('script[type="application/ld+json"]').text());
+
+        const content = $('.story__main');
+
+        fixImage(content);
+        content.find('.player').each((_, elem) => {
+            elem = $(elem);
+            fixVideo(elem);
+        });
+        return {
+            title: data.name,
+            description: content.find('.story__content-inner').html(),
+            pubDate: parseDate(data.dateCreated),
+            author: data.author.name,
+            link: data.url,
+        };
+    });
+
+    ctx.state.data = {
+        title: response.data.title,
+        link: `${baseUrl}/${type}/${name}`,
+        language: 'ru-RU',
+        item: items,
+    };
+};

package/lib/v2/pikabu/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/community/:name': ['TonyRL'],
+    '/tag/:name': ['TonyRL'],
+    '/user/:name': ['TonyRL'],
+};