rsshub 1.0.0-master.f8b1c8d → 1.0.0-master.f91e29d

package/lib/v2/oshwhub/explore.js

@@ -0,0 +1,116 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const md = require('markdown-it')({
+    html: true,
+    linkify: true,
+});
+
+const requestImages = (url, tryGet) =>
+    tryGet(url, async () => {
+        const response = await got({
+            method: 'get',
+            url,
+        });
+
+        const images = {};
+        const boms = [];
+
+        const imgData = response.data;
+        imgData?.result?.boards?.forEach((board) => {
+            const name = board.name;
+            if (images[name] === undefined) {
+                images[name] = [];
+            }
+
+            board.schematic?.documents?.forEach((val) => {
+                if (val.thumb) {
+                    images[name].push(val.thumb);
+                }
+            });
+
+            if (board.pcb_info?.thumb) {
+                images[name].push(board.pcb_info.thumb);
+            }
+            if (board.pcb_info?.boms?.boms) {
+                boms.push(...JSON.parse(board.pcb_info.boms.boms));
+            }
+        });
+        return { images, boms };
+    });
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://oshwhub.com';
+    const sortType = ctx.params.sortType ?? 'updatedTime';
+
+    const url = `${baseUrl}/explore?projectSort=${sortType}`;
+    const response = await got({
+        method: 'get',
+        url,
+    });
+
+    const html = response.data;
+    const $ = cheerio.load(html);
+    const title = $('title').text();
+    const projects = $('a[class="project-link"]');
+
+    const items = await Promise.all(
+        projects.map((id, item) => {
+            const detailUrl = `${baseUrl}${item.attribs.href}`;
+            return ctx.cache.tryGet(detailUrl, async () => {
+                const response = await got({
+                    method: 'get',
+                    url: detailUrl,
+                });
+
+                const html = response.data;
+                const $$ = cheerio.load(html);
+                const projectDetail = $$('div[class="projects-left-top"]');
+
+                const title = $$(projectDetail).find('span[class="title"]').text();
+                const author = $$(projectDetail).find('div[class="member-items"] > ul > li > a > span').first().text().trim();
+                const publishTime = $$(projectDetail).find('div[class="issue-time"] > span').text();
+                const pubDate = timezone(parseDate(publishTime.trim().split('\n')[1].trim(), 'YYYY/MM/DD HH:mm:ss'), 0);
+
+                const coverImg = $$(projectDetail).find('div[class="img-cover"] > img').attr('src').trim();
+                const introduction = $$(projectDetail).find('p[class="introduction"]').text().trim();
+                const content = $$(projectDetail).find('div[class*="html-content"]').attr('data-markdown');
+
+                // request images
+                const dataUuid = $$('div[class="projects-detail"]').attr()['data-uuid'];
+                const { images, boms } = await requestImages(`${baseUrl}/api/project/${dataUuid}/pro_images`, ctx.cache.tryGet);
+
+                const description = art(path.join(__dirname, 'templates/description.art'), {
+                    title,
+                    author,
+                    publishTime,
+                    coverImg,
+                    introduction,
+                    content: md.render(content) || 'No Content',
+                    images,
+                    boms,
+                });
+
+                const single = {
+                    title,
+                    description,
+                    pubDate,
+                    link: detailUrl,
+                    author,
+                };
+
+                return single;
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title,
+        link: url,
+        description: title,
+        item: items,
+    };
+};

package/lib/v2/oshwhub/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:sortType?': ['tylinux'],
+};

package/lib/v2/oshwhub/radar.js

@@ -0,0 +1,16 @@
+module.exports = {
+    'oshwhub.com': {
+        _name: '立创开源硬件平台',
+        '.': [
+            {
+                title: '开源广场',
+                docs: 'https://docs.rsshub.app/other.html#li-chuang-kai-yuan-ying-jian-ping-tai',
+                source: ['/explore'],
+                target: (_, url) => {
+                    const sortType = new URL(url).searchParams.get('projectSort');
+                    return sortType ? `/oshwhub/${sortType}` : '';
+                },
+            },
+        ],
+    },
+};

package/lib/v2/oshwhub/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:sortType?', require('./explore'));
+};

package/lib/v2/oshwhub/templates/description.art

@@ -0,0 +1,58 @@
+<div>
+    <div>
+        <img src={{coverImg}} /><br><br>
+        <span>{{publishTime}}</span><br>
+        <span>作者：{{author}}</span><br>
+        <span>{{introduction}}</span><br>
+    </div>
+
+    <div>
+        <h1> 描述 </h1>
+        {{@ content}}
+    </div>
+
+    {{ if images }}
+    <div>
+        <h1> PCB && schematic </h1>
+        {{each images $imgs $key}}
+            {{each $imgs $img}}
+                <img src={{$img}} /><br>
+            {{/each}}
+        {{/each}}
+    </div>
+    {{/if}}
+
+    {{ if boms }}
+    <div>
+        <h1> BOM </h1>
+        <table class="table">
+            <tr>
+                <th>No</th>
+                <th>Quantity</th>
+                <th>Device</th>
+                <th>Designator</th>
+                <th>Footprint</th>
+                <th>Value</th>
+                <th>Manufacturer Part</th>
+                <th>Manufacturer</th>
+                <th>Supplier Part</th>
+                <th>Supplier</th>
+            </tr>
+            {{each boms}}
+            <tr>
+                <th>{{$value.No}}</th>
+                <th>{{$value.Quantity}}</th>
+                <th>{{$value.Device}}</th>
+                <th>{{$value.Designator}}</th>
+                <th>{{$value.Footprint}}</th>
+                <th>{{$value.Value}}</th>
+                <th>{{$value['Manufacturer Part']}}</th>
+                <th>{{$value.Manufacturer}}</th>
+                <th>{{$value['Supplier Part']}}</th>
+                <th>{{$value.Supplier}}</th>
+            </tr>
+            {{/each}}
+        </table>
+    </div>
+    {{/if}}
+</div>

package/lib/v2/paradigm/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/writing': ['Fatpandac'],
+};

package/lib/v2/paradigm/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'paradigm.xyz': {
+        _name: 'Paradigm',
+        '.': [
+            {
+                title: 'Writing',
+                docs: 'https://docs.rsshub.app/finance.html#paradigm',
+                source: ['/writing'],
+                target: '/paradigm/writing',
+            },
+        ],
+    },
+};

package/lib/v2/paradigm/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/writing', require('./writing'));
+};

package/lib/v2/paradigm/writing.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.paradigm.xyz';
+
+module.exports = async (ctx) => {
+    const url = `${baseUrl}/writing`;
+
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+
+    const nextData = JSON.parse($('#__NEXT_DATA__').text());
+    const buildId = nextData.buildId;
+    const list = nextData.props.pageProps.posts.map((item) => ({
+        title: item.title,
+        link: `${baseUrl}${item.slug}`,
+        api: `${baseUrl}/_next/data/${buildId}${item.slug}.json`,
+        author: item.authors.map((author) => author.name).join(', '),
+        pubDate: parseDate(item.originalDate),
+        category: item.tags,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.api);
+                const $ = cheerio.load(response.data.pageProps.content, null, false);
+
+                // Remove the TOC
+                $('.toc').remove();
+                item.description = $.html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'Paradigm - Writing',
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/patagonia/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/patagonia/new-arrivals/:category': ['NavePnow'],
+};

package/lib/v2/patagonia/new-arrivals.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const cheerio = require('cheerio');
+const host = 'https://www.patagonia.com';
+const categoryMap = {
+    mens: ['mens-new', 'mens-new-arrivals'],
+    womens: ['womens-new', 'womens-new-arrivals'],
+    kids: ['kids-new-arrivals', 'kids-baby-new-arrivals'],
+    luggage: ['luggage-new-arrivals', 'luggage-new-arrivals'],
+};
+function extractSfrmUrl(url) {
+    const urlObj = new URL(url);
+    const sfrmValue = urlObj.searchParams.get('sfrm');
+    urlObj.search = new URLSearchParams({ sfrm: sfrmValue }).toString();
+    return urlObj.toString();
+}
+module.exports = async (ctx) => {
+    const { category } = ctx.params;
+    const url = `${host}/on/demandware.store/Sites-patagonia-us-Site/en_US/Search-LazyGrid`;
+    const response = await got({
+        method: 'get',
+        url,
+        searchParams: {
+            cgid: categoryMap[category][0],
+            isLazyGrid: true,
+        },
+    });
+    const data = response.data;
+
+    const $ = cheerio.load(data);
+    const list = $('.product')
+        .map(function () {
+            const data = {};
+            data.title = $(this).find('.product-tile').data('tealium').product_name[0];
+            let imgUrl = new URL($(this).find('[itemprop="image"]').attr('content'));
+            imgUrl = extractSfrmUrl(imgUrl);
+
+            const price = $(this).find('[itemprop="price"]').eq(0).text();
+            data.link = host + '/' + $(this).find('[itemprop="url"]').attr('href');
+            data.description =
+                price +
+                art(path.join(__dirname, 'templates/product-description.art'), {
+                    imgUrl,
+                });
+            data.category = $(this).find('[itemprop="category"]').attr('content');
+            return data;
+        })
+        .get();
+    ctx.state.data = {
+        title: `Patagonia - New Arrivals - ${category.toUpperCase()}`,
+        link: `${host}/shop/${categoryMap[category][1]}`,
+        description: `Patagonia - New Arrivals - ${category.toUpperCase()}`,
+        item: list.map((item) => ({
+            title: item.title,
+            description: item.description,
+            link: item.link,
+            category: item.category,
+        })),
+    };
+};

package/lib/v2/patagonia/radar.js

@@ -0,0 +1,23 @@
+module.exports = {
+    'patagonia.com': {
+        _name: 'Patagonia',
+        '.': [
+            {
+                title: 'New Arrivals',
+                docs: 'https://docs.rsshub.app/shopping.html#patagonia',
+                source: ['/shop/*new-arrivals'],
+                target: (_, url) => {
+                    const param = new URL(url).pathname.split('/').pop().replace('-new-arrivals', '');
+                    if (param === 'new-arrivals') {
+                        return '';
+                    }
+                    if (param === 'kids-baby') {
+                        return '/patagonia/new-arrivals/kids';
+                    } else {
+                        return `/patagonia/new-arrivals/${param}`;
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/patagonia/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/new-arrivals/:category', require('./new-arrivals'));
+};

package/lib/v2/patagonia/templates/product-description.art

@@ -0,0 +1,4 @@
+<div>
+    <img src={{imgUrl}}>
+</div>
+

package/lib/v2/people/index.js

@@ -3,12 +3,16 @@ const cheerio = require('cheerio');
 const iconv = require('iconv-lite');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const site = ctx.params[0] ?? 'www';
     let category = ctx.params[1] ?? (site === 'www' ? '59476' : '');
     category = site === 'cpc' && category === '24h' ? '87228' : category;
 
+    if (!isValidHost(site)) {
+        throw Error('Invalid site');
+    }
     const rootUrl = `http://${site}.people.com.cn`;
     const currentUrl = `${rootUrl}/GB/${category}/index.html`;
 

package/lib/v2/people/router.js

@@ -1,6 +1,6 @@
 module.exports = function (router) {
     router.get('/liuyan/:id/:state?', require('./liuyan'));
     router.get('/xjpjh/:keyword?/:year?', require('./xjpjh'));
-    router.get(/([\w|\d]+)\/([\w|\d|/]+)?/, require('./index'));
+    router.get(/([\w\d]+)\/([\w\d/]+)?/, require('./index'));
     router.get('/:0?', require('./index'));
 };

package/lib/v2/phoronix/index.js

@@ -0,0 +1,107 @@
+const parser = require('@/utils/rss-parser');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+const baseUrl = 'https://www.phoronix.com';
+
+module.exports = async (ctx) => {
+    const { page, queryOrItem } = ctx.params;
+    const rssUrl = new URL('/rss.php', baseUrl);
+    rssUrl.searchParams.set('page', page);
+    if (queryOrItem) {
+        if (page === 'category') {
+            rssUrl.searchParams.set('item', queryOrItem);
+        } else {
+            rssUrl.searchParams.set('q', queryOrItem);
+        }
+    }
+
+    const feed = await parser.parseURL(rssUrl.toString());
+
+    const items = await Promise.all(
+        feed.items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const html = response.body;
+                const $ = cheerio.load(html);
+                const content = $('.content');
+
+                // Author
+                const authorSelector = $('.author > a');
+                // thel last 2 are the category and comments
+                const author = authorSelector
+                    .slice(0, authorSelector.length - 2)
+                    .toArray()
+                    .map((e) => $(e).text());
+
+                // Maybe it's paginated
+                const links = $('.pagination > a')
+                    .toArray()
+                    .map((pager) => `${baseUrl}${pager.attribs.href}`)
+                    .slice(0, -1); // the last one is "next page"
+
+                if (links.length) {
+                    const pages = await Promise.all(
+                        links.map((link) =>
+                            ctx.cache.tryGet(link, async () => {
+                                const response = await got(link);
+                                const html = response.data;
+                                const $$ = cheerio.load(html);
+                                const page = $$('.content');
+                                return page.html();
+                            })
+                        )
+                    );
+                    content.append(pages);
+                }
+
+                // Summary
+                const summary = $('.content > p:nth-child(1)');
+
+                // High res images
+                content.find('img').each((_, img) => {
+                    if (img.attribs.src.endsWith('_med')) {
+                        img.attribs.src = img.attribs.src.replace('_med', '_show');
+                    }
+                });
+
+                return {
+                    title: item.title,
+                    id: item.guid,
+                    pubDate: item.pubDate,
+                    author: author.join(', '),
+                    link: item.link,
+                    summary: summary.html(),
+                    description: content.html(),
+                    icon: 'https://www.phoronix.com/android-chrome-192x192.png',
+                    logo: 'https://www.phoronix.com/phxcms7-css/phoronix.png',
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: items,
+        language: feed.language,
+        icon: 'https://www.phoronix.com/android-chrome-192x192.png',
+        image: 'https://www.phoronix.com/android-chrome-192x192.png',
+        logo: 'https://www.phoronix.com/phxcms7-css/phoronix.png',
+        // Copied from thier web page metadata
+        category: [
+            'Linux Hardware Reviews',
+            'Linux hardware benchmarks',
+            'Linux Hardware',
+            'Linux benchmarking',
+            'Desktop Linux',
+            'GNU/Linux benchmarks',
+            'Open Source AMD',
+            'Linux How To',
+            'X.Org drivers',
+            'Ubuntu hardware',
+            'Phoronix Test Suite',
+        ],
+    };
+};

package/lib/v2/phoronix/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:page/:queryOrItem?': ['oppliate'],
+};

package/lib/v2/phoronix/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'phoronix.com': {
+        _name: 'Phoronix',
+        '.': [
+            {
+                title: '新闻与评测',
+                docs: 'https://docs.rsshub.app/new-media.html#phoronix',
+                source: ['/*'],
+                target: '/phoronix/news',
+            },
+        ],
+    },
+};

package/lib/v2/phoronix/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:page/:queryOrItem?', require('./index'));
+};

package/lib/v2/picnob/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/user/:id': ['TonyRL'],
+};

package/lib/v2/picnob/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'picnob.com': {
+        _name: 'Picnob',
+        '.': [
+            {
+                title: 'User profile',
+                docs: 'https://docs.rsshub.app/en/social-media.html#picnob',
+                source: ['/profile/:id/*'],
+                target: '/picnob/user/:id',
+            },
+        ],
+    },
+};

package/lib/v2/picnob/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/user/:id', require('./user'));
+};

package/lib/v2/picnob/templates/desc.art

@@ -0,0 +1,13 @@
+{{ if item.type === 'video' }}
+    <video poster="{{ item.pic }}" controls>
+        <source src="{{ item.video }}" type="video/mp4">
+    </video>
+{{ else if item.type === 'img_multi' }}
+    {{ each images i }}
+        <a href="{{ i.ori }}"><img src="{{ i.url }}"></a>
+    {{ /each }}
+{{ else if item.type === 'img_sig' }}
+    <a href="{{ item.pic }}"><img src="{{ item.pic }}"></a>
+{{ /if }}
+<br>
+{{@ item.sum }}

package/lib/v2/picnob/user.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://www.picnob.com';
+    const { id } = ctx.params;
+    const url = `${baseUrl}/profile/${id}/`;
+    const { data: response } = await got(url);
+    const $ = cheerio.load(response);
+
+    const profileName = $('h1.fullname').text();
+    const userId = $('input[name=userid]').attr('value');
+
+    const { data } = await got(`${baseUrl}/api/posts`, {
+        searchParams: {
+            userid: userId,
+        },
+    });
+
+    const list = data.posts.items.map(async (item) => {
+        const { shortcode, type } = item;
+        const link = `${baseUrl}/post/${shortcode}/`;
+        let images = [];
+        if (type === 'img_multi') {
+            images = await ctx.cache.tryGet(link, async () => {
+                const { data } = await got(link);
+                const $ = cheerio.load(data);
+                return [
+                    ...new Set(
+                        $('.post_slide a')
+                            .toArray()
+                            .map((a) => {
+                                a = $(a);
+                                return {
+                                    ori: a.attr('href'),
+                                    url: a.find('img').attr('data-src'),
+                                };
+                            })
+                    ),
+                ];
+            });
+        }
+        return {
+            title: item.sum_pure,
+            description: art(path.join(__dirname, 'templates/desc.art'), { item, images }),
+            link,
+            pubDate: parseDate(item.time, 'X'),
+        };
+    });
+
+    ctx.state.data = {
+        title: `${profileName} (@${id}) - Picnob`,
+        description: $('.info .sum').text(),
+        link: url,
+        image: $('.ava .pic img').attr('src'),
+        item: list,
+    };
+};

package/lib/v2/picuki/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/profile/:id/:functionalFlag?': ['hoilc', 'Rongronggg9'],
+    '/profile/:id/:functionalFlag?': ['hoilc', 'Rongronggg9', 'devinmugen'],
 };