rsshub 1.0.0-master.f961961 → 1.0.0-master.f992838

package/lib/v2/huxiu/utils.js

@@ -1,69 +1,154 @@
 const cheerio = require('cheerio');
-const date = require('@/utils/date');
+const { parseDate } = require('@/utils/parse-date');
 const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const CryptoJS = require('crypto-js');
 
-const ProcessFeed = async (list, cache) => {
-    const host = 'https://www.huxiu.com';
+const accountApi = 'https://account-api.huxiu.com';
+const articleApi = 'https://api-article.huxiu.com';
+const baseUrl = 'https://www.huxiu.com';
+const briefApi = 'https://api-brief.huxiu.com';
+const momentApi = 'https://moment-api.huxiu.com';
+const searchApi = 'https://search-api.huxiu.com';
 
-    const items = await Promise.all(
-        list.map(async (e) => {
-            const link = new URL(e, host).href;
+const ProcessFeed = (list, cache) =>
+    Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await got(item.link, {
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
+                const $ = cheerio.load(response.data);
+                const initialState = parseInitialState($);
 
-            const single = await cache.tryGet(link, async () => {
-                const response = await got(link);
+                if (item.link.startsWith(`${baseUrl}/article/`)) {
+                    fetchArticle(initialState, item);
+                } else if (item.link.startsWith(`${baseUrl}/brief/`)) {
+                    fetchBrief(initialState, item);
+                }
 
-                const $ = cheerio.load(response.data);
+                return item;
+            })
+        )
+    );
 
-                // const __INITIAL_STATE__ = Function(
-                //     'let window={};' +
-                //         $('*')
-                //             .html()
-                //             .match(/window\.__INITIAL_STATE__.*};/gm)[0] +
-                //         'return window.__INITIAL_STATE__'
-                // )();
-
-                $('.article__content')
-                    .find('.lazyImg')
-                    .each((i, e) => {
-                        $(e).attr('src', $(e).attr('_src'));
-                    });
-
-                $('.article__content')
-                    .find('.text-big-title')
-                    .each((i, e) => (e.tagName = 'h3'));
-
-                $('p').each(function () {
-                    if ($(this).find('img').length === 0 && $(this).text().match(/^\s*$/)) {
-                        $(this).remove();
-                    }
-                });
-                $('img.dialog_add_wxxy_qrcode_icon').remove();
+const fetchArticle = (initialState, item) => {
+    const { articleDetail } = initialState.articleDetail;
+    const topImg = art(path.join(__dirname, 'templates/img.art'), {
+        img: articleDetail.pic_path.split('?')[0],
+    });
+    let video = '';
+    if (articleDetail.video_info) {
+        video = art(path.join(__dirname, 'templates/video.art'), {
+            video_info: articleDetail.video_info,
+        });
+    }
 
-                let description = '';
+    const $ = cheerio.load(articleDetail.content, null, false);
 
-                // if ('video_info' in __INITIAL_STATE__.articleDetail.articleDetail) {
-                //     description += `<video height="100%" poster="${__INITIAL_STATE__.articleDetail.articleDetail.video_info.cover}" controls src="${__INITIAL_STATE__.articleDetail.articleDetail.video_info.hd_link}"></video>`;
-                // }
-                if ($('.top-img').html() !== null) {
-                    description += $('.top-img').html();
-                }
+    $('.text-big-title').each((_, e) => (e.tagName = 'h3'));
+    fixImg($);
+    removeExtraLinebreak($);
+    // $('img.dialog_add_wxxy_qrcode_icon').remove();
+
+    item.description = video + topImg + $.html();
+    item.author = articleDetail.author;
+    item.category = articleDetail.tags_info.map((tag) => tag.name);
+};
+
+const fetchBrief = (initialState, item) => {
+    const { brief_detail } = initialState.briefStoreModule;
+    const { brief_column } = brief_detail;
+
+    const description = art(path.join(__dirname, 'templates/brief.art'), {
+        brief: brief_detail.brief,
+    });
 
-                return {
-                    title: $('.article__title').text().trim(),
-                    description: description + $('.article__content').html(),
-                    pubDate: date($('.article__time').text(), 8),
-                    author: $('.author-info__username').text(),
-                    link,
-                };
-            });
-
-            return single;
-        })
+    const $ = cheerio.load(description, null, false);
+
+    $('button.black-dot').remove();
+    fixImg($);
+    removeExtraLinebreak($);
+
+    item.description = $.html();
+    item.author = brief_detail.brief.publisher_list.map((item) => item.username).join(', ');
+    item.pubDate = parseDate(brief_column.update_time, 'X');
+};
+
+const parseInitialState = ($) =>
+    JSON.parse(
+        $('script')
+            .text()
+            .match(/window\.__INITIAL_STATE__=(\{.*?\});\(function\(\)/)[1]
     );
 
-    return items;
+const fixImg = ($) => {
+    $('img.lazyImg, img.js-preview').each((_, e) => {
+        if (e.attribs._src) {
+            e.attribs.src = e.attribs._src.split('?')[0];
+            delete e.attribs._src;
+        }
+        if (e.attribs.src.includes('?')) {
+            e.attribs.src = e.attribs.src.split('?')[0];
+        }
+        if (e.attribs['data-w']) {
+            e.attribs.width = e.attribs['data-w'];
+            delete e.attribs['data-w'];
+        }
+        if (e.attribs['data-h']) {
+            e.attribs.height = e.attribs['data-h'];
+            delete e.attribs['data-h'];
+        }
+    });
+};
+
+const removeExtraLinebreak = ($) => {
+    $('p').each((_, e) => {
+        e = $(e);
+        if (e.find('img').length === 0 && e.text().match(/^\s*$/)) {
+            e.remove();
+        }
+    });
+};
+
+const generateNonce = () => {
+    let nonce = '';
+    const e = 'abcdefghijklmnopqrstuvwxyz1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+    const t = 16;
+    for (let i = 0; i < t; i++) {
+        nonce += e.charAt(Math.floor(Math.random() * e.length));
+    }
+    return nonce;
+};
+
+const generateSignature = () => {
+    const timestamp = Math.round(new Date().getTime() / 1000).toString();
+    // const appSecret = '4nHzGgGt7WX4zFTsTKocpxg4dzU6-wUi';
+    const appSecret = 'hUzaVKtNfDE-6UiyaJdfsmjW-8dwoyVc'; // appid: 'hx_search'
+    const nonce = generateNonce();
+    const r = [appSecret, timestamp, nonce].sort();
+    return {
+        nonce,
+        timestamp,
+        signature: CryptoJS.SHA1(r[0] + r[1] + r[2]).toString(),
+    };
 };
 
 module.exports = {
+    accountApi,
+    articleApi,
+    baseUrl,
+    briefApi,
+    momentApi,
+    searchApi,
     ProcessFeed,
+    fetchArticle,
+    fetchBrief,
+    parseInitialState,
+    fixImg,
+    removeExtraLinebreak,
+    generateSignature,
 };

package/lib/v2/ieee-security/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/security-privacy': ['ZeddYu'],
+};

package/lib/v2/ieee-security/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'ieee-security.org': {
+        _name: 'IEEE Computer Society',
+        '.': [
+            {
+                title: 'IEEE Symposium on Security and Privacy',
+                docs: 'https://docs.rsshub.app/journal.html#ieee-computer-society',
+                source: ['/TC/SP-Index.html', '/'],
+                target: '/ieee-security/security-privacy',
+            },
+        ],
+    },
+};

package/lib/v2/ieee-security/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/security-privacy', require('./sp'));
+};

package/lib/v2/ieee-security/sp.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const url = 'https://www.ieee-security.org/';
+// https://www.ieee-security.org/TC/SP2023/program-papers.html
+
+module.exports = async (ctx) => {
+    const last = new Date().getFullYear() + 1;
+    const urlList = Array.from({ length: last - 2020 }, (_, v) => `${url}TC/SP${v + 2020}/program-papers.html`);
+    const responses = await got.all(urlList.map((url) => got(url)));
+
+    const items = responses.map((response) => {
+        const $ = cheerio.load(response.data);
+        return $('div.panel-body > div.list-group-item')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+                const title = item.find('b').text().trim();
+                const link = response.url;
+                return {
+                    title,
+                    author: item.html().trim().split('<br>')[1].trim(),
+                    link: `${link}#${title}`,
+                    pubDate: parseDate(link.match(/SP(\d{4})/)[1], 'YYYY'),
+                };
+            });
+    });
+
+    ctx.state.data = {
+        title: 'S&P',
+        link: `${url}TC/SP-Index.html`,
+        description: 'IEEE Symposium on Security and Privacy Accepted Papers',
+        allowEmpty: true,
+        item: items.flat(),
+    };
+};

package/lib/v2/iiilab/index.js

@@ -0,0 +1,24 @@
+const buildData = require('@/utils/common-config');
+const baseUrl = 'https://www.iiilab.com/';
+
+module.exports = async (ctx) => {
+    const link = baseUrl;
+    ctx.state.data = await buildData({
+        link,
+        url: link,
+        title: `%title%`,
+        description: `%description%`,
+        params: {
+            title: '发现',
+            description: '人人都是自媒体-发现',
+        },
+        item: {
+            item: '.aw-common-list > div',
+            title: `$('a').first().text()`,
+            link: `$('a').first().attr('href')`,
+            description: `$('.markitup-box').first().text()`,
+            pubDate: `parseDate($('.text-color-999').first().text(), 'YYYY-MM-DD HH:mm')`,
+            guid: Buffer.from(`$('a').attr('href')`).toString('base64'),
+        },
+    });
+};

package/lib/v2/iiilab/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['Joey'],
+};

package/lib/v2/iiilab/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'iiilab.com': {
+        _name: '人人都是自媒体',
+        www: [
+            {
+                title: '发现',
+                docs: 'https://docs.rsshub.app/new-media.html#ren-ren-dou-shi-zi-mei-ti',
+                source: '/',
+                target: '/iiilab',
+            },
+        ],
+    },
+};

package/lib/v2/iiilab/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index'));
+};

package/lib/v2/ikea/gb/new.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const link = 'https://www.ikea.com/gb/en/new/new-products/';
+
+    const {
+        data: { specialPage },
+    } = await got('https://sik.search.blue.cdtapps.com/gb/en/special', {
+        searchParams: {
+            special: 'new_product',
+            // size: 24,
+            // 'subcategories-style': 'tree-navigation',
+            // c: 'lf',
+            // v: 20220826,
+            // sort: 'RELEVANCE',
+        },
+    });
+
+    const {
+        data: {
+            moreProducts: { productWindow },
+        },
+    } = await got('https://sik.search.blue.cdtapps.com/gb/en/special/more-products', {
+        searchParams: {
+            special: 'new_product',
+            start: 24,
+            end: specialPage.productCount,
+            // 'subcategories-style': 'tree-navigation',
+            // c: 'lf',
+            // v: 20220826,
+            // sort: 'RELEVANCE',
+        },
+    });
+
+    const products = [...specialPage.productWindow, ...productWindow];
+
+    const items = products.map((p) => ({
+        title: `${p.name} ${p.typeName}, ${p.itemMeasureReferenceText}`,
+        description: art(path.join(__dirname, '../templates/new.art'), {
+            p,
+        }),
+        link: p.pipUrl,
+        category: p.categoryPath.map((c) => c.name),
+    }));
+
+    ctx.state.data = {
+        title: 'New Products - Browse All New Furniture & Home Decor  - IKEA',
+        link,
+        description: 'New products released by IKEA UK.',
+        item: items,
+    };
+};

package/lib/v2/ikea/gb/offer.js

@@ -0,0 +1,69 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const link = 'https://www.ikea.com/gb/en/offers/';
+    const response = await got(link);
+
+    const $ = cheerio.load(response.data);
+    const carousel = $('.pub__carousel-slide')
+        .toArray()
+        .map((e) => {
+            e = $(e);
+            const title = e.find('h3');
+            const img = e.find('.pub__image').each((_, e) => {
+                e.attribs.src = e.attribs.src.split('?')[0];
+                delete e.attribs.srcset;
+            });
+            const link = new URL(e.find('pub-hide-empty-link a').attr('href'));
+            const { searchParams } = link;
+            searchParams.delete('itm_content');
+            searchParams.delete('itm_element');
+            searchParams.delete('itm_campaign');
+            return {
+                title: title.text(),
+                description: art(path.join(__dirname, '../templates/offer.art'), {
+                    img: img.parent().html(),
+                    desc: title.next().parent().html(),
+                }),
+                link: link.href,
+                guid: `${link.href}#${title.text()}`,
+            };
+        });
+
+    const banner = $('div[data-pub-type="banner"]')
+        .toArray()
+        .map((e) => {
+            e = $(e);
+            const title = e.find('h2');
+            const next = title.next();
+            const img = e.find('.pub__image').each((_, e) => {
+                e.attribs.src = e.attribs.src.split('?')[0];
+                delete e.attribs.srcset;
+            });
+
+            const link = new URL(next.find('a').attr('href'));
+            const { searchParams } = link;
+            searchParams.delete('itm_content');
+            searchParams.delete('itm_element');
+            searchParams.delete('itm_campaign');
+            return {
+                title: title.text(),
+                description: art(path.join(__dirname, '../templates/offer.art'), {
+                    img: img.parent().html(),
+                    desc: title.parent().html(),
+                }),
+                link: link.href,
+                guid: `${link.href}#${title.text()}`,
+            };
+        });
+
+    ctx.state.data = {
+        title: 'IKEA UK - Offers',
+        link,
+        description: 'Offers by IKEA UK.',
+        item: [...carousel, ...banner],
+    };
+};

package/lib/v2/ikea/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/gb/new': ['HenryQW'],
+    '/gb/offer': ['HenryQW'],
+};

package/lib/v2/ikea/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'ikea.com': {
+        _name: 'IKEA 宜家',
+        '.': [
+            {
+                title: '英国 - 商品上新',
+                docs: 'https://docs.rsshub.app/shopping.html#ikea-yi-jia',
+                source: ['/gb/en/new/new-products/', '/'],
+                target: '/ikea/gb/new',
+            },
+            {
+                title: '英国 - 促销',
+                docs: 'https://docs.rsshub.app/shopping.html#ikea-yi-jia',
+                source: ['/gb/en/offers', '/'],
+                target: '/ikea/gb/offer',
+            },
+        ],
+    },
+};

package/lib/v2/ikea/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/gb/new', require('./gb/new'));
+    router.get('/gb/offer', require('./gb/offer'));
+};

package/lib/v2/ikea/templates/new.art

@@ -0,0 +1,7 @@
+<img src="{{ p.mainImageUrl }}" alt="{{ p.mainImageAlt }}">
+<br>
+<b>{{ p.name }}</b>
+<br>
+{{ p.typeName }}, {{ p.itemMeasureReferenceText }}
+<br>
+{{ p.salesPrice.current.prefix }} <b>{{ p.salesPrice.current.wholeNumber }}</b>

package/lib/v2/ikea/templates/offer.art

@@ -0,0 +1,3 @@
+{{@ img }}
+<br>
+{{@ desc }}

package/lib/v2/inewsweek/index.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const iconv = require('iconv-lite');
+
+const rootUrl = 'http://news.inewsweek.cn';
+
+module.exports = async (ctx) => {
+    const channel = ctx.params.channel;
+    const url = `${rootUrl}/${channel}`;
+    const response = await got(url, {
+        responseType: 'buffer',
+    });
+    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+    const items = await Promise.all(
+        $('div.grid-item')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+                const href = item.find('a').attr('href');
+                const articleLink = `${rootUrl}${href}`;
+                return ctx.cache.tryGet(articleLink, async () => {
+                    const response = await got(articleLink, {
+                        responseType: 'buffer',
+                    });
+
+                    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+                    const fullText = $('div.contenttxt').html();
+                    const time = timezone(
+                        parseDate(
+                            $('div.editor')
+                                .html()
+                                .split(/(\s\s+)/)[2]
+                        ),
+                        +8
+                    );
+                    return {
+                        title: item.find('p').text(),
+                        description: fullText,
+                        link: articleLink,
+                        pubDate: time,
+                    };
+                });
+            })
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/inewsweek/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:channel': ['changren-wcr'],
+};

package/lib/v2/inewsweek/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'inewsweek.cn': {
+        _name: '中国新闻周刊',
+        '.': [
+            {
+                title: '栏目',
+                docs: 'https://docs.rsshub.app/traditional-media.html#zhong-guo-xin-wen-zhou-kan',
+                source: ['/:channel', '/'],
+                target: '/inewsweek/:channel',
+            },
+        ],
+    },
+};

package/lib/v2/inewsweek/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:channel', require('./index'));
+};

package/lib/v2/itch/devlog.js

@@ -4,10 +4,14 @@ const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const user = ctx.params.user ?? '';
     const id = ctx.params.id ?? '';
+    if (!isValidHost(user)) {
+        throw Error('Invalid user');
+    }
 
     const rootUrl = `https://${user}.itch.io/${id}/devlog`;
 

package/lib/v2/iwara/index.js

@@ -30,6 +30,7 @@ module.exports = async (ctx) => {
                 author: username,
                 link: rootUrl + $(item).find('h3.title > a').attr('href'),
                 description: `<img src="${imageUrl}">`,
+                guid: (rootUrl + $(item).find('h3.title > a').attr('href')).split('?')[0],
             };
         })
         .get();

package/lib/v2/javbus/index.js

@@ -3,17 +3,27 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const config = require('@/config').value;
 
 const toSize = (raw) => {
     const matches = raw.match(/(\d+(\.\d+)?)(\w+)/);
     return matches[3] === 'GB' ? matches[1] * 1024 : matches[1];
 };
 
+const allowDomain = ['javbus.com', 'javbus.org', 'javsee.icu', 'javsee.one'];
+
 module.exports = async (ctx) => {
     const isWestern = /^\/western/.test(ctx.path);
+    const domain = ctx.query.domain ?? 'javbus.com';
+    const westernDomain = ctx.query.western_domain ?? 'javbus.org';
+
+    const rootUrl = `https://www.${domain}`;
+    const westernUrl = `https://www.${westernDomain}`;
+
+    if (!config.feature.allow_user_supply_unsafe_domain && (!allowDomain.includes(new URL(`https://${domain}/`).hostname) || !allowDomain.includes(new URL(`https://${westernDomain}/`).hostname))) {
+        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+    }
 
-    const rootUrl = `https://www.${ctx.query.domain ?? 'javbus.com'}`;
-    const westernUrl = `https://www.${ctx.query.western_domain ?? 'javbus.org'}`;
     const currentUrl = `${isWestern ? westernUrl : rootUrl}${ctx.path.replace(/^\/western/, '').replace(/\/home/, '')}`;
 
     const response = await got({

package/lib/v2/javdb/utils.js

@@ -1,10 +1,16 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const allowDomain = ['javdb.com', 'javdb36.com', 'javdb007.com'];
 
 module.exports = {
     ProcessItems: async (ctx, currentUrl, title) => {
         const domain = ctx.query.domain ?? 'javdb.com';
+        if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(new URL(`https://${domain}/`).hostname)) {
+            ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+        }
+
         const rootUrl = `https://${domain}`;
 
         const response = await got({

package/lib/v2/jike/maintainer.js

@@ -1,5 +1,5 @@
 module.exports = {
     '/topic/text/:id': ['HenryQW'],
-    '/topic/:id': ['DIYgod', 'prnake'],
+    '/topic/:id/:showUid?': ['DIYgod', 'prnake'],
     '/user/:id': ['DIYgod', 'prnake'],
 };

package/lib/v2/jike/router.js

@@ -1,5 +1,5 @@
 module.exports = (router) => {
     router.get('/topic/text/:id', require('./topicText'));
-    router.get('/topic/:id', require('./topic'));
+    router.get('/topic/:id/:showUid?', require('./topic'));
     router.get('/user/:id', require('./user'));
 };