rsshub 1.0.0-master.fe99193 → 1.0.0-master.fe9c82a

package/lib/v2/huxiu/utils.js

@@ -1,69 +1,154 @@
 const cheerio = require('cheerio');
-const date = require('@/utils/date');
+const { parseDate } = require('@/utils/parse-date');
 const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const CryptoJS = require('crypto-js');
 
-const ProcessFeed = async (list, cache) => {
-    const host = 'https://www.huxiu.com';
+const accountApi = 'https://account-api.huxiu.com';
+const articleApi = 'https://api-article.huxiu.com';
+const baseUrl = 'https://www.huxiu.com';
+const briefApi = 'https://api-brief.huxiu.com';
+const momentApi = 'https://moment-api.huxiu.com';
+const searchApi = 'https://search-api.huxiu.com';
 
-    const items = await Promise.all(
-        list.map(async (e) => {
-            const link = new URL(e, host).href;
+const ProcessFeed = (list, cache) =>
+    Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await got(item.link, {
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
+                const $ = cheerio.load(response.data);
+                const initialState = parseInitialState($);
 
-            const single = await cache.tryGet(link, async () => {
-                const response = await got(link);
+                if (item.link.startsWith(`${baseUrl}/article/`)) {
+                    fetchArticle(initialState, item);
+                } else if (item.link.startsWith(`${baseUrl}/brief/`)) {
+                    fetchBrief(initialState, item);
+                }
 
-                const $ = cheerio.load(response.data);
+                return item;
+            })
+        )
+    );
 
-                // const __INITIAL_STATE__ = Function(
-                //     'let window={};' +
-                //         $('*')
-                //             .html()
-                //             .match(/window\.__INITIAL_STATE__.*};/gm)[0] +
-                //         'return window.__INITIAL_STATE__'
-                // )();
-
-                $('.article__content')
-                    .find('.lazyImg')
-                    .each((i, e) => {
-                        $(e).attr('src', $(e).attr('_src'));
-                    });
-
-                $('.article__content')
-                    .find('.text-big-title')
-                    .each((i, e) => (e.tagName = 'h3'));
-
-                $('p').each(function () {
-                    if ($(this).find('img').length === 0 && $(this).text().match(/^\s*$/)) {
-                        $(this).remove();
-                    }
-                });
-                $('img.dialog_add_wxxy_qrcode_icon').remove();
+const fetchArticle = (initialState, item) => {
+    const { articleDetail } = initialState.articleDetail;
+    const topImg = art(path.join(__dirname, 'templates/img.art'), {
+        img: articleDetail.pic_path.split('?')[0],
+    });
+    let video = '';
+    if (articleDetail.video_info) {
+        video = art(path.join(__dirname, 'templates/video.art'), {
+            video_info: articleDetail.video_info,
+        });
+    }
 
-                let description = '';
+    const $ = cheerio.load(articleDetail.content, null, false);
 
-                // if ('video_info' in __INITIAL_STATE__.articleDetail.articleDetail) {
-                //     description += `<video height="100%" poster="${__INITIAL_STATE__.articleDetail.articleDetail.video_info.cover}" controls src="${__INITIAL_STATE__.articleDetail.articleDetail.video_info.hd_link}"></video>`;
-                // }
-                if ($('.top-img').html() !== null) {
-                    description += $('.top-img').html();
-                }
+    $('.text-big-title').each((_, e) => (e.tagName = 'h3'));
+    fixImg($);
+    removeExtraLinebreak($);
+    // $('img.dialog_add_wxxy_qrcode_icon').remove();
+
+    item.description = video + topImg + $.html();
+    item.author = articleDetail.author;
+    item.category = articleDetail.tags_info.map((tag) => tag.name);
+};
+
+const fetchBrief = (initialState, item) => {
+    const { brief_detail } = initialState.briefStoreModule;
+    const { brief_column } = brief_detail;
+
+    const description = art(path.join(__dirname, 'templates/brief.art'), {
+        brief: brief_detail.brief,
+    });
 
-                return {
-                    title: $('.article__title').text().trim(),
-                    description: description + $('.article__content').html(),
-                    pubDate: date($('.article__time').text(), 8),
-                    author: $('.author-info__username').text(),
-                    link,
-                };
-            });
-
-            return single;
-        })
+    const $ = cheerio.load(description, null, false);
+
+    $('button.black-dot').remove();
+    fixImg($);
+    removeExtraLinebreak($);
+
+    item.description = $.html();
+    item.author = brief_detail.brief.publisher_list.map((item) => item.username).join(', ');
+    item.pubDate = parseDate(brief_column.update_time, 'X');
+};
+
+const parseInitialState = ($) =>
+    JSON.parse(
+        $('script')
+            .text()
+            .match(/window\.__INITIAL_STATE__=(\{.*?\});\(function\(\)/)[1]
     );
 
-    return items;
+const fixImg = ($) => {
+    $('img.lazyImg, img.js-preview').each((_, e) => {
+        if (e.attribs._src) {
+            e.attribs.src = e.attribs._src.split('?')[0];
+            delete e.attribs._src;
+        }
+        if (e.attribs.src.includes('?')) {
+            e.attribs.src = e.attribs.src.split('?')[0];
+        }
+        if (e.attribs['data-w']) {
+            e.attribs.width = e.attribs['data-w'];
+            delete e.attribs['data-w'];
+        }
+        if (e.attribs['data-h']) {
+            e.attribs.height = e.attribs['data-h'];
+            delete e.attribs['data-h'];
+        }
+    });
+};
+
+const removeExtraLinebreak = ($) => {
+    $('p').each((_, e) => {
+        e = $(e);
+        if (e.find('img').length === 0 && e.text().match(/^\s*$/)) {
+            e.remove();
+        }
+    });
+};
+
+const generateNonce = () => {
+    let nonce = '';
+    const e = 'abcdefghijklmnopqrstuvwxyz1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+    const t = 16;
+    for (let i = 0; i < t; i++) {
+        nonce += e.charAt(Math.floor(Math.random() * e.length));
+    }
+    return nonce;
+};
+
+const generateSignature = () => {
+    const timestamp = Math.round(new Date().getTime() / 1000).toString();
+    // const appSecret = '4nHzGgGt7WX4zFTsTKocpxg4dzU6-wUi';
+    const appSecret = 'hUzaVKtNfDE-6UiyaJdfsmjW-8dwoyVc'; // appid: 'hx_search'
+    const nonce = generateNonce();
+    const r = [appSecret, timestamp, nonce].sort();
+    return {
+        nonce,
+        timestamp,
+        signature: CryptoJS.SHA1(r[0] + r[1] + r[2]).toString(),
+    };
 };
 
 module.exports = {
+    accountApi,
+    articleApi,
+    baseUrl,
+    briefApi,
+    momentApi,
+    searchApi,
     ProcessFeed,
+    fetchArticle,
+    fetchBrief,
+    parseInitialState,
+    fixImg,
+    removeExtraLinebreak,
+    generateSignature,
 };

package/lib/v2/ieee-security/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/security-privacy': ['ZeddYu'],
+};

package/lib/v2/ieee-security/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'ieee-security.org': {
+        _name: 'IEEE Computer Society',
+        '.': [
+            {
+                title: 'IEEE Symposium on Security and Privacy',
+                docs: 'https://docs.rsshub.app/journal.html#ieee-computer-society',
+                source: ['/TC/SP-Index.html', '/'],
+                target: '/ieee-security/security-privacy',
+            },
+        ],
+    },
+};

package/lib/v2/ieee-security/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/security-privacy', require('./sp'));
+};

package/lib/v2/ieee-security/sp.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const url = 'https://www.ieee-security.org/';
+// https://www.ieee-security.org/TC/SP2023/program-papers.html
+
+module.exports = async (ctx) => {
+    const last = new Date().getFullYear() + 1;
+    const urlList = Array.from({ length: last - 2020 }, (_, v) => `${url}TC/SP${v + 2020}/program-papers.html`);
+    const responses = await got.all(urlList.map((url) => got(url)));
+
+    const items = responses.map((response) => {
+        const $ = cheerio.load(response.data);
+        return $('div.panel-body > div.list-group-item')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+                const title = item.find('b').text().trim();
+                const link = response.url;
+                return {
+                    title,
+                    author: item.html().trim().split('<br>')[1].trim(),
+                    link: `${link}#${title}`,
+                    pubDate: parseDate(link.match(/SP(\d{4})/)[1], 'YYYY'),
+                };
+            });
+    });
+
+    ctx.state.data = {
+        title: 'S&P',
+        link: `${url}TC/SP-Index.html`,
+        description: 'IEEE Symposium on Security and Privacy Accepted Papers',
+        allowEmpty: true,
+        item: items.flat(),
+    };
+};

package/lib/v2/iiilab/index.js

@@ -0,0 +1,24 @@
+const buildData = require('@/utils/common-config');
+const baseUrl = 'https://www.iiilab.com/';
+
+module.exports = async (ctx) => {
+    const link = baseUrl;
+    ctx.state.data = await buildData({
+        link,
+        url: link,
+        title: `%title%`,
+        description: `%description%`,
+        params: {
+            title: '发现',
+            description: '人人都是自媒体-发现',
+        },
+        item: {
+            item: '.aw-common-list > div',
+            title: `$('a').first().text()`,
+            link: `$('a').first().attr('href')`,
+            description: `$('.markitup-box').first().text()`,
+            pubDate: `parseDate($('.text-color-999').first().text(), 'YYYY-MM-DD HH:mm')`,
+            guid: Buffer.from(`$('a').attr('href')`).toString('base64'),
+        },
+    });
+};

package/lib/v2/iiilab/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['Joey'],
+};

package/lib/v2/iiilab/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'iiilab.com': {
+        _name: '人人都是自媒体',
+        www: [
+            {
+                title: '发现',
+                docs: 'https://docs.rsshub.app/new-media.html#ren-ren-dou-shi-zi-mei-ti',
+                source: '/',
+                target: '/iiilab',
+            },
+        ],
+    },
+};

package/lib/v2/iiilab/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index'));
+};

package/lib/v2/ikea/cn/family_offers.js

@@ -0,0 +1,70 @@
+const got = require('@/utils/got');
+const { generateProductItem } = require('./utils');
+
+const familyPriceProductsRequest = ({ pageIndex = 1 }) =>
+    got({
+        method: 'post',
+        url: 'https://www.ikea.cn/api-host/search/prod/advanced/special',
+        headers: {
+            'X-Client-Platform': 'PcWeb',
+        },
+        json: {
+            contentType: 'PRODUCT,CONTENT,PLANNER',
+            filters: {
+                filters: {},
+                sortOption: 'RELEVANCE',
+            },
+            pageIndex,
+            pageSize: 25,
+            pageType: '',
+            queryContent: 'family_price',
+        },
+    });
+
+const productRequest = async (productIds) => {
+    const response = await got({
+        url: 'https://www.ikea.cn/api-host/content/products',
+        headers: {
+            'X-Client-Platform': 'PcWeb',
+        },
+        searchParams: new URLSearchParams(productIds),
+    });
+    return response.data;
+};
+
+module.exports = async (ctx) => {
+    const familyPriceProductIds = [];
+    const productRequests = [];
+
+    const familyPriceProductsLoadMore = async ({ pageIndex }) => {
+        const response = await familyPriceProductsRequest({ pageIndex });
+        const {
+            data: { productPage, products },
+        } = response.data;
+
+        products.forEach(({ id }) => {
+            familyPriceProductIds.push(['ids', id]);
+        });
+
+        if (productPage.end < productPage.total) {
+            await familyPriceProductsLoadMore({ pageIndex: pageIndex + 1 });
+        }
+        return Promise.resolve();
+    };
+
+    await familyPriceProductsLoadMore({ pageIndex: 1 });
+
+    while (familyPriceProductIds.length) {
+        productRequests.push(productRequest(familyPriceProductIds.splice(0, 25)));
+    }
+
+    const productResponses = await Promise.all(productRequests);
+    const products = productResponses.flat();
+
+    ctx.state.data = {
+        title: 'IKEA 宜家 - 会员特惠',
+        link: 'https://www.ikea.cn/cn/zh/offers/family-offers/',
+        description: '会员特惠',
+        item: products.map(generateProductItem),
+    };
+};

package/lib/v2/ikea/cn/low_price.js

@@ -0,0 +1,22 @@
+const got = require('@/utils/got');
+const { generateRequestHeaders, generateProductItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const response = await got({
+        url: 'https://srv.app.ikea.cn/content/recommendation/v2/product-group/products',
+        headers: generateRequestHeaders(),
+        searchParams: {
+            processOutOfStock: 'SORT',
+            groupId: 'cms_低价好物_cms-商品列表-_0',
+            page: 1,
+            size: 200,
+        },
+    });
+
+    ctx.state.data = {
+        title: 'IKEA 宜家 - 低价优选',
+        link: 'https://www.ikea.cn/cn/zh/campaigns/wo3-men2-de-chao1-zhi2-di1-jia4-pub8b08af40',
+        description: '低价优选',
+        item: response.data.products.map(generateProductItem),
+    };
+};

package/lib/v2/ikea/cn/new.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const { generateRequestHeaders, generateProductItem } = require('./utils');
+
+const request = ({ moreToken = '' }) =>
+    got({
+        method: 'post',
+        url: 'https://srv-mp.app.ikea.cn/content/search/products/advanced',
+        headers: generateRequestHeaders(),
+        searchParams: {
+            keyword: '新品',
+            moreToken,
+        },
+        json: {},
+    });
+
+module.exports = async (ctx) => {
+    const allProductSummaries = [];
+
+    const loadMoreRequest = async ({ moreToken }) => {
+        const response = await request({ moreToken });
+        const { data } = response;
+        allProductSummaries.push(data.productSummaries);
+        if (data.moreToken) {
+            await loadMoreRequest({ moreToken: data.moreToken });
+        }
+        return Promise.resolve();
+    };
+
+    await loadMoreRequest({});
+
+    const products = allProductSummaries.flat();
+
+    ctx.state.data = {
+        title: 'IKEA 宜家 - 当季新品推荐',
+        link: 'https://www.ikea.cn/cn/zh/new/',
+        description: '当季新品推荐',
+        item: products.map(generateProductItem),
+    };
+};

package/lib/v2/ikea/cn/utils.js

@@ -0,0 +1,42 @@
+const { art } = require('@/utils/render');
+const md5 = require('@/utils/md5');
+const path = require('path');
+
+const generateRequestHeaders = () => {
+    const now = Math.round(new Date().getTime() / 1000);
+    return {
+        'X-Client-Platform': 'WechatMiniprogram',
+        'X-Client-DeviceId': md5(now.toString()),
+    };
+};
+
+const generateProductItem = (product) => {
+    const {
+        productFullId,
+        name,
+        productType,
+        measureText,
+        priceDisplay: { currentPrice, originalPrice },
+        images,
+    } = product;
+    const isFamilyOffer = currentPrice && originalPrice;
+
+    return {
+        title: `${name} ${productType} - \u{000A5}${currentPrice}`,
+        description: art(path.join(__dirname, '../templates/cn/product.art'), {
+            isFamilyOffer,
+            name,
+            productType,
+            measureText,
+            currentPrice,
+            originalPrice,
+            images: images.map((image) => image.url),
+        }),
+        link: `https://www.ikea.cn/cn/zh/p/${productFullId}`,
+    };
+};
+
+module.exports = {
+    generateRequestHeaders,
+    generateProductItem,
+};

package/lib/v2/ikea/gb/new.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const link = 'https://www.ikea.com/gb/en/new/new-products/';
+
+    const {
+        data: { specialPage },
+    } = await got('https://sik.search.blue.cdtapps.com/gb/en/special', {
+        searchParams: {
+            special: 'new_product',
+            // size: 24,
+            // 'subcategories-style': 'tree-navigation',
+            // c: 'lf',
+            // v: 20220826,
+            // sort: 'RELEVANCE',
+        },
+    });
+
+    const {
+        data: {
+            moreProducts: { productWindow },
+        },
+    } = await got('https://sik.search.blue.cdtapps.com/gb/en/special/more-products', {
+        searchParams: {
+            special: 'new_product',
+            start: 24,
+            end: specialPage.productCount,
+            // 'subcategories-style': 'tree-navigation',
+            // c: 'lf',
+            // v: 20220826,
+            // sort: 'RELEVANCE',
+        },
+    });
+
+    const products = [...specialPage.productWindow, ...productWindow];
+
+    const items = products.map((p) => ({
+        title: `${p.name} ${p.typeName}, ${p.itemMeasureReferenceText}`,
+        description: art(path.join(__dirname, '../templates/new.art'), {
+            p,
+        }),
+        link: p.pipUrl,
+        category: p.categoryPath.map((c) => c.name),
+    }));
+
+    ctx.state.data = {
+        title: 'New Products - Browse All New Furniture & Home Decor  - IKEA',
+        link,
+        description: 'New products released by IKEA UK.',
+        item: items,
+    };
+};

package/lib/v2/ikea/gb/offer.js

@@ -0,0 +1,69 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const link = 'https://www.ikea.com/gb/en/offers/';
+    const response = await got(link);
+
+    const $ = cheerio.load(response.data);
+    const carousel = $('.pub__carousel-slide')
+        .toArray()
+        .map((e) => {
+            e = $(e);
+            const title = e.find('h3');
+            const img = e.find('.pub__image').each((_, e) => {
+                e.attribs.src = e.attribs.src.split('?')[0];
+                delete e.attribs.srcset;
+            });
+            const link = new URL(e.find('pub-hide-empty-link a').attr('href'));
+            const { searchParams } = link;
+            searchParams.delete('itm_content');
+            searchParams.delete('itm_element');
+            searchParams.delete('itm_campaign');
+            return {
+                title: title.text(),
+                description: art(path.join(__dirname, '../templates/offer.art'), {
+                    img: img.parent().html(),
+                    desc: title.next().parent().html(),
+                }),
+                link: link.href,
+                guid: `${link.href}#${title.text()}`,
+            };
+        });
+
+    const banner = $('div[data-pub-type="banner"]')
+        .toArray()
+        .map((e) => {
+            e = $(e);
+            const title = e.find('h2');
+            const next = title.next();
+            const img = e.find('.pub__image').each((_, e) => {
+                e.attribs.src = e.attribs.src.split('?')[0];
+                delete e.attribs.srcset;
+            });
+
+            const link = new URL(next.find('a').attr('href'));
+            const { searchParams } = link;
+            searchParams.delete('itm_content');
+            searchParams.delete('itm_element');
+            searchParams.delete('itm_campaign');
+            return {
+                title: title.text(),
+                description: art(path.join(__dirname, '../templates/offer.art'), {
+                    img: img.parent().html(),
+                    desc: title.parent().html(),
+                }),
+                link: link.href,
+                guid: `${link.href}#${title.text()}`,
+            };
+        });
+
+    ctx.state.data = {
+        title: 'IKEA UK - Offers',
+        link,
+        description: 'Offers by IKEA UK.',
+        item: [...carousel, ...banner],
+    };
+};

package/lib/v2/ikea/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/cn/family_offers': ['jzhangdev'],
+    '/cn/low_price': ['jzhangdev'],
+    '/cn/new': ['jzhangdev'],
+    '/gb/new': ['HenryQW'],
+    '/gb/offer': ['HenryQW'],
+};

package/lib/v2/ikea/radar.js

@@ -0,0 +1,42 @@
+module.exports = {
+    'ikea.com': {
+        _name: 'IKEA 宜家',
+        '.': [
+            {
+                title: '英国 - 商品上新',
+                docs: 'https://docs.rsshub.app/shopping.html#ikea-yi-jia',
+                source: ['/gb/en/new/new-products/', '/'],
+                target: '/ikea/gb/new',
+            },
+            {
+                title: '英国 - 促销',
+                docs: 'https://docs.rsshub.app/shopping.html#ikea-yi-jia',
+                source: ['/gb/en/offers', '/'],
+                target: '/ikea/gb/offer',
+            },
+        ],
+    },
+    'ikea.cn': {
+        _name: 'IKEA 宜家',
+        '.': [
+            {
+                title: '中国 - 当季新品推荐',
+                docs: 'https://docs.rsshub.app/shopping.html#ikea-yi-jia',
+                source: ['/cn/zh/new/', '/'],
+                target: '/ikea/cn/new',
+            },
+            {
+                title: '中国 - 低价优选',
+                docs: 'https://docs.rsshub.app/shopping.html#ikea-yi-jia',
+                source: ['/cn/zh/campaigns/wo3-men2-de-chao1-zhi2-di1-jia4-pub8b08af40', '/'],
+                target: '/ikea/cn/low_price',
+            },
+            {
+                title: '中国 - 会员特惠',
+                docs: 'https://docs.rsshub.app/shopping.html#ikea-yi-jia',
+                source: ['/cn/zh/offers/family-offers', '/'],
+                target: '/ikea/cn/family_offers',
+            },
+        ],
+    },
+};