rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/lib/v2/huxiu/briefColumn.js

@@ -1,51 +1,41 @@
 const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate, parseRelativeDate } = require('@/utils/parse-date');
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, briefApi, ProcessFeed } = require('./utils');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const link = `https://api-brief.huxiu.com/briefColumn/getContentListByCategoryId`;
-    const response = await got
+    const link = `${briefApi}/briefColumn/getContentListByCategoryId`;
+    const { data: response } = await got
         .post(link, {
-            headers: {
-                'Content-Type': 'application/x-www-form-urlencoded',
+            form: {
+                platform: 'www',
+                brief_column_id: id,
             },
-            body: `platform=www&brief_column_id=${id}&last_id=`,
         })
         .json();
-
-    const list = response.data.datalist;
-
-    const items = await Promise.all(
-        list.map((item) => {
-            const link = `https://www.huxiu.com/brief/${item.brief_id}`;
-
-            return ctx.cache.tryGet(link, async () => {
-                const detailResponse = await got(link);
-                const $ = cheerio.load(detailResponse.data);
-
-                const description = $('#js-part').html();
-
-                return {
-                    title: item.title,
-                    pubDate: item.format_publish_time.includes('前') ? parseRelativeDate(item.format_publish_time) : parseDate(item.format_publish_time),
-                    description,
-                    link,
-                };
-            });
-        })
-    );
-
-    const title = async () => {
-        const res = await got(`https://www.huxiu.com/briefColumn/${id}.html`);
-        const $ = cheerio.load(res.data);
-
-        return $('#top > div:nth-child(1) > div > div.brief-content__name').text();
-    };
+    const {
+        data: { data: briefDetail },
+    } = await got.post(`${briefApi}/briefColumn/detail`, {
+        form: {
+            platform: 'www',
+            brief_column_id: id,
+        },
+    });
+
+    const list = response.datalist.map((item) => ({
+        title: item.title,
+        link: `${baseUrl}/brief/${item.brief_id}`,
+        description: item.preface,
+        pubDate: parseDate(item.publish_time, 'X'),
+    }));
+
+    const items = await ProcessFeed(list, ctx.cache);
 
     ctx.state.data = {
-        title: `虎嗅 - ${await title()}`,
-        link: `https://www.huxiu.com/briefColumn/${id}.html`,
+        title: `虎嗅 - ${briefDetail.name}`,
+        description: briefDetail.summary,
+        image: briefDetail.head_img,
+        link: `${baseUrl}/briefColumn/${id}.html`,
         item: items,
     };
 };

package/lib/v2/huxiu/collection.js

@@ -1,34 +1,39 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const utils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const link = `https://www.huxiu.com/collection/${id}.html`;
-    const response = await got({
-        method: 'get',
-        url: link,
-        headers: {
-            Referer: link,
+    const link = `${utils.baseUrl}/collection/${id}.html`;
+    const { data: response } = await got(link, {
+        https: {
+            rejectUnauthorized: false,
         },
     });
 
-    const data = response.data;
-    const $ = cheerio.load(data);
+    const $ = cheerio.load(response);
 
-    const list = $('.collection-lirt-item > a')
-        .slice(0, 5)
-        .get()
-        .map((e) => $(e).attr('href'));
+    const initialState = utils.parseInitialState($);
+
+    const { collectionDetail } = initialState.collectionDetail;
+    const list = collectionDetail.article_list.datalist.map((e) => ({
+        title: e.title,
+        link: `${utils.baseUrl}/article/${e.aid}.html`,
+        description: e.summary,
+        pubDate: parseDate(e.dateline, 'X'),
+        author: e.user_info.username,
+    }));
 
     const items = await utils.ProcessFeed(list, ctx.cache);
 
-    const info = `虎嗅 - ${$('h1').text()}`;
-    const description = $('p').text();
+    const info = `虎嗅 - ${collectionDetail.name}`;
+
     ctx.state.data = {
         title: info,
+        description: collectionDetail.summary,
+        image: collectionDetail.icon,
         link,
-        description,
         item: items,
     };
 };

package/lib/v2/huxiu/moment.js

@@ -2,11 +2,10 @@ const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { baseUrl: rootUrl, momentApi: apiRootUrl } = require('./utils');
 
 module.exports = async (ctx) => {
-    const rootUrl = 'https://www.huxiu.com';
     const currentUrl = `${rootUrl}/moment`;
-    const apiRootUrl = 'https://moment-api.huxiu.com';
     const apiUrl = `${apiRootUrl}/web-v2/moment/feed`;
 
     const response = await got({

package/lib/v2/huxiu/search.js

@@ -3,18 +3,30 @@ const utils = require('./utils');
 
 module.exports = async (ctx) => {
     const { keyword } = ctx.params;
-    const url = `https://search-api.huxiu.com/api/article?platform=www&s=${encodeURIComponent(keyword)}&sort=&page=1&pagesize=20`;
-    const link = 'https://www.huxiu.com';
+    const url = `${utils.searchApi}/api/article`;
+    const link = utils.baseUrl;
 
-    const response = await got({
-        method: 'post',
-        url,
+    const { data } = await got.post(url, {
         headers: {
-            Referer: link,
+            Referer: utils.baseUrl,
+        },
+        searchParams: {
+            platform: 'www',
+            s: keyword,
+            sort: '',
+            page: 1,
+            pagesize: 20,
+            appid: 'hx_search',
+            ...utils.generateSignature(),
         },
     });
 
-    const list = response.data.data.datalist.slice(0, 10).map((d) => `${link}/article/${d.aid}.html`);
+    const list = data.data.datalist.map((d) => ({
+        title: d.title,
+        link: d.url.includes('article') ? `${d.url}.html` : d.url,
+        description: d.content,
+        author: d.author,
+    }));
 
     const items = await utils.ProcessFeed(list, ctx.cache);
 

package/lib/v2/huxiu/tag.js

@@ -1,23 +1,30 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const utils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const link = `https://www.huxiu.com/tags/${id}.html`;
-    const response = await got({
-        method: 'get',
-        url: link,
-        headers: {
-            Referer: link,
+    const link = `${utils.baseUrl}/tags/${id}.html`;
+    const { data } = await got(link, {
+        https: {
+            rejectUnauthorized: false,
         },
     });
 
-    const data = response.data;
     const $ = cheerio.load(data);
-    const list = $('.related-article li a')
-        .get()
-        .map((e) => $(e).attr('href'));
+    const list = $('.related-article li')
+        .toArray()
+        .map((e) => {
+            e = $(e);
+            const a = e.find('a');
+            return {
+                title: a.text(),
+                link: `${utils.baseUrl}${a.attr('href')}`,
+                pubDate: timezone(parseDate(e.find('.time').text()), 8),
+            };
+        });
 
     const items = await utils.ProcessFeed(list, ctx.cache);
 

package/lib/v2/huxiu/templates/brief.art

@@ -0,0 +1,22 @@
+{{ if brief.preface }}{{@ brief.preface }}<br>{{ /if }}
+
+{{ if brief.audio_info }}
+<figure>
+    <figcaption>{{ brief.audio_title }}</figcaption>
+    <audio controls src="{{ brief.audio_info.audio_path }}"></audio>
+</figure>
+<br>
+{{ /if }}
+
+{{ if brief.part_list }}{{ each brief.part_list part }}
+    {{ if part.title }}<h2>{{ part.title }}</h2>{{ /if }}
+    {{ if part.content_info }}{{ each part.content_info content }}
+        {{ if content.type === 'text' }}{{@ content.text }}{{ /if }}
+        {{ if content.type === 'video' }}
+            <video controls poster="{{ content.video.poster }}"><source src="{{ content.video.url }}" type="video/mp4"></video>
+        {{ /if }}
+        {{ if content.type === 'image'}}{{ /if }}
+    {{ /each }}<br>{{ /if }}
+{{ /each }}{{ /if }}
+
+{{ if brief.peroration }}{{ brief.peroration }}{{ /if }}

package/lib/v2/huxiu/templates/img.art

@@ -0,0 +1,3 @@
+{{ if img }}
+<img src="{{ img }}">
+{{ /if }}

package/lib/v2/huxiu/templates/video.art

@@ -0,0 +1,7 @@
+{{ if video_info }}
+<video controls poster="{{ video_info.cover }}" width="{{ video_info.width }}" height="{{ video_info.height }}">
+{{ if video_info.fhd_link }}<source src="{{ video_info.fhd_link }}" type="video/mp4">{{ /if }}
+{{ if video_info.hd_link }}<source src="{{ video_info.hd_link }}" type="video/mp4">{{ /if }}
+{{ if video_info.sd_link }}<source src="{{ video_info.sd_link }}" type="video/mp4">{{ /if }}
+</video>
+{{ /if }}

package/lib/v2/huxiu/utils.js

@@ -1,69 +1,154 @@
 const cheerio = require('cheerio');
-const date = require('@/utils/date');
+const { parseDate } = require('@/utils/parse-date');
 const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const CryptoJS = require('crypto-js');
 
-const ProcessFeed = async (list, cache) => {
-    const host = 'https://www.huxiu.com';
+const accountApi = 'https://account-api.huxiu.com';
+const articleApi = 'https://api-article.huxiu.com';
+const baseUrl = 'https://www.huxiu.com';
+const briefApi = 'https://api-brief.huxiu.com';
+const momentApi = 'https://moment-api.huxiu.com';
+const searchApi = 'https://search-api.huxiu.com';
 
-    const items = await Promise.all(
-        list.map(async (e) => {
-            const link = new URL(e, host).href;
+const ProcessFeed = (list, cache) =>
+    Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await got(item.link, {
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
+                const $ = cheerio.load(response.data);
+                const initialState = parseInitialState($);
 
-            const single = await cache.tryGet(link, async () => {
-                const response = await got(link);
+                if (item.link.startsWith(`${baseUrl}/article/`)) {
+                    fetchArticle(initialState, item);
+                } else if (item.link.startsWith(`${baseUrl}/brief/`)) {
+                    fetchBrief(initialState, item);
+                }
 
-                const $ = cheerio.load(response.data);
+                return item;
+            })
+        )
+    );
 
-                // const __INITIAL_STATE__ = Function(
-                //     'let window={};' +
-                //         $('*')
-                //             .html()
-                //             .match(/window\.__INITIAL_STATE__.*};/gm)[0] +
-                //         'return window.__INITIAL_STATE__'
-                // )();
-
-                $('.article__content')
-                    .find('.lazyImg')
-                    .each((i, e) => {
-                        $(e).attr('src', $(e).attr('_src'));
-                    });
-
-                $('.article__content')
-                    .find('.text-big-title')
-                    .each((i, e) => (e.tagName = 'h3'));
-
-                $('p').each(function () {
-                    if ($(this).find('img').length === 0 && $(this).text().match(/^\s*$/)) {
-                        $(this).remove();
-                    }
-                });
-                $('img.dialog_add_wxxy_qrcode_icon').remove();
+const fetchArticle = (initialState, item) => {
+    const { articleDetail } = initialState.articleDetail;
+    const topImg = art(path.join(__dirname, 'templates/img.art'), {
+        img: articleDetail.pic_path.split('?')[0],
+    });
+    let video = '';
+    if (articleDetail.video_info) {
+        video = art(path.join(__dirname, 'templates/video.art'), {
+            video_info: articleDetail.video_info,
+        });
+    }
 
-                let description = '';
+    const $ = cheerio.load(articleDetail.content, null, false);
 
-                // if ('video_info' in __INITIAL_STATE__.articleDetail.articleDetail) {
-                //     description += `<video height="100%" poster="${__INITIAL_STATE__.articleDetail.articleDetail.video_info.cover}" controls src="${__INITIAL_STATE__.articleDetail.articleDetail.video_info.hd_link}"></video>`;
-                // }
-                if ($('.top-img').html() !== null) {
-                    description += $('.top-img').html();
-                }
+    $('.text-big-title').each((_, e) => (e.tagName = 'h3'));
+    fixImg($);
+    removeExtraLinebreak($);
+    // $('img.dialog_add_wxxy_qrcode_icon').remove();
+
+    item.description = video + topImg + $.html();
+    item.author = articleDetail.author;
+    item.category = articleDetail.tags_info.map((tag) => tag.name);
+};
+
+const fetchBrief = (initialState, item) => {
+    const { brief_detail } = initialState.briefStoreModule;
+    const { brief_column } = brief_detail;
+
+    const description = art(path.join(__dirname, 'templates/brief.art'), {
+        brief: brief_detail.brief,
+    });
 
-                return {
-                    title: $('.article__title').text().trim(),
-                    description: description + $('.article__content').html(),
-                    pubDate: date($('.article__time').text(), 8),
-                    author: $('.author-info__username').text(),
-                    link,
-                };
-            });
-
-            return single;
-        })
+    const $ = cheerio.load(description, null, false);
+
+    $('button.black-dot').remove();
+    fixImg($);
+    removeExtraLinebreak($);
+
+    item.description = $.html();
+    item.author = brief_detail.brief.publisher_list.map((item) => item.username).join(', ');
+    item.pubDate = parseDate(brief_column.update_time, 'X');
+};
+
+const parseInitialState = ($) =>
+    JSON.parse(
+        $('script')
+            .text()
+            .match(/window\.__INITIAL_STATE__=(\{.*?\});\(function\(\)/)[1]
     );
 
-    return items;
+const fixImg = ($) => {
+    $('img.lazyImg, img.js-preview').each((_, e) => {
+        if (e.attribs._src) {
+            e.attribs.src = e.attribs._src.split('?')[0];
+            delete e.attribs._src;
+        }
+        if (e.attribs.src.includes('?')) {
+            e.attribs.src = e.attribs.src.split('?')[0];
+        }
+        if (e.attribs['data-w']) {
+            e.attribs.width = e.attribs['data-w'];
+            delete e.attribs['data-w'];
+        }
+        if (e.attribs['data-h']) {
+            e.attribs.height = e.attribs['data-h'];
+            delete e.attribs['data-h'];
+        }
+    });
+};
+
+const removeExtraLinebreak = ($) => {
+    $('p').each((_, e) => {
+        e = $(e);
+        if (e.find('img').length === 0 && e.text().match(/^\s*$/)) {
+            e.remove();
+        }
+    });
+};
+
+const generateNonce = () => {
+    let nonce = '';
+    const e = 'abcdefghijklmnopqrstuvwxyz1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+    const t = 16;
+    for (let i = 0; i < t; i++) {
+        nonce += e.charAt(Math.floor(Math.random() * e.length));
+    }
+    return nonce;
+};
+
+const generateSignature = () => {
+    const timestamp = Math.round(new Date().getTime() / 1000).toString();
+    // const appSecret = '4nHzGgGt7WX4zFTsTKocpxg4dzU6-wUi';
+    const appSecret = 'hUzaVKtNfDE-6UiyaJdfsmjW-8dwoyVc'; // appid: 'hx_search'
+    const nonce = generateNonce();
+    const r = [appSecret, timestamp, nonce].sort();
+    return {
+        nonce,
+        timestamp,
+        signature: CryptoJS.SHA1(r[0] + r[1] + r[2]).toString(),
+    };
 };
 
 module.exports = {
+    accountApi,
+    articleApi,
+    baseUrl,
+    briefApi,
+    momentApi,
+    searchApi,
     ProcessFeed,
+    fetchArticle,
+    fetchBrief,
+    parseInitialState,
+    fixImg,
+    removeExtraLinebreak,
+    generateSignature,
 };

package/lib/v2/iiilab/index.js

@@ -0,0 +1,24 @@
+const buildData = require('@/utils/common-config');
+const baseUrl = 'https://www.iiilab.com/';
+
+module.exports = async (ctx) => {
+    const link = baseUrl;
+    ctx.state.data = await buildData({
+        link,
+        url: link,
+        title: `%title%`,
+        description: `%description%`,
+        params: {
+            title: '发现',
+            description: '人人都是自媒体-发现',
+        },
+        item: {
+            item: '.aw-common-list > div',
+            title: `$('a').first().text()`,
+            link: `$('a').first().attr('href')`,
+            description: `$('.markitup-box').first().text()`,
+            pubDate: `parseDate($('.text-color-999').first().text(), 'YYYY-MM-DD HH:mm')`,
+            guid: Buffer.from(`$('a').attr('href')`).toString('base64'),
+        },
+    });
+};

package/lib/v2/iiilab/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['Joey'],
+};

package/lib/v2/iiilab/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'iiilab.com': {
+        _name: '人人都是自媒体',
+        www: [
+            {
+                title: '发现',
+                docs: 'https://docs.rsshub.app/new-media.html#ren-ren-dou-shi-zi-mei-ti',
+                source: '/',
+                target: '/iiilab',
+            },
+        ],
+    },
+};

package/lib/v2/iiilab/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index'));
+};

package/lib/v2/ikea/cn/family_offers.js

@@ -0,0 +1,70 @@
+const got = require('@/utils/got');
+const { generateProductItem } = require('./utils');
+
+const familyPriceProductsRequest = ({ pageIndex = 1 }) =>
+    got({
+        method: 'post',
+        url: 'https://www.ikea.cn/api-host/search/prod/advanced/special',
+        headers: {
+            'X-Client-Platform': 'PcWeb',
+        },
+        json: {
+            contentType: 'PRODUCT,CONTENT,PLANNER',
+            filters: {
+                filters: {},
+                sortOption: 'RELEVANCE',
+            },
+            pageIndex,
+            pageSize: 25,
+            pageType: '',
+            queryContent: 'family_price',
+        },
+    });
+
+const productRequest = async (productIds) => {
+    const response = await got({
+        url: 'https://www.ikea.cn/api-host/content/products',
+        headers: {
+            'X-Client-Platform': 'PcWeb',
+        },
+        searchParams: new URLSearchParams(productIds),
+    });
+    return response.data;
+};
+
+module.exports = async (ctx) => {
+    const familyPriceProductIds = [];
+    const productRequests = [];
+
+    const familyPriceProductsLoadMore = async ({ pageIndex }) => {
+        const response = await familyPriceProductsRequest({ pageIndex });
+        const {
+            data: { productPage, products },
+        } = response.data;
+
+        products.forEach(({ id }) => {
+            familyPriceProductIds.push(['ids', id]);
+        });
+
+        if (productPage.end < productPage.total) {
+            await familyPriceProductsLoadMore({ pageIndex: pageIndex + 1 });
+        }
+        return Promise.resolve();
+    };
+
+    await familyPriceProductsLoadMore({ pageIndex: 1 });
+
+    while (familyPriceProductIds.length) {
+        productRequests.push(productRequest(familyPriceProductIds.splice(0, 25)));
+    }
+
+    const productResponses = await Promise.all(productRequests);
+    const products = productResponses.flat();
+
+    ctx.state.data = {
+        title: 'IKEA 宜家 - 会员特惠',
+        link: 'https://www.ikea.cn/cn/zh/offers/family-offers/',
+        description: '会员特惠',
+        item: products.map(generateProductItem),
+    };
+};

package/lib/v2/ikea/cn/low_price.js

@@ -0,0 +1,22 @@
+const got = require('@/utils/got');
+const { generateRequestHeaders, generateProductItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const response = await got({
+        url: 'https://srv.app.ikea.cn/content/recommendation/v2/product-group/products',
+        headers: generateRequestHeaders(),
+        searchParams: {
+            processOutOfStock: 'SORT',
+            groupId: 'cms_低价好物_cms-商品列表-_0',
+            page: 1,
+            size: 200,
+        },
+    });
+
+    ctx.state.data = {
+        title: 'IKEA 宜家 - 低价优选',
+        link: 'https://www.ikea.cn/cn/zh/campaigns/wo3-men2-de-chao1-zhi2-di1-jia4-pub8b08af40',
+        description: '低价优选',
+        item: response.data.products.map(generateProductItem),
+    };
+};

package/lib/v2/ikea/cn/new.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const { generateRequestHeaders, generateProductItem } = require('./utils');
+
+const request = ({ moreToken = '' }) =>
+    got({
+        method: 'post',
+        url: 'https://srv-mp.app.ikea.cn/content/search/products/advanced',
+        headers: generateRequestHeaders(),
+        searchParams: {
+            keyword: '新品',
+            moreToken,
+        },
+        json: {},
+    });
+
+module.exports = async (ctx) => {
+    const allProductSummaries = [];
+
+    const loadMoreRequest = async ({ moreToken }) => {
+        const response = await request({ moreToken });
+        const { data } = response;
+        allProductSummaries.push(data.productSummaries);
+        if (data.moreToken) {
+            await loadMoreRequest({ moreToken: data.moreToken });
+        }
+        return Promise.resolve();
+    };
+
+    await loadMoreRequest({});
+
+    const products = allProductSummaries.flat();
+
+    ctx.state.data = {
+        title: 'IKEA 宜家 - 当季新品推荐',
+        link: 'https://www.ikea.cn/cn/zh/new/',
+        description: '当季新品推荐',
+        item: products.map(generateProductItem),
+    };
+};