rsshub 1.0.0-master.fcb92f0 → 1.0.0-master.fcbb3f4

package/lib/v2/huxiu/util.js

@@ -0,0 +1,460 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const CryptoJS = require('crypto-js');
+
+const domain = 'huxiu.com';
+const rootUrl = `https://www.${domain}`;
+
+const apiArticleRootUrl = `https://api-article.${domain}`;
+const apiBriefRootUrl = `https://api-brief.${domain}`;
+const apiMemberRootUrl = `https://api-account.${domain}`;
+const apiMomentRootUrl = `https://moment-api.${domain}`;
+const apiSearchRootUrl = `https://search-api.${domain}`;
+
+/**
+ * Cleans up HTML data by removing specific elements and attributes.
+ *
+ * @param {string} data - The HTML data to clean up.
+ * @returns {string} - The cleaned up HTML data.
+ */
+const cleanUpHTML = (data) => {
+    const $ = cheerio.load(data);
+
+    $('div.neirong-shouquan').remove();
+    $('em.vote__bar, div.vote__btn, div.vote__time').remove();
+    $('p img').each((_, e) => {
+        e = $(e);
+        e.parent().replaceWith(
+            art(path.join(__dirname, 'templates/description.art'), {
+                image: {
+                    src: (e.prop('src') ?? e.prop('_src')).split(/\?/)[0],
+                    width: e.prop('data-w'),
+                    height: e.prop('data-h'),
+                },
+            })
+        );
+    });
+    $('p, span').each((_, e) => {
+        e = $(e);
+        if (e.contents().length === 1 && /^\s*$/.test(e.text())) {
+            e.remove();
+        } else {
+            e.removeClass();
+            e.removeAttr('data-check-id label class');
+        }
+    });
+    $('.text-big-title').each((_, e) => {
+        e.tagName = 'h3';
+        e = $(e);
+        e.removeClass();
+        e.removeAttr('class');
+    });
+    $('.text-sm-title').each((_, e) => {
+        e.tagName = 'h4';
+        e = $(e);
+        e.removeClass();
+        e.removeAttr('class');
+    });
+
+    return $.html();
+};
+
+/**
+ * Fetch brief column data for the specified ID.
+ *
+ * @param {string} url - The ID of the brief column to fetch data from.
+ * @returns {Promise<Object>} A promise that resolves to an object containing the fetched data
+ *                            to be added into `ctx.state.data`.
+ */
+const fetchBriefColumnData = async (id) => {
+    const apiBriefColumnUrl = new URL('briefColumn/detail', apiBriefRootUrl).href;
+
+    const {
+        data: { data },
+    } = await got.post(apiBriefColumnUrl, {
+        form: {
+            platform: 'www',
+            brief_column_id: id,
+        },
+    });
+
+    const currentUrl = new URL(`club/${data.club_id}.html`, rootUrl).href;
+
+    const { data: currentResponse } = await got(currentUrl);
+
+    const $ = cheerio.load(currentResponse);
+
+    const subtitle = `${data.name}-${data.sub_name}`;
+    const icon = new URL($('link[rel="apple-touch-icon"]').prop('href'), rootUrl).href;
+    const author = $('meta[name="author"]').prop('content');
+
+    return {
+        title: `${subtitle}-${author}`,
+        link: currentUrl,
+        description: data.summary,
+        language: $('html').prop('lang'),
+        image: data.head_img,
+        icon,
+        logo: icon,
+        subtitle,
+        author,
+        itunes_author: author,
+        itunes_category: 'News',
+        allowEmpty: true,
+    };
+};
+
+/**
+ * Fetches club data for the specified ID and the ID of the default brief column.
+ *
+ * @param {string} id - The ID of the club to fetch data from.
+ * @returns {Promise<Object>} data - A promise that resolves to an object containing the fetched data
+ *                            to be added into `ctx.state.data`.
+ * @returns {string} id - the ID of the default brief column.
+ */
+const fetchClubData = async (id) => {
+    const currentUrl = new URL(`club/${id}.html`, rootUrl).href;
+
+    const { data: currentResponse } = await got(currentUrl);
+
+    const $ = cheerio.load(currentResponse);
+
+    const title = $('title').text();
+    const icon = new URL($('link[rel="apple-touch-icon"]').prop('href'), rootUrl).href;
+    const author = $('meta[name="author"]').prop('content');
+
+    return {
+        data: {
+            title,
+            link: currentUrl,
+            description: $('ul.content-item li.content').text().trim(),
+            language: $('html').prop('lang'),
+            image: $('div.header img.img').prop('data-src')?.split(/\?/)[0] ?? undefined,
+            icon,
+            logo: icon,
+            subtitle: title.split(/-/)[0],
+            author,
+            itunes_author: author,
+            itunes_category: 'News',
+            allowEmpty: true,
+        },
+        briefColumnId: currentResponse.match(/"brief_column_id":"(\d+)",/)[1],
+    };
+};
+
+/**
+ * Fetch data from the specified URL.
+ *
+ * @param {string} url - The URL to fetch data from.
+ * @returns {Promise<Object>} A promise that resolves to an object containing the fetched data
+ *                            to be added into `ctx.state.data`.
+ */
+const fetchData = async (url) => {
+    const { data: response } = await got(url);
+
+    const $ = cheerio.load(response);
+
+    const icon = new URL($('link[rel="apple-touch-icon"]').prop('href'), rootUrl).href;
+    const author = $('meta[name="author"]').prop('content');
+
+    return {
+        title: $('title').text(),
+        link: url,
+        description: $('div.tag-content').text() || $('span.author-intro').text() || $('p.collection__intro').text() || $('meta[name="description"]').prop('content'),
+        language: $('html').prop('lang'),
+        icon,
+        logo: icon,
+        subtitle: $('title').text().split(/-/)[0],
+        author,
+        itunes_author: author,
+        itunes_category: 'News',
+        allowEmpty: true,
+    };
+};
+
+/**
+ * Fetches item data.
+ *
+ * @param {Object} item - The item to fetch data for.
+ * @returns {Promise<Object>} The fetched item data object.
+ */
+const fetchItem = async (item) => {
+    const { data: detailResponse } = await got(item.link);
+
+    const state = parseInitialState(detailResponse);
+    const data = state.briefStoreModule?.brief_detail.brief ?? state.articleDetail?.articleDetail ?? undefined;
+
+    if (!data) {
+        return item;
+    }
+
+    const { processed: audio = undefined, processedItem: audioItem = {} } = processAudioInfo(data.audio_info);
+
+    if (Object.keys(audioItem).length !== 0) {
+        audioItem.itunes_item_image = data.pic_path ?? data.share_info?.share_img ?? undefined;
+    }
+
+    const { processed: video = undefined, processedItem: videoItem = {} } = processVideoInfo(data.video_info);
+
+    item.title = data.title ?? item.title;
+    item.description = art(path.join(__dirname, 'templates/description.art'), {
+        image: {
+            src: data.pic_path,
+        },
+        video,
+        audio,
+        preface: cleanUpHTML(data.content_preface ?? data.preface),
+        summary: data.ai_summary,
+        description: cleanUpHTML(data.content),
+    });
+    item.author = data.user_info?.username ?? item.author;
+    item.category = [data.video_article_tag, data.brief_column?.name ?? undefined, data.club_info?.name ?? undefined, ...(data.tags_info?.map((c) => c.name) ?? []), ...(data.relation_info?.channel?.map((c) => c.name) ?? [])].filter(
+        (c) => c
+    );
+    item.pubDate = parseDate(data.dateline ?? data.publish_time, 'X');
+    item.upvote = data.agreenum ?? item.upvote;
+    item.comments = data.commentnum ?? data.total_comment_num ?? item.comments;
+
+    item.upvote = parseInt(item.upvote, 10);
+    item.comments = parseInt(item.comments, 10);
+
+    return {
+        ...audioItem,
+        ...videoItem,
+        ...item,
+    };
+};
+
+/**
+ * Generates a random nonce string.
+ *
+ * @returns {string} The generated nonce string.
+ */
+const generateNonce = () => {
+    let nonce = '';
+    const e = 'abcdefghijklmnopqrstuvwxyz1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+    const t = 16;
+    for (let i = 0; i < t; i++) {
+        nonce += e.charAt(Math.floor(Math.random() * e.length));
+    }
+    return nonce;
+};
+
+/**
+ * Generates a signature object containing a nonce, timestamp, and signature value.
+ *
+ * @returns {string} nonce - The generated nonce.
+ * @returns {string} timestamp - The timestamp.
+ * @returns {string} signature - The calculated signature value.
+ */
+const generateSignature = () => {
+    const timestamp = Math.round(new Date().getTime() / 1000).toString();
+
+    const appSecret = 'hUzaABtNfDE-6UiyaYhfsmjW-8dnoyVc';
+    const nonce = generateNonce();
+    const r = [appSecret, timestamp, nonce].sort();
+    return {
+        nonce,
+        timestamp,
+        signature: CryptoJS.SHA1(r[0] + r[1] + r[2]).toString(),
+    };
+};
+
+/**
+ * Parses the initial state from the provided data.
+ *
+ * @param {string} data - The data to parse the initial state from.
+ * @returns {Object|undefined} - The parsed initial state object, or undefined if not found.
+ */
+const parseInitialState = (data) => {
+    const matches = data.match(/window\.__INITIAL_STATE__=(\{.*?\});\(function\(\)/);
+    if (matches) {
+        return JSON.parse(matches[1]);
+    }
+    return undefined;
+};
+
+const audioQualities = ['', 'low'];
+
+/**
+ * Processes the audio information and returns the processed data.
+ *
+ * @param {Object} info - The audio information to process.
+ * @returns {Object} - An object containing the processed audio data.
+ */
+const processAudioInfo = (info) => {
+    const quality = info ? audioQualities.find((quality) => info.hasOwnProperty(`audio_${quality === '' ? '' : `${quality}_`}path`)) : undefined;
+
+    if (quality === undefined) {
+        return {
+            processed: undefined,
+            processedItem: {},
+        };
+    }
+
+    const linkKey = `audio_${quality}path`;
+    const sizeKey = `audio_${quality}size`;
+
+    const processed = {
+        duration: info.format_length_new ?? info.format_length,
+        size: info.hasOwnProperty(sizeKey) ? info[sizeKey] : undefined,
+        src: info[linkKey],
+        type: `audio/${info[linkKey].split(/\./).pop()}`,
+    };
+
+    const processedItem = {
+        itunes_duration: processed.duration,
+        enclosure_url: processed.src,
+        enclosure_length: processed.size,
+        enclosure_type: processed.type,
+    };
+
+    return {
+        processed,
+        processedItem,
+    };
+};
+
+/**
+ * Process the item list and return the resulting array.
+ *
+ * @param {Object[]} items - The items to process.
+ * @param {number} limit - The maximum number of items to process.
+ * @param {Function} tryGet   - The tryGet function that handles the retrieval process.
+ * @returns {Promise<Object[]>} - A promise that resolves to an array of processed items.
+ */
+const processItems = async (items, limit, tryGet) => {
+    items = items
+        .map((item) => {
+            let guid = '';
+            let link = '';
+
+            if (item.moment_id) {
+                guid = `huxiu-moment-${item.moment_id}`;
+                if (item.url) {
+                    link = item.url;
+                } else {
+                    link = new URL(`moment/${item.moment_id}.html`, rootUrl).href;
+                }
+            } else if (item.brief_id || /huxiu\.com\/brief\//.test(item.url)) {
+                item.brief_id = item.brief_id ?? item.aid;
+                guid = `huxiu-brief-${item.brief_id}`;
+                link = new URL(`brief/${item.brief_id}.html`, rootUrl).href;
+            } else if (item.aid) {
+                guid = `huxiu-article-${item.aid}`;
+                link = new URL(`article/${item.aid}.html`, rootUrl).href;
+            } else {
+                return undefined;
+            }
+
+            const { processed: audio = undefined, processedItem: audioItem = {} } = processAudioInfo(item.audio_info);
+
+            if (Object.keys(audioItem).length !== 0) {
+                audioItem.itunes_item_image = item.pic_path ?? item.share_info?.share_img ?? undefined;
+            }
+
+            const { processed: video = undefined, processedItem: videoItem = {} } = processVideoInfo(item.video_info);
+
+            const upvotes = item.count_info?.agree ?? item.count_info?.favtimes ?? item.agree_num ?? 0;
+            const downvotes = item.count_info?.disagree ?? 0;
+            const comments = item.count_info?.total_comment_num ?? item.count_info?.commentnum ?? item.total_comment_num ?? item.commentnum ?? 0;
+
+            return {
+                ...audioItem,
+                ...videoItem,
+                title: (item.title ?? item.summary ?? item.content)?.replace(/<\/?(?:em|br)?>/g, ''),
+                link,
+                description: art(path.join(__dirname, 'templates/description.art'), {
+                    image: {
+                        src: item.origin_pic_path ?? item.pic_path ?? item.big_pic_path?.split(/\?/)[0] ?? undefined,
+                    },
+                    audio,
+                    video,
+                    summary: item.summary ?? item.content ?? item.preface,
+                }),
+                author: item.user_info?.username ?? item.brief_column?.name ?? item.author_info?.username ?? item.author,
+                guid,
+                pubDate: item.publish_time ?? item.dateline ? parseDate(item.publish_time ?? item.dateline, 'X') : undefined,
+                upvotes: parseInt(upvotes, 10),
+                downvotes: parseInt(downvotes, 10),
+                comments: parseInt(comments, 10),
+            };
+        })
+        .filter((item) => item)
+        .slice(0, limit);
+
+    return await Promise.all(
+        items.map((item) =>
+            tryGet(item.guid, async () => {
+                if (!new RegExp(domain, 'i').test(new URL(item.link).hostname)) {
+                    return item;
+                } else if (!item.guid.startsWith('huxiu-moment')) {
+                    return await fetchItem(item);
+                }
+
+                return item;
+            })
+        )
+    );
+};
+
+const videoQualities = ['fhd', 'fhd_medium', 'wifi', 'fhd_low', 'flow', 'hd', 'sd'];
+
+/**
+ * Processes the video information and returns the processed data.
+ *
+ * @param {Object} info - The video information to process.
+ * @returns {Object} - An object containing the processed video data.
+ */
+const processVideoInfo = (info) => {
+    const quality = info ? videoQualities.find((quality) => info.hasOwnProperty(`${quality}_link`)) : undefined;
+
+    if (quality === undefined) {
+        return {
+            processed: undefined,
+            processedItem: {},
+        };
+    }
+
+    const linkKey = `${quality}_link`;
+    const sizeKey = `origin_${quality}_size`;
+
+    const processed = {
+        duration: info.duration ?? info.origin_duration,
+        poster: info.cover ?? info.custom_cover_path ?? info.gif_path,
+        size: info.hasOwnProperty(sizeKey) ? info[sizeKey] : undefined,
+        src: info[linkKey],
+        type: `video/${info[linkKey].split(/\./).pop()}`,
+    };
+
+    const processedItem = {
+        itunes_item_image: processed.poster,
+        itunes_duration: processed.duration,
+        enclosure_url: processed.src,
+        enclosure_length: processed.size,
+        enclosure_type: processed.type,
+    };
+
+    return {
+        processed,
+        processedItem,
+    };
+};
+
+module.exports = {
+    rootUrl,
+    apiArticleRootUrl,
+    apiBriefRootUrl,
+    apiMemberRootUrl,
+    apiMomentRootUrl,
+    apiSearchRootUrl,
+
+    fetchBriefColumnData,
+    fetchClubData,
+    fetchData,
+    generateSignature,
+    processItems,
+};

package/lib/v2/ianspriggs/index.js

@@ -0,0 +1,90 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { category = 'portraits' } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30;
+
+    const author = 'Ian Spriggs';
+    const rootUrl = 'https://ianspriggs.com';
+    const currentUrl = new URL(category, rootUrl).href;
+
+    const { data: response } = await got(currentUrl);
+
+    const $ = cheerio.load(response);
+
+    let items = $('div.work-item')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const image = item.find('img').first();
+
+            return {
+                title: item.find('div.work-info').text(),
+                link: item.find('a').prop('href'),
+                description: art(path.join(__dirname, 'templates/description.art'), {
+                    images: image?.prop('src')
+                        ? [
+                              {
+                                  src: image.prop('src').replace(/_thumbnail\./, '.'),
+                                  alt: image.prop('alt'),
+                              },
+                          ]
+                        : undefined,
+                }),
+                author,
+                pubDate: parseDate(item.find('div.work-info p').last(), 'YYYY'),
+                enclosure_url: image?.prop('src') ?? undefined,
+                enclosure_type: image?.prop('src') ? 'image/jpeg' : undefined,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: detailResponse } = await got(item.link);
+
+                const content = cheerio.load(detailResponse);
+
+                const images = content('div.work-item img')
+                    .toArray()
+                    .map((item) => {
+                        item = content(item);
+
+                        return {
+                            src: item.prop('src').replace(/-\d+x\d+\./, '.'),
+                            alt: item.prop('alt'),
+                        };
+                    });
+
+                item.title = content('div.project-title').text();
+                item.description += art(path.join(__dirname, 'templates/description.art'), {
+                    images,
+                    description: content('div.nectar-fancy-ul').html(),
+                });
+                item.pubDate = parseDate(content('span.subheader').last().text(), 'YYYY');
+
+                return item;
+            })
+        )
+    );
+
+    const icon = new URL('favicon.ico', rootUrl).href;
+
+    ctx.state.data = {
+        item: items,
+        title: $('title').text(),
+        link: currentUrl,
+        description: $('meta[property="og:description"]').prop('content'),
+        language: $('html').prop('lang'),
+        icon,
+        logo: icon,
+        subtitle: $('a[aria-current="page"] span.menu-title-text').text(),
+        author,
+    };
+};

package/lib/v2/ianspriggs/radar.js

@@ -0,0 +1,17 @@
+module.exports = {
+    'ianspriggs.com': {
+        _name: 'Ian Spriggs',
+        '.': [
+            {
+                title: 'Category',
+                docs: 'https://docs.rsshub.app/routes/blog#ian-spriggs-category',
+                source: ['/:category'],
+                target: (params) => {
+                    const category = params.category;
+
+                    return `/ianspriggs${category ? `/${category}` : ''}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/ianspriggs/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:category?', require('./'));
+};

package/lib/v2/ianspriggs/templates/description.art

@@ -0,0 +1,13 @@
+{{ each images image }}
+  <figure>
+    <img
+      {{ if image.alt }}
+        alt="{{ image.alt }}"
+      {{ /if }}
+    src="{{ image.src }}">
+  </figure>
+{{ /each }}
+
+{{ if description }}
+  {{@ description }}
+{{ /if }}

package/lib/v2/icac/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '新闻公布',
-                docs: 'https://docs.rsshub.app/government.html#xiang-gang-lian-zheng-gong-shu',
+                docs: 'https://docs.rsshub.app/routes/government#xiang-gang-lian-zheng-gong-shu',
                 source: ['/:lang/press/index.html'],
                 target: '/icac/news/:lang',
             },

package/lib/v2/icbc/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/whpj/:format?': ['leoleoasd'],
+};

package/lib/v2/icbc/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'icbc.com.cn': {
+        _name: '中国工商银行',
+        '.': [
+            {
+                title: '外汇牌价',
+                docs: 'https://docs.rsshub.app/routes/other#zhong-guo-gong-shang-yin-hang',
+                source: ['/column/1438058341489590354.html'],
+                target: '/icbc/whpj',
+            },
+        ],
+    },
+};

package/lib/v2/icbc/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/whpj/:format?', require('./whpj'));
+};

package/lib/v2/icbc/whpj.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    // yes, insecure
+    // their openssl version isn't safe anyway (https://stackoverflow.com/questions/75763525/curl-35-error0a000152ssl-routinesunsafe-legacy-renegotiation-disabled)
+    const { data } = await got('http://papi.icbc.com.cn/exchanges/ns/getLatest');
+    const format = ctx.params.format;
+
+    ctx.state.data = {
+        title: '中国工商银行外汇牌价',
+        link: 'https://www.icbc.com.cn/column/1438058341489590354.html',
+        item: data.data.map((item) => {
+            const name = `${item.currencyCHName} ${item.currencyENName}`;
+
+            const xhmr = `现汇买入价：${item.foreignBuy}`;
+
+            const xcmr = `现钞买入价：${item.cashBuy}`;
+
+            const xhmc = `现汇卖出价：${item.foreignSell}`;
+
+            const xcmc = `现钞卖出价：${item.cashSell}`;
+
+            const zs = `参考价：${item.reference}`;
+
+            const content = `${xhmr} ${xcmr} ${xhmc} ${xcmc} ${zs}`;
+
+            const formatTitle = () => {
+                switch (format) {
+                    case 'short':
+                        return name;
+                    case 'xh':
+                        return `${name} ${xhmr} ${xhmc}`;
+                    case 'xc':
+                        return `${name} ${xcmr} ${xcmc}`;
+                    case 'zs':
+                        return `${name} ${zs}`;
+                    case 'xhmr':
+                        return `${name} ${xhmr}`;
+                    case 'xhmc':
+                        return `${name} ${xhmc}`;
+                    case 'xcmr':
+                        return `${name} ${xcmr}`;
+                    case 'xcmc':
+                        return `${name} ${xcmc}`;
+                    default:
+                        return `${name} ${content}`;
+                }
+            };
+            return {
+                title: formatTitle(),
+                pubDate: new Date(`${item.publishDate} ${item.publishTime}`),
+                description: content.replace(/\s/g, '<br>'),
+                guid: `${name} ${item.publishDate} ${item.publishTime}`,
+            };
+        }),
+    };
+};

package/lib/v2/idolypride/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '新闻',
-                docs: 'https://docs.rsshub.app/anime.html#ou-xiang-rong-yao',
+                docs: 'https://docs.rsshub.app/routes/anime#ou-xiang-rong-yao',
                 source: '/news',
                 target: '/idolypride/news',
             },