rsshub 1.0.0-master.fe99193 → 1.0.0-master.fe9c82a

package/lib/v2/ikea/router.js

@@ -0,0 +1,7 @@
+module.exports = (router) => {
+    router.get('/cn/family_offers', require('./cn/family_offers'));
+    router.get('/cn/low_price', require('./cn/low_price'));
+    router.get('/cn/new', require('./cn/new'));
+    router.get('/gb/new', require('./gb/new'));
+    router.get('/gb/offer', require('./gb/offer'));
+};

package/lib/v2/ikea/templates/cn/product.art

@@ -0,0 +1,14 @@
+<p>名称：{{name}}</p>
+<p>类型：{{productType}}</p>
+<p>尺寸：{{measureText}}</p>
+{{if isFamilyOffer}}
+    <p>会员价格：&yen;{{currentPrice}}</p>
+    <p>非会员价格：&yen;{{originalPrice}}</p>
+{{else}}
+    <p>价格：&yen;{{currentPrice}}</p>
+{{/if}}
+<p>
+    {{each images}}
+        <img src="{{$value}}" />
+    {{/each}}
+</p>

package/lib/v2/ikea/templates/new.art

@@ -0,0 +1,7 @@
+<img src="{{ p.mainImageUrl }}" alt="{{ p.mainImageAlt }}">
+<br>
+<b>{{ p.name }}</b>
+<br>
+{{ p.typeName }}, {{ p.itemMeasureReferenceText }}
+<br>
+{{ p.salesPrice.current.prefix }} <b>{{ p.salesPrice.current.wholeNumber }}</b>

package/lib/v2/ikea/templates/offer.art

@@ -0,0 +1,3 @@
+{{@ img }}
+<br>
+{{@ desc }}

package/lib/v2/inewsweek/index.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const iconv = require('iconv-lite');
+
+const rootUrl = 'http://news.inewsweek.cn';
+
+module.exports = async (ctx) => {
+    const channel = ctx.params.channel;
+    const url = `${rootUrl}/${channel}`;
+    const response = await got(url, {
+        responseType: 'buffer',
+    });
+    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+    const items = await Promise.all(
+        $('div.grid-item')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+                const href = item.find('a').attr('href');
+                const articleLink = `${rootUrl}${href}`;
+                return ctx.cache.tryGet(articleLink, async () => {
+                    const response = await got(articleLink, {
+                        responseType: 'buffer',
+                    });
+
+                    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+                    const fullText = $('div.contenttxt').html();
+                    const time = timezone(
+                        parseDate(
+                            $('div.editor')
+                                .html()
+                                .split(/(\s\s+)/)[2]
+                        ),
+                        +8
+                    );
+                    return {
+                        title: item.find('p').text(),
+                        description: fullText,
+                        link: articleLink,
+                        pubDate: time,
+                    };
+                });
+            })
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/inewsweek/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:channel': ['changren-wcr'],
+};

package/lib/v2/inewsweek/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'inewsweek.cn': {
+        _name: '中国新闻周刊',
+        '.': [
+            {
+                title: '栏目',
+                docs: 'https://docs.rsshub.app/traditional-media.html#zhong-guo-xin-wen-zhou-kan',
+                source: ['/:channel', '/'],
+                target: '/inewsweek/:channel',
+            },
+        ],
+    },
+};

package/lib/v2/inewsweek/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:channel', require('./index'));
+};

package/lib/v2/inoreader/index.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const type = 'html';
+    const user = ctx.params.user;
+    const tag = ctx.params.tag;
+    const num = ctx.query.limit || 20;
+    const rootUrl = 'https://www.inoreader.com/stream';
+    const currentUrl = `${rootUrl}/user/${user}/tag/${tag}/view/${type}?n=${num}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+    const data = response.data;
+    const $ = cheerio.load(data);
+    const entries = $('#snip_body>.article_content');
+
+    ctx.state.data = {
+        title: $('.header_text').text().trim(),
+        link: currentUrl,
+        item: entries
+            .map((idx, item) => {
+                const content = $(item).clone();
+                const header = $(item).prev();
+                const pubDate = $('div.article_author .au1', header)
+                    .contents()
+                    .filter((_, e) => e.nodeType === 3)
+                    .text()
+                    .replace(/posted (on|at) /, '')
+                    .replace(/UTC.*/, '');
+                return {
+                    title: $('a.title_link', header).text().trim(),
+                    link: $('a.title_link', header).attr('href'),
+                    author: $('div.article_author span span', header).text().trim() + ' via ' + $('div.article_author a.feed_link', header).text().trim(),
+                    pubDate: parseDate(pubDate, ['MMM DD YYYY HH:mm:ss', 'HH:mm:ss']),
+                    description: $(content).html(),
+                };
+            })
+            .get(),
+        allowEmpty: true,
+    };
+    ctx.state.json = ctx.state.data;
+};

package/lib/v2/inoreader/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/html_clip/:user/:tag/:num?': ['BeautyyuYanli'],
+};

package/lib/v2/inoreader/radar.js

@@ -0,0 +1,17 @@
+module.exports = {
+    'inoreader.com': {
+        _name: 'Inoreader',
+        '.': [
+            {
+                title: 'HTML Clip',
+                docs: 'https://docs.rsshub.app/reading.html#inoreader',
+                source: ['/stream/user/:user/tag/:tag/*'],
+                target: (params, url) => {
+                    const origin = new URL(url);
+                    const limit = origin.searchParams.get('n');
+                    return `/inoreader/html_clip/${params.user}/${params.tag}` + (limit ? `?limit=${limit}` : '');
+                },
+            },
+        ],
+    },
+};

package/lib/v2/inoreader/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/html_clip/:user/:tag', require('./index'));
+};

package/lib/v2/instagram/common-utils.js

@@ -0,0 +1,59 @@
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const renderItems = (items) =>
+    items.map((item) => {
+        const { product_type } = item; // carousel_container, feed, clips, igtv
+        // Content
+        const summary = item.caption?.text ?? '';
+
+        let description = '';
+        switch (product_type) {
+            case 'carousel_container': {
+                const images = item.carousel_media.map((i) => i.image_versions2.candidates[0]);
+                description = art(path.join(__dirname, 'templates/images.art'), {
+                    summary,
+                    images,
+                });
+                break;
+            }
+            case 'clips':
+            case 'igtv':
+                description = art(path.join(__dirname, 'templates/video.art'), {
+                    summary,
+                    image: item.image_versions2.candidates[0].url,
+                    video: item.video_versions[0],
+                });
+                break;
+            case 'feed': {
+                const images = [item.image_versions2.candidates[0]];
+                description = art(path.join(__dirname, 'templates/images.art'), {
+                    summary,
+                    images,
+                });
+                break;
+            }
+            default:
+                throw Error(`Instagram: Unhandled feed type: ${product_type}`);
+        }
+
+        // Metadata
+        const url = `https://www.instagram.com/p/${item.code}/`;
+        const pubDate = parseDate(item.taken_at, 'X');
+        const title = summary.split('\n')[0];
+
+        return {
+            title,
+            id: item.pk,
+            pubDate,
+            author: item.user.username,
+            link: url,
+            summary,
+            description,
+        };
+    });
+
+module.exports = {
+    renderItems,
+};

package/lib/v2/instagram/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/:category/:key': ['oppilate', 'DIYgod'],
+    '/2/:category/:key': ['TonyRL'],
+};

package/lib/v2/instagram/private-api/index.js

@@ -0,0 +1,94 @@
+const { ig, login } = require('./utils');
+const logger = require('@/utils/logger');
+const config = require('@/config').value;
+const { renderItems } = require('../common-utils');
+
+// loadContent pulls the desired user/tag/etc
+async function loadContent(category, nameOrId, tryGet) {
+    let feedTitle, feedLink, feedDescription, feedLogo;
+    let itemsRaw;
+
+    switch (category) {
+        case 'user': {
+            let userInfo, username, id;
+            if (!isNaN(nameOrId)) {
+                id = nameOrId;
+                userInfo = await tryGet(`instagram:userInfo:${id}`, () => ig.user.info(id));
+                username = userInfo.username;
+            } else {
+                username = nameOrId;
+                id = await tryGet(`instagram:getIdByUsername:${username}`, () => ig.user.getIdByUsername(username), 31536000); // 1 year since it will never change
+                userInfo = await tryGet(`instagram:userInfo:${id}`, () => ig.user.info(id));
+            }
+
+            feedDescription = userInfo.biography;
+            // exists in web api ?? exist in private api ?? exist in both
+            feedLogo = userInfo.profile_pic_url_hd ?? userInfo.hd_profile_pic_url_info?.url ?? userInfo.profile_pic_url;
+            const fullName = userInfo.full_name;
+            feedTitle = `${fullName} (@${username}) - Instagram`;
+            feedLink = `https://www.instagram.com/${username}`;
+
+            itemsRaw = await tryGet(`instagram:feed:${id}`, () => ig.feed.user(id).items(), config.cache.routeExpire, false);
+            break;
+        }
+        case 'tags': {
+            const tag = nameOrId;
+
+            feedTitle = `#${tag} - Instagram`;
+            feedLink = `https://www.instagram.com/explore/tags/${tag}`;
+
+            itemsRaw = await tryGet(`instagram:tags:${tag}`, () => ig.feed.tags(tag, 'recent').items(), config.cache.routeExpire, false);
+            break;
+        }
+        default:
+            break;
+    }
+
+    return {
+        feedTitle,
+        feedLink,
+        feedDescription,
+        feedLogo,
+        itemsRaw,
+    };
+}
+
+module.exports = async (ctx) => {
+    // https://github.com/dilame/instagram-private-api#feeds
+    // const availableCategories = ["accountFollowers", "accountFollowing", "news",
+    //     "discover", "pendingFriendships", "blockedUsers", "directInbox", "directPending",
+    //     "directThread", "user", "tag", "location", "mediaComments", "reelsMedia", "reelsTray",
+    //     "timeline", "musicTrending", "musicSearch", "musicGenre", "musicMood", "usertags", "saved"];
+    const availableCategories = ['user', 'tags'];
+    // Unique key for a feed category
+    // e.g. username for user feed
+    const { category, key } = ctx.params;
+    if (!availableCategories.includes(category)) {
+        throw Error('Such feed is not supported.');
+    }
+
+    if (config.instagram && config.instagram.proxy) {
+        ig.state.proxyUrl = config.instagram.proxy;
+    }
+
+    await login(ig, ctx.cache);
+
+    let data;
+    try {
+        data = await loadContent(category, key, ctx.cache.tryGet);
+    } catch (e) {
+        logger.error(`Instagram error: ${e}`);
+        throw e;
+    }
+
+    ctx.state.data = {
+        title: data.feedTitle,
+        link: data.feedLink,
+        description: data.feedDescription,
+        item: renderItems(data.itemsRaw),
+        icon: 'https://www.instagram.com/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png',
+        logo: data.feedLogo,
+        image: data.feedLogo,
+        allowEmpty: true,
+    };
+};

package/lib/v2/instagram/private-api/utils.js

@@ -0,0 +1,37 @@
+const { IgApiClient } = require('instagram-private-api');
+const logger = require('@/utils/logger');
+const config = require('@/config').value;
+
+const ig = new IgApiClient();
+
+async function login(ig, cache) {
+    if (!config.instagram || !config.instagram.username || !config.instagram.password) {
+        throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
+    }
+    const LOGIN_CACHE_KEY = 'instagram:login';
+    const { username, password } = config.instagram;
+    const state = await cache.get(LOGIN_CACHE_KEY);
+    if (state) {
+        ig.state.deserialize(state);
+    } else {
+        ig.state.generateDevice(username);
+        // try {
+        //     await ig.simulate.preLoginFlow();
+        // } catch (error) {
+        //     logger.info('Instagram preLoginFlow fail: ' + error);
+        // }
+        await ig.account.login(username, password);
+        process.nextTick(() => ig.simulate.postLoginFlow());
+        logger.debug('Instagram login success.');
+    }
+    // Post request hook
+    ig.request.end$.subscribe(async () => {
+        const loginState = await ig.state.serialize();
+        delete loginState.constants;
+        // lifetime is defined in
+        // https://github.com/dilame/instagram-private-api/blob/6c160b2defcf8b0790b5a80033aab3e3084d0114/docs/classes/index/State.md#clientsessionidlifetime
+        await cache.set(LOGIN_CACHE_KEY, loginState, 1200000);
+    });
+}
+
+module.exports = { ig, login };

package/lib/v2/instagram/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'instagram.com': {
+        _name: 'Instagram',
+        www: [
+            {
+                title: '用户',
+                docs: 'https://docs.rsshub.app/social-media.html#instagram',
+                source: ['/:id'],
+                target: '/instagram/user/:id',
+            },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/social-media.html#instagram',
+                source: ['/explore/tags/:key'],
+                target: '/instagram/tags/:key',
+            },
+        ],
+    },
+};

package/lib/v2/instagram/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/:category/:key', require('./private-api/index'));
+    router.get('/2/:category/:key', require('./web-api/index'));
+};

package/lib/v2/instagram/templates/images.art

@@ -0,0 +1,5 @@
+{{@ summary.replace(/\n/g, '<br>') }}
+
+{{ each images i }}
+<img src="{{ i.url }}" height="{{ i.height }}" width="{{ i.width }}">
+{{ /each }}

package/lib/v2/instagram/templates/video.art

@@ -0,0 +1,5 @@
+{{@ summary.replace(/\n/g, '<br>') }}
+
+<video controls preload="none" poster="{{ image }}" width="{{ video.width }}">
+    <source src="{{ video.url }}" type="video/mp4">
+</video>

package/lib/v2/instagram/web-api/index.js

@@ -0,0 +1,70 @@
+const { CookieJar } = require('tough-cookie');
+const config = require('@/config').value;
+const { renderItems } = require('../common-utils');
+const { baseUrl, COOKIE_URL, getUserInfo, getUserFeedItems, getTagsFeedItems } = require('./utils');
+
+module.exports = async (ctx) => {
+    if (!config.instagram || !config.instagram.cookie) {
+        throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
+    }
+    const availableCategories = ['user', 'tags'];
+    const { category, key } = ctx.params;
+    const { cookie } = config.instagram;
+    if (!availableCategories.includes(category)) {
+        throw Error('Such feed is not supported.');
+    }
+
+    let cookieJar = await ctx.cache.get('instagram:cookieJar');
+    const cacheMiss = !cookieJar;
+    if (cacheMiss) {
+        cookieJar = new CookieJar();
+        for await (const c of cookie.split('; ')) {
+            await cookieJar.setCookie(c, COOKIE_URL);
+        }
+    } else {
+        cookieJar = CookieJar.fromJSON(cookieJar);
+    }
+
+    let feedTitle, feedLink, feedDescription, feedLogo;
+    let items;
+    switch (category) {
+        case 'user': {
+            const userInfo = await getUserInfo(key, cookieJar, ctx.cache);
+
+            // User feed metadata
+            const { biography, full_name, id, username } = userInfo;
+            feedTitle = `${full_name} (@${username}) - Instagram`;
+            feedDescription = biography;
+            // exists in web api ?? exist in private api ?? exist in both
+            feedLogo = userInfo.profile_pic_url_hd ?? userInfo.hd_profile_pic_url_info?.url ?? userInfo.profile_pic_url;
+            feedLink = `${baseUrl}/${username}`;
+
+            items = await getUserFeedItems(id, username, cookieJar, ctx.cache.tryGet);
+            break;
+        }
+        case 'tags': {
+            const tag = key;
+
+            feedTitle = `#${tag} - Instagram`;
+            feedLink = `${baseUrl}/explore/tags/${tag}`;
+
+            items = await getTagsFeedItems(tag, 'recent', cookieJar, ctx.cache.tryGet);
+            break;
+        }
+        default:
+            break;
+    }
+
+    await ctx.cache.set('instagram:cookieJar', cookieJar.toJSON(), 31536000);
+
+    ctx.state.data = {
+        title: feedTitle,
+        link: feedLink,
+        description: feedDescription,
+        item: renderItems(items),
+        icon: `${baseUrl}/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png`,
+        logo: feedLogo,
+        image: feedLogo,
+        allowEmpty: true,
+    };
+};

package/lib/v2/instagram/web-api/utils.js

@@ -0,0 +1,99 @@
+const got = require('@/utils/got');
+const config = require('@/config').value;
+
+const baseUrl = 'https://www.instagram.com';
+const COOKIE_URL = 'https://instagram.com';
+let igWwwClaim;
+
+const getCSRFTokenFromJar = async (cookieJar) => {
+    const cookieString = await cookieJar.getCookieString(COOKIE_URL);
+    return cookieString.match(/csrftoken=([^;]+)/)?.[1];
+};
+
+const getHeaders = async (cookieJar) => ({
+    'X-ASBD-ID': 198387,
+    'X-CSRFToken': await getCSRFTokenFromJar(cookieJar),
+    'X-IG-App-ID': 936619743392459,
+    'X-IG-WWW-Claim': igWwwClaim,
+});
+
+const getUserInfo = async (username, cookieJar, cache) => {
+    let webProfileInfo;
+    let id = await cache.get(`instagram:getIdByUsername:${username}`);
+    let userInfoCache = await cache.get(`instagram:userInfo:${id}`);
+
+    if (!userInfoCache) {
+        const response = await got(`${baseUrl}/api/v1/users/web_profile_info/`, {
+            cookieJar,
+            headers: await getHeaders(cookieJar),
+            searchParams: {
+                username,
+            },
+        });
+        if (response.url.includes('/accounts/login/')) {
+            throw Error('Invalid cookie');
+        }
+        igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+        webProfileInfo = response.data.data.user;
+        id = webProfileInfo.id;
+
+        await cache.set(`instagram:getIdByUsername:${username}`, id, 31536000); // 1 year since it will never change
+        await cache.set(`instagram:userInfo:${id}`, webProfileInfo);
+    }
+
+    userInfoCache = typeof userInfoCache === 'string' ? JSON.parse(userInfoCache) : userInfoCache;
+
+    return userInfoCache || webProfileInfo;
+};
+
+const getUserFeedItems = (id, username, cookieJar, tryGet) =>
+    tryGet(
+        `instagram:feed:${id}`,
+        async () => {
+            const response = await got(`${baseUrl}/api/v1/feed/user/${username}/username/`, {
+                cookieJar,
+                headers: await getHeaders(cookieJar),
+                searchParams: {
+                    count: 30,
+                },
+            });
+            // 401 Unauthorized if cookie does not match with IP
+            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+            return response.data.items;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+const getTagsFeedItems = (tag, tab, cookieJar, tryGet) =>
+    tryGet(
+        `instagram:tags:${tag}`,
+        async () => {
+            const response = await got(`${baseUrl}/api/v1/tags/web_info/`, {
+                // cookieJar, cookieJar is behaving weirdly here, so we use cookie header instead
+                headers: {
+                    cookie: await cookieJar.getCookieString(COOKIE_URL),
+                    ...(await getHeaders(cookieJar)),
+                },
+                searchParams: {
+                    tag_name: tag,
+                },
+            });
+            // Looks like cookie IP check is not applied to tags
+            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+            return response.data.data[tab].sections.flatMap((section) => section.layout_content.medias.map((media) => media.media));
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+module.exports = {
+    baseUrl,
+    COOKIE_URL,
+    getUserInfo,
+    getUserFeedItems,
+    getTagsFeedItems,
+};

package/lib/v2/iqiyi/album.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+
+    const { data: response } = await got(`https://www.iq.com/album/${id}`);
+
+    const $ = cheerio.load(response);
+    const nextData = JSON.parse($('#__NEXT_DATA__').text());
+    const { album } = nextData.props.initialState;
+
+    const {
+        data: { data: baseInfo },
+    } = await got(`https://pcw-api.iqiyi.com/album/album/baseinfo/${album.videoAlbumInfo.albumId}`);
+
+    if (Object.keys(album.cacheAlbumList).length === 0) {
+        throw Error(`${baseInfo.name} is not available in this server region.`);
+    }
+
+    let pos = 1;
+    let hasMore = false;
+    let epgs = [];
+    do {
+        const {
+            data: { data },
+            // eslint-disable-next-line no-await-in-loop
+        } = await got(`https://pcw-api.iq.com/api/v2/episodeListSource/${album.videoAlbumInfo.albumId}`, {
+            searchParams: {
+                platformId: 3,
+                modeCode: 'intl',
+                langCode: 'zh_cn',
+                endOrder: album.videoAlbumInfo.maxOrder,
+                startOrder: pos,
+            },
+        });
+        epgs = [...epgs, ...data.epg];
+        pos = data.pos;
+        hasMore = data.hasMore;
+    } while (hasMore);
+
+    const items = epgs.map((item) => ({
+        title: item.name,
+        description: art(path.join(__dirname, 'templates/album.art'), {
+            item,
+        }),
+        link: `https://www.iq.com/play/${item.playLocSuffix}`,
+        pubDate: parseDate(item.initIssueTime),
+    }));
+
+    ctx.state.data = {
+        title: baseInfo.name,
+        description: baseInfo.description,
+        image: album.videoAlbumInfo.albumFocus1024,
+        link: `https://www.iq.com/album/${album.videoAlbumInfo.albumLocSuffix}`,
+        item: items,
+        allowEmpty: true,
+    };
+};

package/lib/v2/iqiyi/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/album/:id': ['TonyRL'],
+    '/user/video/:uid': ['talengu'],
+};