rsshub 1.0.0-master.f992838 → 1.0.0-master.f9a482d

package/lib/v2/ikea/templates/cn/product.art

@@ -0,0 +1,14 @@
+<p>名称：{{name}}</p>
+<p>类型：{{productType}}</p>
+<p>尺寸：{{measureText}}</p>
+{{if isFamilyOffer}}
+    <p>会员价格：&yen;{{currentPrice}}</p>
+    <p>非会员价格：&yen;{{originalPrice}}</p>
+{{else}}
+    <p>价格：&yen;{{currentPrice}}</p>
+{{/if}}
+<p>
+    {{each images}}
+        <img src="{{$value}}" />
+    {{/each}}
+</p>

package/lib/v2/infoq/utils.js

@@ -21,10 +21,11 @@ const ProcessFeed = async (list, cache) => {
                 const data = resp.data.data;
                 const author = data.author ? data.author.map((p) => p.nickname).join(',') : data.no_author;
                 const category = e.topic.map((t) => t.name).concat(e.label.map((l) => l.name));
+                const content = data.content_url ? (await got(data.content_url)).body : data.content;
 
                 return {
                     title: data.article_title,
-                    description: parseContent(data.content),
+                    description: parseContent(content),
                     pubDate: parseDate(e.publish_time, 'x'),
                     category,
                     author,

package/lib/v2/inoreader/index.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const type = 'html';
+    const user = ctx.params.user;
+    const tag = ctx.params.tag;
+    const num = ctx.query.limit || 20;
+    const rootUrl = 'https://www.inoreader.com/stream';
+    const currentUrl = `${rootUrl}/user/${user}/tag/${tag}/view/${type}?n=${num}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+    const data = response.data;
+    const $ = cheerio.load(data);
+    const entries = $('#snip_body>.article_content');
+
+    ctx.state.data = {
+        title: $('.header_text').text().trim(),
+        link: currentUrl,
+        item: entries
+            .map((idx, item) => {
+                const content = $(item).clone();
+                const header = $(item).prev();
+                const pubDate = $('div.article_author .au1', header)
+                    .contents()
+                    .filter((_, e) => e.nodeType === 3)
+                    .text()
+                    .replace(/posted (on|at) /, '')
+                    .replace(/UTC.*/, '');
+                return {
+                    title: $('a.title_link', header).text().trim(),
+                    link: $('a.title_link', header).attr('href'),
+                    author: $('div.article_author span span', header).text().trim() + ' via ' + $('div.article_author a.feed_link', header).text().trim(),
+                    pubDate: parseDate(pubDate, ['MMM DD YYYY HH:mm:ss', 'HH:mm:ss']),
+                    description: $(content).html(),
+                };
+            })
+            .get(),
+        allowEmpty: true,
+    };
+    ctx.state.json = ctx.state.data;
+};

package/lib/v2/inoreader/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/html_clip/:user/:tag/:num?': ['BeautyyuYanli'],
+    '/rss/:user/:tag': ['NavePnow'],
+};

package/lib/v2/inoreader/radar.js

@@ -0,0 +1,23 @@
+module.exports = {
+    'inoreader.com': {
+        _name: 'Inoreader',
+        '.': [
+            {
+                title: 'HTML Clip',
+                docs: 'https://docs.rsshub.app/reading.html#inoreader',
+                source: ['/stream/user/:user/tag/:tag/*'],
+                target: (params, url) => {
+                    const origin = new URL(url);
+                    const limit = origin.searchParams.get('n');
+                    return `/inoreader/html_clip/${params.user}/${params.tag}` + (limit ? `?limit=${limit}` : '');
+                },
+            },
+            {
+                title: 'RSS',
+                docs: 'https://docs.rsshub.app/reading.html#inoreader',
+                source: ['/stream/user/:user/tag/:tag'],
+                target: (params) => `/inoreader/rss/${params.user}/${params.tag}`,
+            },
+        ],
+    },
+};

package/lib/v2/inoreader/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/html_clip/:user/:tag', require('./index'));
+    router.get('/rss/:user/:tag', require('./rss'));
+};

package/lib/v2/inoreader/rss.js

@@ -0,0 +1,47 @@
+const parser = require('@/utils/rss-parser');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const user = ctx.params.user;
+    const tag = ctx.params.tag;
+    const rootUrl = 'https://www.inoreader.com/stream';
+    const rssUrl = `${rootUrl}/user/${user}/tag/${tag}`;
+    const feed = await parser.parseURL(rssUrl);
+    feed.items = feed.items.map((item) => {
+        if (item && item.enclosure && item.enclosure.type.includes('audio')) {
+            // output podcast rss
+            // get first image in content
+            let firstImgSrc = '';
+            if (item.content !== null) {
+                const $ = cheerio.load(item.content);
+                firstImgSrc = $('img').first().attr('src');
+            }
+            return {
+                title: item.title,
+                pubDate: item.pubDate,
+                link: item.link,
+                description: item.content,
+                category: item.categories,
+                itunes_item_image: firstImgSrc,
+                enclosure_url: item.enclosure.url,
+                enclosure_length: item.enclosure.length,
+                enclosure_type: item.enclosure.type,
+            };
+        }
+        return {
+            title: item?.title ?? '',
+            pubDate: item?.pubDate ?? '',
+            link: item?.link ?? '',
+            description: item?.content ?? '',
+            category: item?.categories ?? [],
+        };
+    });
+    ctx.state.data = {
+        title: feed.title,
+        itunes_author: 'Inoreader',
+        image: 'https://www.inoreader.com/brand/img/ino_app_icon.png',
+        link: feed.link,
+        description: feed.description,
+        item: feed.items,
+    };
+};

package/lib/v2/instagram/common-utils.js

@@ -0,0 +1,59 @@
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const renderItems = (items) =>
+    items.map((item) => {
+        const { product_type } = item; // carousel_container, feed, clips, igtv
+        // Content
+        const summary = item.caption?.text ?? '';
+
+        let description = '';
+        switch (product_type) {
+            case 'carousel_container': {
+                const images = item.carousel_media.map((i) => i.image_versions2.candidates[0]);
+                description = art(path.join(__dirname, 'templates/images.art'), {
+                    summary,
+                    images,
+                });
+                break;
+            }
+            case 'clips':
+            case 'igtv':
+                description = art(path.join(__dirname, 'templates/video.art'), {
+                    summary,
+                    image: item.image_versions2.candidates[0].url,
+                    video: item.video_versions[0],
+                });
+                break;
+            case 'feed': {
+                const images = [item.image_versions2.candidates[0]];
+                description = art(path.join(__dirname, 'templates/images.art'), {
+                    summary,
+                    images,
+                });
+                break;
+            }
+            default:
+                throw Error(`Instagram: Unhandled feed type: ${product_type}`);
+        }
+
+        // Metadata
+        const url = `https://www.instagram.com/p/${item.code}/`;
+        const pubDate = parseDate(item.taken_at, 'X');
+        const title = summary.split('\n')[0];
+
+        return {
+            title,
+            id: item.pk,
+            pubDate,
+            author: item.user.username,
+            link: url,
+            summary,
+            description,
+        };
+    });
+
+module.exports = {
+    renderItems,
+};

package/lib/v2/instagram/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/:category/:key': ['oppilate', 'DIYgod'],
+    '/2/:category/:key': ['TonyRL'],
+};

package/lib/v2/instagram/private-api/index.js

@@ -0,0 +1,94 @@
+const { ig, login } = require('./utils');
+const logger = require('@/utils/logger');
+const config = require('@/config').value;
+const { renderItems } = require('../common-utils');
+
+// loadContent pulls the desired user/tag/etc
+async function loadContent(category, nameOrId, tryGet) {
+    let feedTitle, feedLink, feedDescription, feedLogo;
+    let itemsRaw;
+
+    switch (category) {
+        case 'user': {
+            let userInfo, username, id;
+            if (!isNaN(nameOrId)) {
+                id = nameOrId;
+                userInfo = await tryGet(`instagram:userInfo:${id}`, () => ig.user.info(id));
+                username = userInfo.username;
+            } else {
+                username = nameOrId;
+                id = await tryGet(`instagram:getIdByUsername:${username}`, () => ig.user.getIdByUsername(username), 31536000); // 1 year since it will never change
+                userInfo = await tryGet(`instagram:userInfo:${id}`, () => ig.user.info(id));
+            }
+
+            feedDescription = userInfo.biography;
+            // exists in web api ?? exist in private api ?? exist in both
+            feedLogo = userInfo.profile_pic_url_hd ?? userInfo.hd_profile_pic_url_info?.url ?? userInfo.profile_pic_url;
+            const fullName = userInfo.full_name;
+            feedTitle = `${fullName} (@${username}) - Instagram`;
+            feedLink = `https://www.instagram.com/${username}`;
+
+            itemsRaw = await tryGet(`instagram:feed:${id}`, () => ig.feed.user(id).items(), config.cache.routeExpire, false);
+            break;
+        }
+        case 'tags': {
+            const tag = nameOrId;
+
+            feedTitle = `#${tag} - Instagram`;
+            feedLink = `https://www.instagram.com/explore/tags/${tag}`;
+
+            itemsRaw = await tryGet(`instagram:tags:${tag}`, () => ig.feed.tags(tag, 'recent').items(), config.cache.routeExpire, false);
+            break;
+        }
+        default:
+            break;
+    }
+
+    return {
+        feedTitle,
+        feedLink,
+        feedDescription,
+        feedLogo,
+        itemsRaw,
+    };
+}
+
+module.exports = async (ctx) => {
+    // https://github.com/dilame/instagram-private-api#feeds
+    // const availableCategories = ["accountFollowers", "accountFollowing", "news",
+    //     "discover", "pendingFriendships", "blockedUsers", "directInbox", "directPending",
+    //     "directThread", "user", "tag", "location", "mediaComments", "reelsMedia", "reelsTray",
+    //     "timeline", "musicTrending", "musicSearch", "musicGenre", "musicMood", "usertags", "saved"];
+    const availableCategories = ['user', 'tags'];
+    // Unique key for a feed category
+    // e.g. username for user feed
+    const { category, key } = ctx.params;
+    if (!availableCategories.includes(category)) {
+        throw Error('Such feed is not supported.');
+    }
+
+    if (config.instagram && config.instagram.proxy) {
+        ig.state.proxyUrl = config.instagram.proxy;
+    }
+
+    await login(ig, ctx.cache);
+
+    let data;
+    try {
+        data = await loadContent(category, key, ctx.cache.tryGet);
+    } catch (e) {
+        logger.error(`Instagram error: ${e}`);
+        throw e;
+    }
+
+    ctx.state.data = {
+        title: data.feedTitle,
+        link: data.feedLink,
+        description: data.feedDescription,
+        item: renderItems(data.itemsRaw),
+        icon: 'https://www.instagram.com/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png',
+        logo: data.feedLogo,
+        image: data.feedLogo,
+        allowEmpty: true,
+    };
+};

package/lib/v2/instagram/private-api/utils.js

@@ -0,0 +1,37 @@
+const { IgApiClient } = require('instagram-private-api');
+const logger = require('@/utils/logger');
+const config = require('@/config').value;
+
+const ig = new IgApiClient();
+
+async function login(ig, cache) {
+    if (!config.instagram || !config.instagram.username || !config.instagram.password) {
+        throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
+    }
+    const LOGIN_CACHE_KEY = 'instagram:login';
+    const { username, password } = config.instagram;
+    const state = await cache.get(LOGIN_CACHE_KEY);
+    if (state) {
+        ig.state.deserialize(state);
+    } else {
+        ig.state.generateDevice(username);
+        // try {
+        //     await ig.simulate.preLoginFlow();
+        // } catch (error) {
+        //     logger.info('Instagram preLoginFlow fail: ' + error);
+        // }
+        await ig.account.login(username, password);
+        process.nextTick(() => ig.simulate.postLoginFlow());
+        logger.debug('Instagram login success.');
+    }
+    // Post request hook
+    ig.request.end$.subscribe(async () => {
+        const loginState = await ig.state.serialize();
+        delete loginState.constants;
+        // lifetime is defined in
+        // https://github.com/dilame/instagram-private-api/blob/6c160b2defcf8b0790b5a80033aab3e3084d0114/docs/classes/index/State.md#clientsessionidlifetime
+        await cache.set(LOGIN_CACHE_KEY, loginState, 1200000);
+    });
+}
+
+module.exports = { ig, login };

package/lib/v2/instagram/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'instagram.com': {
+        _name: 'Instagram',
+        www: [
+            {
+                title: '用户',
+                docs: 'https://docs.rsshub.app/social-media.html#instagram',
+                source: ['/:id'],
+                target: '/instagram/user/:id',
+            },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/social-media.html#instagram',
+                source: ['/explore/tags/:key'],
+                target: '/instagram/tags/:key',
+            },
+        ],
+    },
+};

package/lib/v2/instagram/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/:category/:key', require('./private-api/index'));
+    router.get('/2/:category/:key', require('./web-api/index'));
+};

package/lib/v2/instagram/templates/images.art

@@ -0,0 +1,5 @@
+{{@ summary.replace(/\n/g, '<br>') }}
+
+{{ each images i }}
+<img src="{{ i.url }}" height="{{ i.height }}" width="{{ i.width }}">
+{{ /each }}

package/lib/v2/instagram/templates/video.art

@@ -0,0 +1,5 @@
+{{@ summary.replace(/\n/g, '<br>') }}
+
+<video controls preload="none" poster="{{ image }}" width="{{ video.width }}">
+    <source src="{{ video.url }}" type="video/mp4">
+</video>

package/lib/v2/instagram/web-api/index.js

@@ -0,0 +1,70 @@
+const { CookieJar } = require('tough-cookie');
+const config = require('@/config').value;
+const { renderItems } = require('../common-utils');
+const { baseUrl, COOKIE_URL, getUserInfo, getUserFeedItems, getTagsFeedItems } = require('./utils');
+
+module.exports = async (ctx) => {
+    if (!config.instagram || !config.instagram.cookie) {
+        throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
+    }
+    const availableCategories = ['user', 'tags'];
+    const { category, key } = ctx.params;
+    const { cookie } = config.instagram;
+    if (!availableCategories.includes(category)) {
+        throw Error('Such feed is not supported.');
+    }
+
+    let cookieJar = await ctx.cache.get('instagram:cookieJar');
+    const cacheMiss = !cookieJar;
+    if (cacheMiss) {
+        cookieJar = new CookieJar();
+        for await (const c of cookie.split('; ')) {
+            await cookieJar.setCookie(c, COOKIE_URL);
+        }
+    } else {
+        cookieJar = CookieJar.fromJSON(cookieJar);
+    }
+
+    let feedTitle, feedLink, feedDescription, feedLogo;
+    let items;
+    switch (category) {
+        case 'user': {
+            const userInfo = await getUserInfo(key, cookieJar, ctx.cache);
+
+            // User feed metadata
+            const { biography, full_name, id, username } = userInfo;
+            feedTitle = `${full_name} (@${username}) - Instagram`;
+            feedDescription = biography;
+            // exists in web api ?? exist in private api ?? exist in both
+            feedLogo = userInfo.profile_pic_url_hd ?? userInfo.hd_profile_pic_url_info?.url ?? userInfo.profile_pic_url;
+            feedLink = `${baseUrl}/${username}`;
+
+            items = await getUserFeedItems(id, username, cookieJar, ctx.cache.tryGet);
+            break;
+        }
+        case 'tags': {
+            const tag = key;
+
+            feedTitle = `#${tag} - Instagram`;
+            feedLink = `${baseUrl}/explore/tags/${tag}`;
+
+            items = await getTagsFeedItems(tag, 'recent', cookieJar, ctx.cache.tryGet);
+            break;
+        }
+        default:
+            break;
+    }
+
+    await ctx.cache.set('instagram:cookieJar', cookieJar.toJSON(), 31536000);
+
+    ctx.state.data = {
+        title: feedTitle,
+        link: feedLink,
+        description: feedDescription,
+        item: renderItems(items),
+        icon: `${baseUrl}/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png`,
+        logo: feedLogo,
+        image: feedLogo,
+        allowEmpty: true,
+    };
+};

package/lib/v2/instagram/web-api/utils.js

@@ -0,0 +1,99 @@
+const got = require('@/utils/got');
+const config = require('@/config').value;
+
+const baseUrl = 'https://www.instagram.com';
+const COOKIE_URL = 'https://instagram.com';
+let igWwwClaim;
+
+const getCSRFTokenFromJar = async (cookieJar) => {
+    const cookieString = await cookieJar.getCookieString(COOKIE_URL);
+    return cookieString.match(/csrftoken=([^;]+)/)?.[1];
+};
+
+const getHeaders = async (cookieJar) => ({
+    'X-ASBD-ID': 198387,
+    'X-CSRFToken': await getCSRFTokenFromJar(cookieJar),
+    'X-IG-App-ID': 936619743392459,
+    'X-IG-WWW-Claim': igWwwClaim,
+});
+
+const getUserInfo = async (username, cookieJar, cache) => {
+    let webProfileInfo;
+    let id = await cache.get(`instagram:getIdByUsername:${username}`);
+    let userInfoCache = await cache.get(`instagram:userInfo:${id}`);
+
+    if (!userInfoCache) {
+        const response = await got(`${baseUrl}/api/v1/users/web_profile_info/`, {
+            cookieJar,
+            headers: await getHeaders(cookieJar),
+            searchParams: {
+                username,
+            },
+        });
+        if (response.url.includes('/accounts/login/')) {
+            throw Error('Invalid cookie');
+        }
+        igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+        webProfileInfo = response.data.data.user;
+        id = webProfileInfo.id;
+
+        await cache.set(`instagram:getIdByUsername:${username}`, id, 31536000); // 1 year since it will never change
+        await cache.set(`instagram:userInfo:${id}`, webProfileInfo);
+    }
+
+    userInfoCache = typeof userInfoCache === 'string' ? JSON.parse(userInfoCache) : userInfoCache;
+
+    return userInfoCache || webProfileInfo;
+};
+
+const getUserFeedItems = (id, username, cookieJar, tryGet) =>
+    tryGet(
+        `instagram:feed:${id}`,
+        async () => {
+            const response = await got(`${baseUrl}/api/v1/feed/user/${username}/username/`, {
+                cookieJar,
+                headers: await getHeaders(cookieJar),
+                searchParams: {
+                    count: 30,
+                },
+            });
+            // 401 Unauthorized if cookie does not match with IP
+            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+            return response.data.items;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+const getTagsFeedItems = (tag, tab, cookieJar, tryGet) =>
+    tryGet(
+        `instagram:tags:${tag}`,
+        async () => {
+            const response = await got(`${baseUrl}/api/v1/tags/web_info/`, {
+                // cookieJar, cookieJar is behaving weirdly here, so we use cookie header instead
+                headers: {
+                    cookie: await cookieJar.getCookieString(COOKIE_URL),
+                    ...(await getHeaders(cookieJar)),
+                },
+                searchParams: {
+                    tag_name: tag,
+                },
+            });
+            // Looks like cookie IP check is not applied to tags
+            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+            return response.data.data[tab].sections.flatMap((section) => section.layout_content.medias.map((media) => media.media));
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+module.exports = {
+    baseUrl,
+    COOKIE_URL,
+    getUserInfo,
+    getUserFeedItems,
+    getTagsFeedItems,
+};

package/lib/v2/instructables/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/projects/:category?': ['wolfg1969'],
+};

package/lib/v2/instructables/projects.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'all';
+
+    const siteDomain = 'www.instructables.com';
+    const apiKey = 'NU5CdGwyRDdMVnVmM3l4cWNqQzFSVzJNZU5jaUxFU3dGK3J2L203MkVmVT02ZWFYeyJleGNsdWRlX2ZpZWxkcyI6WyJvdXRfb2YiLCJzZWFyY2hfdGltZV9tcyIsInN0ZXBCb2R5Il0sInBlcl9wYWdlIjo1MH0=';
+
+    let pathPrefix, projectFilter;
+    if (category === 'all') {
+        pathPrefix = '';
+        projectFilter = '';
+    } else {
+        pathPrefix = `${category}/`;
+        const filterValue = `${category.charAt(0).toUpperCase()}${category.slice(1)}`;
+        projectFilter = category !== 'teachers' ? ` && category:=${filterValue}` : `&& teachers:=${filterValue}`;
+    }
+
+    const link = `https://${siteDomain}/${pathPrefix}projects?projects=all`;
+
+    const response = await got({
+        method: 'get',
+        url: `https://${siteDomain}/api_proxy/search/collections/projects/documents/search`,
+        headers: {
+            Referer: link,
+            Host: siteDomain,
+            'x-typesense-api-key': apiKey,
+        },
+        searchParams: {
+            q: '*',
+            query_by: 'title,stepBody,screenName',
+            page: 1,
+            per_page: 50,
+            sort_by: 'publishDate:desc',
+            include_fields: 'title,urlString,coverImageUrl,screenName,publishDate,favorites,views,primaryClassification,featureFlag,prizeLevel,IMadeItCount',
+            filter_by: `featureFlag:=true${projectFilter}`,
+        },
+    });
+
+    const data = response.data;
+
+    ctx.state.data = {
+        title: 'Instructables Projects', // 项目的标题
+        link, // 指向项目的链接
+        description: 'Instructables Projects', // 描述项目
+        language: 'en', // 频道语言
+        item: data.hits.map((item) => ({
+            title: item.document.title,
+            link: `https://${siteDomain}/${item.document.urlString}`,
+            author: item.document.screenName,
+            description: `<img src="${item.document.coverImageUrl}?auto=webp&crop=1.2%3A1&frame=1&width=500" width="500">`,
+            pubDate: new Date(item.document.publishDate).toUTCString(),
+            category: item.document.primaryClassification,
+        })),
+    };
+};

package/lib/v2/instructables/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'instructables.com': {
+        _name: 'Instructables',
+        '.': [
+            {
+                title: 'All Projects - Instructables',
+                docs: 'https://docs.rsshub.app/other.html#instructables',
+                source: ['/projects'],
+                target: '/instructables/projects',
+            },
+            {
+                title: 'Projects - Instructables',
+                docs: 'https://docs.rsshub.app/other.html#instructables',
+                source: ['/:category/projects'],
+                target: (params) => `/instructables/projects/${params.category}`,
+            },
+        ],
+    },
+};

package/lib/v2/instructables/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/projects/:category?', require('./projects'));
+};