rsshub 1.0.0-master.f8b1c8d → 1.0.0-master.f91e29d

package/lib/v2/instagram/web-api/index.js

@@ -0,0 +1,70 @@
+const { CookieJar } = require('tough-cookie');
+const config = require('@/config').value;
+const { renderItems } = require('../common-utils');
+const { baseUrl, COOKIE_URL, getUserInfo, getUserFeedItems, getTagsFeedItems } = require('./utils');
+
+module.exports = async (ctx) => {
+    if (!config.instagram || !config.instagram.cookie) {
+        throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
+    }
+    const availableCategories = ['user', 'tags'];
+    const { category, key } = ctx.params;
+    const { cookie } = config.instagram;
+    if (!availableCategories.includes(category)) {
+        throw Error('Such feed is not supported.');
+    }
+
+    let cookieJar = await ctx.cache.get('instagram:cookieJar');
+    const cacheMiss = !cookieJar;
+    if (cacheMiss) {
+        cookieJar = new CookieJar();
+        for await (const c of cookie.split('; ')) {
+            await cookieJar.setCookie(c, COOKIE_URL);
+        }
+    } else {
+        cookieJar = CookieJar.fromJSON(cookieJar);
+    }
+
+    let feedTitle, feedLink, feedDescription, feedLogo;
+    let items;
+    switch (category) {
+        case 'user': {
+            const userInfo = await getUserInfo(key, cookieJar, ctx.cache);
+
+            // User feed metadata
+            const { biography, full_name, id, username } = userInfo;
+            feedTitle = `${full_name} (@${username}) - Instagram`;
+            feedDescription = biography;
+            // exists in web api ?? exist in private api ?? exist in both
+            feedLogo = userInfo.profile_pic_url_hd ?? userInfo.hd_profile_pic_url_info?.url ?? userInfo.profile_pic_url;
+            feedLink = `${baseUrl}/${username}`;
+
+            items = await getUserFeedItems(id, username, cookieJar, ctx.cache.tryGet);
+            break;
+        }
+        case 'tags': {
+            const tag = key;
+
+            feedTitle = `#${tag} - Instagram`;
+            feedLink = `${baseUrl}/explore/tags/${tag}`;
+
+            items = await getTagsFeedItems(tag, 'recent', cookieJar, ctx.cache.tryGet);
+            break;
+        }
+        default:
+            break;
+    }
+
+    await ctx.cache.set('instagram:cookieJar', cookieJar.toJSON(), 31536000);
+
+    ctx.state.data = {
+        title: feedTitle,
+        link: feedLink,
+        description: feedDescription,
+        item: renderItems(items),
+        icon: `${baseUrl}/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png`,
+        logo: feedLogo,
+        image: feedLogo,
+        allowEmpty: true,
+    };
+};

package/lib/v2/instagram/web-api/utils.js

@@ -0,0 +1,99 @@
+const got = require('@/utils/got');
+const config = require('@/config').value;
+
+const baseUrl = 'https://www.instagram.com';
+const COOKIE_URL = 'https://instagram.com';
+let igWwwClaim;
+
+const getCSRFTokenFromJar = async (cookieJar) => {
+    const cookieString = await cookieJar.getCookieString(COOKIE_URL);
+    return cookieString.match(/csrftoken=([^;]+)/)?.[1];
+};
+
+const getHeaders = async (cookieJar) => ({
+    'X-ASBD-ID': 198387,
+    'X-CSRFToken': await getCSRFTokenFromJar(cookieJar),
+    'X-IG-App-ID': 936619743392459,
+    'X-IG-WWW-Claim': igWwwClaim,
+});
+
+const getUserInfo = async (username, cookieJar, cache) => {
+    let webProfileInfo;
+    let id = await cache.get(`instagram:getIdByUsername:${username}`);
+    let userInfoCache = await cache.get(`instagram:userInfo:${id}`);
+
+    if (!userInfoCache) {
+        const response = await got(`${baseUrl}/api/v1/users/web_profile_info/`, {
+            cookieJar,
+            headers: await getHeaders(cookieJar),
+            searchParams: {
+                username,
+            },
+        });
+        if (response.url.includes('/accounts/login/')) {
+            throw Error('Invalid cookie');
+        }
+        igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+        webProfileInfo = response.data.data.user;
+        id = webProfileInfo.id;
+
+        await cache.set(`instagram:getIdByUsername:${username}`, id, 31536000); // 1 year since it will never change
+        await cache.set(`instagram:userInfo:${id}`, webProfileInfo);
+    }
+
+    userInfoCache = typeof userInfoCache === 'string' ? JSON.parse(userInfoCache) : userInfoCache;
+
+    return userInfoCache || webProfileInfo;
+};
+
+const getUserFeedItems = (id, username, cookieJar, tryGet) =>
+    tryGet(
+        `instagram:feed:${id}`,
+        async () => {
+            const response = await got(`${baseUrl}/api/v1/feed/user/${username}/username/`, {
+                cookieJar,
+                headers: await getHeaders(cookieJar),
+                searchParams: {
+                    count: 30,
+                },
+            });
+            // 401 Unauthorized if cookie does not match with IP
+            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+            return response.data.items;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+const getTagsFeedItems = (tag, tab, cookieJar, tryGet) =>
+    tryGet(
+        `instagram:tags:${tag}`,
+        async () => {
+            const response = await got(`${baseUrl}/api/v1/tags/web_info/`, {
+                // cookieJar, cookieJar is behaving weirdly here, so we use cookie header instead
+                headers: {
+                    cookie: await cookieJar.getCookieString(COOKIE_URL),
+                    ...(await getHeaders(cookieJar)),
+                },
+                searchParams: {
+                    tag_name: tag,
+                },
+            });
+            // Looks like cookie IP check is not applied to tags
+            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+
+            return response.data.data[tab].sections.flatMap((section) => section.layout_content.medias.map((media) => media.media));
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+module.exports = {
+    baseUrl,
+    COOKIE_URL,
+    getUserInfo,
+    getUserFeedItems,
+    getTagsFeedItems,
+};

package/lib/v2/instructables/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/projects/:category?': ['wolfg1969'],
+};

package/lib/v2/instructables/projects.js

@@ -0,0 +1,55 @@
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'all';
+
+    const siteDomain = 'www.instructables.com';
+    const apiKey = 'NU5CdGwyRDdMVnVmM3l4cWNqQzFSVzJNZU5jaUxFU3dGK3J2L203MkVmVT02ZWFYeyJleGNsdWRlX2ZpZWxkcyI6WyJvdXRfb2YiLCJzZWFyY2hfdGltZV9tcyIsInN0ZXBCb2R5Il0sInBlcl9wYWdlIjo1MH0=';
+
+    let pathPrefix, projectFilter;
+    if (category === 'all') {
+        pathPrefix = '';
+        projectFilter = '';
+    } else {
+        pathPrefix = `${category}/`;
+        const filterValue = `${category.charAt(0).toUpperCase()}${category.slice(1)}`;
+        projectFilter = category !== 'teachers' ? ` && category:=${filterValue}` : `&& teachers:=${filterValue}`;
+    }
+
+    const link = `https://${siteDomain}/${pathPrefix}projects?projects=all`;
+
+    const response = await got({
+        method: 'get',
+        url: `https://${siteDomain}/api_proxy/search/collections/projects/documents/search`,
+        headers: {
+            Referer: link,
+            Host: siteDomain,
+            'x-typesense-api-key': apiKey,
+        },
+        searchParams: {
+            q: '*',
+            query_by: 'title,stepBody,screenName',
+            page: 1,
+            per_page: 50,
+            sort_by: 'publishDate:desc',
+            include_fields: 'title,urlString,coverImageUrl,screenName,publishDate,favorites,views,primaryClassification,featureFlag,prizeLevel,IMadeItCount',
+            filter_by: `featureFlag:=true${projectFilter}`,
+        },
+    });
+
+    const data = response.data;
+
+    ctx.state.data = {
+        title: 'Instructables Projects', // 项目的标题
+        link, // 指向项目的链接
+        description: 'Instructables Projects', // 描述项目
+        language: 'en', // 频道语言
+        item: data.hits.map((item) => ({
+            title: item.document.title,
+            link: `https://${siteDomain}/${item.document.urlString}`,
+            author: item.document.screenName,
+            pubDate: new Date(item.document.publishDate).toUTCString(),
+            category: item.document.primaryClassification,
+        })),
+    };
+};

package/lib/v2/instructables/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'instructables.com': {
+        _name: 'Instructables',
+        '.': [
+            {
+                title: 'All Projects - Instructables',
+                docs: 'https://docs.rsshub.app/other.html#instructables',
+                source: ['/projects'],
+                target: '/instructables/projects',
+            },
+            {
+                title: 'Projects - Instructables',
+                docs: 'https://docs.rsshub.app/other.html#instructables',
+                source: ['/:category/projects'],
+                target: (params) => `/instructables/projects/${params.category}`,
+            },
+        ],
+    },
+};

package/lib/v2/instructables/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/projects/:category?', require('./projects'));
+};

package/lib/v2/iqiyi/album.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+
+    const { data: response } = await got(`https://www.iq.com/album/${id}`);
+
+    const $ = cheerio.load(response);
+    const nextData = JSON.parse($('#__NEXT_DATA__').text());
+    const { album } = nextData.props.initialState;
+
+    const {
+        data: { data: baseInfo },
+    } = await got(`https://pcw-api.iqiyi.com/album/album/baseinfo/${album.videoAlbumInfo.albumId}`);
+
+    if (Object.keys(album.cacheAlbumList).length === 0) {
+        throw Error(`${baseInfo.name} is not available in this server region.`);
+    }
+
+    let pos = 1;
+    let hasMore = false;
+    let epgs = [];
+    do {
+        const {
+            data: { data },
+            // eslint-disable-next-line no-await-in-loop
+        } = await got(`https://pcw-api.iq.com/api/v2/episodeListSource/${album.videoAlbumInfo.albumId}`, {
+            searchParams: {
+                platformId: 3,
+                modeCode: 'intl',
+                langCode: 'zh_cn',
+                endOrder: album.videoAlbumInfo.maxOrder,
+                startOrder: pos,
+            },
+        });
+        epgs = [...epgs, ...data.epg];
+        pos = data.pos;
+        hasMore = data.hasMore;
+    } while (hasMore);
+
+    const items = epgs.map((item) => ({
+        title: item.name,
+        description: art(path.join(__dirname, 'templates/album.art'), {
+            item,
+        }),
+        link: `https://www.iq.com/play/${item.playLocSuffix}`,
+        pubDate: parseDate(item.initIssueTime),
+    }));
+
+    ctx.state.data = {
+        title: baseInfo.name,
+        description: baseInfo.description,
+        image: album.videoAlbumInfo.albumFocus1024,
+        link: `https://www.iq.com/album/${album.videoAlbumInfo.albumLocSuffix}`,
+        item: items,
+        allowEmpty: true,
+    };
+};

package/lib/v2/iqiyi/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/album/:id': ['TonyRL'],
+    '/user/video/:uid': ['talengu'],
+};

package/lib/v2/iqiyi/radar.js

@@ -0,0 +1,24 @@
+module.exports = {
+    'iq.com': {
+        _name: '爱奇艺',
+        '.': [
+            {
+                title: '剧集',
+                docs: 'https://docs.rsshub.app/multimedia.html#ai-qi-yi',
+                source: ['/album/:id'],
+                target: '/iqiyi/:category/:id',
+            },
+        ],
+    },
+    'iqiyi.com': {
+        _name: '爱奇艺',
+        '.': [
+            {
+                title: '用户视频',
+                docs: 'https://docs.rsshub.app/multimedia.html#ai-qi-yi',
+                source: ['/u/:uid/*'],
+                target: '/iqiyi/user/video/:uid',
+            },
+        ],
+    },
+};

package/lib/v2/iqiyi/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/album/:id', require('./album'));
+    router.get('/user/video/:uid', require('./video'));
+};

package/lib/v2/iqiyi/templates/album.art

@@ -0,0 +1 @@
+<img src="https:{{ item.pic.replace('http:', 'https:') }}">

package/lib/v2/issuehunt/funded.js

@@ -0,0 +1,28 @@
+const got = require('@/utils/got');
+const MarkdownIt = require('markdown-it');
+
+module.exports = async (ctx) => {
+    const { username, repo } = ctx.params;
+    const response = await got(`https://issuehunt.io/apis/pages/repos/show?repositoryOwnerName=${username}&repositoryName=${repo}`);
+
+    const { issues } = response.data;
+    if (issues === undefined) {
+        throw new Error('没有获取到数据');
+    }
+
+    const md = MarkdownIt({
+        html: true,
+    });
+    ctx.state.data = {
+        title: `Issue Hunt 的悬赏 -- ${username}/${repo}`,
+        link: `https://issuehunt.io/r/${username}/${repo}`,
+        description: ``,
+        item: issues.map((item) => ({
+            title: item.title,
+            description: md.render(item.body),
+            pubDate: item.fundedAt,
+            link: `https://issuehunt.io/r/${username}/${repo}/issues/${item.number}`,
+            author: item.userName,
+        })),
+    };
+};

package/lib/v2/issuehunt/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/funded/:username/:repo': ['running-grass'],
+};

package/lib/v2/issuehunt/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'issuehunt.io': {
+        _name: 'Issue Hunt',
+        '.': [
+            {
+                title: '项目悬赏',
+                docs: 'https://docs.rsshub.app/programming.html#issue-hunt-xiang-mu-xuan-shang',
+                source: '/r/:username/:repo',
+                target: ({ username, repo }) => `/issuehunt/funded/${username}/${repo}`,
+            },
+        ],
+    },
+};

package/lib/v2/issuehunt/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/funded/:username/:repo', require('./funded'));
+};

package/lib/v2/itch/devlog.js

@@ -4,10 +4,14 @@ const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const user = ctx.params.user ?? '';
     const id = ctx.params.id ?? '';
+    if (!isValidHost(user)) {
+        throw Error('Invalid user');
+    }
 
     const rootUrl = `https://${user}.itch.io/${id}/devlog`;
 

package/lib/v2/iwara/index.js

@@ -30,6 +30,7 @@ module.exports = async (ctx) => {
                 author: username,
                 link: rootUrl + $(item).find('h3.title > a').attr('href'),
                 description: `<img src="${imageUrl}">`,
+                guid: (rootUrl + $(item).find('h3.title > a').attr('href')).split('?')[0],
             };
         })
         .get();

package/lib/v2/jandan/index.js

@@ -1,64 +1,30 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const timezone = require('@/utils/timezone');
-const { parseDate } = require('@/utils/parse-date');
-const { art } = require('@/utils/render');
-const path = require('path');
+const parser = require('@/utils/rss-parser');
 
 module.exports = async (ctx) => {
     const rootUrl = 'http://i.jandan.net';
-
-    const response = await got({
-        method: 'get',
-        url: rootUrl,
-    });
-
-    const $ = cheerio.load(response.data);
-
-    let items = $('.post')
-        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 15)
-        .toArray()
-        .map((item) => {
-            item = $(item);
-
-            const a = item.find('.thetitle a');
-
-            return {
-                title: a.text(),
-                link: a.attr('href'),
-                description: art(path.join(__dirname, 'templates/description.art'), {
-                    summary: item.find('.indexs').text(),
-                    image: item
-                        .find('img')
-                        .attr('data-original')
-                        .replace(/!square/, ''),
-                }),
-            };
-        });
-
-    items = await Promise.all(
-        items.map((item) =>
+    const feed = await parser.parseURL(`${rootUrl}/feed/`);
+    const items = await Promise.all(
+        feed.items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-
-                const content = cheerio.load(detailResponse.data);
-
-                content('.wechat-hide').prev().nextAll().remove();
-
-                item.description += content('.entry').html();
-                item.author = content('.postinfo strong').text();
-                item.pubDate = timezone(parseDate(detailResponse.data.match(/"pubDate": "(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})"/)[1]), +8);
-
-                return item;
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                $('.wechat-hide').prev().nextAll().remove();
+                const single = {
+                    title: item.title,
+                    description: $('.entry').html(),
+                    pubDate: item.pubDate,
+                    link: item.link,
+                    author: item['dc:creator'],
+                    category: item.categories,
+                };
+                return single;
             })
         )
     );
-
     ctx.state.data = {
-        title: $('title').text(),
+        title: '煎蛋',
         link: rootUrl,
         item: items,
     };

package/lib/v2/jandan/maintainer.js

@@ -1,4 +1,4 @@
 module.exports = {
     '/:category?': ['nczitzk'],
-    '/': ['nczitzk'],
+    '/': ['nczitzk', 'bigfei'],
 };

package/lib/v2/javbus/index.js

@@ -3,12 +3,27 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const config = require('@/config').value;
+
+const toSize = (raw) => {
+    const matches = raw.match(/(\d+(\.\d+)?)(\w+)/);
+    return matches[3] === 'GB' ? matches[1] * 1024 : matches[1];
+};
+
+const allowDomain = ['javbus.com', 'javbus.org', 'javsee.icu', 'javsee.one'];
 
 module.exports = async (ctx) => {
     const isWestern = /^\/western/.test(ctx.path);
+    const domain = ctx.query.domain ?? 'javbus.com';
+    const westernDomain = ctx.query.western_domain ?? 'javbus.org';
+
+    const rootUrl = `https://www.${domain}`;
+    const westernUrl = `https://www.${westernDomain}`;
+
+    if (!config.feature.allow_user_supply_unsafe_domain && (!allowDomain.includes(new URL(`https://${domain}/`).hostname) || !allowDomain.includes(new URL(`https://${westernDomain}/`).hostname))) {
+        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+    }
 
-    const rootUrl = `https://www.${ctx.query.domain ?? 'javbus.com'}`;
-    const westernUrl = `https://www.${ctx.query.western_domain ?? 'javbus.org'}`;
     const currentUrl = `${isWestern ? westernUrl : rootUrl}${ctx.path.replace(/^\/western/, '').replace(/\/home/, '')}`;
 
     const response = await got({
@@ -19,7 +34,7 @@ module.exports = async (ctx) => {
     const $ = cheerio.load(response.data);
 
     let items = $('.movie-box')
-        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 30)
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
         .toArray()
         .map((item) => {
             item = $(item);
@@ -98,11 +113,12 @@ module.exports = async (ctx) => {
                                 link: td.first().attr('href'),
                                 size: td.eq(1).text().trim(),
                                 date: td.last().text().trim(),
+                                score: content(tr).find('a').length ** 8 * toSize(td.eq(1).text().trim()),
                             };
                         });
 
                     if (magnets) {
-                        item.enclosure_url = magnets[0].link;
+                        item.enclosure_url = magnets.sort((a, b) => b.score - a.score)[0].link;
                         item.enclosure_type = 'application/x-bittorrent';
                     }
                 } catch (e) {

package/lib/v2/javbus/templates/description.art

@@ -9,9 +9,20 @@
 {{/if}}
 <br>
 <h4>磁力連結投稿</h4>
+<table>
+<tr>
+<th>磁力名稱</th>
+<th>檔案大小</th>
+<th>分享日期</th>
+</tr>
 {{each magnets magnet}}
-<li><a href="{{ magnet.link }}">{{ magnet.title }} [{{ magnet.size }}] ({{ magnet.date }})</a></li>
+<tr>
+<td><a href="{{ magnet.link }}">{{ magnet.title }}</a></td>
+<td>{{ magnet.size }}</td>
+<td>{{ magnet.date }}</td>
+</tr>
 {{/each}}
+</table>
 <h4>樣品圖像</h4>
 {{each thumbs}}
 <img src="{{ $value }}">

package/lib/v2/javdb/utils.js

@@ -1,10 +1,16 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const allowDomain = ['javdb.com', 'javdb36.com', 'javdb007.com'];
 
 module.exports = {
     ProcessItems: async (ctx, currentUrl, title) => {
         const domain = ctx.query.domain ?? 'javdb.com';
+        if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(new URL(`https://${domain}/`).hostname)) {
+            ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+        }
+
         const rootUrl = `https://${domain}`;
 
         const response = await got({

package/lib/v2/jiaoliudao/index.js

@@ -0,0 +1,35 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const baseUrl = 'https://www.jiaoliudao.com';
+
+module.exports = async (ctx) => {
+    const { data } = await got(`${baseUrl}/wp-json/wp/v2/posts`, {
+        searchParams: {
+            per_page: ctx.query.limit ? parseInt(ctx.query.limit) : 100,
+        },
+    });
+
+    const items = data.map((item) => {
+        const $ = cheerio.load(item.content.rendered, null, false);
+        $('img').each((_, img) => {
+            if (img.attribs['data-original']) {
+                img.attribs.src = img.attribs['data-original'];
+                delete img.attribs['data-original'];
+            }
+        });
+        return {
+            title: item.title.rendered,
+            description: $.html(),
+            pubDate: parseDate(item.date_gmt),
+            link: item.link,
+        };
+    });
+
+    ctx.state.data = {
+        title: '交流岛资源网-专注网络资源收集',
+        image: `${baseUrl}/favicon.ico`,
+        link: baseUrl,
+        item: items,
+    };
+};

package/lib/v2/jiaoliudao/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['TonyRL'],
+};