rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f812e91

package/lib/v2/instagram/web-api/index.js

@@ -1,12 +1,12 @@
 const { CookieJar } = require('tough-cookie');
 const config = require('@/config').value;
 const { renderItems } = require('../common-utils');
-const { baseUrl, COOKIE_URL, getUserInfo, getUserFeedItems, getTagsFeedItems } = require('./utils');
+const { baseUrl, COOKIE_URL, checkLogin, getUserInfo, getUserFeedItems, getTagsFeedItems, getLoggedOutTagsFeedItems, renderGuestItems } = require('./utils');
 
 module.exports = async (ctx) => {
-    if (!config.instagram || !config.instagram.cookie) {
-        throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
-    }
+    // if (!config.instagram || !config.instagram.cookie) {
+    //     throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
+    // }
     const availableCategories = ['user', 'tags'];
     const { category, key } = ctx.params;
     const { cookie } = config.instagram;
@@ -15,16 +15,26 @@ module.exports = async (ctx) => {
     }
 
     let cookieJar = await ctx.cache.get('instagram:cookieJar');
+    const wwwClaimV2 = await ctx.cache.get('instagram:wwwClaimV2');
     const cacheMiss = !cookieJar;
+
     if (cacheMiss) {
         cookieJar = new CookieJar();
-        for await (const c of cookie.split('; ')) {
-            await cookieJar.setCookie(c, COOKIE_URL);
+        if (cookie) {
+            for await (const c of cookie.split('; ')) {
+                await cookieJar.setCookie(c, COOKIE_URL);
+            }
         }
     } else {
         cookieJar = CookieJar.fromJSON(cookieJar);
     }
 
+    if (!wwwClaimV2 && cookie) {
+        if (!(await checkLogin(cookieJar, ctx.cache))) {
+            throw Error('Invalid cookie');
+        }
+    }
+
     let feedTitle, feedLink, feedDescription, feedLogo;
     let items;
     switch (category) {
@@ -39,7 +49,12 @@ module.exports = async (ctx) => {
             feedLogo = userInfo.profile_pic_url_hd ?? userInfo.hd_profile_pic_url_info?.url ?? userInfo.profile_pic_url;
             feedLink = `${baseUrl}/${username}`;
 
-            items = await getUserFeedItems(id, username, cookieJar, ctx.cache.tryGet);
+            if (cookie) {
+                items = await getUserFeedItems(id, username, cookieJar, ctx.cache);
+            } else {
+                items = [...userInfo.edge_felix_video_timeline.edges, ...userInfo.edge_owner_to_timeline_media.edges];
+            }
+
             break;
         }
         case 'tags': {
@@ -48,7 +63,13 @@ module.exports = async (ctx) => {
             feedTitle = `#${tag} - Instagram`;
             feedLink = `${baseUrl}/explore/tags/${tag}`;
 
-            items = await getTagsFeedItems(tag, 'recent', cookieJar, ctx.cache.tryGet);
+            if (cookie) {
+                items = await getTagsFeedItems(tag, 'recent', cookieJar, ctx.cache.tryGet);
+            } else {
+                const hashTag = await getLoggedOutTagsFeedItems(tag, cookieJar);
+                items = [...hashTag.edge_hashtag_to_media.edges, ...hashTag.edge_hashtag_to_top_posts.edges];
+            }
+
             break;
         }
         default:
@@ -61,7 +82,7 @@ module.exports = async (ctx) => {
         title: feedTitle,
         link: feedLink,
         description: feedDescription,
-        item: renderItems(items),
+        item: cookie ? renderItems(items) : renderGuestItems(items),
         icon: `${baseUrl}/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png`,
         logo: feedLogo,
         image: feedLogo,

package/lib/v2/instagram/web-api/utils.js

@@ -1,9 +1,11 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 const config = require('@/config').value;
+const { art } = require('@/utils/render');
+const path = require('path');
 
 const baseUrl = 'https://www.instagram.com';
 const COOKIE_URL = 'https://instagram.com';
-let igWwwClaim;
 
 const getCSRFTokenFromJar = async (cookieJar) => {
     const cookieString = await cookieJar.getCookieString(COOKIE_URL);
@@ -14,32 +16,60 @@ const getHeaders = async (cookieJar) => ({
     'X-ASBD-ID': 198387,
     'X-CSRFToken': await getCSRFTokenFromJar(cookieJar),
     'X-IG-App-ID': 936619743392459,
-    'X-IG-WWW-Claim': igWwwClaim,
+    'X-IG-WWW-Claim': undefined,
 });
 
+const checkLogin = async (cookieJar, cache) => {
+    const response = await got.post(`${baseUrl}/api/v1/web/fxcal/ig_sso_users/`, {
+        // cookieJar,
+        headers: {
+            cookie: await cookieJar.getCookieString(COOKIE_URL),
+            ...(await getHeaders(cookieJar)),
+            'X-IG-WWW-Claim': '0',
+        },
+    });
+
+    const wwwClaimV2 = response.headers['x-ig-set-www-claim'];
+
+    if (wwwClaimV2) {
+        cache.set('instagram:wwwClaimV2', wwwClaimV2);
+    }
+
+    return Boolean(response.data.status === 'ok');
+};
+
 const getUserInfo = async (username, cookieJar, cache) => {
     let webProfileInfo;
     let id = await cache.get(`instagram:getIdByUsername:${username}`);
     let userInfoCache = await cache.get(`instagram:userInfo:${id}`);
 
     if (!userInfoCache) {
-        const response = await got(`${baseUrl}/api/v1/users/web_profile_info/`, {
-            cookieJar,
-            headers: await getHeaders(cookieJar),
-            searchParams: {
-                username,
-            },
-        });
-        if (response.url.includes('/accounts/login/')) {
-            throw Error('Invalid cookie');
-        }
-        igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+        try {
+            const response = await got(`${baseUrl}/api/v1/users/web_profile_info/`, {
+                cookieJar,
+                headers: {
+                    ...(await getHeaders(cookieJar)),
+                    'X-IG-WWW-Claim': (await cache.get('instagram:wwwClaimV2')) ?? undefined,
+                },
+                searchParams: {
+                    username,
+                },
+            });
+            if (response.url.includes('/accounts/login/')) {
+                throw Error('Invalid cookie');
+            }
 
-        webProfileInfo = response.data.data.user;
-        id = webProfileInfo.id;
+            webProfileInfo = response.data.data.user;
+            id = webProfileInfo.id;
 
-        await cache.set(`instagram:getIdByUsername:${username}`, id, 31536000); // 1 year since it will never change
-        await cache.set(`instagram:userInfo:${id}`, webProfileInfo);
+            await cache.set(`instagram:getIdByUsername:${username}`, id, 31536000); // 1 year since it will never change
+            await cache.set(`instagram:userInfo:${id}`, webProfileInfo);
+        } catch (e) {
+            if (e.message.includes("Cookie not in this host's domain")) {
+                throw Error('Invalid cookie');
+            }
+            throw e;
+        }
     }
 
     userInfoCache = typeof userInfoCache === 'string' ? JSON.parse(userInfoCache) : userInfoCache;
@@ -47,19 +77,26 @@ const getUserInfo = async (username, cookieJar, cache) => {
     return userInfoCache || webProfileInfo;
 };
 
-const getUserFeedItems = (id, username, cookieJar, tryGet) =>
-    tryGet(
+const getUserFeedItems = (id, username, cookieJar, cache) =>
+    cache.tryGet(
         `instagram:feed:${id}`,
         async () => {
             const response = await got(`${baseUrl}/api/v1/feed/user/${username}/username/`, {
-                cookieJar,
-                headers: await getHeaders(cookieJar),
+                // cookieJar,
+                headers: {
+                    cookie: await cookieJar.getCookieString(COOKIE_URL),
+                    ...(await getHeaders(cookieJar)),
+                    // 401 Unauthorized if cookie does not match with IP
+                    'X-IG-WWW-Claim': await cache.get('instagram:wwwClaimV2'),
+                },
                 searchParams: {
                     count: 30,
                 },
             });
-            // 401 Unauthorized if cookie does not match with IP
-            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+            if (response.url.includes('/accounts/login/')) {
+                throw Error(`Invalid cookie.
+                Please also check if your account is being blocked by Instagram.`);
+            }
 
             return response.data.items;
         },
@@ -81,8 +118,6 @@ const getTagsFeedItems = (tag, tab, cookieJar, tryGet) =>
                     tag_name: tag,
                 },
             });
-            // Looks like cookie IP check is not applied to tags
-            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
 
             return response.data.data[tab].sections.flatMap((section) => section.layout_content.medias.map((media) => media.media));
         },
@@ -90,10 +125,99 @@ const getTagsFeedItems = (tag, tab, cookieJar, tryGet) =>
         false
     );
 
+const getLoggedOutTagsFeedItems = async (tag, cookieJar) => {
+    const response = await got(`${baseUrl}/api/v1/tags/logged_out_web_info/`, {
+        // cookieJar, cookieJar is behaving weirdly here, so we use cookie header instead
+        headers: {
+            cookie: await cookieJar.getCookieString(COOKIE_URL),
+            ...(await getHeaders(cookieJar)),
+            'X-IG-WWW-Claim': '0',
+        },
+        searchParams: {
+            tag_name: tag,
+        },
+    });
+
+    const cookies = response.headers['set-cookie'];
+    if (cookies) {
+        for await (const cookie of cookies) {
+            await cookieJar.setCookie(cookie, COOKIE_URL);
+        }
+    }
+
+    return response.data.data.hashtag;
+};
+
+const renderGuestItems = (items) => {
+    const renderVideo = (node, summary) =>
+        art(path.join(__dirname, '../templates/video.art'), {
+            summary,
+            image: node.display_url,
+            video: {
+                url: node.video_url,
+                height: node.dimensions.height,
+                width: node.dimensions.width,
+            },
+        });
+    const renderImages = (node, summary) =>
+        art(path.join(__dirname, '../templates/images.art'), {
+            summary,
+            images: [{ url: node.display_url, height: node.dimensions.height, width: node.dimensions.width }],
+        });
+
+    return items.map(({ node }) => {
+        const type = node.__typename;
+        const summary = node.edge_media_to_caption.edges[0]?.node.text ?? '';
+
+        let description = '';
+        switch (type) {
+            // carousel, can include GraphVideo and GraphImage
+            case 'GraphSidecar':
+                description = node.edge_sidecar_to_children
+                    ? node.edge_sidecar_to_children.edges
+                          .map(({ node }, i) => {
+                              const _type = node.__typename;
+                              switch (_type) {
+                                  case 'GraphVideo':
+                                      return renderVideo(node, i === 0 ? summary : '');
+                                  case 'GraphImage':
+                                      return renderImages(node, i === 0 ? summary : '');
+                                  default:
+                                      throw Error(`Instagram: Unhandled carousel type: ${_type}`);
+                              }
+                          })
+                          .join('')
+                    : renderImages(node, summary);
+                break;
+            case 'GraphVideo':
+                description = renderVideo(node, summary);
+                break;
+            case 'GraphImage':
+                description = renderImages(node, summary);
+                break;
+            default:
+                throw Error(`Instagram: Unhandled feed type: ${type}`);
+        }
+
+        return {
+            title: summary.split('\n')[0],
+            id: node.id,
+            pubDate: parseDate(node.taken_at_timestamp, 'X'),
+            author: node.owner.username,
+            link: `${baseUrl}/p/${node.shortcode}/`,
+            summary,
+            description,
+        };
+    });
+};
+
 module.exports = {
     baseUrl,
     COOKIE_URL,
+    checkLogin,
     getUserInfo,
     getUserFeedItems,
     getTagsFeedItems,
+    getLoggedOutTagsFeedItems,
+    renderGuestItems,
 };

package/lib/v2/jianshu/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/collection/:id': ['DIYgod', 'HenryQW', 'JimenezLi'],
+    '/home': ['DIYgod', 'HenryQW', 'JimenezLi'],
+    '/user/:id': ['DIYgod', 'HenryQW', 'JimenezLi'],
+};

package/lib/v2/jianshu/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'jianshu.com': {
+        _name: '简书',
+        www: [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/routes/social-media#jian-shu',
+                source: '/',
+                target: '/jianshu/home',
+            },
+            {
+                title: '专题',
+                docs: 'https://docs.rsshub.app/routes/social-media#jian-shu',
+                source: '/c/:id',
+                target: '/jianshu/collection/:id',
+            },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/routes/social-media#jian-shu',
+                source: '/u/:id',
+                target: '/jianshu/user/:id',
+            },
+        ],
+    },
+};

package/lib/v2/jianshu/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/collection/:id', require('./collection'));
+    router.get('/home', require('./home'));
+    router.get('/user/:id', require('./user'));
+};

package/lib/{routes → v2}/jianshu/utils.js

@@ -1,18 +1,14 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const url = require('url');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 // 加载文章页
-async function load(link) {
+const load = async (link) => {
     const response = await got.get(link);
     const $ = cheerio.load(response.data);
-
     // 解析日期
-    const date = new Date($('time').attr('datetime'));
-    const timeZone = 8;
-    const serverOffset = date.getTimezoneOffset() / 60;
-    const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
-
+    const pubDate = timezone(parseDate($('time').attr('datetime')), +8);
     // 去除样式
     $('.image-container, .image-container-fill').removeAttr('style');
     // 处理视频
@@ -30,33 +26,29 @@ async function load(link) {
     const description = $('article').html();
 
     return { description, pubDate };
-}
+};
 
 const ProcessFeed = (list, caches) => {
     const host = 'https://www.jianshu.com';
 
     return Promise.all(
-        list.map(async (item) => {
+        list.map((item) => {
             const $ = cheerio.load(item);
-
             const $title = $('.title');
             // 还原相对链接为绝对链接
-            const itemUrl = url.resolve(host, $title.attr('href'));
-
-            // 列表上提取到的信息
-            const single = {
-                title: $title.text(),
-                link: itemUrl,
-                author: $('.nickname').text(),
-                guid: itemUrl,
-            };
-
+            const itemUrl = new URL($title.attr('href'), host).toString();
             // 使用tryGet方法从缓存获取内容。
-            // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
-            const other = await caches.tryGet(itemUrl, () => load(itemUrl));
-
-            // 合并解析后的结果集作为该篇文章最终的输出结果
-            return Promise.resolve(Object.assign({}, single, other));
+            // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容
+            return caches.tryGet(itemUrl, async () => {
+                const other = await load(itemUrl);
+                // 合并解析后的结果集作为该篇文章最终的输出结果
+                return {
+                    title: $title.text(),
+                    link: itemUrl,
+                    author: $('.nickname').text(),
+                    ...other,
+                };
+            });
         })
     );
 };

package/lib/v2/kemono/index.js

@@ -8,7 +8,7 @@ module.exports = async (ctx) => {
     const source = ctx.params.source ?? '';
     const id = ctx.params.id;
 
-    const rootUrl = 'https://kemono.party';
+    const rootUrl = 'https://kemono.su';
     const apiUrl = `${rootUrl}/api/v1/discord/channel/lookup/${id}`;
     const currentUrl = `${rootUrl}/${source ? `${source}/${source === 'discord' ? `server/${id}` : `user/${id}`}` : 'posts'}`;
 
@@ -23,7 +23,8 @@ module.exports = async (ctx) => {
     });
 
     let items = [],
-        title = '';
+        title = '',
+        image = undefined;
 
     if (source === 'discord') {
         title = `Posts of ${id} from Discord | Kemono`;
@@ -56,6 +57,7 @@ module.exports = async (ctx) => {
         const $ = cheerio.load(response.data);
 
         title = $('title').text();
+        image = $('.user-header__avatar img[src]').attr('src');
 
         items = await Promise.all(
             $('.card-list__items')
@@ -87,8 +89,29 @@ module.exports = async (ctx) => {
                         item.description = content('.post__body').html();
                         item.author = content('.post__user-name').text();
                         item.title = content('.post__title span').first().text();
+                        item.guid = item.link.replace('kemono.su', 'kemono.party');
                         item.pubDate = parseDate(content('.timestamp').attr('datetime'));
 
+                        // find the first attachment with a file extension we
+                        // want, and set it as the enclosure
+                        content('.post__attachment-link[href][download]').each(function () {
+                            const extension = content(this).attr('download').replace(/.*\./, '').toLowerCase();
+                            const mimeType =
+                                {
+                                    m4a: 'audio/mp4',
+                                    mp3: 'audio/mpeg',
+                                    mp4: 'video/mp4',
+                                }[extension] || null;
+                            if (mimeType === null) {
+                                return;
+                            }
+
+                            item.enclosure_url = content(this).attr('href');
+                            item.enclosure_type = mimeType;
+
+                            return false;
+                        });
+
                         return item;
                     })
                 )
@@ -97,6 +120,7 @@ module.exports = async (ctx) => {
 
     ctx.state.data = {
         title,
+        image,
         link: currentUrl,
         item: items,
     };

package/lib/v2/kemono/radar.js

@@ -1,5 +1,5 @@
 module.exports = {
-    'kemono.party': {
+    'kemono.su': {
         _name: 'Kemono',
         '.': [
             {

package/lib/v2/kepu/live.js

@@ -0,0 +1,89 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 50;
+
+    const rootUrl = 'https://live.kepu.net.cn';
+    const apiRootUrl = 'https://live.kepu.net.cn:8089';
+
+    const currentUrl = new URL('replay/index', rootUrl).href;
+    const apiUrl = new URL('index.php/front/index/replay_list', apiRootUrl).href;
+
+    const { data: response } = await got.post(apiUrl, {
+        form: {
+            page: 1,
+            size: limit,
+        },
+    });
+
+    let items = response.data.data.slice(0, limit).map((item) => ({
+        title: item.title,
+        link: new URL(`live/index?id=${item.id}`, rootUrl).href,
+        description: item.desc,
+        author: item.company,
+        guid: item.id,
+        pubDate: timezone(parseDate(item.live_start_time ?? item.start_time), +8),
+        updated: timezone(parseDate(item.live_end_time ?? item.end_time), +8),
+        itunes_item_image: new URL(item.cover, apiRootUrl).href,
+        comments: item.display_comment ?? 0,
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const apiReplayUrl = new URL('index.php/front/live/replay_url', apiRootUrl).href;
+
+                const { data: detailResponse } = await got.post(apiReplayUrl, {
+                    form: {
+                        id: item.guid,
+                    },
+                });
+
+                item.guid = `kepu-live#${item.guid}`;
+                item.enclosure_url = detailResponse.data.RecordIndexInfoList.RecordIndexInfo.pop()?.RecordUrl;
+
+                if (item.enclosure_url) {
+                    item.enclosure_type = `video/${item.enclosure_url.split(/\./).pop()}`;
+                }
+
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    image: {
+                        src: item.itunes_item_image,
+                        alt: item.title,
+                    },
+                    video: {
+                        src: item.enclosure_url,
+                        type: item.enclosure_type,
+                        poster: item.itunes_item_image,
+                    },
+                    description: item.description,
+                });
+
+                return item;
+            })
+        )
+    );
+
+    const icon = new URL('favicon.ico', rootUrl).href;
+    const author = '中国科普博览';
+    const subtitle = '直播回看';
+
+    ctx.state.data = {
+        item: items,
+        title: `${author} - ${subtitle}`,
+        link: currentUrl,
+        description: '科学直播(live.kepu.net.cn)',
+        language: 'zh',
+        icon,
+        logo: icon,
+        subtitle,
+        author,
+        itunes_author: author,
+        itunes_category: 'Science',
+        allowEmpty: true,
+    };
+};

package/lib/v2/kepu/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/live': ['nczitzk'],
+};

package/lib/v2/kepu/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'kepu.net.cn': {
+        _name: '中国科普博览',
+        live: [
+            {
+                title: '直播回看',
+                docs: 'https://docs.rsshub.app/routes/new-media#zhong-guo-ke-pu-bo-lan-zhi-bo-hui-kan',
+                source: ['/replay/index'],
+                target: '/kepu/live',
+            },
+        ],
+    },
+};

package/lib/v2/kepu/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/live', require('./live'));
+};

package/lib/v2/kepu/templates/description.art

@@ -0,0 +1,33 @@
+{{ if image?.src }}
+  <figure>
+    <img 
+      {{ if image.alt }}
+        alt="{{ image.alt }}"
+      {{ /if }}
+    src="{{ image.src }}">
+  </figure>
+{{ /if }}
+
+{{ if intro }}
+  <p>{{ intro }}</p>
+{{ /if }}
+
+{{ if video?.src }}
+  <video
+    {{ set poster = video.poster || image?.src }}
+    {{ if poster }}
+      poster="{{ poster }}"
+    {{ /if }}
+    controls>
+    <source
+      src="{{ video.src }}"
+      type="{{ video.type }}">
+    <object data="{{ video.src }}">
+      <embed src="{{ video.src }}">
+    </object>
+  </video>
+{{ /if }}
+
+{{ if description }}
+  <p>{{ description }}</p>
+{{ /if }}

package/lib/v2/leetcode/maintainer.js

@@ -1,6 +1,6 @@
 module.exports = {
     '/articles': ['LogicJake'],
-    '/dailyquestion/:lang': ['NavePnow'],
+    '/dailyquestion/:lang': ['EthanWng97'],
     '/dailyquestion/solution/:lang': ['woaidouya123'],
     '/submission/:country/:user': ['NathanDai'],
 };