rsshub 1.0.0-master.f7add90 → 1.0.0-master.f7bfacc

package/lib/v2/ibc/radio.js

@@ -0,0 +1,73 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? 'maitta';
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 150;
+
+    const rootUrl = 'https://www.ibc.co.jp';
+    const currentUrl = `${rootUrl}/radio/${id}/audio/`;
+
+    let response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    let $ = cheerio.load(response.data);
+
+    const image = $('meta[property="og:image"]').attr('content');
+
+    let radioId = $('link[title="RSS"]').attr('href')?.split('/rss/').pop().replace(/\/$/, '') ?? '';
+
+    let items = $('.audio_list section').toArray();
+
+    if (!radioId) {
+        const item = $(items[0]);
+
+        response = await got({
+            method: 'get',
+            url: new URL(item.find('a').attr('href'), rootUrl).href,
+        });
+
+        $ = cheerio.load(response.data);
+
+        radioId = $('audio')
+            .attr('src')
+            .match(/\/sound\/(\d+)_/)[1];
+    }
+
+    items = items.slice(0, limit).map((item) => {
+        item = $(item);
+
+        const link = new URL(item.find('a').attr('href'), rootUrl).href;
+        const audioId = link.match(/\/aitem\/(\d+)/)[1];
+        const audio = `https://media-data.cdn.ibc.co.jp/out/sound/${radioId}_${audioId}/${radioId}_${audioId}.mp3`;
+
+        return {
+            link,
+            title: item.find('h3').text(),
+            pubDate: timezone(parseDate(item.find('.update_date').text(), 'YYYY年MM月DD日'), +9),
+            enclosure_url: audio,
+            enclosure_type: 'audio/mpeg',
+            itunes_item_image: image,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                description: item.find('.audio_intro').text(),
+                audio,
+            }),
+        };
+    });
+
+    ctx.state.data = {
+        title: $('title').text() || `IBC岩手放送｜${$('.programinfo h3').text()}`,
+        link: currentUrl,
+        item: items,
+        description: $('meta[property="og:description"]').attr('content'),
+        itunes_author: 'IBC岩手放送',
+        image,
+        allowEmpty: true,
+    };
+};

package/lib/v2/ibc/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/maitta', require('./maitta'));
+    router.get('/radio/:id?', require('./radio'));
+};

package/lib/v2/ibc/templates/description.art

@@ -0,0 +1,6 @@
+{{ if description }}
+<p>{{ description }}</p>
+{{ /if }}
+{{ if audio }}
+<audio controls src="{{ audio }}"></audio>
+{{ /if }}

package/lib/v2/instructables/projects.js

@@ -48,7 +48,7 @@ module.exports = async (ctx) => {
             title: item.document.title,
             link: `https://${siteDomain}/${item.document.urlString}`,
             author: item.document.screenName,
-            description: `<img src="${item.document.coverImageUrl}">`,
+            description: `<img src="${item.document.coverImageUrl}?auto=webp&crop=1.2%3A1&frame=1&width=500" width="500">`,
             pubDate: new Date(item.document.publishDate).toUTCString(),
             category: item.document.primaryClassification,
         })),

package/lib/v2/ithome/radar.js

@@ -17,19 +17,19 @@ module.exports = {
             {
                 title: '24 小时阅读榜',
                 docs: 'https://docs.rsshub.app/new-media.html#it-zhi-jia-re-bang',
-                source: ['', '/*'],
+                source: ['/*'],
                 target: '/ithome/ranking/24h',
             },
             {
                 title: '7 天最热',
                 docs: 'https://docs.rsshub.app/new-media.html#it-zhi-jia-re-bang',
-                source: ['', '/*'],
+                source: ['/*'],
                 target: '/ithome/ranking/7days',
             },
             {
                 title: '月榜',
                 docs: 'https://docs.rsshub.app/new-media.html#it-zhi-jia-re-bang',
-                source: ['', '/*'],
+                source: ['/*'],
                 target: '/ithome/ranking/monthly',
             },
         ],

package/lib/v2/iwara/index.js

@@ -1,43 +1,55 @@
 const got = require('@/utils/got');
-const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
 
-const rootUrl = 'https://ecchi.iwara.tv';
+const rootUrl = 'https://www.iwara.tv';
+const apiRootUrl = 'https://api.iwara.tv';
+const imageRootUrl = 'https://i.iwara.tv';
 
 const typeMap = {
     video: 'Videos',
     image: 'Images',
 };
 
-const selectorMap = {
-    video: 'div.node.node-video.node-teaser.node-teaser.clearfix',
-    image: 'div.node.node-image.node-teaser.node-teaser.clearfix',
+const apiUrlMap = {
+    video: `${apiRootUrl}/videos`,
+    image: `${apiRootUrl}/images`,
+};
+
+const parseThumbnail = (type, item) => {
+    if (type === 'image') {
+        return `<img src="${imageRootUrl}/image/original/${item.thumbnail.id}/${item.thumbnail.name}">`;
+    }
+
+    if (item.embedUrl === null) {
+        return `<img src="${imageRootUrl}/image/original/${item.file.id}/thumbnail-${String(item.thumbnail).padStart(2, '0')}.jpg">`;
+    }
+
+    // regex borrowed from https://stackoverflow.com/a/3726073
+    const match = /http(?:s?):\/\/(?:www\.)?youtu(?:be\.com\/watch\?v=|\.be\/)([\w\-_]*)(&(amp;)?[\w?=]*)?/.exec(item.embedUrl);
+    if (match) {
+        return `<img src="${imageRootUrl}/image/embed/original/youtube/${match[1]}">`;
+    }
+
+    return '';
 };
 
 module.exports = async (ctx) => {
     const username = ctx.params.username;
+    const id = await ctx.cache.tryGet(`${apiRootUrl}/profile/${username}`, async () => (await got(`${apiRootUrl}/profile/${username}`)).data.user.id);
     const type = ctx.params.type ?? 'video';
-    const userUrl = `${rootUrl}/users/${username}`;
-
-    const response = await got.get(userUrl);
-    const $ = cheerio.load(response.data);
-    const list = $(selectorMap[type]);
-
-    const items = list
-        .map((_, item) => {
-            const imageUrl = 'https:' + $(item).find('img').attr('src');
-            return {
-                title: $(item).find('h3.title').text(),
-                author: username,
-                link: rootUrl + $(item).find('h3.title > a').attr('href'),
-                description: `<img src="${imageUrl}">`,
-                guid: (rootUrl + $(item).find('h3.title > a').attr('href')).split('?')[0],
-            };
-        })
-        .get();
+    const items = (await ctx.cache.tryGet(`${apiUrlMap[type]}?user=${id}`, async () => (await got(`${apiUrlMap[type]}?user=${id}`)).data.results, config.cache.routeExpire, false)).map((item) => ({
+        title: item.title,
+        author: username,
+        link: `${rootUrl}/${type}/${item.id}/${item.slug}`,
+        category: item.tags.map((i) => i.id),
+        description: parseThumbnail(type, item),
+        pubDate: parseDate(item.createdAt),
+    }));
 
     ctx.state.data = {
         title: `${username}'s iwara - ${typeMap[type]}`,
-        link: userUrl,
+        link: `${rootUrl}/users/${username}`,
         item: items,
     };
 };

package/lib/v2/iwara/radar.js

@@ -17,7 +17,7 @@ module.exports = {
             {
                 title: '用户订阅列表',
                 docs: 'https://docs.rsshub.app/anime.html#iwara',
-                source: '',
+                source: '/',
                 target: '/iwara/subscriptions',
             },
         ],

package/lib/v2/iwara/subscriptions.js

@@ -1,48 +1,147 @@
 const got = require('@/utils/got');
 const config = require('@/config').value;
-const cheerio = require('cheerio');
 const { art } = require('@/utils/render');
+const { parseDate } = require('@/utils/parse-date');
 const path = require('path');
+const md = require('markdown-it')({
+    html: true,
+});
 
 module.exports = async (ctx) => {
-    const rootUrl = `https://ecchi.iwara.tv`;
-    const subUrl = `https://ecchi.iwara.tv/subscriptions`;
-    const cookie = config.iwara.cookie;
-
-    if (cookie === undefined) {
+    if (!config.iwara || !config.iwara.username || !config.iwara.password) {
         throw Error('Iwara subscription RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
     }
 
-    const response = await got({
-        method: 'get',
-        url: subUrl,
-        headers: {
-            Cookie: cookie,
-        },
-    });
-    const $ = cheerio.load(response.data);
-    const username = $('a.btn.btn-info.btn-sm.user').text().trim();
-    const list = $('div.views-column');
-
-    const items = list
-        .map((_, item) => {
-            const imageUrl = 'https:' + $(item).find('img').attr('src');
-            const type = $(item).find('div.field').hasClass('field-type-video') ? 'Video' : 'Image';
-            return {
-                title: $(item).find('h3.title').text(),
-                author: $(item).find('a.username').text(),
-                link: rootUrl + $(item).find('h3.title > a').attr('href'),
-                category: type,
-                description: art(path.join(__dirname, 'templates/subscriptions.art'), {
-                    type,
-                    imageUrl,
+    const rootUrl = `https://www.iwara.tv`;
+    const username = config.iwara.username;
+    const password = config.iwara.password;
+
+    // get refresh token
+    const refreshHeaders = await ctx.cache.tryGet(
+        'iwara:token',
+        async () => {
+            const loginResponse = await got({
+                method: 'post',
+                url: 'https://api.iwara.tv/user/login',
+                headers: {
+                    'content-type': 'application/json',
+                },
+                data: JSON.stringify({
+                    email: username,
+                    password,
                 }),
+            });
+            return {
+                authorization: 'Bearer ' + loginResponse.data.token,
             };
-        })
-        .get();
+        },
+        30 * 24 * 60 * 60,
+        false
+    );
+
+    // get subscription list
+    const videoSubUrl = 'https://api.iwara.tv/videos?page=0&limit=30&subscribed=true';
+    const imageSubUrl = 'https://api.iwara.tv/images?page=0&limit=30&subscribed=true';
+
+    // get access token
+    const accessResponse = await got({
+        method: 'post',
+        url: 'https://api.iwara.tv/user/token',
+        headers: refreshHeaders,
+    });
+
+    const authHeaders = {
+        authorization: 'Bearer ' + accessResponse.data.accessToken,
+    };
+
+    const videoResponse = await got({
+        method: 'get',
+        url: videoSubUrl,
+        headers: authHeaders,
+    });
+
+    const imageResponse = await got({
+        method: 'get',
+        url: imageSubUrl,
+        headers: authHeaders,
+    });
+
+    const videoList = videoResponse.data.results.map((item) => {
+        let img_path;
+        if (item.private === true) {
+            img_path = 'https://i.iwara.tv/image/original/';
+        } else {
+            img_path = 'https://i.iwara.tv/image/thumbnail/';
+        }
+        const imageUrl = item.file ? img_path + item.file.id.toString().padStart(2, '0') + '/thumbnail-' + item.thumbnail.toString().padStart(2, '0') + '.jpg' : '';
+
+        return {
+            title: item.title,
+            author: item.user.name,
+            link: rootUrl + '/video/' + item.id,
+            category: 'Video',
+            imageUrl,
+            pubDate: parseDate(item.createdAt),
+            private: item.private,
+        };
+    });
+
+    const imageList = imageResponse.data.results.map((item) => {
+        const imageUrl = item.thumbnail ? 'https://i.iwara.tv/image/thumbnail/' + item.thumbnail.id + '/' + item.thumbnail.id + '.jpg' : '';
+        return {
+            title: item.title,
+            author: item.user.name,
+            link: rootUrl + '/image/' + item.id,
+            category: 'Image',
+            imageUrl,
+            pubDate: parseDate(item.createdAt),
+        };
+    });
+
+    // fulltext
+    const list = videoList.concat(imageList);
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                let description = art(path.join(__dirname, 'templates/subscriptions.art'), {
+                    type: item.category,
+                    imageUrl: item.imageUrl,
+                });
+
+                if (item.private === true) {
+                    description += 'private';
+                    return {
+                        title: item.title,
+                        author: item.author,
+                        link: item.link,
+                        category: item.category,
+                        pubDate: item.pubDate,
+                        description,
+                    };
+                }
+                const link = item.link.replace('www.iwara.tv', 'api.iwara.tv');
+                const response = await got({
+                    method: 'get',
+                    url: link,
+                    headers: authHeaders,
+                });
+                const body = response.data.body ? md.render(response.data.body) : '';
+                description += body;
+
+                return {
+                    title: item.title,
+                    author: item.author,
+                    link: item.link,
+                    category: item.category,
+                    pubDate: item.pubDate,
+                    description,
+                };
+            })
+        )
+    );
 
     ctx.state.data = {
-        title: `${username}'s iwara Subscription`,
+        title: `Iwara Subscription`,
         link: rootUrl,
         item: items,
     };

package/lib/v2/iwara/templates/subscriptions.art

@@ -1 +1 @@
-<h3>{{ type }}</h3><br><img src="{{ imageUrl }}">
+<h3>{{ type }}</h3><br><img src="{{ imageUrl }}"><br>

package/lib/v2/line/publisher.js

@@ -46,7 +46,7 @@ module.exports = async (ctx) => {
     ctx.state.data = {
         title: `${publisherInfo.data.name} - Line Today`,
         description: publisherInfo.data.introduction,
-        image: `https://obs.line-scdn.net/${publisherInfo.data.icon.hash}`,
+        image: publisherInfo.data.icon ? `https://obs.line-scdn.net/${publisherInfo.data.icon.hash}` : undefined,
         link: `${baseUrl}/${edition}/v2/publisher/${id}`,
         item: items,
     };

package/lib/v2/medium/following.js

@@ -0,0 +1,31 @@
+const config = require('@/config').value;
+
+const parseArticle = require('./parse-article.js');
+const { getFollowingFeedQuery } = require('./graphql.js');
+
+module.exports = async (ctx) => {
+    const user = ctx.params.user;
+
+    const cookie = config.medium.cookies[user];
+    if (cookie === undefined) {
+        throw Error(`缺少 Medium 用户 ${user} 登录后的 Cookie 值`);
+    }
+
+    const posts = await getFollowingFeedQuery(user, cookie);
+    ctx.state.json = posts;
+
+    if (!posts) {
+        // login failed
+        throw Error(`Medium 用户 ${user} 的 Cookie 无效或已过期`);
+    }
+
+    const urls = posts.items.map((data) => data.post.mediumUrl);
+
+    const parsedArticles = await Promise.all(urls.map((url) => parseArticle(ctx, url)));
+
+    ctx.state.data = {
+        title: `${user} Medium Following`,
+        link: 'https://medium.com/?feed=following',
+        item: parsedArticles,
+    };
+};

package/lib/v2/medium/for-you.js

@@ -0,0 +1,31 @@
+const config = require('@/config').value;
+
+const parseArticle = require('./parse-article.js');
+const { getWebInlineRecommendedFeedQuery } = require('./graphql.js');
+
+module.exports = async (ctx) => {
+    const user = ctx.params.user;
+
+    const cookie = config.medium.cookies[user];
+    if (cookie === undefined) {
+        throw Error(`缺少 Medium 用户 ${user} 登录后的 Cookie 值`);
+    }
+
+    const posts = await getWebInlineRecommendedFeedQuery(user, cookie);
+    ctx.state.json = posts;
+
+    if (!posts) {
+        // login failed
+        throw Error(`Medium 用户 ${user} 的 Cookie 无效或已过期`);
+    }
+
+    const urls = posts.items.map((data) => data.post.mediumUrl);
+
+    const parsedArticles = await Promise.all(urls.map((url) => parseArticle(ctx, url)));
+
+    ctx.state.data = {
+        title: `${user} Medium For You`,
+        link: 'https://medium.com/',
+        item: parsedArticles,
+    };
+};

package/lib/v2/medium/graphql.js

@@ -0,0 +1,190 @@
+const got = require('@/utils/got');
+
+async function graphqlRequest(body, cookie) {
+    const { data } = await got('https://medium.com/_/graphql', {
+        method: 'POST',
+        headers: {
+            accept: '*/*',
+            'accept-language': 'en-US,en;q=0.9,zh;q=0.8,zh-CN;q=0.7',
+            'apollographql-client-name': 'lite',
+            'apollographql-client-version': 'main-20230505-195233-209f54c418',
+            'cache-control': 'no-cache',
+            'content-type': 'application/json',
+            'medium-frontend-app': 'lite/main-20230505-195233-209f54c418',
+            'medium-frontend-path': '/',
+            'medium-frontend-route': 'homepage',
+            'ot-tracer-sampled': 'true',
+            'ot-tracer-spanid': '2db0b0d7263ffad8',
+            'ot-tracer-traceid': '679eb621b33147c4',
+            pragma: 'no-cache',
+            'sec-ch-ua': '"Chromium";v="113", "Not-A.Brand";v="24"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"macOS"',
+            'sec-fetch-dest': 'empty',
+            'sec-fetch-mode': 'cors',
+            'sec-fetch-site': 'same-origin',
+            'graphql-operation': body.operationName,
+            cookie,
+        },
+        data: JSON.stringify([body]),
+    });
+    return data[0].data;
+}
+
+async function getFollowingFeedQuery(user, cookie, pagingLimit = 20) {
+    return (await graphqlRequest(newFollowingFeedQuery(pagingLimit), cookie))?.followingFeed;
+}
+
+async function getWebInlineRecommendedFeedQuery(user, cookie, pagingLimit = 20) {
+    return (await graphqlRequest(newWebInlineRecommendedFeedQuery(pagingLimit), cookie))?.webRecommendedFeed;
+}
+
+async function getWebInlineTopicFeedQuery(user, tagSlug, cookie, pagingLimit = 20) {
+    return (await graphqlRequest(newWebInlineTopicFeedQuery(tagSlug, pagingLimit), cookie))?.personalisedTagFeed;
+}
+
+async function getUserCatalogMainContentQuery(user, catalogId, cookie, pagingLimit = 20) {
+    return (await graphqlRequest(newUserCatalogMainContentQuery(catalogId, pagingLimit), cookie))?.catalogById;
+}
+
+module.exports = {
+    getWebInlineRecommendedFeedQuery,
+    getFollowingFeedQuery,
+    getWebInlineTopicFeedQuery,
+    getUserCatalogMainContentQuery,
+};
+
+function newFollowingFeedQuery(pagingLimit = 5) {
+    return {
+        operationName: 'FollowingFeedQuery',
+        variables: {
+            paging: {
+                limit: pagingLimit,
+            },
+        },
+        query: `query FollowingFeedQuery($paging: PagingOptions) {
+            followingFeed(paging: $paging) {
+              items {
+                feedId
+                post {
+                  mediumUrl
+                  __typename
+                }
+                __typename
+              }
+              pagingInfo {
+                next {
+                  to
+                  from
+                  limit
+                  source
+                  __typename
+                }
+                __typename
+              }
+              __typename
+            }
+        }`,
+    };
+}
+
+function newWebInlineRecommendedFeedQuery(pagingLimit = 5) {
+    return {
+        operationName: 'WebInlineRecommendedFeedQuery',
+        variables: {
+            forceRank: true,
+            paging: {
+                limit: pagingLimit,
+            },
+        },
+        query: `query WebInlineRecommendedFeedQuery($paging: PagingOptions, $forceRank: Boolean) {
+            webRecommendedFeed(paging: $paging, forceRank: $forceRank) {
+              items {
+                feedId
+                post {
+                  mediumUrl
+                  __typename
+                }
+                __typename
+              }
+              pagingInfo {
+                next {
+                  limit
+                  to
+                  source
+                  __typename
+                }
+                __typename
+              }
+              __typename
+            }
+        }`,
+    };
+}
+
+function newWebInlineTopicFeedQuery(tagSlug, pagingLimit = 5) {
+    return {
+        operationName: 'WebInlineTopicFeedQuery',
+        variables: {
+            tagSlug,
+            paging: {
+                limit: pagingLimit,
+            },
+            skipCache: true,
+        },
+        query: `query WebInlineTopicFeedQuery($tagSlug: String!, $paging: PagingOptions!, $skipCache: Boolean) {
+            personalisedTagFeed(tagSlug: $tagSlug, paging: $paging, skipCache: $skipCache) {
+              items {
+                feedId
+                post {
+                  mediumUrl
+                  __typename
+                }
+                __typename
+              }
+              pagingInfo {
+                next {
+                  source
+                  limit
+                  from
+                  to
+                  __typename
+                }
+                __typename
+              }
+              __typename
+            }
+        }`,
+    };
+}
+
+function newUserCatalogMainContentQuery(catalogId, pagingLimit = 20) {
+    return {
+        operationName: 'UserCatalogMainContentQuery',
+        variables: {
+            catalogId,
+            pagingOptions: {
+                limit: pagingLimit,
+            },
+        },
+        query: `query UserCatalogMainContentQuery($catalogId: ID!, $pagingOptions: CatalogPagingOptionsInput!) {
+            catalogById(catalogId: $catalogId) {
+              __typename
+              ... on Catalog {
+                name
+                itemsConnection(pagingOptions: $pagingOptions) {
+                  items {
+                    entity {
+                      ... on Post {
+                        mediumUrl
+                      }
+                    }
+                    __typename
+                  }
+                  __typename
+                }
+              }
+            }
+          }`,
+    };
+}