rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/v2/picuki/profile.js

@@ -4,8 +4,12 @@ const chrono = require('chrono-node');
 const { art } = require('@/utils/render');
 const path = require('path');
 const config = require('@/config').value;
+const { puppeteerGet } = require('./utils');
 
 module.exports = async (ctx) => {
+    // use Puppeteer due to the obstacle by cloudflare challenge
+    const browser = await require('@/utils/puppeteer')();
+
     const id = ctx.params.id;
     const displayVideo = ctx.params.functionalFlag !== '0';
     const includeStories = ctx.params.functionalFlag === '10';
@@ -15,8 +19,8 @@ module.exports = async (ctx) => {
     const data = await ctx.cache.tryGet(
         profileUrl,
         async () => {
-            const _r = await got(profileUrl);
-            return _r.data;
+            const _r = await puppeteerGet(profileUrl, browser);
+            return _r;
         },
         config.cache.routeExpire,
         false
@@ -31,12 +35,8 @@ module.exports = async (ctx) => {
 
     async function getMedia(url) {
         const getPost = async () => {
-            const response = await got(url, {
-                headers: {
-                    Referer: profileUrl,
-                },
-            });
-            return response.data;
+            const response = await puppeteerGet(url, browser);
+            return response;
         };
         let data = await ctx.cache.tryGet(url, getPost);
         if (Object.prototype.toString.call(data) === '[object Object]') {
@@ -48,8 +48,20 @@ module.exports = async (ctx) => {
         const $ = cheerio.load(data);
         // Instagram 允许最多 10 条图片/视频任意混合于一条 post，picuki 在所有情况下都会将它（们）置于 .single-photo 内
         let html = '';
-        $('.single-photo img,video').each((_, item) => {
-            html += $(item).toString(); // 可能是视频，也可能是图片，格式都比较友好，这里直接添加
+        $('.single-photo img').each((_, item) => {
+            html += $(item).toString();
+        });
+        $('.single-photo video').each((_, item) => {
+            item = $(item);
+            let videoSrc = item.attr('src');
+            if (videoSrc === undefined) {
+                videoSrc = item.children().attr('src');
+            }
+            const videoPoster = item.attr('poster');
+            html += art(path.join(__dirname, 'templates/video.art'), {
+                videoSrc,
+                videoPoster,
+            });
         });
         return html;
     }
@@ -166,6 +178,8 @@ module.exports = async (ctx) => {
         }
     }
 
+    await browser.close();
+
     ctx.state.data = {
         title: `${profileName} (@${id}) - Picuki`,
         link: profileUrl,

package/lib/v2/picuki/templates/video.art

@@ -0,0 +1,3 @@
+<video poster="{{ videoPoster }}" controls>
+    <source src="{{ videoSrc }}" type="video/mp4">
+</video>

package/lib/v2/picuki/utils.js

@@ -0,0 +1,21 @@
+const puppeteerGet = async (url, browser) => {
+    const page = await browser.newPage();
+    // await page.setExtraHTTPHeaders({ referer: host });
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    await page.waitForSelector('.wrapper');
+
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    return html;
+};
+
+module.exports = {
+    puppeteerGet,
+};

package/lib/{routes → v2}/pixiv/api/getBookmarks.js

@@ -10,9 +10,7 @@ const queryString = require('query-string');
  * @returns {Promise<got.AxiosResponse<{illusts: illust[]}>>}
  */
 module.exports = function getBookmarks(user_id, token) {
-    return got({
-        method: 'get',
-        url: 'https://app-api.pixiv.net/v1/user/bookmarks/illust',
+    return got('https://app-api.pixiv.net/v1/user/bookmarks/illust', {
         headers: {
             ...maskHeader,
             Authorization: 'Bearer ' + token,

package/lib/{routes → v2}/pixiv/api/getIllustFollows.js

@@ -8,9 +8,7 @@ const queryString = require('query-string');
  * @returns {Promise<got.AxiosResponse<{illusts: illust[]}>>}
  */
 module.exports = function getUserIllustFollows(token) {
-    return got({
-        method: 'get',
-        url: 'https://app-api.pixiv.net/v2/illust/follow',
+    return got('https://app-api.pixiv.net/v2/illust/follow', {
         headers: {
             ...maskHeader,
             Authorization: 'Bearer ' + token,

package/lib/{routes → v2}/pixiv/api/getIllusts.js

@@ -9,9 +9,7 @@ const queryString = require('query-string');
  * @returns {Promise<got.AxiosResponse<{illusts: illust[]}>>}
  */
 module.exports = function getIllusts(user_id, token) {
-    return got({
-        method: 'get',
-        url: 'https://app-api.pixiv.net/v1/user/illusts',
+    return got('https://app-api.pixiv.net/v1/user/illusts', {
         headers: {
             ...maskHeader,
             Authorization: 'Bearer ' + token,

package/lib/{routes → v2}/pixiv/api/getRanking.js

@@ -3,7 +3,7 @@ const maskHeader = require('../constants').maskHeader;
 const assert = require('assert');
 const queryString = require('query-string');
 
-const allowMode = ['day', 'week', 'month', 'day_male', 'day_female', 'week_original', 'week_rookie', 'day_r18', 'day_male_r18', 'day_female_r18', 'week_r18', 'week_r18g'];
+const allowMode = ['day', 'week', 'month', 'day_male', 'day_female', 'day_ai', 'week_original', 'week_rookie', 'day_r18', 'day_male_r18', 'day_female_r18', 'week_r18', 'week_r18g'];
 
 /**
  * 获取某天的排行榜
@@ -14,9 +14,7 @@ const allowMode = ['day', 'week', 'month', 'day_male', 'day_female', 'week_origi
  */
 module.exports = function getRanking(mode, date, token) {
     assert(allowMode.includes(mode), 'Mode not allow.');
-    return got({
-        method: 'get',
-        url: 'https://app-api.pixiv.net/v1/illust/ranking',
+    return got('https://app-api.pixiv.net/v1/illust/ranking', {
         headers: {
             ...maskHeader,
             Authorization: 'Bearer ' + token,

package/lib/{routes → v2}/pixiv/api/getUserDetail.js

@@ -15,9 +15,7 @@ const queryString = require('query-string');
  * @returns {Promise<got.AxiosResponse<userDetail>>}
  */
 module.exports = function getUserDetail(user_id, token) {
-    return got({
-        method: 'get',
-        url: 'https://app-api.pixiv.net/v1/user/detail',
+    return got('https://app-api.pixiv.net/v1/user/detail', {
         headers: {
             ...maskHeader,
             Authorization: 'Bearer ' + token,

package/lib/{routes → v2}/pixiv/api/searchIllust.js

@@ -9,9 +9,7 @@ const queryString = require('query-string');
  * @returns {Promise<got.AxiosResponse<{illusts: illust[]}>>}
  */
 module.exports = function searchIllust(keyword, token) {
-    return got({
-        method: 'get',
-        url: 'https://app-api.pixiv.net/v1/search/illust',
+    return got('https://app-api.pixiv.net/v1/search/illust', {
         headers: {
             ...maskHeader,
             Authorization: 'Bearer ' + token,

package/lib/{routes → v2}/pixiv/api/searchPopularIllust.js

@@ -9,9 +9,7 @@ const queryString = require('query-string');
  * @returns {Promise<got.AxiosResponse<{illusts: illust[]}>>}
  */
 module.exports = function searchPopularIllust(keyword, token) {
-    return got({
-        method: 'get',
-        url: 'https://app-api.pixiv.net/v1/search/popular-preview/illust',
+    return got('https://app-api.pixiv.net/v1/search/popular-preview/illust', {
         headers: {
             ...maskHeader,
             Authorization: 'Bearer ' + token,

package/lib/{routes → v2}/pixiv/bookmarks.js

@@ -1,8 +1,9 @@
-const getToken = require('./token');
+const { getToken } = require('./token');
 const getBookmarks = require('./api/getBookmarks');
 const getUserDetail = require('./api/getUserDetail');
 const config = require('@/config').value;
 const pixivUtils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     if (!config.pixiv || !config.pixiv.refreshToken) {
@@ -11,7 +12,7 @@ module.exports = async (ctx) => {
 
     const id = ctx.params.id;
 
-    const token = await getToken();
+    const token = await getToken(ctx.cache.tryGet);
     if (!token) {
         throw 'pixiv not login';
     }
@@ -30,7 +31,7 @@ module.exports = async (ctx) => {
             return {
                 title: illust.title,
                 author: illust.user.name,
-                pubDate: new Date(illust.create_date).toUTCString(),
+                pubDate: parseDate(illust.create_date),
                 description: `<p>画师：${illust.user.name} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}</p>${images.join('')}`,
                 link: `https://www.pixiv.net/artworks/${illust.id}`,
             };

package/lib/{routes → v2}/pixiv/illustfollow.js

@@ -1,15 +1,16 @@
-const getToken = require('./token');
+const { getToken } = require('./token');
 const getIllustFollows = require('./api/getIllustFollows');
 const config = require('@/config').value;
 const pixivUtils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     if (!config.pixiv || !config.pixiv.refreshToken) {
         throw 'pixiv RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>';
     }
 
-    const token = await getToken();
-    if (!(await getToken())) {
+    const token = await getToken(ctx.cache.tryGet);
+    if (!token) {
         throw 'pixiv not login';
     }
 
@@ -24,7 +25,7 @@ module.exports = async (ctx) => {
             return {
                 title: illust.title,
                 author: illust.user.name,
-                pubDate: new Date(illust.create_date).toUTCString(),
+                pubDate: parseDate(illust.create_date),
                 description: `<p>画师：${illust.user.name} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}</p>${images.join('')}`,
                 link: `https://www.pixiv.net/artworks/${illust.id}`,
             };

package/lib/v2/pixiv/maintainer.js

@@ -0,0 +1,8 @@
+module.exports = {
+    '/ranking/:mode/:date?': ['EYHN'],
+    '/search/:keyword/:order?/:mode?': ['DIYgod'],
+    '/user/bookmarks/:id': ['EYHN'],
+    '/user/illustfollows': ['ClarkeCheng'],
+    '/user/novels/:id': ['TonyRL'],
+    '/user/:id': ['DIYgod'],
+};

package/lib/v2/pixiv/novels.js

@@ -0,0 +1,35 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const baseUrl = 'https://www.pixiv.net';
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/all`);
+
+    const items = [
+        ...data.body.novelSeries.map((item) => ({
+            title: item.title,
+            description: item.caption,
+            link: `https://www.pixiv.net/novel/series/${item.id}`,
+            author: item.userName,
+            pubDate: parseDate(item.createDate),
+            updated: parseDate(item.updateDate),
+            category: item.tags,
+        })),
+        ...data.body.pickup.map((item) => ({
+            title: item.title,
+            description: item.description,
+            link: item.contentUrl,
+            author: item.userName,
+            pubDate: parseDate(item.createDate),
+            updated: parseDate(item.updateDate),
+            category: item.tags,
+        })),
+    ];
+
+    ctx.state.data = {
+        title: `${data.body.novelSeries[0].userName || data.body.pickup[0].userName} 的小说`,
+        link: `${baseUrl}/users/${id}/novels`,
+        item: items,
+    };
+};

package/lib/{routes → v2}/pixiv/pixiv-got.js

@@ -6,7 +6,7 @@ const pixivConfig = require('@/config').value.pixiv;
 
 async function dohResolve(name, jsonDohEndpoint) {
     try {
-        const response = await got.get(jsonDohEndpoint, {
+        const response = await got(jsonDohEndpoint, {
             searchParams: {
                 name,
                 type: 'A',

package/lib/v2/pixiv/radar.js

@@ -0,0 +1,43 @@
+module.exports = {
+    'pixiv.net': {
+        _name: 'Pixiv',
+        www: [
+            {
+                title: '用户收藏',
+                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
+                source: '/users/:id/bookmarks/artworks',
+                target: '/pixiv/user/bookmarks/:id',
+            },
+            {
+                title: '用户动态',
+                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
+                source: '/users/:id',
+                target: '/pixiv/user/:id',
+            },
+            {
+                title: '用户小说',
+                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
+                source: '/users/:id/novels',
+                target: '/pixiv/user/novels/:id',
+            },
+            {
+                title: '排行榜',
+                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
+                source: '/ranking.php',
+                target: (params, url) => `/pixiv/ranking/${new URL(url).searchParams.get('mode') || 'daily'}`,
+            },
+            {
+                title: '关键词',
+                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
+                source: ['/tags/:keyword', '/tags/:keyword/:type?'],
+                target: (params, url) => `/pixiv/search/:keyword/${new URL(url).searchParams.get('order')}/${new URL(url).searchParams.get('mode')}`,
+            },
+            {
+                title: '关注的新作品',
+                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
+                source: '/bookmark_new_illust.php',
+                target: '/pixiv/user/illustfollows',
+            },
+        ],
+    },
+};

package/lib/{routes → v2}/pixiv/ranking.js

@@ -1,7 +1,8 @@
-const getToken = require('./token');
+const { getToken } = require('./token');
 const getRanking = require('./api/getRanking');
 const config = require('@/config').value;
 const pixivUtils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
 
 const titles = {
     day: 'pixiv 日排行',
@@ -16,6 +17,7 @@ const titles = {
     day_female_r18: 'pixiv R-18 受女性欢迎排行',
     week_r18: 'pixiv R-18 周排行',
     week_r18g: 'pixiv R-18G 排行',
+    day_ai: 'AI 生成作品排行榜',
 };
 
 const links = {
@@ -24,6 +26,7 @@ const links = {
     month: 'https://www.pixiv.net/ranking.php?mode=monthly',
     day_male: 'https://www.pixiv.net/ranking.php?mode=male',
     day_female: 'https://www.pixiv.net/ranking.php?mode=female',
+    day_ai: 'https://www.pixiv.net/ranking.php?mode=daily_ai',
     week_original: 'https://www.pixiv.net/ranking.php?mode=original',
     week_rookie: 'https://www.pixiv.net/ranking.php?mode=rookie',
     day_r18: 'https://www.pixiv.net/ranking.php?mode=daily_r18',
@@ -39,6 +42,7 @@ const alias = {
     monthly: 'month',
     male: 'day_male',
     female: 'day_female',
+    daily_ai: 'day_ai',
     original: 'week_original',
     rookie: 'week_rookie',
     daily_r18: 'day_r18',
@@ -56,7 +60,7 @@ module.exports = async (ctx) => {
     const mode = alias[ctx.params.mode] ? alias[ctx.params.mode] : ctx.params.mode;
     const date = ctx.params.date ? new Date(ctx.params.date) : new Date();
 
-    const token = await getToken();
+    const token = await getToken(ctx.cache.tryGet);
     if (!token) {
         throw 'pixiv not login';
     }
@@ -75,7 +79,7 @@ module.exports = async (ctx) => {
             const images = pixivUtils.getImgs(illust);
             return {
                 title: `#${index + 1} ${illust.title}`,
-                pubDate: new Date(illust.create_date).toUTCString(),
+                pubDate: parseDate(illust.create_date),
                 description: `<p>画师：${illust.user.name} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}</p><br>${images.join('')}`,
                 link: `https://www.pixiv.net/artworks/${illust.id}`,
                 author: illust.user.name,

package/lib/v2/pixiv/router.js

@@ -0,0 +1,8 @@
+module.exports = (router) => {
+    router.get('/ranking/:mode/:date?', require('./ranking'));
+    router.get('/search/:keyword/:order?/:mode?', require('./search'));
+    router.get('/user/bookmarks/:id', require('./bookmarks'));
+    router.get('/user/illustfollows', require('./illustfollow'));
+    router.get('/user/novels/:id', require('./novels'));
+    router.get('/user/:id', require('./user'));
+};

package/lib/{routes → v2}/pixiv/search.js

@@ -1,8 +1,9 @@
-const getToken = require('./token');
+const { getToken } = require('./token');
 const searchPopularIllust = require('./api/searchPopularIllust');
 const searchIllust = require('./api/searchIllust');
 const config = require('@/config').value;
 const pixivUtils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     if (!config.pixiv || !config.pixiv.refreshToken) {
@@ -13,7 +14,7 @@ module.exports = async (ctx) => {
     const order = ctx.params.order || 'date';
     const mode = ctx.params.mode;
 
-    const token = await getToken();
+    const token = await getToken(ctx.cache.tryGet);
     if (!token) {
         throw 'pixiv not login';
     }
@@ -40,7 +41,7 @@ module.exports = async (ctx) => {
             return {
                 title: illust.title,
                 author: illust.user.name,
-                pubDate: new Date(illust.create_date).toUTCString(),
+                pubDate: parseDate(illust.create_date),
                 description: `<p>画师：${illust.user.name} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}</p>${images.join('')}`,
                 link: `https://www.pixiv.net/artworks/${illust.id}`,
             };

package/lib/v2/pixiv/token.js

@@ -0,0 +1,86 @@
+const config = require('@/config').value;
+const logger = require('@/utils/logger');
+const got = require('./pixiv-got');
+const { maskHeader } = require('./constants');
+
+let token = null;
+
+const authorizationInfo = {
+    client_id: 'MOBrBDS8blbauoSck0ZfDbtuzpyT',
+    client_secret: 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj',
+    hash_secret: '28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c',
+};
+
+const refreshToken = (tryGet) =>
+    tryGet(
+        'pixiv:accessToken',
+        () =>
+            got
+                .post('https://oauth.secure.pixiv.net/auth/token', {
+                    form: {
+                        ...authorizationInfo,
+                        get_secure_url: 1,
+                        grant_type: 'refresh_token',
+                        refresh_token: config.pixiv.refreshToken,
+                    },
+                    headers: {
+                        ...maskHeader,
+                    },
+                    responseType: 'json',
+                    resolveBodyOnly: true,
+                })
+                .catch((e) => {
+                    logger.error('Pixiv refresh token failed.');
+                    logger.error(e);
+                }),
+        3600,
+        false
+    );
+
+async function getToken(tryGet) {
+    const result = await refreshToken(tryGet);
+    // let expireTime;
+    if (result && result.access_token) {
+        logger.debug('Pixiv refresh token success.');
+        token = result.access_token;
+        // expireTime = result.expires_in;
+    }
+    // } else {
+    // expireTime = 30;
+    // logger.error(`Pixiv refresh token failed, retry in ${expireTime} seconds.`);
+    // }
+    // If is package, the mock server should be closed after request, needn't to refresh
+    // if (!config.isPackage) {
+    //     setTimeout(tickToken, expireTime * 1000);
+    // }
+    return token;
+}
+
+module.exports = {
+    getToken,
+};
+
+// let tickTokenStarted = false;
+
+// async function startTickToken(tryGet) {
+//     if (!tickTokenStarted) {
+//         // 如果tickToken没启动
+//         tickTokenStarted = true;
+//         await tickToken(tryGet); // 启动tickToken
+//     }
+// }
+
+// async function waitForToken() {
+//     while (!token) {
+//         // eslint-disable-next-line no-await-in-loop
+//         await new Promise((resolve) => setTimeout(resolve, 0));
+//     }
+//     return token;
+// }
+
+// module.exports = async (tryGet) => {
+//     await startTickToken(tryGet);
+//     return waitForToken();
+// };
+
+// module.exports.tickToken = tickToken;

package/lib/{routes → v2}/pixiv/user.js

@@ -1,7 +1,8 @@
-const getToken = require('./token');
+const { getToken } = require('./token');
 const getIllusts = require('./api/getIllusts');
 const config = require('@/config').value;
 const pixivUtils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     if (!config.pixiv || !config.pixiv.refreshToken) {
@@ -9,7 +10,7 @@ module.exports = async (ctx) => {
     }
 
     const id = ctx.params.id;
-    const token = await getToken();
+    const token = await getToken(ctx.cache.tryGet);
     if (!token) {
         throw 'pixiv not login';
     }
@@ -28,7 +29,7 @@ module.exports = async (ctx) => {
             return {
                 title: illust.title,
                 author: username,
-                pubDate: new Date(illust.create_date).toUTCString(),
+                pubDate: parseDate(illust.create_date),
                 description: `<p>画师：${username} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}</p>${images.join('')}`,
                 link: `https://www.pixiv.net/artworks/${illust.id}`,
             };

package/lib/v2/producthunt/templates/descImg.art

@@ -1,5 +1,5 @@
 <br>
-{{ each descData.structuredData.json.screenshot }}
-<img src="{{ $value }}">
+{{ each descData.structuredData.screenshot }}
+    <img src="{{ $value }}">
 {{ /each }}
 

package/lib/v2/producthunt/today.js

@@ -5,22 +5,25 @@ const { art } = require('@/utils/render');
 const path = require('path');
 
 module.exports = async (ctx) => {
-    const response = await got.get('https://www.producthunt.com/');
+    const response = await got('https://www.producthunt.com/');
 
     const data = JSON.parse(cheerio.load(response.data)('#__NEXT_DATA__').html());
 
-    const list = Object.values(data.props.apolloState).filter((o) => o.__typename === 'Post');
+    const list = Object.values(data.props.apolloState)
+        .filter((o) => o.__typename === 'Post')
+        // only includes new post, not product
+        .filter((o) => o.hasOwnProperty('redirectToProduct') && o.redirectToProduct === null);
 
     const items = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.slug, async () => {
-                const detailresponse = await got.get(`https://www.producthunt.com/posts/${item.slug}`);
+                const detailresponse = await got(`https://www.producthunt.com/posts/${item.slug}`);
 
                 const data = JSON.parse(cheerio.load(detailresponse.data)('#__NEXT_DATA__').html());
-                const descData = Object.values(data.props.apolloState)[0];
+                const descData = data.props.apolloState[`Post${item.id}`];
 
                 return {
-                    title: `${descData.slug} - ${item.tagline}`,
+                    title: `${item.slug} - ${item.tagline}`,
                     description:
                         descData.description +
                         art(path.join(__dirname, 'templates/descImg.art'), {

package/lib/v2/pts/index.js

@@ -38,7 +38,10 @@ module.exports = async (ctx) => {
 
                 const content = cheerio.load(detailResponse.data);
 
-                item.author = content('meta[property="article:author"]').attr('content').split(' / ')[0];
+                item.author = content('meta[name="author"]')
+                    .attr('content')
+                    .replace(/文／/, '')
+                    .split(/／|\/|編譯|報導/)[0];
                 item.pubDate = timezone(parseDate(content('meta[property="article:published_time"]').attr('content')), +8);
                 item.category = content('.tag-list')
                     .first()