rsshub 1.0.0-master.fe4f650 → 1.0.0-master.fe560d2

package/lib/v2/paradigm/writing.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.paradigm.xyz';
+
+module.exports = async (ctx) => {
+    const url = `${baseUrl}/writing`;
+
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+
+    const nextData = JSON.parse($('#__NEXT_DATA__').text());
+    const buildId = nextData.buildId;
+    const list = nextData.props.pageProps.posts.map((item) => ({
+        title: item.title,
+        link: `${baseUrl}${item.slug}`,
+        api: `${baseUrl}/_next/data/${buildId}${item.slug}.json`,
+        author: item.authors.map((author) => author.name).join(', '),
+        pubDate: parseDate(item.originalDate),
+        category: item.tags,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.api);
+                const $ = cheerio.load(response.data.pageProps.content, null, false);
+
+                // Remove the TOC
+                $('.toc').remove();
+                item.description = $.html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'Paradigm - Writing',
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/picuki/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/profile/:id/:functionalFlag?': ['hoilc', 'Rongronggg9'],
+    '/profile/:id/:functionalFlag?': ['hoilc', 'Rongronggg9', 'devinmugen'],
 };

package/lib/v2/picuki/profile.js

@@ -4,8 +4,12 @@ const chrono = require('chrono-node');
 const { art } = require('@/utils/render');
 const path = require('path');
 const config = require('@/config').value;
+const { puppeteerGet } = require('./utils');
 
 module.exports = async (ctx) => {
+    // use Puppeteer due to the obstacle by cloudflare challenge
+    const browser = await require('@/utils/puppeteer')();
+
     const id = ctx.params.id;
     const displayVideo = ctx.params.functionalFlag !== '0';
     const includeStories = ctx.params.functionalFlag === '10';
@@ -15,8 +19,8 @@ module.exports = async (ctx) => {
     const data = await ctx.cache.tryGet(
         profileUrl,
         async () => {
-            const _r = await got(profileUrl);
-            return _r.data;
+            const _r = await puppeteerGet(profileUrl, browser);
+            return _r;
         },
         config.cache.routeExpire,
         false
@@ -31,12 +35,8 @@ module.exports = async (ctx) => {
 
     async function getMedia(url) {
         const getPost = async () => {
-            const response = await got(url, {
-                headers: {
-                    Referer: profileUrl,
-                },
-            });
-            return response.data;
+            const response = await puppeteerGet(url, browser);
+            return response;
         };
         let data = await ctx.cache.tryGet(url, getPost);
         if (Object.prototype.toString.call(data) === '[object Object]') {
@@ -178,6 +178,8 @@ module.exports = async (ctx) => {
         }
     }
 
+    await browser.close();
+
     ctx.state.data = {
         title: `${profileName} (@${id}) - Picuki`,
         link: profileUrl,

package/lib/v2/picuki/utils.js

@@ -0,0 +1,21 @@
+const puppeteerGet = async (url, browser) => {
+    const page = await browser.newPage();
+    // await page.setExtraHTTPHeaders({ referer: host });
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    await page.waitForSelector('.wrapper');
+
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    return html;
+};
+
+module.exports = {
+    puppeteerGet,
+};

package/lib/v2/pixiv/api/getRanking.js

@@ -3,7 +3,7 @@ const maskHeader = require('../constants').maskHeader;
 const assert = require('assert');
 const queryString = require('query-string');
 
-const allowMode = ['day', 'week', 'month', 'day_male', 'day_female', 'week_original', 'week_rookie', 'day_r18', 'day_male_r18', 'day_female_r18', 'week_r18', 'week_r18g'];
+const allowMode = ['day', 'week', 'month', 'day_male', 'day_female', 'day_ai', 'week_original', 'week_rookie', 'day_r18', 'day_male_r18', 'day_female_r18', 'week_r18', 'week_r18g'];
 
 /**
  * 获取某天的排行榜

package/lib/v2/pixiv/ranking.js

@@ -17,6 +17,7 @@ const titles = {
     day_female_r18: 'pixiv R-18 受女性欢迎排行',
     week_r18: 'pixiv R-18 周排行',
     week_r18g: 'pixiv R-18G 排行',
+    day_ai: 'AI 生成作品排行榜',
 };
 
 const links = {
@@ -25,6 +26,7 @@ const links = {
     month: 'https://www.pixiv.net/ranking.php?mode=monthly',
     day_male: 'https://www.pixiv.net/ranking.php?mode=male',
     day_female: 'https://www.pixiv.net/ranking.php?mode=female',
+    day_ai: 'https://www.pixiv.net/ranking.php?mode=daily_ai',
     week_original: 'https://www.pixiv.net/ranking.php?mode=original',
     week_rookie: 'https://www.pixiv.net/ranking.php?mode=rookie',
     day_r18: 'https://www.pixiv.net/ranking.php?mode=daily_r18',
@@ -40,6 +42,7 @@ const alias = {
     monthly: 'month',
     male: 'day_male',
     female: 'day_female',
+    daily_ai: 'day_ai',
     original: 'week_original',
     rookie: 'week_rookie',
     daily_r18: 'day_r18',

package/lib/v2/producthunt/templates/descImg.art

@@ -1,5 +1,5 @@
 <br>
-{{ each descData.structuredData.json.screenshot }}
-<img src="{{ $value }}">
+{{ each descData.structuredData.screenshot }}
+    <img src="{{ $value }}">
 {{ /each }}
 

package/lib/v2/qlu/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/notice': ['SunBK201'],
+};

package/lib/v2/qlu/notice.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+const host = 'https://www.qlu.edu.cn';
+
+module.exports = async (ctx) => {
+    const response = await got({
+        method: 'get',
+        url: `${host}/tzggsh/list1.htm`,
+    });
+
+    const $ = cheerio.load(response.data);
+    const list = $('ul.news_list.list2').children();
+
+    const items = await Promise.all(
+        list.map((i, item) => {
+            item = $(item);
+            const itemTitle = item.find('.news_title').children().text();
+            const itemDate = item.find('.news_year').text() + item.find('.news_days').text();
+            const path = item.find('.news_title').children().attr('href');
+            let itemUrl = '';
+            if (!path.startsWith('https://') && !path.startsWith('http://')) {
+                itemUrl = host + path;
+            } else {
+                itemUrl = path;
+            }
+            return ctx.cache.tryGet(itemUrl, async () => {
+                let description = '';
+                if (!path.startsWith('https://') && !path.startsWith('http://')) {
+                    const result = await got(itemUrl);
+                    const $ = cheerio.load(result.data);
+                    description = $('.read').html().trim();
+                } else {
+                    description = itemTitle;
+                }
+                return {
+                    title: itemTitle,
+                    link: itemUrl,
+                    pubDate: timezone(parseDate(itemDate), 8),
+                    description,
+                };
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: `齐鲁工业大学 - 通知公告`,
+        link: `${host}/tzggsh/list1.htm`,
+        description: '齐鲁工业大学 - 通知公告',
+        item: items,
+    };
+};

package/lib/v2/qlu/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'qlu.edu.cn': {
+        _name: '齐鲁工业大学',
+        '.': [
+            {
+                title: '通知公告',
+                docs: 'https://docs.rsshub.app/university.html#qi-lu-gong-ye-da-xue',
+                source: ['/tzggsh/list1.htm'],
+                target: '/qlu/notice',
+            },
+        ],
+    },
+};

package/lib/v2/qlu/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/notice', require('./notice'));
+};

package/lib/v2/reactnewsletter/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '': ['meixger'],
+    '/': ['meixger'],
+};

package/lib/v2/reactnewsletter/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'bytes.dev': {
+        _name: 'ui.dev',
+        '.': [
+            {
+                title: 'React Newsletter',
+                docs: 'https://docs.rsshub.app/en/programming.html#ui-dev-bytes',
+                source: ['/issues', '/'],
+                target: '/reactnewsletter',
+            },
+        ],
+    },
+};

package/lib/v2/reactnewsletter/reactnewsletter.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const currentURL = 'https://reactnewsletter.com/issues';
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const resp = await got(currentURL);
+    const $ = cheerio.load(resp.data);
+    const text = $('script#__NEXT_DATA__').text();
+    const json = JSON.parse(text);
+
+    const items = json.props.pageProps.issues.map((item) => ({
+        title: item.title,
+        pubDate: parseDate(item.date),
+        description: item.summary,
+        link: `/issues/${item.slug}`,
+    }));
+
+    ctx.state.data = {
+        title: 'reactnewsletter.dev',
+        description: 'Stay up to date on the latest React news, tutorials, resources, and more. Delivered every Tuesday, for free.',
+        link: currentURL,
+        item: items,
+    };
+};

package/lib/v2/reactnewsletter/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./reactnewsletter'));
+};

package/lib/v2/researchgate/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/publications/:username': ['nczitzk'],
+};

package/lib/v2/researchgate/publications.js

@@ -0,0 +1,70 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const rootUrl = 'https://www.researchgate.net';
+    const currentUrl = `${rootUrl}/profile/${id}`;
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(currentUrl);
+    const response = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+
+    const $ = cheerio.load(response);
+
+    const list = $('div[itemprop="headline"] a')
+        .toArray()
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 15)
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await page.goto(item.link);
+                const detailResponse = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
+                const content = cheerio.load(detailResponse);
+
+                item.doi = content('meta[property="citation_doi"]').attr('content');
+                item.pubDate = parseDate(content('meta[property="citation_publication_date"]').attr('content'));
+
+                const authors = [];
+
+                content('meta[property="citation_author"]').each(function () {
+                    authors.push(content(this).attr('content'));
+                });
+
+                item.author = authors.join(', ');
+
+                item.description = content('div[itemprop="description"]').html();
+
+                return item;
+            })
+        )
+    );
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: `${$('meta[property="profile:username"]').attr('content')}'s Publications - ResearchGate`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/researchgate/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'researchgate.net': {
+        _name: 'ResearchGate',
+        '.': [
+            {
+                title: 'Publications',
+                docs: 'https://docs.rsshub.app/study.html#researchgate',
+                source: ['/profile/:username'],
+                target: '/researchgate/publications/:username',
+            },
+        ],
+    },
+};

package/lib/v2/researchgate/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/publications/:id', require('./publications'));
+};

package/lib/v2/saraba1st/digest.js

@@ -0,0 +1,79 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const tid = ctx.params.tid;
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const res = await got('https://bbs.saraba1st.com/2b/' + tid + '.html', {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const $ = cheerio.load(res.data);
+    const title = $('head title').text().replace(/-.*/, '');
+    const list = $('#threadlisttableid tbody[id^="normalthread_"] tr');
+    const count = list
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 20)
+        .toArray()
+        .map((each) => {
+            each = $(each);
+            const floor = each.find('th.new a.s.xst').text();
+            const floorUrl = each.find('th.new a.s.xst').attr('href');
+            return {
+                title: `${title}:${floor}`,
+                link: new URL(floorUrl, 'https://bbs.saraba1st.com/2b/').href,
+                author: each.find('td.by cite').text(),
+                pubDate: timezone(parseDate(each.find('td.by em').first().text()), +8),
+            };
+        });
+
+    const resultItems = await Promise.all(
+        count.map((i) =>
+            ctx.cache.tryGet(i.link, async () => {
+                i.description = await fetchContent(i.link);
+                return i;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `Stage1 论坛 - ${title}`,
+        link: `https://bbs.saraba1st.com/2b/${tid}.html`,
+        //        item: await resultItems,
+        item: resultItems,
+    };
+};
+
+async function fetchContent(url) {
+    // Fetch the subpageinof
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const subres = await got(url, {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const subind = cheerio.load(subres.data);
+    const stubS = subind('<div>');
+    subind('#postlist')
+        .find('div[id*="post_"] ')
+        .each(function () {
+            if (subind(this).find('td[id*="postmessage_"]').length > 0) {
+                const section = art(path.join(__dirname, 'templates/digest.art'), {
+                    author: {
+                        link: subind(this).find('.pls.favatar div.authi a').attr('href'),
+                        name: subind(this).find('.pls.favatar div.authi').text(),
+                        postinfo: subind(this).find('div.authi em[id*=authorposton]').text(),
+                    },
+                    msg: subind(this).find('td[id*="postmessage_"]').html(),
+                });
+                stubS.append(section);
+            }
+        });
+
+    return stubS.html();
+}

package/lib/v2/saraba1st/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
+    '/digest/:tid': ['shinemoon'],
     '/thread/:tid': ['zengxs'],
 };

package/lib/v2/saraba1st/radar.js

@@ -3,12 +3,21 @@ module.exports = {
         _name: 'Saraba1st',
         bbs: [
             {
-                title: '帖子',
+                title: '论坛',
                 docs: 'https://docs.rsshub.app/bbs.html#saraba1st',
                 source: '/2b/:id',
                 target: (params) => {
-                    const id = params.id.includes('thread') ? params.id.split('-')[1] : '';
-                    return id ? `/saraba1st/thread/${id}` : '';
+                    let id = params.id;
+                    // For Digest
+                    if (id.match('forum') !== null) {
+                        id = id.substring(0, id.length - 5);
+                        return `/saraba1st/digest/${id}`;
+                    }
+                    // For Thread
+                    else if (id.match('thread') !== null) {
+                        id = params.id.includes('thread') ? params.id.split('-')[1] : '';
+                        return id ? `/saraba1st/thread/${id}` : '';
+                    }
                 },
             },
         ],

package/lib/v2/saraba1st/router.js

@@ -1,3 +1,4 @@
 module.exports = (router) => {
     router.get('/thread/:tid', require('./thread'));
+    router.get('/digest/:tid', require('./digest'));
 };

package/lib/v2/saraba1st/templates/digest.art

@@ -0,0 +1,5 @@
+<div class="quoted" style="background:rgba(220,220,220,0.3);">
+    <a style="text-decoration: none;" href="https://bbs.saraba1st.com/2b/{{ author.link }}">{{ author.name }}</a>
+    <span style="text-decoration: none;color:#aaa;">{{ author.postinfo }}</span>
+</div>
+<div class="content" style="margin-bottom:20px!important;">{{@ msg }}</div>

package/lib/v2/sctv/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/programme/:id?': ['nczitzk'],
+    '/programme/:id?/:limit?/:isFull?': ['nczitzk'],
 };

package/lib/v2/sctv/programme.js

@@ -6,11 +6,13 @@ const path = require('path');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id ?? '1';
+    const limit = ctx.params.limit ? parseInt(ctx.params.limit) : 15;
+    const isFull = /t|y/i.test(ctx.params.isFull ?? 'true');
 
     const rootUrl = 'https://www.sctv.com';
     const apiRootUrl = 'https://kscgc.sctv-tf.com';
-    const apiUrl = `${apiRootUrl}//sctv/lookback/${id}/index.json`;
-    const listUrl = `${apiRootUrl}//sctv/lookback/index/lookbackList.json`;
+    const apiUrl = `${apiRootUrl}/sctv/lookback/${id}/date.json`;
+    const listUrl = `${apiRootUrl}/sctv/lookback/index/lookbackList.json`;
     const currentUrl = `${rootUrl}/column/detail?programmeIndex=/sctv/lookback/${id}/index.json`;
 
     let response = await got({
@@ -18,23 +20,43 @@ module.exports = async (ctx) => {
         url: apiUrl,
     });
 
-    const programmeUrl = `${apiRootUrl}/${response.data.data.programmeUrl}`;
+    let items = [];
 
-    response = await got({
-        method: 'get',
-        url: programmeUrl,
-    });
-
-    const items = response.data.data.programmeList.map((item) => ({
-        title: item.programmeTitle,
-        link: item.programmeUrl,
-        pubDate: timezone(parseDate(item.pubTime), +8),
-        description: art(path.join(__dirname, 'templates/description.art'), {
-            cover: item.programmeImage,
-            video: item.programmeUrl,
-        }),
+    const array = response.data.data.programmeArray.slice(0, limit).map((list) => ({
+        guid: list.id,
+        link: `${apiRootUrl}${list.programmeListUrl}`,
     }));
 
+    await Promise.all(
+        array.map((list) =>
+            ctx.cache.tryGet(list.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: list.link,
+                });
+
+                const currentItems = detailResponse.data.data.programmeList.map((item) => ({
+                    guid: item.id,
+                    title: item.programmeTitle,
+                    link: item.programmeUrl,
+                    pubDate: timezone(parseDate(item.pubTime), +8),
+                    description: art(path.join(__dirname, 'templates/description.art'), {
+                        cover: item.programmeImage,
+                        video: item.programmeUrl,
+                    }),
+                }));
+
+                let currentFullItems = [];
+
+                if (isFull) {
+                    currentFullItems = currentItems.filter((item) => /（\d{4}\.\d{2}\.\d{2}）/.test(item.title));
+                }
+
+                items = [...items, ...(currentFullItems.length === 0 ? currentItems : currentFullItems)];
+            })
+        )
+    );
+
     response = await got({
         method: 'get',
         url: listUrl,
@@ -52,7 +74,7 @@ module.exports = async (ctx) => {
     ctx.state.data = {
         title: `四川广播电视台 - ${name}`,
         link: currentUrl,
-        item: items,
+        item: items.slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 100),
         image: cover,
     };
 };

package/lib/v2/sctv/router.js

@@ -1,3 +1,3 @@
 module.exports = function (router) {
-    router.get('/programme/:id?', require('./programme'));
+    router.get('/programme/:id?/:limit?/:isFull?', require('./programme'));
 };